Merge pull request #150 from Yif-Yang/test/assert-fractional-degradation-score

test(sleep): assert fractional candidate score

.env.example

@@ -8,6 +8,7 @@ export AZURE_OPENAI_API_VERSION=2024-12-01-preview
 # Authentication: choose one method
 #   Option 1: API Key
 export AZURE_OPENAI_API_KEY=
+export AZURE_OPENAI_AUTH_MODE=api_key
 #   Option 2: Azure CLI (no API key needed, recommended on Azure VMs)
 # export AZURE_OPENAI_AUTH_MODE=azure_cli
 #   Option 3: Managed Identity
@@ -15,20 +16,45 @@ export AZURE_OPENAI_API_KEY=
 # export AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID=your-client-id
 
 # ── OpenAI-compatible endpoints ──────────────────────────────────────
-# Set AUTH_MODE to openai_compatible and reuse AZURE_OPENAI_ENDPOINT / _API_KEY.
+# Path 1: generic research backend. Select openai_compatible explicitly as
-# The plain OpenAI client is used; no Azure auth, no api-version header.
+# model.optimizer_backend and/or model.target_backend.
+# export OPENAI_COMPATIBLE_BASE_URL=https://api.deepseek.com/v1
+# export OPENAI_COMPATIBLE_API_KEY=sk-...
+# export OPENAI_COMPATIBLE_MODEL=deepseek-chat
+# Per-role overrides use OPTIMIZER_OPENAI_COMPATIBLE_* and
+# TARGET_OPENAI_COMPATIBLE_* (BASE_URL, API_KEY, MODEL, TEMPERATURE,
+# MAX_TOKENS, TIMEOUT_SECONDS).
+# For scripts/train.py and scripts/eval_only.py, also set model.optimizer and
+# model.target in YAML (or via --cfg-options). Those role model values are
+# applied after backend initialization and override *_MODEL environment values.
+
+# Path 2: research openai_chat compatibility mode. This reuses the Azure-family
+# variables but creates a plain OpenAI client (no Azure auth or api-version).
 # export AZURE_OPENAI_ENDPOINT=https://api.openai.com/v1
 # export AZURE_OPENAI_API_KEY=sk-...
 # export AZURE_OPENAI_AUTH_MODE=openai_compatible
 
-# ── Anthropic / Claude (for claude_chat backend) ─────────────────────
+# Path 3: SkillOpt-Sleep. `skillopt-sleep run --backend azure_openai` uses the
+# same three AZURE_* variables from path 2. Optional Sleep-only controls:
+# export SKILLOPT_SLEEP_COMPAT_MAX_TOKENS=8192
+# export SKILLOPT_SLEEP_CHAT_EXTRA_BODY='{"provider_option": true}'
+
+# ── Claude Code CLI (for claude_chat backend) ─────────────────────────
+# Install and authenticate the `claude` CLI before use. For a non-default path:
+# export CLAUDE_CLI_BIN=/path/to/claude
+# ANTHROPIC_API_KEY is one authentication option understood by the CLI; SkillOpt
+# does not create a direct Anthropic API client for this backend.
 # export ANTHROPIC_API_KEY=sk-ant-...
 
 # ── Qwen Local Model (for qwen_chat backend) ────────────────────────
 # export QWEN_CHAT_BASE_URL=http://localhost:8000/v1
 # export QWEN_CHAT_MODEL=Qwen/Qwen3.5-4B
+# The train/eval entry points likewise override this model with
+# model.optimizer/model.target for the selected Qwen roles.
 
 # ── MiniMax (for minimax_chat backend) ──────────────────────────────
 # export MINIMAX_BASE_URL=https://api.minimax.io/v1
 # export MINIMAX_API_KEY=...
-# export MINIMAX_MODEL=MiniMax-M2.7
+# When MiniMax is the target, set model.minimax_model in YAML. The current
+# adapter shares one deployment across MiniMax roles; mixed-backend runs cannot
+# independently select a MiniMax optimizer model and a different target model.

.gitignore

@@ -22,6 +22,11 @@ data/*
 outputs/
 logs/
 external/
+# SkillOpt-Sleep runtime state (staging proposals, config, diagnostics, cron logs)
+.skillopt-sleep/
+# SkillOpt-Sleep handoff-backend round data (prompts/answers derived from transcripts)
+.skillopt-sleep-handoff/
+.skillopt-sleep-handoff.night*.done/
 
 /BabyVision/
 /MMRB/
@@ -54,3 +59,8 @@ docs/render_ablation_paper_tables.py
 docs/让*
 .gradio/
 .venv
+
+# Local experiment launchers — contain machine-specific endpoints/identities, never commit
+tests/run_*.sh
+tests/launch_*.py
+*.launch.log

CHANGELOG.md

@@ -0,0 +1,171 @@
+# Changelog
+
+All notable changes to SkillOpt are documented here. This project adheres to
+[Semantic Versioning](https://semver.org/) and the format is based on
+[Keep a Changelog](https://keepachangelog.com/).
+
+## [Unreleased]
+
+### Added
+- **Handoff backend** (`--backend handoff`) for SkillOpt-Sleep — runs the
+  sleep cycle with no model subprocess or API key: the engine writes each
+  pending model call to `PROMPTS.md`/`pending.json` (exit code 3) and the
+  user's own agent session answers into `answers/<id>.md`; re-running the
+  same command resumes statelessly from the answers (typically 3–6 rounds
+  per night). Mined tasks are pinned per night so answering sessions cannot
+  shift the task set. Ships a `/skillopt-sleep-handoff` Claude Code command
+  that automates the loop with fresh-context subagents to protect the
+  held-out gate (thanks @dimitarvdenev, #125).
+- **Generic OpenAI-compatible research backend** for optimizer and target
+  calls, with configurable base URL, API key, model, and timeout (thanks
+  @nankingjing, #115).
+- **OpenAI-compatible SkillOpt-Sleep endpoint support** for providers such as
+  DeepSeek and self-hosted vLLM servers (thanks @Alphaxalchemy, #129; hardened
+  in #138).
+- End-to-end wiring for the documented reflection `--preferences` option
+  (thanks @AKhozya, #131).
+
+### Changed
+- Claude Code's Sleep plugin can now use a `pip`/`uv`-installed
+  `skillopt-sleep` when no repository checkout is present (thanks
+  @ichoosetoaccept, #107).
+- Qwen reasoning-model requests now use `max_completion_tokens` and omit
+  unsupported temperature parameters (thanks @chirag127, #128).
+- Configuration files are read explicitly as UTF-8 (thanks @nankingjing,
+  #124).
+
+### Fixed
+- Preserve fractional rollout hard scores instead of coercing them to binary
+  values (thanks @zixuanguo786-ctrl, #104).
+- Reject duplicate and overlapping IDs while materializing SearchQA manifests
+  (thanks @zixuanguo786-ctrl, #105).
+- Make JSON-array extraction robust to unmatched braces and keep malformed
+  scans linear-time (thanks @zixuanguo786-ctrl, #103; follow-up #136).
+- Package Markdown prompt assets in wheels and tolerate Windows temporary-file
+  cleanup failures (thanks @nankingjing, #135; follow-up #137).
+- Exclude sub-agent transcripts and plugin-generated sessions from Sleep task
+  mining (thanks @codeL1985, #99).
+- Normalize validation-gate density against the proposed edits and handle
+  zero-edit candidates safely (thanks @SparshGarg999, #102).
+- Route optimizer-role MiniMax calls through the MiniMax backend (thanks
+  @jcforever1, #116).
+- Surface Claude CLI spawn failures instead of silently turning them into zero
+  scores (thanks @Phoenix0531-sudo, #126).
+- Improve Claude CLI behavior on Windows, including `.cmd` resolution and
+  long-prompt handling (thanks @codeL1985, #98).
+- Preserve the scheduler's established annealing contract while expanding its
+  endpoint and sequence coverage (thanks @nankingjing, #123; follow-up #133).
+
+### Security
+- Prevent managed-identity credentials from being sent to non-Azure or
+  non-HTTPS endpoints, and isolate compatible-provider request extensions
+  from native Azure mode in SkillOpt-Sleep (#138, following
+  @Alphaxalchemy's #129).
+
+### Tests
+- Strengthen SkillOpt-Sleep verifier-discipline assertions, including recorded
+  scores and gate actions (thanks @Tanmay9223, #96).
+- Add focused coverage for the validation-gate decision core and edit-budget
+  schedulers (thanks @nankingjing, #122, #123).
+
+### Acknowledgements 🙏
+Thank you to the contributors behind this unreleased work:
+@AKhozya, @Alphaxalchemy, @Phoenix0531-sudo, @SparshGarg999,
+@Tanmay9223, @chirag127, @codeL1985, @dimitarvdenev,
+@ichoosetoaccept, @jcforever1, @nankingjing, and
+@zixuanguo786-ctrl.
+
+## [0.2.0] — 2026-07-02
+
+The headline of this release is **SkillOpt-Sleep**: a nightly offline
+self-evolution engine that harvests a coding agent's real session
+transcripts, mines recurring tasks, replays them offline, and consolidates
+short-term experience into long-term memory and skills — all behind the same
+held-out validation gate that keeps SkillOpt training honest. It ships as a
+decoupled top-level package (`skillopt_sleep/`, zero dependency on the
+research code) and as the new `skillopt-sleep` CLI.
+
+### Added
+- **SkillOpt-Sleep engine** — nightly offline self-evolution cycle
+  (harvest → mine → replay → consolidate) behind a validation gate, exposed
+  as the `skillopt-sleep` console script and `python -m skillopt_sleep`.
+  - Multi-objective reward (accuracy / tokens / latency) with user preferences.
+  - Multi-rollout contrastive reflection under a token/time budget.
+  - Experience replay + controllable dream rollouts (opt-in).
+  - Slow-update long-term memory field (runs even with the gate off).
+  - 3-way train/val/test split with `gate_mode on|off`.
+  - Verifier-discipline validation gate, with a stress-test suite
+    (thanks @Tanmay9223, #87).
+- **Cross-tool backends & plugin shells** for Claude Code, Codex, Copilot,
+  Devin, and OpenClaw:
+  - Codex Desktop transcript harvesting, skill-first Codex integration, and a
+    reviewed task-file flow (thanks @Kirchberg, #48, #49, #60).
+  - GitHub Copilot backend (`CopilotCliBackend`) + research-engine MCP plugin
+    (thanks @Dongbumlee, #50).
+  - Devin plugin: MCP server + ATIF-v1.7 harvest (thanks @xerxes-y, #88).
+  - OpenClaw shell for SkillOpt-Sleep (thanks @Elzlxx, #59).
+- **SearchQA** split materialization helper and fail-fast on systemic rollout
+  failures, with a `searchqa` install extra (thanks @summerview1997,
+  #63, #64, #65).
+- WebUI environment loading and backend preflight (thanks @summerview1997, #63).
+
+### Changed
+- Decoupled the Sleep engine into a standalone top-level `skillopt_sleep/`
+  package with zero dependency on the research code.
+- Made `EnvAdapter.reflect` a shared default so reflect kwargs are no longer
+  dropped (thanks @imshunsuke, #44).
+- English-only pass across the engine, plugins, and docs.
+
+### Fixed
+- Windows robustness for the Claude/Codex backends, plus a hardened JSON
+  fallback path (thanks @Yif-Yang, #79).
+- Reject prose pseudo-JSON wrapped in single quotes/backticks (#82).
+- Surface Codex auth/model/version failures instead of silently scoring 0
+  (thanks @dmmdea, #92).
+- Redact secrets before persisting cycle diagnostics.
+- Configure the `qwen_chat`/`minimax` backends so local LLM endpoints work
+  (thanks @imrehg, #85).
+- Forward the Qwen target timeout and gate `enable_thinking` for vLLM targets
+  (thanks @mvanhorn, #40).
+- Make `--bare` conditional on `ANTHROPIC_API_KEY` (#68), add a
+  `SKILLOPT_SLEEP_PYTHON` override with a lookback-hours first-run fallback
+  (#74), and fix ALFWorld gamefile paths relative to `ALFWORLD_DATA`.
+
+### Packaging
+- Bump `skillopt`, `skillopt.__version__`, and `skillopt_sleep.__version__`
+  to `0.2.0`.
+- Restore `skillopt_webui` to the built wheel (it was dropped when the
+  `packages.find` include list was made explicit).
+- Add the `searchqa` extra and include `json_repair` in the `claude`, `qwen`,
+  and `all` extras.
+
+### Acknowledgements 🙏
+v0.2.0 landed thanks to our community contributors — thank you!
+
+- @Kirchberg — Codex Desktop harvesting, skill-first Codex integration,
+  reviewed task-file flow (#48, #49, #60)
+- @Dongbumlee — GitHub Copilot backend + research-engine MCP plugin (#50)
+- @summerview1997 — SearchQA materialization, rollout fail-fast, WebUI
+  preflight (#63, #64, #65)
+- @xerxes-y — Devin plugin: MCP server + ATIF-v1.7 harvest (#88)
+- @Elzlxx — OpenClaw shell for SkillOpt-Sleep (#59)
+- @imshunsuke — shared `EnvAdapter.reflect` default + docs fixes (#43, #44)
+- @mvanhorn — Qwen timeout forwarding + `enable_thinking` gating (#40)
+- @dmmdea — surface Codex auth/model/version failures (#92)
+- @Tanmay9223 — verifier-discipline stress test (#87)
+- @imrehg — `configure_qwen_chat` for local LLM endpoints (#85)
+- @samuelgoofus-boop — community contributions
+
+Special thanks to @Yif-Yang for driving the SkillOpt-Sleep engine.
+
+**Full changelog:** https://github.com/microsoft/SkillOpt/compare/v0.1.0...v0.2.0
+
+## [0.1.0] — 2026-06-02
+
+Initial public release: the full training loop (rollout → reflect →
+aggregate → select → update → evaluate), multi-backend support
+(OpenAI / Azure / Claude / Qwen / MiniMax), six built-in benchmarks, and the
+WebUI dashboard.
+
+[0.2.0]: https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0
+[0.1.0]: https://github.com/microsoft/SkillOpt/releases/tag/v0.1.0

CONTRIBUTING.md

@@ -7,7 +7,7 @@ Thank you for your interest in contributing! SkillOpt welcomes contributions of
 ```bash
 git clone https://github.com/microsoft/SkillOpt.git
 cd SkillOpt
-pip install -e ".[dev]"
+python -m pip install -e ".[dev,docs]"
 ```
 
 ## How to Contribute
@@ -16,23 +16,28 @@ pip install -e ".[dev]"
 Open a GitHub issue with reproduction steps, expected/actual behavior, and your config file (remove API keys).
 
 ### 🔧 Add a Benchmark
-See the [guide](docs/guide/new-benchmark.md) and use the scaffold at `skillopt/envs/_template/`.
+See the [guide](docs/guide/new-benchmark.md) and use the scaffold at
+`skillopt/envs/_template/`. Register the adapter lazily in both
+`scripts/train.py` and `scripts/eval_only.py`, and add focused tests.
 
 ### 🤖 Add a Model Backend
-See the [guide](docs/guide/new-backend.md).
+First check whether the built-in `openai_compatible` backend covers the
+provider. Otherwise follow the function-based backend contract in the
+[backend guide](docs/guide/new-backend.md), including routing, configuration,
+token accounting, and no-network tests.
 
 ### 📝 Improve Documentation
 ```bash
-pip install -e ".[docs]"
+python -m mkdocs serve   # Preview at http://localhost:8000
-mkdocs serve   # Preview at http://localhost:8000
 ```
 
 ## Pull Request Process
 
 1. Fork the repo and create a feature branch
-2. Make changes and test with an existing benchmark
+2. Make changes and run focused tests plus `python -m pytest -q`
 3. Submit a PR with a clear description
-4. Ensure CI passes
+4. For documentation changes, run `python -m mkdocs build --strict`
+5. Ensure CI passes
 
 ## Code Style
 - Follow existing patterns in the codebase

README.md

@@ -2,7 +2,22 @@
 
 *Train agent skills like you train neural networks — with epochs, (mini-)batchsize, learning rates, and validation gates — but without touching model weights.*
 
-[![Project Page](https://img.shields.io/badge/Project%20Page-SkillOpt-8dbb3c)](https://microsoft.github.io/SkillOpt/) [![Paper](https://img.shields.io/badge/Paper-arXiv-b31b1b)](https://arxiv.org/abs/2605.23904) [![Project Video](https://img.shields.io/badge/Project%20Video-Watch%20Demo-ff0000)](https://youtu.be/JUBMDTCiM0M) [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-blue.svg)](https://www.python.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![Project Page](https://img.shields.io/badge/Project%20Page-SkillOpt-8dbb3c)](https://microsoft.github.io/SkillOpt/) [![Paper](https://img.shields.io/badge/Paper-arXiv-b31b1b)](https://arxiv.org/abs/2605.23904) [![Project Video](https://img.shields.io/badge/Project%20Video-Watch%20Demo-ff0000)](https://youtu.be/JUBMDTCiM0M) [![PyPI](https://img.shields.io/badge/PyPI-skillopt-green.svg)](https://pypi.org/project/skillopt/) [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-blue.svg)](https://www.python.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+<p align="center">
+  <a href="https://trendshift.io/repositories/38498?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-38498" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/38498/daily?language=Python" alt="microsoft%2FSkillOpt | Trendshift" width="250" height="55"/></a>
+  <a href="https://trendshift.io/repositories/38498?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-38498" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/38498/weekly?language=Python" alt="microsoft%2FSkillOpt | Trendshift" width="250" height="55"/></a>
+</p>
+
+> 📖 **For installation, data preparation, training/eval commands, configuration, and framework internals, start with the versioned [SkillOpt documentation](https://github.com/microsoft/SkillOpt/blob/main/docs/index.md). A concise rendered overview is available in the [Documentation & Reproduction Guide](https://microsoft.github.io/SkillOpt/docs/guideline.html), and longer-form engineering analysis appears on the [Technical Blog](https://microsoft.github.io/SkillOpt/blog/). We also maintain a [Changelog](CHANGELOG.md) for released and unreleased changes.**
+
+---
+
+## News 🔥🔥🔥
+- **[2026-07-02]** 🚀 **SkillOpt [v0.2.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) is out on [PyPI](https://pypi.org/project/skillopt/)!** Headline feature: **SkillOpt-Sleep**, a nightly offline self-evolution engine (harvest → mine → replay → consolidate behind a held-out validation gate), now shipped as the `skillopt-sleep` CLI. It also includes experimental multi-objective, replay, and dream-rollout controls; the main CLI keeps conservative defaults and does not expose every experiment-harness control as a flag. The release source adds integration shells for **Claude Code, Codex, Copilot, and Devin**, plus an **OpenClaw reference adaptation**; these plugin/MCP files live in the repository rather than the PyPI wheel. It also adds SearchQA split materialization, Windows robustness, and hardened JSON parsing. See the [release notes](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) for full release details and contributor acknowledgements.
+- **[2026-06-15]** 😴 **SkillOpt-Sleep (preview)** — a nightly offline self-evolution companion for local coding agents (Claude Code / Codex / Copilot): review past sessions, replay recurring tasks, and consolidate validated skills behind a held-out gate. See **[`docs/sleep/README.md`](docs/sleep/README.md)** for what it is, how to use it, and results.
+- **[2026-06-03]** 🎉 **[gbrain](https://github.com/garrytan/gbrain), [gbrain-evals](https://github.com/garrytan/gbrain-evals/blob/main/docs/benchmarks/2026-06-03-skillopt.md), and [darwin-skill](https://github.com/alchaincyf/darwin-skill) have all integrated SkillOpt.**
+- **[2026-06-02]** 🎉 **SkillOpt [v0.1.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.1.0) is now available on [PyPI](https://pypi.org/project/skillopt/)!** Install with `pip install skillopt`. This initial release includes the full training loop (rollout → reflect → aggregate → select → update → evaluate), multi-backend support (OpenAI / Azure / Claude / Qwen / MiniMax), six built-in benchmarks, and WebUI dashboard.
 
 ---
 
@@ -16,9 +31,9 @@ which reliably improves over its starting point under feedback.
 **SkillOpt treats the skill document as the trainable state of a frozen
 agent**, and trains it with the discipline that makes weight-space
 optimization reproducible. A separate optimizer model turns scored rollouts
-into bounded add / delete / replace edits on a single skill document; a
+into bounded add / delete / replace edits on a single skill document; in the
-candidate edit is accepted only when it strictly improves a held-out
+default paper-style path, a candidate edit is accepted only when it strictly
-validation score. A textual learning-rate budget, a rejected-edit buffer,
+improves a held-out validation score. A textual learning-rate budget, a rejected-edit buffer,
 and an epoch-wise slow / meta update make skill training stable while
 adding **zero inference-time model calls** at deployment.
 
@@ -44,312 +59,14 @@ https://github.com/user-attachments/assets/eb12d3bc-371c-467f-904d-91b61f339ed7
 
 ---
 
-## Install
-
-### Requirements
-
-- Python 3.10+
-
-```bash
-git clone https://github.com/microsoft/SkillOpt.git
-cd SkillOpt
-pip install -e .
-
-# For the ALFWorld benchmark (optional):
-pip install -e ".[alfworld]"
-alfworld-download
-```
-
-### Configure API Credentials
-
-```bash
-cp .env.example .env
-# Edit .env with your API credentials, then:
-source .env
-```
-
-#### Azure OpenAI *(recommended)*
-
-```bash
-export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
-# Option 1: API key auth
-export AZURE_OPENAI_API_KEY="your-key"
-# Option 2: Azure CLI auth (no API key needed)
-export AZURE_OPENAI_AUTH_MODE="azure_cli"
-```
-
-> **Note:** `AZURE_OPENAI_ENDPOINT` is required for all three modes (`api_key`, `azure_cli`, `openai_compatible`). Without it, all LLM calls will fail.
-
-#### OpenAI-compatible endpoints
-
-```bash
-export AZURE_OPENAI_ENDPOINT="https://api.openai.com/v1"
-export AZURE_OPENAI_API_KEY="sk-..."
-export AZURE_OPENAI_AUTH_MODE="openai_compatible"
-```
-
-This routes all calls through the plain OpenAI Python client (no Azure auth, no `api-version` header).
-
-> **Note:** SkillOpt reuses the `AZURE_OPENAI_*` env var names even in this mode — there is no separate `OPENAI_API_KEY` knob.
-
-#### Anthropic Claude
-
-```bash
-export ANTHROPIC_API_KEY="sk-ant-..."
-```
-
-#### Qwen *(local vLLM)*
-
-```bash
-export QWEN_CHAT_BASE_URL="http://localhost:8000/v1"
-export QWEN_CHAT_MODEL="Qwen/Qwen3.5-4B"
-```
-
-`qwen_chat` can also be used as the optimizer backend. When optimizer and
-target should point to different local vLLM services, use the role-specific
-settings:
-
-```bash
-python scripts/train.py \
-    --config configs/searchqa/default.yaml \
-    --optimizer_backend qwen_chat \
-    --target_backend qwen_chat \
-    --optimizer_model Qwen/Qwen3.5-4B \
-    --target_model Qwen/Qwen3.5-4B \
-    --optimizer_qwen_chat_base_url http://localhost:8001/v1 \
-    --target_qwen_chat_base_url http://localhost:8000/v1
-```
-
-#### MiniMax
-
-```bash
-export MINIMAX_BASE_URL="https://api.minimax.io/v1"
-export MINIMAX_API_KEY="..."
-export MINIMAX_MODEL="MiniMax-M2.7"
-```
-
----
-
-## Quick Start
-
-### Training
-
-```bash
-# Minimal example — train on SearchQA:
-python scripts/train.py \
-    --config configs/searchqa/default.yaml \
-    --split_dir /path/to/your/searchqa_split \
-    --azure_openai_endpoint https://your-resource.openai.azure.com/ \
-    --optimizer_model gpt-5.5 \
-    --target_model gpt-5.5
-
-# Train on LiveMathematicianBench:
-python scripts/train.py \
-    --config configs/livemathematicianbench/default.yaml \
-    --split_dir /path/to/your/livemath_split \
-    --azure_openai_endpoint https://your-resource.openai.azure.com/ \
-    --optimizer_model gpt-5.5 \
-    --target_model gpt-5.5
-
-# Train on ALFWorld:
-python scripts/train.py \
-    --config configs/alfworld/default.yaml \
-    --split_dir data/alfworld_path_split \
-    --azure_openai_endpoint https://your-resource.openai.azure.com/ \
-    --optimizer_model gpt-5.5 \
-    --target_model gpt-5.5
-```
-
-Key CLI arguments:
-
-| Argument | Description | Example |
-|---|---|---|
-| `--config` | Benchmark config YAML | `configs/searchqa/default.yaml` |
-| `--split_dir` | Path to data split directory | `/path/to/split` |
-| `--azure_openai_endpoint` | Azure OpenAI endpoint URL | `https://your-resource.openai.azure.com/` |
-| `--optimizer_model` | Optimizer model deployment name | `gpt-5.5` |
-| `--target_model` | Target model deployment name | `gpt-5.5` |
-| `--num_epochs` | Number of training epochs | `4` |
-| `--batch_size` | Batch size per step | `40` |
-| `--workers` | Parallel rollout workers | `8` |
-| `--out_root` | Output directory | `outputs/my_run` |
-
-### Eval Only
-
-Evaluate a trained skill on specific data splits without training:
-
-```bash
-# Evaluate the packaged GPT-5.5 SearchQA skill on the test split:
-python scripts/eval_only.py \
-  --config configs/searchqa/default.yaml \
-  --skill ckpt/searchqa/gpt5.5_skill.md \
-  --split valid_unseen \
-  --split_dir /path/to/searchqa_split \
-  --azure_openai_endpoint https://your-resource.openai.azure.com/
-
-# Evaluate on all splits (train + val + test):
-python scripts/eval_only.py \
-  --config configs/searchqa/default.yaml \
-  --skill ckpt/searchqa/gpt5.5_skill.md \
-  --split all \
-  --split_dir /path/to/searchqa_split \
-  --azure_openai_endpoint https://your-resource.openai.azure.com/
-```
-
-To evaluate a skill produced by your own training run, replace `--skill` with that run's best-skill path, for example `outputs/my_run/best_skill.md`.
-
-| Split | Description |
-|---|---|
-| `valid_unseen` | Test set |
-| `valid_seen` | Validation set |
-| `train` | Training set |
-| `all` | All splits combined (default) |
-
-### Output Structure
-
-Each training run writes to a structured output directory:
-
-```
-outputs/<run_name>/
-├── config.json              # Flattened runtime config
-├── history.json             # Per-step training history
-├── runtime_state.json       # Resume checkpoint
-├── best_skill.md            # Best validated skill document
-├── skills/skill_vXXXX.md   # Skill snapshot per step
-├── steps/step_XXXX/        # Per-step artifacts (patches, evals)
-├── slow_update/epoch_XX/   # Slow update logs
-└── meta_skill/epoch_XX/    # Meta skill logs
-```
-
-Re-running the same command auto-resumes from the last completed step.
-
-### Pretrained Skill Artifacts
-
-We provide a subset of the paper's main Table 1 GPT-5.5 optimized skills in
-[`ckpt/`](ckpt/) as reference artifacts. Use them with `scripts/eval_only.py`
-to evaluate the provided skills on a matching data split without re-running
-training. See [`ckpt/README.md`](ckpt/README.md) for the full per-benchmark
-command. This is the first artifact batch; we plan to continue uploading
-the remaining optimized skills and benchmark split manifests as they are
-cleaned and verified.
-
----
-
-## Data Preparation
-
-### Directory layout
-
-SkillOpt expects data in a **split directory** with `train/`, `val/`, `test/` subdirectories, each containing a JSON file (e.g., `items.json`):
-
-```
-data/my_split/
-├── train/items.json
-├── val/items.json
-└── test/items.json
-```
-
-Each JSON file is an array of task items. The required fields depend on the benchmark. For example, SearchQA items look like:
-
-```json
-[
-  {
-    "id": "unique_item_id",
-    "question": "Who wrote the novel ...",
-    "context": "[DOC] relevant passage text ...",
-    "answers": ["expected answer"]
-  }
-]
-```
-
-See `skillopt/envs/<benchmark>/dataloader.py` for the exact format each benchmark expects.
-
-> **Note:** Most benchmark datasets are not included in this repository. Prepare your own data following the format above. The exact SearchQA split used in the paper is provided at [`data/searchqa_id_split/`](data/searchqa_id_split) (400 train / 200 val / 1400 test). We are preparing the remaining benchmark split manifests for upload.
-
-### Supported Benchmarks
-
-| Benchmark | Type | Config |
-|---|---|---|
-| SearchQA | QA | `configs/searchqa/default.yaml` |
-| ALFWorld | Embodied agent | `configs/alfworld/default.yaml` |
-| DocVQA | Document QA | `configs/docvqa/default.yaml` |
-| LiveMathematicianBench | Math | `configs/livemathematicianbench/default.yaml` |
-| SpreadsheetBench | Code generation | `configs/spreadsheetbench/default.yaml` |
-| OfficeQA | Tool-augmented QA | `configs/officeqa/default.yaml` |
-
----
-
-## Configuration
-
-### Default settings and paper-reproduction knobs
-
-`configs/_base_/default.yaml` is the single source of truth for SkillOpt's
-runtime knobs. Out of the box, every included benchmark config inherits
-from it and keeps the paper protocol visible: 4 epochs, rollout batch 40,
-reflection minibatch 8, textual learning rate 4 with cosine decay, strict
-hard validation gating, and slow-update + meta-skill enabled. One detail to
-watch is slow-update acceptance: the current `main` default is the newer
-post-submission force-accept mode, while the paper protocol and the
-paper-aligned skills under `ckpt/` use the gated semantics described in
-paper Section 3.6.
-
-### Slow-update acceptance mode
-
-The epoch-boundary slow / meta update can be applied two ways, controlled
-by `optimizer.slow_update_gate_with_selection`:
-
-```yaml
-optimizer:
-  slow_update_gate_with_selection: false   # current main default
-```
-
-- **`false`** *(current `main` default)*: force-accept. The
-  slow-update guidance is injected into both `current_skill` and
-  `best_skill` unconditionally at the epoch boundary. This is the newer
-  post-submission behavior on `main`.
-- **`true`** *(paper / ckpt-skill reproduction)*: gated, matching paper
-  Section 3.6 verbatim. The slow-update candidate is evaluated on the
-  selection split and accepted only if it passes the same validation gate
-  as a step-level edit. Use this setting when re-running optimization to
-  match the paper protocol and the provenance of the provided `ckpt/` skills.
-
-The trainer prints which mode is active at startup
-(`[slow update] acceptance=...`). See issue #22 for the discussion that
-led to the flag.
-
-### Gate metric (`hard` / `soft` / `mixed`)
-
-The validation gate compares candidate vs. current skills on the selection
-split using `gate_metric`:
-
-- **`hard`** *(default, paper)*: exact-match accuracy, strictly greater
-  than the current score is required.
-- **`soft`**: per-item soft / partial-credit score. Useful when the
-  selection split is small (e.g. ≤10 items) and the reward is continuous,
-  where the discrete hard gate often rejects every candidate.
-- **`mixed`**: weighted average, `(1 - w) * hard + w * soft`, with `w`
-  set by `gate_mixed_weight` (default `0.5`).
-
-Default is `hard`. Use the optional feature config below to switch.
-
-### Optional feature configs
-
-These are **not** default SkillOpt settings — they are optional feature configs
-contributed by users for specific scenarios. The paper-reported numbers
-were obtained with the default settings, not these.
-
-- **[`configs/features/soft_gate.yaml`](configs/features/soft_gate.yaml)**
-  *(PR #25, contributed by [@lvbaocheng](https://github.com/lvbaocheng))* —
-  switches `gate_metric` to `soft` (or `mixed`). See the comment at the
-  top of the file for when to use and when not to.
-
----
-
 ## Extensibility & WebUI
 
 ### Adding a new backend
 
 A backend = a chat / exec target (e.g. `openai_chat`, `claude_chat`,
-`qwen_chat`, `minimax_chat`, `codex_exec`, `claude_code_exec`). See
+`qwen_chat`, `minimax_chat`, `openai_compatible`, `codex_exec`,
+`claude_code_exec`). If a provider implements the OpenAI Chat Completions
+protocol, try the built-in `openai_compatible` backend before adding code. See
 [`docs/guide/new-backend.md`](docs/guide/new-backend.md) for the full
 contract; in short you add a `skillopt/model/<name>_backend.py` module,
 register it in `skillopt/model/common.py` + `backend_config.py`, and wire
@@ -358,8 +75,9 @@ and `minimax_backend.py` are good templates.
 
 ### Adding a new benchmark
 
-A benchmark = a `skillopt/envs/<name>/` package with a `dataloader.py`, a
+A benchmark = a `skillopt/envs/<name>/` package with an adapter, a data loader,
-`rollout.py`, and an `initial.md` seed skill. See
+a scored rollout helper, a YAML config, and optionally an initial seed skill.
+See
 [`docs/guide/new-benchmark.md`](docs/guide/new-benchmark.md) for the full
 contract; the simplest reference is `skillopt/envs/searchqa/`.
 
@@ -378,18 +96,18 @@ python -m skillopt_webui.app
 | `--host` | `0.0.0.0` | Bind address |
 | `--share` | off | Create a public Gradio share link |
 
+The default host listens on every network interface. Use
+`--host 127.0.0.1` for local-only access.
+
 ---
 
 ## Citation
 
 ```bibtex
-@misc{yang2026skilloptexecutivestrategyselfevolving,
+@article{yang2026skillopt,
-      title={SkillOpt: Executive Strategy for Self-Evolving Agent Skills}, 
+  title={Skillopt: Executive strategy for self-evolving agent skills},
-      author={Yifan Yang and Ziyang Gong and Weiquan Huang and Qihao Yang and Ziwei Zhou and Zisu Huang and Yan Li and Xuemei Gao and Qi Dai and Bei Liu and Kai Qiu and Yuqing Yang and Dongdong Chen and Xue Yang and Chong Luo},
+  author={Yang, Yifan and Gong, Ziyang and Huang, Weiquan and Yang, Qihao and Zhou, Ziwei and Huang, Zisu and Li, Yan and Gao, Xuemei and Dai, Qi and Liu, Bei and others},
-      year={2026},
+  journal={arXiv preprint arXiv:2605.23904},
-      eprint={2605.23904},
+  year={2026}
-      archivePrefix={arXiv},
-      primaryClass={cs.AI},
-      url={https://arxiv.org/abs/2605.23904}
 }
 ```

blog/index.html

@@ -0,0 +1,149 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>SkillOpt Technical Blog</title>
+  <meta name="description" content="Technical notes from the SkillOpt team on skill optimization, evaluation, safety, and agent learning systems.">
+  <link rel="canonical" href="https://microsoft.github.io/SkillOpt/blog/">
+  <meta property="og:type" content="website">
+  <meta property="og:site_name" content="SkillOpt">
+  <meta property="og:title" content="SkillOpt Technical Blog">
+  <meta property="og:description" content="Technical notes on skill optimization, evaluation, safety, and agent learning systems.">
+  <meta property="og:url" content="https://microsoft.github.io/SkillOpt/blog/">
+  <meta property="og:image" content="https://microsoft.github.io/SkillOpt/skillopt-assets/teaser-1.png">
+  <meta name="twitter:card" content="summary_large_image">
+  <meta name="twitter:title" content="SkillOpt Technical Blog">
+  <meta name="twitter:description" content="Technical notes on skill optimization, evaluation, safety, and agent learning systems.">
+  <meta name="twitter:image" content="https://microsoft.github.io/SkillOpt/skillopt-assets/teaser-1.png">
+  <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Crect width='100' height='100' rx='16' fill='%23245fc7'/%3E%3Ctext x='50' y='68' text-anchor='middle' font-size='58' font-family='Arial' font-weight='700' fill='white'%3ES%3C/text%3E%3C/svg%3E">
+  <style>
+    :root {
+      --ink: #172033;
+      --muted: #596579;
+      --line: #d9deea;
+      --paper: #fff;
+      --wash: #f7f9fc;
+      --blue: #245fc7;
+      --blue-soft: #eaf1ff;
+      --shadow: 0 14px 44px rgba(23, 32, 51, .08);
+    }
+    * { box-sizing: border-box; }
+    body {
+      margin: 0;
+      color: var(--ink);
+      background: var(--wash);
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
+      line-height: 1.6;
+    }
+    a { color: var(--blue); }
+    .skip-link {
+      position: fixed;
+      left: 16px;
+      top: -80px;
+      z-index: 100;
+      padding: 10px 14px;
+      border-radius: 8px;
+      color: #fff;
+      background: var(--blue);
+    }
+    .skip-link:focus { top: 12px; }
+    header {
+      border-bottom: 1px solid var(--line);
+      background: rgba(255, 255, 255, .95);
+    }
+    .header-inner {
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+      gap: 24px;
+      max-width: 1040px;
+      margin: 0 auto;
+      padding: 16px 24px;
+    }
+    .brand {
+      color: var(--ink);
+      font-weight: 750;
+      text-decoration: none;
+    }
+    nav { display: flex; flex-wrap: wrap; gap: 16px; font-size: 14px; }
+    nav a { color: var(--muted); text-decoration: none; }
+    nav a:hover, nav a:focus-visible { color: var(--blue); text-decoration: underline; }
+    main { max-width: 1040px; margin: 0 auto; padding: 72px 24px 96px; }
+    .eyebrow {
+      color: var(--blue);
+      font-size: 13px;
+      font-weight: 750;
+      letter-spacing: .08em;
+      text-transform: uppercase;
+    }
+    h1 { margin: 10px 0 16px; font-size: clamp(40px, 7vw, 68px); line-height: 1.02; }
+    .intro { max-width: 760px; color: var(--muted); font-size: 20px; }
+    .post-list { margin-top: 52px; }
+    .post-card {
+      display: grid;
+      grid-template-columns: minmax(0, 1fr) auto;
+      gap: 30px;
+      padding: 34px;
+      border: 1px solid var(--line);
+      border-radius: 16px;
+      background: var(--paper);
+      box-shadow: var(--shadow);
+    }
+    .post-card h2 { margin: 8px 0 12px; font-size: clamp(25px, 4vw, 36px); line-height: 1.15; }
+    .post-card h2 a { color: var(--ink); text-decoration: none; }
+    .post-card h2 a:hover, .post-card h2 a:focus-visible { color: var(--blue); text-decoration: underline; }
+    .meta { color: var(--muted); font-size: 14px; }
+    .summary { max-width: 720px; margin: 16px 0 0; color: #344054; }
+    .read-link { align-self: center; white-space: nowrap; font-weight: 700; }
+    footer {
+      max-width: 1040px;
+      margin: 0 auto;
+      padding: 26px 24px 42px;
+      border-top: 1px solid var(--line);
+      color: var(--muted);
+      font-size: 14px;
+    }
+    @media (max-width: 720px) {
+      .header-inner { align-items: flex-start; flex-direction: column; }
+      main { padding-top: 48px; }
+      .post-card { grid-template-columns: 1fr; padding: 24px; }
+      .read-link { justify-self: start; }
+    }
+  </style>
+</head>
+<body>
+  <a class="skip-link" href="#main-content">Skip to content</a>
+  <header>
+    <div class="header-inner">
+      <a class="brand" href="../">SkillOpt</a>
+      <nav aria-label="Site navigation">
+        <a href="../">Project</a>
+        <a href="../docs/guideline.html">Documentation</a>
+        <a href="https://arxiv.org/abs/2605.23904">Paper</a>
+        <a href="https://github.com/microsoft/SkillOpt">Code</a>
+      </nav>
+    </div>
+  </header>
+
+  <main id="main-content">
+    <span class="eyebrow">Microsoft Research · SkillOpt</span>
+    <h1>Technical Blog</h1>
+    <p class="intro">Scoped experiments and engineering notes on optimizing agent skills, evaluating text-space updates, and deploying self-improving systems responsibly.</p>
+
+    <section class="post-list" aria-labelledby="latest-posts">
+      <h2 id="latest-posts">Latest posts</h2>
+      <article class="post-card">
+        <div>
+          <div class="meta"><time datetime="2026-07-14">July 14, 2026</time> · Ziwei Zhou, Ziyang Gong, and Yifan Yang</div>
+          <h2><a href="gating-reflection-safe-updates/">Expanded SkillOpt Ablations, Skill-Aware Reflection, and SkillOpt-Sleep</a></h2>
+          <p class="summary">A three-part report with expanded SkillOpt ablations, a skill-aware reflection design with memory consolidation, and a controlled study of the SkillOpt-Sleep plugin.</p>
+        </div>
+        <a class="read-link" href="gating-reflection-safe-updates/" aria-label="Read Expanded SkillOpt Ablations, Skill-Aware Reflection, and SkillOpt-Sleep">Read article →</a>
+      </article>
+    </section>
+  </main>
+
+  <footer>SkillOpt Technical Blog · <a href="https://github.com/microsoft/SkillOpt">github.com/microsoft/SkillOpt</a></footer>
+</body>
+</html>

ckpt/README.md

@@ -10,9 +10,10 @@ provided skills on a given split without re-running the training loop.
 > skills as portable artifacts. If you want to *train* your own skill,
 > use `scripts/train.py` per the top-level README.
 >
-> This is the first artifact batch. We plan to continue uploading the
+> This is the first optimized-skill artifact batch. We plan to continue
-> remaining optimized skills and benchmark split manifests as they are
+> uploading remaining paper artifacts as they are cleaned and verified. All
-> cleaned and verified.
+> six lightweight ID/path split manifests are already checked in under
+> `data/`; most still require materializing their upstream benchmark payload.
 
 ## What's here
 
@@ -35,12 +36,17 @@ longitudinal guidance — that's expected, not a formatting issue.
 invoking the optimizer. Example for SearchQA against the test split:
 
 ```bash
+# The checked-in SearchQA split is ID-only; materialize full examples first.
+python -m pip install -e ".[searchqa]"
+python scripts/materialize_searchqa.py
+
 python scripts/eval_only.py \
   --config configs/searchqa/default.yaml \
   --skill ckpt/searchqa/gpt5.5_skill.md \
   --split valid_unseen \
-  --split_dir data/searchqa_id_split \
+  --split_dir data/searchqa_split \
   --azure_openai_endpoint https://your-resource.openai.azure.com/ \
+  --azure_openai_auth_mode api_key \
   --target_model gpt-5.5
 ```
 
@@ -52,13 +58,16 @@ is the selection / validation split, `train` is the training split, and
 ## On comparing to the paper numbers
 
 To compare against the paper-reported cells, use the same dataset split and
-scorer. SearchQA's split is checked in at `data/searchqa_id_split/` (400
+scorer. SearchQA's ID manifest is checked in at `data/searchqa_id_split/` (400
-train / 200 selection / 1400 test). For the other benchmarks, point
+train / 200 selection / 1400 test); the materializer writes the runnable
-`--split_dir` at your own materialized split; the loader is deterministic
+payload to `data/searchqa_split/`. All six lightweight split manifests are
-from `split_seed` (default `42`) + `split_ratio` (default `2:1:7`) when
+checked in under `data/`. ALFWorld's manifest records game-file paths; the
-`split_mode: ratio` is used, so a given `data_path` + seed reproduces
+other ID manifests still require you to materialize the corresponding
-across machines. Explicit per-benchmark split manifests are being prepared
+upstream benchmark payload into the documented `split_dir`. See
-for upload — see issues #14 and #21.
+[`data/README.md`](../data/README.md) for the exact status of each benchmark.
+When using `split_mode: ratio` instead, the loader is deterministic from
+`split_seed` (default `42`) + `split_ratio` (default `2:1:7`), so a given
+`data_path` + seed reproduces across machines.
 
 ## Why force-accept vs. gated slow-update matters
 
@@ -74,6 +83,5 @@ Current `main` defaults to `false` (force-accept mode), a newer
 post-submission behavior where the slow-update guidance is written into
 `current_skill` and `best_skill` unconditionally at the epoch boundary. If
 you re-train with the current default, you may produce a *different*
-`best_skill.md` than the one checked in here. Both modes are supported;
+`best_skill.md` than the one checked in here. Both modes are supported; see
-see the top-level README's "Configuration -> Slow-update acceptance mode"
+the [configuration reference](../docs/reference/config.md).
-section.

configs/_base_/default.yaml

@@ -81,6 +81,9 @@ optimizer:
   slow_update_gate_with_selection: false
   longitudinal_pair_policy: mixed  # mixed / changed / unchanged
   use_meta_skill: true
+  use_skill_aware_reflection: false  # EmbodiSkill: split failures into SKILL_DEFECT (edit body) vs EXECUTION_LAPSE (protected appendix)
+  skill_aware_appendix_source: both  # both = success+failure emit appendix notes; failure_only = only EXECUTION_LAPSE (paper-faithful)
+  skill_aware_consolidate_threshold: 0  # 0 = off; >0 = LLM-consolidate the appendix when its note count exceeds N
 
 evaluation:
   use_gate: true

data/README.md

@@ -138,6 +138,20 @@ ALFWorld:
 `searchqa_id_split/` is an ID-only manifest. Each released `id` exactly matches
 the `key` field in `lucadiliello/searchqa`.
 
+To materialize the runnable SearchQA split used by
+`configs/searchqa/default.yaml`, install the optional dependency and run:
+
+```bash
+python -m pip install 'skillopt[searchqa]'
+python scripts/materialize_searchqa.py
+```
+
+This writes full examples to:
+
+```text
+data/searchqa_split
+```
+
 Materialized examples must include the fields consumed by the SearchQA
 environment, including:
 

docs/contributing.md

@@ -15,6 +15,7 @@ pip install -e ".[dev]"
 ### 🐛 Bug Reports
 
 Open an issue with:
+
 - Steps to reproduce
 - Expected vs actual behavior
 - Config file used (sanitize API keys)
@@ -25,21 +26,29 @@ Open an issue with:
 See [Add a New Benchmark](guide/new-benchmark.md) for the implementation guide.
 
 **Checklist:**
-- [ ] Data loader in `skillopt/envs/<benchmark>/loader.py`
+
-- [ ] Environment adapter in `skillopt/envs/<benchmark>/env.py`
+- [ ] Data loader in `skillopt/envs/<benchmark>/dataloader.py`
+- [ ] Scored rollout implementation in `skillopt/envs/<benchmark>/rollout.py`
+- [ ] Per-item `predictions/<id>/conversation.json` artifacts for shared reflection
+- [ ] Environment adapter in `skillopt/envs/<benchmark>/adapter.py`
 - [ ] Config file in `configs/<benchmark>/default.yaml`
-- [ ] Registration in `skillopt/envs/__init__.py`
+- [ ] Lazy registration in `scripts/train.py` and `scripts/eval_only.py`
-- [ ] Documentation page in `docs/`
+- [ ] Focused tests and an optional seed skill referenced by `env.skill_init`
+- [ ] Documentation update
 
 ### 🤖 New Model Backend
 
 See [Add a New Model Backend](guide/new-backend.md) for the implementation guide.
 
 **Checklist:**
-- [ ] Backend in `skillopt/model/<backend>.py`
+
-- [ ] Registration in `skillopt/model/__init__.py`
+- [ ] Function-based backend module in `skillopt/model/<name>_backend.py`
-- [ ] API key entry in `.env.example`
+- [ ] Alias and default model in `skillopt/model/common.py`
-- [ ] Documentation update
+- [ ] Optimizer/target whitelist entries in `skillopt/model/backend_config.py`
+- [ ] Dispatch, token tracking, and setter forwarding in `skillopt/model/__init__.py`
+- [ ] YAML/CLI wiring when the backend exposes structured config fields
+- [ ] Focused routing, configuration, tool-call, and token-accounting tests
+- [ ] `.env.example` and backend/configuration reference updates
 
 ### 📝 Documentation
 
@@ -59,9 +68,9 @@ mkdocs serve  # Preview at http://localhost:8000
 ## Pull Request Process
 
 1. Fork the repository
-2. Create a feature branch: `git checkout -b feature/my-benchmark`
+2. Create a feature branch: `git switch -c feature/my-benchmark`
 3. Make your changes
-4. Test with an existing benchmark config
+4. Run focused tests, the full test suite, and `mkdocs build --strict` when docs change
 5. Submit a PR with a clear description
 
 ## License

docs/guide/configuration.md

@@ -20,15 +20,53 @@ Benchmark configs inherit from `_base_/default.yaml` and override specific value
 
 ## Key Parameters
 
-### Model
+### Model Backends
+
+`optimizer_backend` controls reflection and skill editing;
+`target_backend` controls task rollout. The legacy `backend` field remains for
+backward compatibility, but explicit role fields are the clearest configuration.
 
 ```yaml
 model:
-  backend: azure_openai          # azure_openai | openai_chat | claude_code_exec | qwen
+  backend: azure_openai          # High-level compatibility label
-  optimizer: gpt-5.5               # Optimizer model (for reflection)
+  optimizer_backend: openai_chat
-  target: gpt-5.5               # Target model (for rollout)
+  target_backend: openai_chat
+  optimizer: gpt-5.5             # Optimizer deployment/model
+  target: gpt-5.5                # Target deployment/model
+  azure_openai_auth_mode: api_key
 ```
 
+| Backend | Optimizer | Target | Configuration |
+|---|:---:|:---:|---|
+| `openai_chat` | ✓ | ✓ | Azure OpenAI, or its explicit compatibility auth mode |
+| `openai_compatible` | ✓ | ✓ | Generic OpenAI Chat Completions endpoint |
+| `claude_chat` | ✓ | ✓ | Claude Code CLI (`claude -p`) |
+| `qwen_chat` | ✓ | ✓ | Qwen served through an OpenAI-compatible local endpoint |
+| `minimax_chat` | ✓ | ✓ | MiniMax API |
+| `codex_exec` | — | ✓ | Codex CLI execution harness |
+| `claude_code_exec` | — | ✓ | Claude Code CLI execution harness |
+
+The current MiniMax adapter has one shared deployment. Set
+`model.minimax_model` when MiniMax is the target; a mixed-backend run cannot
+independently select a MiniMax optimizer model and a different target model.
+
+For a generic compatible provider, select the role backends explicitly rather
+than relying on a high-level shorthand:
+
+```yaml
+model:
+  optimizer_backend: openai_compatible
+  target_backend: openai_compatible
+  optimizer: deepseek-chat
+  target: deepseek-chat
+```
+
+The train/eval entry points apply `model.optimizer` and `model.target` after
+backend initialization. For the selected roles, these YAML values override
+`OPENAI_COMPATIBLE_MODEL`, `QWEN_CHAT_MODEL`, and their per-role environment
+forms. The environment model variables mainly seed direct library use; always
+set the role models in a training or evaluation config.
+
 ### Training
 
 ```yaml
@@ -61,14 +99,51 @@ optimizer:
   use_meta_skill: true           # Cross-epoch strategy memory
 ```
 
+### Skill-Aware Reflection (optional, off by default)
+
+EmbodiSkill-style failure routing: the failure analyst classifies each
+failure pattern as **SKILL_DEFECT** (the rule is wrong or missing → normal
+gated body edit) or **EXECUTION_LAPSE** (a valid rule exists but was not
+followed → a short reminder appended to a protected appendix region inside
+the skill that step-level edits can never modify).
+
+```yaml
+optimizer:
+  use_skill_aware_reflection: false    # Master switch (default off = baseline-identical)
+  skill_aware_appendix_source: both    # both | failure_only (paper-faithful S_app)
+  skill_aware_consolidate_threshold: 0 # >0: LLM-compact the appendix past N notes (experimental)
+```
+
+Notes:
+
+- The switch is resolved process-wide from the config
+  (`configure_skill_aware_reflection`), so it applies to every benchmark
+  with no per-adapter wiring.
+- `failure_only` restricts appendix notes to the failure analyst, matching
+  the original S_app formulation; `both` additionally lets the success
+  analyst re-emphasize existing rules.
+- Appendix notes bypass the validation gate by design and accumulate with
+  order-preserving dedup; lapse-only steps (no body edits) still flush
+  their notes.
+- Not supported together with `skill_update_mode=rewrite_from_suggestions`
+  or the full-rewrite modes: whole-document rewrites can drop the appendix
+  region.
+
 ### Evaluation
 
 ```yaml
 evaluation:
   use_gate: true                 # Validation gating (accept/reject updates)
+  gate_metric: hard              # hard | soft | mixed
+  gate_mixed_weight: 0.5         # Soft-score weight when metric=mixed
+  use_semantic_density: false    # Optional instruction-density bonus
   eval_test: true                # Run test evaluation after training
 ```
 
+The default and paper-style setting is `use_gate: true`. Setting it to `false`
+still records selection scores but force-accepts every candidate, so it changes
+the optimization semantics and should be reported explicitly.
+
 ### Environment (Data)
 
 ```yaml
@@ -87,6 +162,7 @@ Override any config value from the command line:
 ```bash
 python scripts/train.py \
   --config configs/searchqa/default.yaml \
+  --cfg-options \
     optimizer.learning_rate=16 \
     optimizer.lr_scheduler=linear \
     gradient.analyst_workers=8
@@ -98,11 +174,38 @@ Model credentials are loaded from environment variables:
 
 | Variable | Backend | Description |
 |---|---|---|
-| `AZURE_OPENAI_ENDPOINT` | azure_openai | Azure resource endpoint |
+| `AZURE_OPENAI_ENDPOINT` | `openai_chat` | Azure resource URL, or compatibility-mode base URL |
-| `AZURE_OPENAI_API_KEY` | azure_openai | Azure API key |
+| `AZURE_OPENAI_API_VERSION` | `openai_chat` | Azure API version |
-| `OPENAI_API_KEY` | openai | OpenAI API key |
+| `AZURE_OPENAI_AUTH_MODE` | `openai_chat` | `api_key`, `azure_cli`, `managed_identity`, or `openai_compatible` |
-| `ANTHROPIC_API_KEY` | claude | Anthropic API key |
+| `AZURE_OPENAI_API_KEY` | `openai_chat` | Required when auth mode is `api_key` or `openai_compatible` |
-| `QWEN_API_BASE` | qwen | Local Qwen vLLM endpoint |
+| `OPENAI_COMPATIBLE_BASE_URL` | `openai_compatible` | Generic Chat Completions base URL |
+| `OPENAI_COMPATIBLE_API_KEY` | `openai_compatible` | Provider API key; optional for local servers |
+| `OPENAI_COMPATIBLE_MODEL` | `openai_compatible` | Shared provider model ID for direct library use; train/eval YAML role models take precedence |
+| `CLAUDE_CLI_BIN` | `claude_chat` | Optional path to the `claude` executable; defaults to `claude` |
+| `ANTHROPIC_API_KEY` | `claude_chat` | Optional authentication method understood by the Claude CLI, not a direct SkillOpt API client |
+| `QWEN_CHAT_BASE_URL` | `qwen_chat` | Local Qwen/vLLM endpoint |
+| `QWEN_CHAT_MODEL` | `qwen_chat` | Served model name for direct library use; train/eval YAML role models take precedence |
+| `MINIMAX_BASE_URL` | `minimax_chat` | MiniMax-compatible base URL |
+| `MINIMAX_API_KEY` | `minimax_chat` | MiniMax API key |
+
+`OPTIMIZER_` and `TARGET_` prefixes provide per-role overrides for the
+Azure, OpenAI-compatible, and Qwen variable families. See the
+[Configuration Reference](../reference/config.md) for exact names.
+
+`claude_chat` launches the installed Claude Code CLI with `claude -p`; install
+and authenticate that CLI before use. Setting `ANTHROPIC_API_KEY` is one way
+the CLI may authenticate, but SkillOpt does not call the Anthropic API
+directly through this backend.
+
+### Three OpenAI-compatible paths
+
+- Research, generic provider: select `openai_compatible` and use
+  `OPENAI_COMPATIBLE_*`.
+- Research, Azure-family compatibility mode: keep `openai_chat`, set
+  `AZURE_OPENAI_AUTH_MODE=openai_compatible`, and use `AZURE_OPENAI_*`.
+- SkillOpt-Sleep: run with `--backend azure_openai` and use the same
+  compatibility-mode `AZURE_OPENAI_*` variables. Sleep does not read the
+  research backend's role-specific variables.
 
 ## Full Reference
 

docs/guide/dl-analogy.md

@@ -14,10 +14,9 @@ SkillOpt is designed around a core insight: **optimizing natural-language prompt
 | **Gradient aggregation** | Patch aggregation | Merge similar edits |
 | **Gradient clipping** | Edit selection | Cap max edits per step |
 | **Learning rate** | `learning_rate` | Max number of edits applied per step |
-| **LR scheduler** | `lr_scheduler` | Decay schedule: cosine, linear, constant |
+| **LR scheduler** | `lr_scheduler` | Edit-budget schedule: cosine, linear, constant, or autonomous |
 | **SGD step** | Skill update | Apply selected patches to document |
 | **Validation set** | Selection split | Gate checks improvement before accepting |
-| **Early stopping** | Gate patience | Reject updates that don't improve |
 | **Training step** | Step | One rollout → reflect → update cycle |
 | **Epoch** | Epoch | Full pass with slow update + meta memory |
 | **Momentum** | Slow update | Longitudinal comparison at epoch boundary |
@@ -34,7 +33,10 @@ SkillOpt is designed around a core insight: **optimizing natural-language prompt
 
 1. **Familiar mental model**: ML practitioners immediately understand how to tune SkillOpt
 2. **Principled hyperparameter search**: Grid search over `learning_rate` × `lr_scheduler` works just like in DL
-3. **Proven mechanisms**: Gating ≈ validation-based selection, patience ≈ early stopping, slow update ≈ momentum — all with strong theoretical motivation
+3. **Reusable mechanisms**: Gating provides validation-based model selection, while slow update plays a momentum-like role across epochs
+
+The gate is a per-candidate accept/reject decision. SkillOpt does not implement
+a gate-patience counter or stop training after a run of rejected candidates.
 
 ## Hyperparameter Transfer Rules
 

docs/guide/first-experiment.md

@@ -4,17 +4,43 @@ This guide walks through running a complete SkillOpt training on SearchQA.
 
 ## 1. Choose a Benchmark
 
-SkillOpt includes ready-to-use configs for several benchmarks:
+SkillOpt includes ready-to-use configs for several benchmarks. End-to-end
+runtime depends on the chosen models, provider latency, worker limits, and
+dataset size, so the project does not promise fixed wall-clock estimates.
 
-| Benchmark | Difficulty | Typical Runtime |
+| Benchmark | Modality | Additional setup |
 |---|---|---|
-| SearchQA | ⭐ Easy | ~30 min |
+| SearchQA | Text QA | Materialize the released ID manifest |
-| DocVQA | ⭐⭐ Medium | ~2 hours |
+| DocVQA | Document/image QA | Obtain and materialize images and examples |
-| ALFWorld | ⭐⭐⭐ Hard | ~3 hours |
+| ALFWorld | Embodied agent | Install ALFWorld and download its assets |
 
-We'll use **SearchQA** as it's the fastest to complete.
+We'll use **SearchQA** because it is the simplest text-only walkthrough.
 
-## 2. Configure
+## 2. Install and Materialize SearchQA
+
+The repository contains a stable SearchQA ID manifest, not the full runnable
+examples. From a source checkout, install the data extra and materialize the
+split once:
+
+```bash
+python -m pip install -e ".[searchqa]"
+python scripts/materialize_searchqa.py
+```
+
+By default, the materializer reads `data/searchqa_id_split/` and writes the
+train/validation/test payloads expected by the config to
+`data/searchqa_split/`; both paths have command-line overrides.
+
+## 3. Configure
+
+Configure and export one model backend as described in
+[Installation](installation.md#environment-variables). For example:
+
+```bash
+cp .env.example .env
+# Edit .env, choose one authentication mode, then export it:
+set -a; source .env; set +a
+```
 
 Review the config file:
 
@@ -42,56 +68,55 @@ evaluation:
   use_gate: true          # (validation gating)
 ```
 
-## 3. Train
+## 4. Train
 
 ```bash
-python scripts/train.py --config configs/searchqa/default.yaml
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --out_root outputs/searchqa_first_run
 ```
 
-You'll see output like:
+The command prints the resolved backend/data configuration, per-step rollout
+and gate progress, and the generated output directory.
+
+## 5. Monitor
+
+The explicit `--out_root` above creates this run directory:
 
 ```
-[Step 1/8] Rollout: 20 items, 4 workers...
+outputs/searchqa_first_run/
-[Step 1/8] Score: 0.65 → Reflect...
+├── config.json
-[Step 1/8] 6 edit patches generated
+├── runtime_state.json
-[Step 1/8] Selected 4 edits (lr=8, cosine → 7.7)
-[Step 1/8] Gate: val score 0.68 > 0.65 ✓ ACCEPT
-[Step 2/8] ...
-```
-
-## 4. Monitor
-
-Training outputs are saved to `outputs/<benchmark>/<run_id>/`:
-
-```
-outputs/searchqa/2024-01-15_10-30-00/
-├── steps/
-│   ├── step_0001/
-│   │   ├── candidate_skill.md
-│   │   ├── step_record.json
-│   │   └── trajectory_digest.json
-│   └── step_0002/
-├── slow_update/
-│   └── epoch_02/
-├── meta_skill/
-│   └── epoch_02/
-├── skills/
-│   └── step_0001.md
-├── best_skill.md
 ├── history.json
-└── config.yaml
+├── best_skill.md
+├── skills/
+│   └── skill_vXXXX.md
+├── steps/
+│   └── step_XXXX/
+│       ├── candidate_skill.md
+│       ├── step_record.json
+│       └── trajectory_digest.json
+├── slow_update/
+│   └── epoch_XX/
+└── meta_skill/
+    └── epoch_XX/
 ```
 
-## 5. Evaluate
+## 6. Evaluate
 
 Evaluate the best skill on the test split:
 
 ```bash
 python scripts/eval_only.py \
   --config configs/searchqa/default.yaml \
-  --skill outputs/searchqa/<run_id>/skills/best_skill.md
+  --skill outputs/searchqa_first_run/best_skill.md \
+  --split valid_unseen
 ```
 
+The `--skill` path above is the training artifact. Evaluation writes
+`eval_summary.json` to its own timestamped `outputs/eval_.../` directory unless
+you pass an explicit `--out_root`; it does not overwrite the training run.
+
 ## WebUI
 
 Prefer a graphical interface? Launch the WebUI:
@@ -101,7 +126,9 @@ pip install -e ".[webui]"
 python -m skillopt_webui.app
 ```
 
-Then open `http://localhost:7860` in your browser to configure parameters and launch training.
+Then open `http://localhost:7860` in your browser to configure parameters and
+launch training. The default host is `0.0.0.0`; pass `--host 127.0.0.1` for a
+local-only dashboard.
 
 ## Next Steps
 

docs/guide/installation.md

@@ -3,16 +3,44 @@
 ## Requirements
 
 - Python ≥ 3.10
-- At least one model API key (Azure OpenAI, OpenAI, Anthropic, or local Qwen)
+- For research training/evaluation, access to at least one configured model
+  backend (hosted API, local server, or an installed execution CLI)
+- The SkillOpt-Sleep `mock` backend needs no credentials
 
-## Quick Install
+## Choose an Install
+
+### PyPI
+
+Use PyPI for the Python packages and installed commands:
+
+```bash
+python -m pip install skillopt
+skillopt-sleep --help
+```
+
+This installs `skillopt-train`, `skillopt-eval`, and `skillopt-sleep`. The wheel
+does not include the repository's benchmark configs, data materializers,
+agent-integration shells/MCP servers, or development tests; use a source
+checkout for those files.
+
+!!! important "PyPI versus `main`"
+    These docs track the latest `main`. The current PyPI release is `0.2.0`.
+    The generic research `openai_compatible` backend, SkillOpt-Sleep handoff,
+    Sleep support for non-Azure OpenAI-compatible endpoints, and the Sleep
+    `--preferences` flag landed after that release and require a source install
+    from `main` until the next release.
+
+### Source checkout
 
 ```bash
 git clone https://github.com/microsoft/SkillOpt.git
 cd SkillOpt
-pip install -e .
+python -m pip install -e .
 ```
 
+Use the source checkout for paper reproduction, built-in benchmark configs,
+and contributions.
+
 ## Optional Dependencies
 
 Install extras for specific benchmarks or backends:
@@ -20,68 +48,115 @@ Install extras for specific benchmarks or backends:
 === "ALFWorld"
 
     ```bash
-    pip install -e ".[alfworld]"
+    python -m pip install -e ".[alfworld]"
     ```
 
-=== "Claude Backend"
+=== "Claude agent SDK (optional)"
 
     ```bash
-    pip install -e ".[claude]"
+    python -m pip install -e ".[claude]"
     ```
 
+    This extra does not install the `claude` executable. The research
+    `claude_chat` backend launches `claude -p`, so install and authenticate the
+    Claude Code CLI separately. The SDK extra is only needed when selecting an
+    SDK-backed Claude Code exec path.
+
 === "Qwen (Local)"
 
     ```bash
-    pip install -e ".[qwen]"
+    python -m pip install -e ".[qwen]"
+    ```
+
+=== "SearchQA data"
+
+    ```bash
+    python -m pip install -e ".[searchqa]"
     ```
 
 === "WebUI"
 
     ```bash
-    pip install -e ".[webui]"
+    python -m pip install -e ".[webui]"
     ```
 
 === "Development"
 
     ```bash
-    pip install -e ".[dev]"
+    python -m pip install -e ".[dev]"
     ```
 
 === "All"
 
     ```bash
-    pip install -e ".[alfworld,claude,qwen,webui,dev]"
+    python -m pip install -e ".[alfworld,claude,qwen,searchqa,webui,docs,dev]"
     ```
 
 ## Environment Variables
 
-Copy the example `.env` file and fill in your credentials:
+From a source checkout, copy the template and fill in only the backend you
+will use:
 
 ```bash
 cp .env.example .env
 ```
 
-Edit `.env` with your API keys:
+SkillOpt does not automatically load `.env`; export it into the current shell
+before running commands:
 
-```ini
+```bash
-# Azure OpenAI (default backend)
+set -a
-AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+source .env
-AZURE_OPENAI_API_KEY=your-key
+set +a
-
-# Or use OpenAI directly
-OPENAI_API_KEY=sk-...
-
-# Or Anthropic Claude
-ANTHROPIC_API_KEY=sk-ant-...
 ```
 
+For Azure OpenAI with API-key authentication, the minimum settings are:
+
+```ini
+AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+AZURE_OPENAI_API_VERSION=2024-12-01-preview
+AZURE_OPENAI_API_KEY=your-key
+AZURE_OPENAI_AUTH_MODE=api_key
+```
+
+Use `AZURE_OPENAI_AUTH_MODE=azure_cli` for Azure CLI credentials, or
+`managed_identity` with an optional
+`AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID`.
+
+The research `claude_chat` backend is a Claude Code CLI adapter, not a direct
+Anthropic API client. Install and authenticate `claude`, and set
+`CLAUDE_CLI_BIN` only if the executable is not available as `claude` on
+`PATH`. `ANTHROPIC_API_KEY` is one authentication option the CLI may consume.
+
+OpenAI-compatible servers have three distinct entry points:
+
+1. The research engine's generic `openai_compatible` backend uses
+   `OPENAI_COMPATIBLE_BASE_URL`, `OPENAI_COMPATIBLE_API_KEY`, and
+   `OPENAI_COMPATIBLE_MODEL`.
+2. The research `openai_chat` backend can use
+   `AZURE_OPENAI_AUTH_MODE=openai_compatible` with
+   `AZURE_OPENAI_ENDPOINT` and `AZURE_OPENAI_API_KEY`.
+3. SkillOpt-Sleep uses the same Azure-family variables as item 2 with
+   `skillopt-sleep run --backend azure_openai`.
+
+For research train/eval commands, `model.optimizer` and `model.target` in the
+YAML config are applied after backend initialization. They override model-name
+environment variables such as `OPENAI_COMPATIBLE_MODEL` and
+`QWEN_CHAT_MODEL`; set both role models explicitly when selecting those
+backends.
+
 !!! tip
-    You only need credentials for the backend you plan to use. Azure OpenAI is the default.
+    You only need to configure the backend you plan to use. See
+    [Configuration](configuration.md#model-backends) for exact backend names
+    and role-specific overrides.
 
 ## Verify Installation
 
 ```bash
 python -c "import skillopt; print('SkillOpt ready!')"
+skillopt-train --help
+skillopt-eval --help
+skillopt-sleep --help
 ```
 
 ## Next Steps

docs/guide/local-env-smoke.md

@@ -35,9 +35,14 @@ Use the split names your adapter maps to SkillOpt phases:
 - `val` or `valid_seen` for selection/gating
 - `test` or `valid_unseen` for final evaluation
 
-## 2. Support an offline mock mode
+## 2. Support a genuinely offline mock mode
 
-Add a configuration flag such as `mock: true` to your adapter. In mock mode, `rollout()` should return deterministic responses without calling external model APIs.
+Add a configuration flag such as `mock: true` to your adapter. In mock mode,
+`rollout()` should return deterministic responses without calling external
+model APIs. The inherited `EnvAdapter.reflect()` does call the configured
+optimizer backend, so a no-credential smoke test must also override
+`reflect()` in mock mode to return a small, schema-valid deterministic patch
+(and delegate to `super().reflect(...)` otherwise).
 
 This lets you verify the SkillOpt loop with a fast command such as:
 
@@ -46,13 +51,14 @@ python scripts/train.py \
   --config configs/myenv/tiny_mock.yaml
 ```
 
-Mock mode should still write the same artifacts as a real run, for example:
+Mock mode should still exercise the trainer's normal artifact paths, including:
 
-- `responses.json`
+- `config.json`, `runtime_state.json`, and `history.json`
-- `rollout_results.json`
+- `skills/skill_vXXXX.md`
-- `ranked_edits.json`
+- `steps/step_XXXX/ranked_edits.json`
-- `candidate_skill.md`
+- `steps/step_XXXX/candidate_skill.md`
-- `summary.json`
+- `steps/step_XXXX/step_record.json`
+- the final `summary.json`
 
 ## 3. Keep the smoke config tiny
 
@@ -127,7 +133,7 @@ For the real tiny run, verify that:
 
 - the run completes
 - `summary.json` is written
-- `ranked_edits.json` contains the expected ranking metadata
+- the step directory's `ranked_edits.json` contains the expected ranking metadata
 - any optimizer bridge log marks the response schema as valid
 - no generated files are written outside `out_root`
 

docs/guide/new-backend.md

@@ -1,130 +1,200 @@
 # Add a New Model Backend
 
-SkillOpt supports multiple LLM backends. This guide shows how to add your own.
+SkillOpt's model layer is function-based: each chat backend is a Python module
+that exposes the call, token-tracking, and deployment-setting functions used by
+`skillopt.model`. There is no backend base class or registry object to subclass.
 
-## Backend Architecture
+## Built-in: the generic OpenAI-compatible backend
 
-```
+!!! note "Version requirement"
-skillopt/model/
+    This backend landed after v0.2.0. Install from the latest `main` until it is
-├── base.py           # Abstract base class
+    included in the next release.
-├── azure_openai.py   # Azure OpenAI backend
-├── openai_model.py   # Direct OpenAI backend
-├── claude.py         # Anthropic Claude backend
-├── qwen.py           # Local Qwen (vLLM) backend
-└── your_backend.py   # Your new backend
-```
 
-## Step 1: Create the Backend
+Before writing a new backend, check whether your provider already speaks the
+OpenAI Chat Completions protocol. Most do, in which case you can use the
+built-in **`openai_compatible`** backend
+(`skillopt/model/openai_compatible_backend.py`) with no code changes.
 
-Create `skillopt/model/your_backend.py`:
+A single `base_url` + `api_key` pair lets you point SkillOpt at, for example:
+
+| Provider | `base_url` | Example model |
+|---|---|---|
+| DeepSeek | `https://api.deepseek.com/v1` | `deepseek-chat` |
+| Groq | `https://api.groq.com/openai/v1` | `llama-3.3-70b-versatile` |
+| Together AI | `https://api.together.xyz/v1` | `meta-llama/Llama-3.3-70B-Instruct-Turbo` |
+| Ollama (local) | `http://localhost:11434/v1` | `qwen2.5:7b` |
+| vLLM / SGLang / TGI | `http://localhost:8000/v1` | your served model |
+| LiteLLM proxy | `http://localhost:4000` | any proxied model |
+| OpenRouter / Fireworks / xAI / … | provider base URL | provider model id |
+
+### Python API
+
+Select and configure the backend directly when embedding SkillOpt as a Python
+library:
 
 ```python
-from skillopt.model.base import ModelBackend, ModelResponse
+import skillopt.model as model
 
-class YourBackend(ModelBackend):
+# Use the generic backend for both optimizer and target calls.
-    """Your custom model backend."""
+model.set_backend("openai_compatible")
-    
+model.configure_openai_compatible(
-    def __init__(self, cfg: dict):
+    base_url="https://api.deepseek.com/v1",
-        super().__init__(cfg)
+    api_key="sk-...",
-        self.model_name = cfg.get('model_name', 'your-default-model')
+    model="deepseek-chat",
-        self.api_key = os.environ.get('YOUR_API_KEY', '')
-        self.client = self._init_client()
-    
-    def _init_client(self):
-        """Initialize API client."""
-        # TODO: Set up your API client
-        pass
-    
-    async def generate(
-        self,
-        messages: list[dict],
-        temperature: float = 0.7,
-        max_tokens: int = 4096,
-        **kwargs
-    ) -> ModelResponse:
-        """
-        Generate a completion.
-        
-        Args:
-            messages: Chat messages [{"role": "...", "content": "..."}]
-            temperature: Sampling temperature
-            max_tokens: Maximum tokens in response
-            
-        Returns:
-            ModelResponse with content, usage, and metadata
-        """
-        response = await self.client.chat(
-            model=self.model_name,
-            messages=messages,
-            temperature=temperature,
-            max_tokens=max_tokens,
 )
-        
-        return ModelResponse(
-            content=response.text,
-            usage={
-                'prompt_tokens': response.usage.input,
-                'completion_tokens': response.usage.output,
-            },
-            model=self.model_name,
-        )
-    
-    async def generate_with_tools(
-        self,
-        messages: list[dict],
-        tools: list[dict],
-        **kwargs
-    ) -> ModelResponse:
-        """Generate with tool/function calling support."""
-        # Optional: implement if your model supports tool use
-        raise NotImplementedError("Tool use not supported")
 ```
 
-## Step 2: Register the Backend
+`configure_openai_compatible()` also accepts `optimizer_*` and `target_*`
+arguments when the two roles use different endpoints or models.
 
-Add to `skillopt/model/__init__.py`:
+### Environment variables
 
-```python
+The shared variables below configure both roles. Role-specific
-from .your_backend import YourBackend
+`OPTIMIZER_OPENAI_COMPATIBLE_*` and `TARGET_OPENAI_COMPATIBLE_*` variables take
+precedence:
 
-BACKEND_REGISTRY = {
+```bash
-    # ... existing backends ...
+export OPENAI_COMPATIBLE_BASE_URL="https://api.groq.com/openai/v1"
-    'your_backend': YourBackend,
+export OPENAI_COMPATIBLE_API_KEY="gsk_..."
-}
+export OPENAI_COMPATIBLE_MODEL="llama-3.3-70b-versatile"
+# Optional: OPENAI_COMPATIBLE_TEMPERATURE, _MAX_TOKENS, _TIMEOUT_SECONDS
 ```
 
-## Step 3: Configure
+For direct library use, `OPTIMIZER_BACKEND=openai_compatible` and/or
-
+`TARGET_BACKEND=openai_compatible` select the role. The training and evaluation
-Use your backend in any config:
+scripts resolve backend selection from their config, so set the split fields
+explicitly there:
 
 ```yaml
 model:
-  backend: your_backend
+  optimizer_backend: openai_compatible
-  model_name: your-model-id
+  target_backend: openai_compatible
-  temperature: 0.7
+  optimizer: llama-3.3-70b-versatile
-  max_tokens: 4096
+  target: llama-3.3-70b-versatile
 ```
 
-Set credentials via environment variable:
+Equivalently, override those fields on the command line:
 
 ```bash
-export YOUR_API_KEY="your-key"
+python scripts/train.py --config configs/searchqa/default.yaml \
+  --cfg-options \
+  model.optimizer_backend=openai_compatible \
+  model.target_backend=openai_compatible \
+  model.optimizer=llama-3.3-70b-versatile \
+  model.target=llama-3.3-70b-versatile
 ```
 
-## Required Interface
+Do not rely on the legacy high-level `model.backend` label to replace the two
+role-specific fields in a structured config.
 
-Your backend must implement these methods:
+The generic backend uses the official `openai` SDK and the Chat Completions
+API. It records token usage through the shared tracker, supports provider tool
+calling through `chat_*_messages(..., tools=...)`, and exposes `count_tokens()`
+(tiktoken when available, with a character-based fallback). Provider-specific
+Responses API features are outside this backend's contract.
 
-| Method | Required | Description |
+Only write a new backend when the provider is not compatible with this surface
-|---|---|---|
+or requires behavior that cannot be expressed by its configuration.
-| `generate()` | ✅ | Basic text generation |
-| `generate_with_tools()` | Optional | Tool/function calling |
-| `count_tokens()` | Optional | Token counting for context management |
 
-## Tips
+## Backend architecture
 
-!!! tip
+The active split optimizer/target dispatcher is the public
-    - Test your backend with `python -c "from skillopt.model.your_backend import YourBackend"` first
+`skillopt/model/__init__.py` module:
-    - Use `async` methods for all API calls — SkillOpt uses asyncio throughout
+
-    - Implement retry logic with exponential backoff for production use
+```text
-    - Add your API key to `.env.example` when submitting a PR
+skillopt/model/
+├── common.py                       # aliases, default models, token/response helpers
+├── backend_config.py               # optimizer/target whitelists and runtime selection
+├── __init__.py                     # public API and split-role dispatch
+├── openai_compatible_backend.py    # generic Chat Completions example
+├── qwen_backend.py                 # raw-HTTP chat example with per-role config
+├── minimax_backend.py              # compact raw-HTTP chat example
+├── codex_harness.py                # target-only exec harnesses
+└── router.py                       # legacy single-backend compatibility surface
+```
+
+`router.py` is not the dispatcher used by the current training loop. Update it
+only if the new backend must also be exposed through that legacy single-backend
+API.
+
+## Step 1: implement the module contract
+
+Create a module such as `skillopt/model/your_backend.py`. Copy the signatures
+from `openai_compatible_backend.py` or `qwen_backend.py`; model calls in the
+current framework are synchronous.
+
+For a chat backend that supports both roles, the public module surface is:
+
+| Function | Purpose |
+|---|---|
+| `chat_optimizer(...)` | Optimizer system/user call; returns `(text, usage)` |
+| `chat_target(...)` | Target system/user call; returns `(text, usage)` |
+| `chat_optimizer_messages(...)` | Optimizer message-list call, including optional tools |
+| `chat_target_messages(...)` | Target message-list call, including optional tools |
+| `get_token_summary()` | Return per-stage counters plus `_total` |
+| `reset_token_tracker()` | Clear this backend's counters |
+| `set_optimizer_deployment(name)` | Change the optimizer model at runtime |
+| `set_target_deployment(name)` | Change the target model at runtime |
+| `set_reasoning_effort(effort)` | Apply or safely ignore the shared reasoning setting |
+
+Every call returns a usage dict with `prompt_tokens`, `completion_tokens`, and
+`total_tokens`. Use `TokenTracker` from `skillopt.model.common` and record each
+call exactly once. Message-list calls that accept tools should return the
+compatibility message objects from `common.py` when `return_message=True`.
+
+Provider-specific configuration helpers and `count_tokens()` are optional, but
+their state must be safe to update while calls may run concurrently. Keep
+credentials out of logs and persisted artifacts.
+
+Exec-style targets do not implement this chat contract. They are target-only
+and are integrated through `codex_harness.py` plus environment-specific rollout
+code.
+
+## Step 2: register and route the backend
+
+A new backend normally requires all of the following:
+
+1. Add its canonical name, aliases, and default model to
+   `skillopt/model/common.py`.
+2. Add the canonical name to the appropriate optimizer and/or target whitelist
+   in `skillopt/model/backend_config.py`. Do not advertise a role the module
+   cannot execute.
+3. Import the module in `skillopt/model/__init__.py` and add dispatch branches
+   for every supported call surface.
+4. Include its counters in `get_token_summary()` / `reset_token_tracker()` and
+   forward the shared deployment/reasoning setters where applicable.
+5. If it has YAML settings, add structured-to-flat mappings in
+   `skillopt/config.py`, wire them through `scripts/train.py` and
+   `scripts/eval_only.py`, and document their precedence over environment
+   variables.
+6. Update `router.py` only when legacy single-backend compatibility is part of
+   the intended feature.
+
+Backend selection in `scripts/train.py` must use
+`model.optimizer_backend` and `model.target_backend`. A high-level
+`model.backend` alias alone is not a substitute for this explicit split.
+
+## Step 3: test the integration
+
+Add focused tests under `tests/` that do not call a live provider. At minimum,
+cover:
+
+- optimizer and target whitelist validation;
+- routing for text and message-list calls;
+- role-specific configuration precedence;
+- tool-call compatibility, if supported;
+- deployment/reasoning setters;
+- token accounting, including a single correct `_total`;
+- actionable errors for missing credentials or invalid responses.
+
+Then run the focused test, the full suite, and the documentation build:
+
+```bash
+python -m pytest tests/test_your_backend.py -q
+python -m pytest tests/ -q
+mkdocs build --strict
+```
+
+Also update `.env.example`, the configuration reference, and the backend table
+in the API reference. Add an optional dependency extra only when the backend
+requires a package that is not already a core dependency.

docs/guide/new-benchmark.md

@@ -14,16 +14,18 @@ To add a benchmark you implement four things:
 
 1. **A `SplitDataLoader` subclass** — knows how to load train / val / test
    item dicts from disk.
-2. **A rollout helper** — runs the target model on a batch of items
+2. **A rollout helper** — runs the target model on a batch of items, scores
-   under the current skill and scores each prediction.
+   each prediction, and persists the per-item conversation consumed by the
+   shared reflection stage.
 3. **An `EnvAdapter` subclass** — wires the loader + rollout helper into
-   SkillOpt's lifecycle (`build_*_env`, `rollout`, `reflect`,
+   SkillOpt's lifecycle (`build_*_env`, `rollout`, and `get_task_types`).
-   `get_task_types`).
+   The shared `reflect()` implementation is inherited unless the benchmark
+   needs custom reflection logic.
 4. **A YAML config** — references your env name plus the standard
    train / optimizer / gradient knobs.
 
-Then one line in `scripts/train.py`'s `_register_builtins()` makes it
+Then lazy registration in the training and evaluation scripts makes it
-discoverable.
+discoverable without importing optional dependencies at startup.
 
 ---
 
@@ -36,7 +38,7 @@ touch skillopt/envs/docfaithful/__init__.py
 
 ## Step 2 — Implement the data loader
 
-`skillopt/envs/docfaithful/loader.py`:
+`skillopt/envs/docfaithful/dataloader.py`:
 
 ```python
 from __future__ import annotations
@@ -99,8 +101,8 @@ def _score(prediction: str, ground_truth: str) -> tuple[int, float]:
     return hard, soft
 
 
-def _rollout_one(item: dict, skill_content: str,
+def _rollout_one(item: dict, skill_content: str, *, prediction_dir: Path,
-                 *, max_completion_tokens: int) -> dict:
+                 max_completion_tokens: int) -> dict:
     system = skill_content
     user = (
         f"Question: {item['question']}\n\n"
@@ -113,14 +115,33 @@ def _rollout_one(item: dict, skill_content: str,
         max_completion_tokens=max_completion_tokens,
     )
     hard, soft = _score(prediction, item.get("ground_truth", ""))
+
+    # EnvAdapter.reflect() reads this exact trajectory path. Keep item IDs
+    # unique and filesystem-safe.
+    task_dir = prediction_dir / str(item["id"])
+    task_dir.mkdir(parents=True, exist_ok=True)
+    conversation = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+        {"role": "assistant", "content": prediction},
+    ]
+    (task_dir / "conversation.json").write_text(
+        json.dumps(conversation, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
     return {
         "id": str(item["id"]),
         "hard": hard,
         "soft": soft,
         "predicted_answer": prediction,
+        "task_description": item.get("question", ""),
         "question": item.get("question", ""),
         "reference_text": item.get("reference_text", ""),
         "task_type": item.get("task_type", "docfaithful"),
+        "target_system_prompt": system,
+        "target_user_prompt": user,
+        "n_turns": 1,
     }
 
 
@@ -128,15 +149,18 @@ def run_batch(*, items: list[dict], skill_content: str, out_root: str,
               workers: int = 4, max_completion_tokens: int = 4096) -> list[dict]:
     """Run a batch of episodes sequentially or with a thread pool."""
     os.makedirs(out_root, exist_ok=True)
+    prediction_dir = Path(out_root, "predictions")
     # For brevity we go sequentially — swap in concurrent.futures.ThreadPoolExecutor
     # when network / model latency dominates.
     results = [
         _rollout_one(item, skill_content,
+                     prediction_dir=prediction_dir,
                      max_completion_tokens=max_completion_tokens)
         for item in items
     ]
     Path(out_root, "rollouts.json").write_text(
-        json.dumps(results, ensure_ascii=False, indent=2)
+        json.dumps(results, ensure_ascii=False, indent=2),
+        encoding="utf-8",
     )
     return results
 ```
@@ -150,9 +174,17 @@ Two design points worth flagging:
 - **Use `skillopt.model.chat_target`**, not raw OpenAI/Claude calls.
   That routes through whichever **chat** target backend the user
   configured (`openai_chat` / `claude_chat` / `qwen_chat` /
-  `minimax_chat`) without your adapter caring. Exec-style backends
+  `minimax_chat` / `openai_compatible`) without your adapter caring.
-  (`codex_exec`, `claude_code_exec`) need env-specific rollout code —
+  Exec-style backends (`codex_exec`, `claude_code_exec`) need
-  see `skillopt/envs/swebench/` for an example.
+  environment-specific rollout code —
+  see `skillopt/model/codex_harness.py` together with the rollout modules in
+  `skillopt/envs/searchqa/`, `skillopt/envs/docvqa/`, or
+  `skillopt/envs/officeqa/` for working examples.
+- **Persist a conversation for reflection.** The shared `EnvAdapter.reflect()`
+  looks under `<rollout_dir>/predictions/<result-id>/conversation.json` and
+  skips results whose trajectory is absent or empty. Returning `hard`/`soft`
+  scores alone is sufficient for evaluation, but it cannot produce learning
+  patches.
 
 ## Step 4 — Implement the environment adapter
 
@@ -161,13 +193,10 @@ Two design points worth flagging:
 ```python
 from __future__ import annotations
 
-import os
-
 from skillopt.datasets.base import BatchSpec
 from skillopt.envs.base import EnvAdapter
-from skillopt.envs.docfaithful.loader import DocFaithfulDataLoader
+from skillopt.envs.docfaithful.dataloader import DocFaithfulDataLoader
 from skillopt.envs.docfaithful.rollout import run_batch
-from skillopt.gradient.reflect import run_minibatch_reflect
 
 
 class DocFaithfulAdapter(EnvAdapter):
@@ -234,7 +263,7 @@ class DocFaithfulAdapter(EnvAdapter):
         )
         return self.build_env_from_batch(batch, **kwargs)
 
-    # ── The two real action methods ─────────────────────────────────────
+    # ── The rollout method (reflect is inherited) ───────────────────────
 
     def rollout(self, env_manager, skill_content: str,
                 out_dir: str, **kwargs) -> list[dict]:
@@ -247,27 +276,9 @@ class DocFaithfulAdapter(EnvAdapter):
             max_completion_tokens=self.max_completion_tokens,
         )
 
-    def reflect(self, results: list[dict], skill_content: str,
+    # reflect() is inherited from EnvAdapter — it delegates to
-                out_dir: str, **kwargs) -> list[dict | None]:
+    # run_minibatch_reflect with your analyst_error_* / analyst_success_*
-        return run_minibatch_reflect(
+    # prompts. Override it only if you need custom reflection logic.
-            results=results,
-            skill_content=skill_content,
-            prediction_dir=kwargs.get(
-                "prediction_dir", os.path.join(out_dir, "predictions")
-            ),
-            patches_dir=kwargs.get(
-                "patches_dir", os.path.join(out_dir, "patches")
-            ),
-            workers=self.analyst_workers,
-            failure_only=self.failure_only,
-            minibatch_size=self.minibatch_size,
-            edit_budget=self.edit_budget,
-            random_seed=kwargs.get("random_seed"),
-            error_system=self.get_error_minibatch_prompt(),
-            success_system=self.get_success_minibatch_prompt(),
-            step_buffer_context=kwargs.get("step_buffer_context", ""),
-            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
-        )
 
     def get_task_types(self) -> list[str]:
         seen: list[str] = []
@@ -298,9 +309,11 @@ the answer against `item["ground_truth"]`, and returns a list of dicts:
 ]
 ```
 
-The trainer only requires `id`, `hard`, `soft`. The rest is preserved on
+The trainer requires `id`, `hard`, and `soft` for scoring. The remaining fields
-`RolloutResult.extras` (see `skillopt/types.py`) and is what your
+are preserved on `RolloutResult.extras` (see `skillopt/types.py`). The shared
-`reflect()` consumes via `run_minibatch_reflect`.
+reflection implementation combines those fields with each persisted
+`predictions/<id>/conversation.json`; without that file the result is omitted
+from reflection.
 
 ## Step 5 — Register the adapter
 
@@ -315,9 +328,11 @@ and add to `_register_builtins()`:
         pass  # docfaithful deps not installed — skip
 ```
 
-There is **no `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`** —
+Mirror the same lazy registration in
-the registry lives in `scripts/train.py` and is populated lazily so that
+[`scripts/eval_only.py`](https://github.com/microsoft/SkillOpt/blob/main/scripts/eval_only.py)
-optional deps don't break `--help`.
+so standalone evaluation can resolve the environment too. There is **no
+`BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`**; both entry points
+keep a small lazy registry so optional dependencies do not break `--help`.
 
 ## Step 6 — Create the YAML config
 
@@ -343,9 +358,7 @@ optimizer:
 
 env:
   name: docfaithful
-  # Optional: a seed skill document. Create this file (or any markdown
+  # Point to an existing Markdown file. Use an empty file to start blank.
-  # file) yourself before the first run, or omit the key to let SkillOpt
-  # start from an empty skill.
   skill_init: skillopt/envs/docfaithful/skills/initial.md
   split_mode: split_dir
   split_dir: data/docfaithful_split
@@ -362,7 +375,7 @@ env:
 ## Step 7 — Run
 
 ```bash
-# If you set skill_init above, create the seed skill first:
+# Create the file referenced by env.skill_init before the first run:
 #   mkdir -p skillopt/envs/docfaithful/skills
 #   echo "# DocFaithful initial skill" > skillopt/envs/docfaithful/skills/initial.md
 
@@ -373,9 +386,8 @@ If you get `ValueError: Unknown environment 'docfaithful'. Available: [...]`,
 you forgot Step 5.
 
 If you get `TypeError: Can't instantiate abstract class DocFaithfulAdapter`,
-you forgot to implement one of the five abstract methods on `EnvAdapter`:
+you forgot to implement one of the four abstract methods on `EnvAdapter`:
-`build_train_env`, `build_eval_env`, `rollout`, `reflect`,
+`build_train_env`, `build_eval_env`, `rollout`, `get_task_types`.
-`get_task_types`.
 
 ## Tips
 
@@ -386,8 +398,11 @@ you forgot to implement one of the five abstract methods on `EnvAdapter`:
   `hard` / `soft`.
 - Noisy scoring kills the optimizer. Spend time on `run_batch`'s scoring
   before you spend time on prompts.
+- If training repeatedly reports `skip_no_patches`, first verify that every
+  rollout result has a non-empty
+  `rollout/predictions/<id>/conversation.json` using the same `id` string.
 - If your benchmark needs heavy optional deps (selenium, vllm, ...),
-  wrap the registration block with `try / except ImportError` (Step 5)
+  wrap both registration blocks with `try / except ImportError` (Step 5)
   so people without those deps can still `--help`.
 - Copy `skillopt/envs/_template/` as a starting skeleton — it now
   implements the real abstract methods.

docs/guide/skill-document.md

@@ -38,23 +38,45 @@ During training, the skill document is modified by **edit patches**:
 2. **Modifications**: Refining existing rules that are partially correct
 3. **Deletions**: Removing rules that consistently lead to errors
 
-Each edit is validated through the **gate** mechanism before being permanently accepted.
+Selected edits are applied together to produce a candidate skill. With the
+validation gate enabled, that candidate replaces the current skill only when
+its score on the selection split strictly improves.
+
+SkillOpt may maintain two protected, machine-managed regions:
+
+```markdown
+<!-- SLOW_UPDATE_START -->
+... epoch-level longitudinal guidance ...
+<!-- SLOW_UPDATE_END -->
+
+<!-- APPENDIX_START -->
+... skill-aware execution reminders ...
+<!-- APPENDIX_END -->
+```
+
+Normal edit patches cannot modify either region. Slow update owns the first;
+optional skill-aware reflection owns the second. Preserve these markers when
+copying or manually inspecting a trained skill.
 
 ## Initial Skill
 
 You can start training with:
 
-- **Empty skill**: The system learns everything from scratch
+- **Empty skill**: Point `env.skill_init` to an empty Markdown file
 - **Seed skill**: Provide initial instructions to bootstrap training
 - **Pre-trained skill**: Transfer a skill from a related benchmark
 
 Configure the initial skill in your YAML:
 
 ```yaml
-train:
+env:
-  init_skill: "path/to/initial_skill.md"  # or omit for empty
+  skill_init: path/to/initial_skill.md
 ```
 
+To start from scratch, create an empty Markdown file and use its path. A missing
+path currently also starts blank, so using an explicit file avoids silently
+treating a typo as an empty skill.
+
 ## Skill Quality Metrics
 
 Track your skill's evolution through:
@@ -62,15 +84,16 @@ Track your skill's evolution through:
 - **Validation score**: Primary metric on the selection split
 - **Test score**: Final metric on held-out test data
 - **Skill length**: Total tokens in the document
-- **Edit acceptance rate**: Fraction of proposed edits that pass gating
+- **Candidate acceptance rate**: Fraction of candidate skill updates that pass
+  gating; multiple proposed edits can be combined into one candidate
 
 ## Best Practices
 
 !!! tip "Tips for better skills"
     1. **Start with a seed skill** (`env.skill_init`) if you have domain knowledge — it converges faster
     2. **Use cosine LR schedule** — aggressive early exploration + careful late refinement
-    3. **Enable slow update** (`use_slow_update: true`) to prevent forgetting across epochs
+    3. **Enable slow update** (`optimizer.use_slow_update: true`) to counter forgetting across epochs
-    4. **Enable meta skill** (`use_meta_skill: true`) so the optimizer accumulates strategy memory
+    4. **Enable meta skill** (`optimizer.use_meta_skill: true`) so the optimizer accumulates strategy memory
 
 ## Next Steps
 

docs/guide/training-loop.md

@@ -37,12 +37,11 @@ scores = evaluate(predictions, ground_truth)
 
 ### 2. Reflect (Backward Pass)
 
-The **optimizer** model analyzes failed trajectories and produces **edit patches** — structured suggestions for improving the skill document.
+The **optimizer** model analyzes trajectory minibatches and produces **edit
-
+patches** — structured suggestions for improving the skill document. Failure
-Two modes:
+minibatches are always eligible for analysis; successful trajectories are also
-
+analyzed unless `gradient.failure_only` is enabled. Independent minibatches can
-- **Shallow**: Analyze each trajectory independently
+run concurrently according to `gradient.analyst_workers`.
-- **Deep**: Cross-reference multiple failures to find systemic issues
 
 ```python
 # Analogy: computing gradients
@@ -74,17 +73,31 @@ Selected edits are applied to the skill document, producing a new version.
 
 ### 6. Gate (Validation)
 
-The updated skill is evaluated on a **selection split** (analogous to a validation set). The update is only accepted if performance improves.
+The updated skill is evaluated on a **selection split** (analogous to a
+validation set). With the gate enabled, the candidate is accepted only when its
+configured gate score (`hard`, `soft`, or `mixed`) is strictly higher than the
+current skill's score. With `evaluation.use_gate: false`, validation is still
+recorded but candidates are force-accepted.
 
 ## Epoch Boundary Mechanisms
 
 ### Slow Update
 
-At the end of each epoch (starting from epoch 2), the system performs a **longitudinal comparison**: it rolls out both the previous epoch's skill and the current skill on the same samples, categorizes items as improved/regressed/persistent_fail/stable_success, then generates high-level **guidance** that is injected into the skill document. This prevents catastrophic forgetting of earlier improvements.
+At the end of each epoch (starting from epoch 2), the system performs a
+**longitudinal comparison**: it rolls out both the previous epoch's skill and
+the current skill on the same samples, categorizes items as
+improved/regressed/persistent-fail/stable-success, then generates high-level
+**guidance** for the skill document. Depending on
+`optimizer.slow_update_gate_with_selection`, that guidance is either checked on
+the selection split or applied unconditionally. Its purpose is to counter
+cross-epoch forgetting.
 
 ### Meta Skill
 
-A **meta-skill memory** accumulates high-level strategy notes across the entire training run. At the end of each epoch, the optimizer reflects on what changed between epochs and produces a compact memory that is provided as additional context during future reflection steps.
+A **meta-skill memory** accumulates high-level strategy notes across the training
+run. Starting at the end of epoch 2, the optimizer compares the previous and
+current epoch, writes a compact memory, and provides the prior epoch's memory as
+additional context during later reflection and update stages.
 
 ## Next Steps
 

docs/guideline.html

@@ -0,0 +1,550 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>SkillOpt — Documentation &amp; Reproduction Guide</title>
+<meta name="description" content="Accurate entry points for installing, configuring, running, and extending SkillOpt and SkillOpt-Sleep.">
+<link rel="icon" type="image/svg+xml" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 23 23'%3E%3Crect width='10' height='10' fill='%23F25022'/%3E%3Crect x='13' width='10' height='10' fill='%237FBA00'/%3E%3Crect y='13' width='10' height='10' fill='%2300A4EF'/%3E%3Crect x='13' y='13' width='10' height='10' fill='%23FFB900'/%3E%3C/svg%3E">
+<style>
+  :root {
+    --bg: #fff;
+    --soft: #f7f8fb;
+    --ink: #1f2733;
+    --muted: #5b6675;
+    --quiet: #7c8797;
+    --line: #e2e7ef;
+    --brand: #4f46e5;
+    --brand-soft: #eef0ff;
+    --green: #047857;
+    --amber: #a16207;
+    --code-bg: #0f172a;
+    --code-ink: #e2e8f0;
+    --mono: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
+    --sans: Inter, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+  }
+  * { box-sizing: border-box; }
+  html { scroll-behavior: smooth; }
+  body {
+    margin: 0;
+    color: var(--ink);
+    background: var(--bg);
+    font: 15px/1.65 var(--sans);
+    -webkit-font-smoothing: antialiased;
+  }
+  header {
+    position: sticky;
+    top: 0;
+    z-index: 10;
+    display: flex;
+    align-items: center;
+    gap: 12px;
+    height: 58px;
+    padding: 0 24px;
+    background: rgba(255,255,255,.94);
+    border-bottom: 1px solid var(--line);
+    backdrop-filter: blur(8px);
+  }
+  header svg { width: 22px; }
+  header strong { letter-spacing: -.01em; }
+  header strong span { color: var(--brand); }
+  header .spacer { flex: 1; }
+  header a {
+    color: var(--muted);
+    text-decoration: none;
+    font-size: 13px;
+    font-weight: 600;
+  }
+  header a:hover { color: var(--brand); }
+  .layout {
+    display: grid;
+    grid-template-columns: 240px minmax(0, 880px);
+    justify-content: center;
+    align-items: start;
+  }
+  nav {
+    position: sticky;
+    top: 58px;
+    height: calc(100vh - 58px);
+    padding: 32px 26px;
+    overflow-y: auto;
+    border-right: 1px solid var(--line);
+  }
+  nav strong {
+    display: block;
+    margin: 18px 0 6px;
+    color: var(--quiet);
+    font-size: 11px;
+    letter-spacing: .08em;
+    text-transform: uppercase;
+  }
+  nav strong:first-child { margin-top: 0; }
+  nav a {
+    display: block;
+    padding: 4px 0;
+    color: var(--muted);
+    text-decoration: none;
+    font-size: 13px;
+  }
+  nav a:hover { color: var(--brand); }
+  main { padding: 46px 54px 100px; min-width: 0; }
+  section { scroll-margin-top: 76px; }
+  h1 {
+    margin: 4px 0 10px;
+    font-size: clamp(30px, 4vw, 42px);
+    line-height: 1.12;
+    letter-spacing: -.035em;
+  }
+  h2 {
+    margin: 54px 0 14px;
+    padding-bottom: 9px;
+    border-bottom: 1px solid var(--line);
+    font-size: 24px;
+    letter-spacing: -.02em;
+  }
+  h3 { margin: 28px 0 8px; font-size: 18px; }
+  p { margin: 10px 0; }
+  a { color: var(--brand); }
+  code {
+    padding: 2px 5px;
+    border-radius: 5px;
+    background: #eef1f6;
+    color: #9d174d;
+    font: .88em var(--mono);
+  }
+  pre {
+    overflow-x: auto;
+    margin: 14px 0;
+    padding: 16px 18px;
+    border: 1px solid #1e293b;
+    border-radius: 11px;
+    background: var(--code-bg);
+    color: var(--code-ink);
+    font: 13px/1.6 var(--mono);
+  }
+  pre code { padding: 0; background: transparent; color: inherit; font-size: inherit; }
+  ul, ol { padding-left: 23px; }
+  li { margin: 5px 0; }
+  .eyebrow {
+    color: var(--brand);
+    font-size: 12px;
+    font-weight: 750;
+    letter-spacing: .09em;
+    text-transform: uppercase;
+  }
+  .lead { color: var(--muted); font-size: 17px; }
+  .notice {
+    margin: 22px 0;
+    padding: 14px 17px;
+    border: 1px solid #c7d2fe;
+    border-radius: 10px;
+    background: var(--brand-soft);
+  }
+  .notice.warn { border-color: #fde68a; background: #fffbeb; }
+  .notice strong { display: block; color: var(--brand); }
+  .notice.warn strong { color: var(--amber); }
+  .cards {
+    display: grid;
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+    gap: 14px;
+    margin: 18px 0;
+  }
+  .card {
+    padding: 17px;
+    border: 1px solid var(--line);
+    border-radius: 11px;
+    background: var(--soft);
+  }
+  .card h3 { margin: 0 0 6px; font-size: 17px; }
+  .card p { margin: 0; color: var(--muted); font-size: 14px; }
+  .table {
+    margin: 16px 0;
+    overflow-x: auto;
+    border: 1px solid var(--line);
+    border-radius: 11px;
+  }
+  table { width: 100%; border-collapse: collapse; font-size: 13.5px; }
+  th, td { padding: 9px 12px; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; }
+  th { background: var(--soft); white-space: nowrap; }
+  tr:last-child td { border-bottom: 0; }
+  footer {
+    margin-top: 60px;
+    padding-top: 20px;
+    border-top: 1px solid var(--line);
+    color: var(--quiet);
+    font-size: 13px;
+  }
+  @media (max-width: 850px) {
+    header { padding: 0 16px; }
+    header .optional { display: none; }
+    .layout { display: block; }
+    nav { display: none; }
+    main { padding: 34px 20px 80px; }
+    .cards { grid-template-columns: 1fr; }
+  }
+</style>
+</head>
+<body>
+<header>
+  <svg viewBox="0 0 23 23" aria-hidden="true"><rect width="10" height="10" fill="#F25022"/><rect x="13" width="10" height="10" fill="#7FBA00"/><rect y="13" width="10" height="10" fill="#00A4EF"/><rect x="13" y="13" width="10" height="10" fill="#FFB900"/></svg>
+  <strong>Skill<span>Opt</span></strong>
+  <span class="spacer"></span>
+  <a class="optional" href="https://arxiv.org/abs/2605.23904">Paper</a>
+  <a class="optional" href="https://microsoft.github.io/SkillOpt/blog/">Blog</a>
+  <a href="https://github.com/microsoft/SkillOpt">GitHub</a>
+</header>
+
+<div class="layout">
+<nav aria-label="Guide sections">
+  <strong>Start here</strong>
+  <a href="#overview">Overview</a>
+  <a href="#choose">Choose a workflow</a>
+  <a href="#install">Install</a>
+  <a href="#credentials">Credentials</a>
+
+  <strong>Research engine</strong>
+  <a href="#research">First experiment</a>
+  <a href="#backends">Model backends</a>
+  <a href="#research-docs">Reference map</a>
+
+  <strong>SkillOpt-Sleep</strong>
+  <a href="#sleep">Safe first run</a>
+  <a href="#sleep-plugins">Agent integrations</a>
+  <a href="#sleep-replay">Advanced controls</a>
+  <a href="#safety">Data and safety</a>
+
+  <strong>Project</strong>
+  <a href="#contributing">Contributing</a>
+</nav>
+
+<main>
+  <span class="eyebrow">Microsoft Research · documentation hub</span>
+  <h1>SkillOpt Documentation &amp; Reproduction Guide</h1>
+  <p class="lead">Improve frozen agents by optimizing the Markdown skills that guide them—using reflective updates and held-out validation instead of weight training.</p>
+
+  <div class="notice">
+    <strong>How this guide stays accurate</strong>
+    This page is a stable, concise entry point. Detailed commands, defaults,
+    and APIs live in the versioned
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/index.md">Markdown documentation</a>
+    beside the code. For exact behavior in a checkout, the command's
+    <code>--help</code>, the selected YAML config, and the code are authoritative.
+  </div>
+
+  <section id="overview">
+    <h2>Overview</h2>
+    <p><strong>SkillOpt</strong> treats a natural-language skill document as the
+    trainable state of an agent. A target model executes tasks, an optimizer
+    reflects on the resulting trajectories, bounded edits form a candidate
+    skill, and a validation gate decides whether to keep it.</p>
+    <div class="cards">
+      <div class="card">
+        <h3>Research engine</h3>
+        <p>Run reproducible training and evaluation over benchmark splits.
+        Six released benchmark configurations cover QA, document QA, embodied
+        agents, math, spreadsheets, and tool-augmented QA.</p>
+      </div>
+      <div class="card">
+        <h3>SkillOpt-Sleep preview</h3>
+        <p>Harvest supported coding-agent sessions, mine replayable tasks, and
+        stage proposed memory or skill updates for review. It is a separate,
+        evolving deployment companion—not the paper's benchmark runner.</p>
+      </div>
+    </div>
+    <p>The optimizer and target are separate roles and may use different
+    backends. Validation gating is the research default and the paper-style
+    setting; deliberately disabling it force-accepts candidates and changes the
+    experiment semantics. SkillOpt-Sleep stages updates by default; automatic
+    adoption is opt-in.</p>
+  </section>
+
+  <section id="choose">
+    <h2>Choose the right workflow</h2>
+    <div class="table">
+      <table>
+        <thead><tr><th>Goal</th><th>Start with</th></tr></thead>
+        <tbody>
+          <tr><td>Reproduce paper-style benchmark training</td><td><a href="#research">Research first experiment</a></td></tr>
+          <tr><td>Evaluate an existing skill without training</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/reference/cli.md">Evaluation CLI reference</a></td></tr>
+          <tr><td>Add a benchmark adapter</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/new-benchmark.md">New benchmark guide</a></td></tr>
+          <tr><td>Connect another model provider</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/new-backend.md">Backend guide</a></td></tr>
+          <tr><td>Improve a coding-agent skill from local sessions</td><td><a href="#sleep">SkillOpt-Sleep</a></td></tr>
+        </tbody>
+      </table>
+    </div>
+  </section>
+
+  <section id="install">
+    <h2>Install</h2>
+    <p>SkillOpt requires Python 3.10 or newer.</p>
+<pre><code># Published package
+python -m pip install skillopt
+
+# Latest source and development workflow
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+python -m pip install -e .
+
+# Install only the extras you need
+python -m pip install -e ".[searchqa]"  # SearchQA materialization
+python -m pip install -e ".[alfworld]"  # ALFWorld
+python -m pip install -e ".[claude]"    # optional Claude agent SDK support
+python -m pip install -e ".[webui]"     # Gradio dashboard
+python -m pip install -e ".[dev]"       # tests and linting</code></pre>
+    <div class="notice warn">
+      <strong>Release boundary</strong>
+      This guide tracks <code>main</code>. PyPI currently serves 0.2.0; the
+      generic research <code>openai_compatible</code> backend, Sleep handoff,
+      SkillOpt-Sleep support for non-Azure OpenAI-compatible endpoints, and the
+      Sleep <code>--preferences</code> flag require a source install from
+      <code>main</code> until the next release.
+    </div>
+    <p>See the <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/installation.md">installation guide</a>
+    for platform notes and dependency boundaries.</p>
+  </section>
+
+  <section id="credentials">
+    <h2>Credentials and endpoint families</h2>
+    <p>Copy <code>.env.example</code>, fill only the backend you use, and load
+    it into your shell. Do not commit the resulting <code>.env</code>.</p>
+<pre><code>cp .env.example .env
+set -a
+source .env
+set +a</code></pre>
+
+    <h3>Azure OpenAI</h3>
+<pre><code>export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
+export AZURE_OPENAI_API_VERSION="2024-12-01-preview"
+export AZURE_OPENAI_AUTH_MODE="api_key"
+export AZURE_OPENAI_API_KEY="your-key"</code></pre>
+    <p>For keyless Azure authentication, use <code>azure_cli</code> or
+    <code>managed_identity</code> and follow the
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/configuration.md">configuration guide</a>.
+    Setting an API key without <code>AZURE_OPENAI_AUTH_MODE=api_key</code> does
+    not change the default authentication mode.</p>
+
+    <h3>Generic OpenAI-compatible research backend</h3>
+<pre><code>export OPENAI_COMPATIBLE_BASE_URL="https://api.example.com/v1"
+export OPENAI_COMPATIBLE_API_KEY="your-key"
+export OPENAI_COMPATIBLE_MODEL="provider-model"
+
+python scripts/train.py --config configs/searchqa/default.yaml &#92;
+  --cfg-options &#92;
+  model.optimizer_backend=openai_compatible &#92;
+  model.target_backend=openai_compatible &#92;
+  model.optimizer=provider-model &#92;
+  model.target=provider-model</code></pre>
+    <p>This provider-neutral backend is distinct from Azure OpenAI. Per-role
+    overrides use <code>OPTIMIZER_OPENAI_COMPATIBLE_*</code> and
+    <code>TARGET_OPENAI_COMPATIBLE_*</code>. Train/eval applies the YAML role
+    models after backend initialization, so they override model-name
+    environment variables.</p>
+
+    <h3>OpenAI-compatible endpoints in SkillOpt-Sleep</h3>
+    <p>The Sleep CLI exposes this compatibility path through its
+    <code>azure_openai</code> backend for backward compatibility, so it uses a
+    different environment-variable family:</p>
+<pre><code>export AZURE_OPENAI_ENDPOINT="https://api.example.com/v1"
+export AZURE_OPENAI_API_KEY="your-key"
+export AZURE_OPENAI_AUTH_MODE="openai_compatible"
+
+skillopt-sleep run --backend azure_openai --model provider-model</code></pre>
+    <p>Do not mix this mode with Azure CLI or managed-identity settings. See
+    the dedicated
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/sleep/openai-compatible-endpoints.md">Sleep endpoint guide</a>.</p>
+  </section>
+
+  <section id="research">
+    <h2>Research engine: first experiment</h2>
+    <p>The repository ships deterministic ID manifests, not the benchmark
+    examples themselves. Materialize the SearchQA examples once, then run its
+    checked-in config:</p>
+<pre><code>python -m pip install -e ".[searchqa]"
+python scripts/materialize_searchqa.py
+
+# Load model credentials first, then:
+python scripts/train.py --config configs/searchqa/default.yaml</code></pre>
+    <p>The run directory contains <code>best_skill.md</code>,
+    <code>runtime_state.json</code>, <code>history.json</code>, versioned files
+    under <code>skills/</code>, and step-level artifacts. Re-running with the
+    same output root resumes from persisted state.</p>
+<pre><code>python scripts/eval_only.py &#92;
+  --config configs/searchqa/default.yaml &#92;
+  --skill outputs/&lt;run&gt;/best_skill.md &#92;
+  --split valid_unseen</code></pre>
+    <div class="notice warn">
+      <strong>Reproduction boundary</strong>
+      Use the released train/validation/test manifests and record the exact
+      model deployment, config, seed, and source revision. Provider behavior
+      can change independently of this repository.
+    </div>
+    <p>Continue with the
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/first-experiment.md">first-experiment guide</a>
+    and <a href="https://github.com/microsoft/SkillOpt/blob/main/data/README.md">dataset manifest documentation</a>.</p>
+  </section>
+
+  <section id="backends">
+    <h2>Research model backends</h2>
+    <div class="table">
+      <table>
+        <thead><tr><th>Backend</th><th>Optimizer</th><th>Target</th><th>Notes</th></tr></thead>
+        <tbody>
+          <tr><td><code>openai_chat</code></td><td>Yes</td><td>Yes</td><td>Azure OpenAI plus its explicit authentication modes.</td></tr>
+          <tr><td><code>openai_compatible</code></td><td>Yes</td><td>Yes</td><td>Provider-neutral chat-completions endpoint.</td></tr>
+          <tr><td><code>claude_chat</code></td><td>Yes</td><td>Yes</td><td>Runs an installed, authenticated Claude Code CLI via <code>claude -p</code>; not a direct Anthropic API client.</td></tr>
+          <tr><td><code>qwen_chat</code></td><td>Yes</td><td>Yes</td><td>Local or hosted Qwen-compatible server.</td></tr>
+          <tr><td><code>minimax_chat</code></td><td>Yes</td><td>Yes</td><td>MiniMax chat endpoint.</td></tr>
+          <tr><td><code>codex_exec</code></td><td>Yes</td><td>Supported adapters only</td><td>Executes Codex for optimizer calls and as a target agent where supported.</td></tr>
+          <tr><td><code>claude_code_exec</code></td><td>No</td><td>Supported adapters only</td><td>Executes Claude Code as a target agent.</td></tr>
+        </tbody>
+      </table>
+    </div>
+    <p>Prefer the structured <code>model.optimizer_backend</code> and
+    <code>model.target_backend</code> settings. Legacy <code>--backend</code>
+    aliases do not expose every role-specific combination. Exec backends are
+    not generic chat replacements and require adapter support.</p>
+  </section>
+
+  <section id="research-docs">
+    <h2>Research documentation map</h2>
+    <div class="table">
+      <table>
+        <thead><tr><th>Reference</th><th>Use it for</th></tr></thead>
+        <tbody>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/configuration.md">Configuration</a></td><td>Authentication, structured YAML, role-specific backends, and overrides.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/reference/cli.md">CLI</a></td><td>Current train/eval entry points and exact output paths.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/reference/config.md">Config reference</a></td><td>Supported sections, defaults, and validation constraints.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/training-loop.md">Training loop</a></td><td>Rollout, reflection, edit selection, gating, slow update, and meta skill.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/skill-document.md">Skill document</a></td><td>Skill structure and protected regions.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/docs/reference/api.md">Python API</a></td><td>Stable public imports and low-level/internal boundaries.</td></tr>
+          <tr><td><a href="https://github.com/microsoft/SkillOpt/blob/main/CHANGELOG.md">Changelog</a></td><td>Recently merged capabilities, fixes, and contributor credits.</td></tr>
+        </tbody>
+      </table>
+    </div>
+  </section>
+
+  <section id="sleep">
+    <h2>SkillOpt-Sleep: safe first run</h2>
+    <p>SkillOpt-Sleep is a preview deployment companion. Its default
+    <code>mock</code> backend is useful for testing control flow without API
+    spend; it is not evidence that a real model's quality improved.</p>
+<pre><code># Deterministic engine proof; no model credentials required
+python -m skillopt_sleep.experiments.run_experiment &#92;
+  --persona researcher --assert-improves
+
+# Inspect local session handling without adopting any update
+skillopt-sleep dry-run &#92;
+  --project "$PWD" &#92;
+  --source auto &#92;
+  --backend mock
+
+# A real run: explicitly identify the skill to evolve
+skillopt-sleep run &#92;
+  --project "$PWD" &#92;
+  --target-skill-path path/to/SKILL.md &#92;
+  --source auto &#92;
+  --backend claude
+
+skillopt-sleep status --project "$PWD"
+skillopt-sleep adopt --project "$PWD"</code></pre>
+    <p><code>--project</code> scopes collection but does not automatically
+    choose a project's skill file. Use <code>--target-skill-path</code> when
+    you intend to evolve a particular <code>SKILL.md</code>. Transcript source
+    (<code>claude</code>, <code>codex</code>, or <code>auto</code>) and replay
+    backend are independent settings.</p>
+    <p>For subscription-based workflows that should not launch an API or model
+    subprocess, use <code>--backend handoff</code> and follow the generated
+    prompt/answer loop. Read the
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/sleep/README.md">complete Sleep guide</a>
+    before a real run.</p>
+  </section>
+
+  <section id="sleep-plugins">
+    <h2>Agent integrations</h2>
+    <div class="table">
+      <table>
+        <thead><tr><th>Agent</th><th>Integration status</th><th>Guide</th></tr></thead>
+        <tbody>
+          <tr><td>Claude Code</td><td>Shared-engine plugin and handoff command</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/claude-code/README.md">README</a></td></tr>
+          <tr><td>Codex</td><td>Shared-engine skill shell</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/codex/README.md">README</a></td></tr>
+          <tr><td>GitHub Copilot</td><td>Shared-engine Sleep MCP plus a separate research MCP</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/copilot/README.md">README</a></td></tr>
+          <tr><td>Devin</td><td>Shared-engine MCP with Devin transcript conversion</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/devin/README.md">README</a></td></tr>
+          <tr><td>OpenClaw</td><td>Independent community/reference adaptation; review locally before use</td><td><a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/openclaw/README.md">README</a></td></tr>
+        </tbody>
+      </table>
+    </div>
+    <p>The <a href="https://github.com/microsoft/SkillOpt/blob/main/plugins/README.md">plugin overview</a>
+    records which integrations use the shared engine and which require local
+    adaptation.</p>
+  </section>
+
+  <section id="sleep-replay">
+    <h2>Advanced Sleep controls</h2>
+    <p>The main CLI exposes project/source selection, backend/model selection,
+    bounded task and edit counts, preferences, reviewed task files, and
+    staged adoption. Additional JSON configuration fields include:</p>
+    <div class="table">
+      <table>
+        <thead><tr><th>Field</th><th>Default</th><th>Status</th></tr></thead>
+        <tbody>
+          <tr><td><code>dream_rollouts</code></td><td>1</td><td>Single rollout by default; values above 1 enable experimental contrastive replay.</td></tr>
+          <tr><td><code>dream_factor</code></td><td>0</td><td>Synthetic task variants are off by default.</td></tr>
+          <tr><td><code>recall_k</code></td><td>0</td><td>Historical associative recall is off by default.</td></tr>
+        </tbody>
+      </table>
+    </div>
+    <p>These are configuration fields, not current <code>skillopt-sleep run</code>
+    flags. Treat multi-rollout, recall, synthetic dreaming, and experimental
+    reward/budget controls as advanced features that require task-specific
+    validation. The reported experiments and their exact settings are in
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/sleep/RESULTS.md">RESULTS.md</a>.</p>
+  </section>
+
+  <section id="safety">
+    <h2>Data, privacy, and adoption safety</h2>
+    <ul>
+      <li>Real Sleep backends may send session-derived prompts, mined tasks,
+      trajectories, and candidate edits to the selected provider. Review the
+      source data and provider policy before use.</li>
+      <li>Secret redaction for persisted diagnostics is defense in depth; it
+      is not a guarantee that every outbound model prompt is free of sensitive
+      content. In particular, do not treat raw coding-agent transcripts as
+      pre-sanitized.</li>
+      <li>Updates are staged for review by default. Use
+      <code>--auto-adopt</code> only when you have an independent rollback and
+      validation process.</li>
+      <li>A held-out gate reduces regressions on its measured tasks; it is not
+      a security boundary or a proof of general improvement.</li>
+      <li>Use a temporary clone and synthetic transcripts when validating a
+      new backend or plugin integration.</li>
+    </ul>
+  </section>
+
+  <section id="contributing">
+    <h2>Contributing and extending</h2>
+    <p>Before proposing a change, run the focused tests for the affected area,
+    then the full suite where practical. Documentation changes should pass a
+    strict MkDocs build and should be checked against actual CLI
+    <code>--help</code> output.</p>
+<pre><code>python -m pip install -e ".[dev,docs]"
+python -m pytest -q
+python -m mkdocs build --strict</code></pre>
+    <p>See <a href="https://github.com/microsoft/SkillOpt/blob/main/CONTRIBUTING.md">CONTRIBUTING.md</a>,
+    the <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/contributing.md">documentation workflow</a>,
+    and the focused guides for
+    <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/new-benchmark.md">benchmarks</a>
+    and <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/guide/new-backend.md">model backends</a>.</p>
+  </section>
+
+  <footer>
+    SkillOpt · <a href="https://github.com/microsoft/SkillOpt">github.com/microsoft/SkillOpt</a>
+    · <a href="https://microsoft.github.io/SkillOpt/blog/">Technical Blog</a>
+    · <a href="https://arxiv.org/abs/2605.23904">arXiv:2605.23904</a><br>
+    This public overview intentionally avoids duplicating the complete,
+    fast-changing configuration surface. Follow the linked versioned
+    references for details.
+  </footer>
+</main>
+</div>
+</body>
+</html>

docs/index.md

@@ -18,6 +18,19 @@ hide:
 
 ---
 
+## Two Complementary Workflows
+
+| Workflow | Package / command | Use it for |
+|---|---|---|
+| **Research engine** | `skillopt`, `skillopt-train`, `skillopt-eval` | Train and evaluate skill documents on explicit benchmark splits. |
+| **SkillOpt-Sleep (preview)** | `skillopt_sleep`, `skillopt-sleep` | Review supported coding-agent sessions and stage proposed memory/skill updates for human adoption. |
+
+They share the idea of bounded text updates and validation, but they are
+separate entry points with different configs and safety boundaries. Start with
+the [SkillOpt-Sleep overview](sleep/README.md) before using real session data.
+
+---
+
 ## How It Works
 
 <div class="pipeline-container" markdown>
@@ -106,29 +119,52 @@ SkillOpt brings the familiar deep-learning training paradigm to agentic prompt o
 | **ALFWorld** | Embodied AI | `configs/alfworld/` |
 | **OfficeQA** | Enterprise QA | `configs/officeqa/` |
 | **SearchQA** | Open-domain QA | `configs/searchqa/` |
-| **LiveMathBench** | Math reasoning | `configs/livemathematicianbench/` |
+| **LiveMathematicianBench** | Math reasoning | `configs/livemathematicianbench/` |
-| **SWEBench** | Software Engineering | `configs/swebench/` |
+| **SpreadsheetBench** | Spreadsheet editing | `configs/spreadsheetbench/` |
-| + 5 more | Various | See [docs](guide/first-experiment.md) |
+
+---
+
+## Model Backends
+
+Optimizer and target roles are configured separately. Chat backends include
+Azure OpenAI (`openai_chat`), the provider-neutral
+`openai_compatible` backend, the Claude Code CLI (`claude_chat`), Qwen, and
+MiniMax. Codex and Claude Code exec harnesses are target-only and require
+adapter support. Despite its name, `claude_chat` launches `claude -p`; it is
+not a direct Anthropic API client.
+
+If a provider implements OpenAI Chat Completions, begin with the
+[built-in compatible backend](guide/new-backend.md#built-in-the-generic-openai-compatible-backend)
+instead of adding a new integration. See [Configuration](guide/configuration.md)
+for authentication and per-role overrides.
 
 ---
 
 ## Quick Example
 
 ```bash
-# Install
+# Clone and install the research checkout plus the SearchQA data extra
-pip install -e .
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+python -m pip install -e ".[searchqa]"
 
-# Configure credentials
+# Configure credentials (choose one auth mode in .env)
-export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
+cp .env.example .env
-export AZURE_OPENAI_API_KEY="your-key"
+set -a; source .env; set +a
 
-# Train on SearchQA
+# Materialize the runnable split from the checked-in ID manifest
-python scripts/train.py --config configs/searchqa/default.yaml
+python scripts/materialize_searchqa.py
+
+# Train on SearchQA into a predictable output directory
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --out_root outputs/searchqa_quickstart
 
 # Evaluate best skill
 python scripts/eval_only.py \
   --config configs/searchqa/default.yaml \
-  --skill outputs/best_skill.md
+  --skill outputs/searchqa_quickstart/best_skill.md \
+  --split valid_unseen
 ```
 
 ---
@@ -167,4 +203,13 @@ python scripts/eval_only.py \
 
     [:octicons-arrow-right-24: WebUI Guide](guide/first-experiment.md#webui)
 
+-   :material-weather-night:{ .lg .middle } **SkillOpt-Sleep**
+
+    ---
+
+    Test the deployment companion with the no-provider mock path, then review
+    its data boundary before selecting a real backend.
+
+    [:octicons-arrow-right-24: Sleep Overview](sleep/README.md)
+
 </div>

docs/reference/api.md

@@ -17,7 +17,9 @@ browse [`skillopt/envs/`](https://github.com/microsoft/SkillOpt/tree/main/skillo
 
 `skillopt/envs/base.py` — abstract adapter that connects the SkillOpt
 trainer to an environment (benchmark, simulator, REST API, ...).
-Subclasses **must** implement the five abstract methods below.
+Subclasses **must** implement the four abstract methods below. Reflection has a
+shared default implementation and only needs to be overridden for
+environment-specific behavior.
 
 ```python
 from abc import ABC, abstractmethod
@@ -30,6 +32,10 @@ class EnvAdapter(ABC):
     def setup(self, cfg: dict) -> None: ...
     def get_dataloader(self) -> BaseDataLoader | None: ...
     def requires_ray(self) -> bool: ...                 # default False
+    def reflect(self, results: list[dict], skill_content: str,
+                out_dir: str, **kwargs) -> list[dict | None]:
+        """Delegate to the shared minibatch reflection pipeline."""
+        ...
 
     # ── Abstract methods (subclasses MUST implement) ────────────────────
 
@@ -53,26 +59,21 @@ class EnvAdapter(ABC):
         It MAY contain env-specific extra keys (parsed into RolloutResult.extras).
         """
 
-    @abstractmethod
-    def reflect(self, results: list[dict], skill_content: str,
-                out_dir: str, **kwargs) -> list[dict | None]:
-        """Turn rollout results into a list of raw patch dicts.
-
-        Each dict (or None to drop the slot) MUST contain:
-          - "patch":       {"edits": [...]}     a Patch.to_dict() payload
-          - "source_type": "failure" | "success"
-        """
-
     @abstractmethod
     def get_task_types(self) -> list[str]:
         """Distinct task-type strings used for stratified sampling."""
 ```
 
-The trainer also calls a few default-implemented helpers on every adapter:
+The default `reflect()` delegates to `run_minibatch_reflect` and returns raw
+patch dicts with a `patch` payload plus a `failure` or `success` source type.
+It expects each rollout to persist a non-empty trajectory at
+`<rollout_dir>/predictions/<result-id>/conversation.json`; results without that
+file can be scored but are skipped during reflection.
+The trainer also calls several default-implemented helpers on every adapter:
 `build_reference_text`, `get_reference_metadata`, `attach_reference_context`,
 `select_representative_items`, and `build_env_from_batch`. Read the docstrings
 in `skillopt/envs/base.py` if you need to override any of these — most
-benchmarks don't.
+benchmarks do not.
 
 ### `BaseDataLoader` / `SplitDataLoader`
 
@@ -161,16 +162,17 @@ into `RolloutResult.extras`.
 ### `GateResult` / `GateAction`
 
 `skillopt/evaluation/gate.py` — the validation-gate decision types
-returned each epoch.
+returned for each candidate optimization step, and optionally for a separate
+epoch-end slow-update candidate.
 
 ---
 
 ## Registering an environment
 
 Environments are not registered via decorators or a `BENCHMARK_REGISTRY`
-dict. The trainer keeps a lazy registry inside `scripts/train.py` —
+dict. The training and standalone-evaluation entry points each keep a lazy
-`_ENV_REGISTRY` — populated by `_register_builtins()`. To add a new env
+`_ENV_REGISTRY`, populated by `_register_builtins()` in `scripts/train.py` and
-you append a `try / except ImportError` block there. See
+`scripts/eval_only.py`. Add the environment to both entry points. See
 [Add a New Benchmark](../guide/new-benchmark.md) for the full step-by-step.
 
 ---
@@ -187,7 +189,8 @@ not via a base class subclass. Supported values (as of this writing):
 | `claude_chat` | ✓ | ✓ |
 | `qwen_chat` | ✓ | ✓ |
 | `minimax_chat` | ✓ | ✓ |
-| `codex_exec` | — | ✓ |
+| `openai_compatible` | ✓ | ✓ |
+| `codex_exec` | ✓ | ✓ |
 | `claude_code_exec` | — | ✓ |
 
 See `skillopt/model/backend_config.py` for the live whitelist and

docs/reference/cli.md

@@ -1,9 +1,17 @@
 # CLI Reference
 
+> **Version note.** This reference tracks `main`. PyPI 0.2.0 does not yet
+> include the generic research `openai_compatible` backend, Sleep handoff,
+> Sleep support for non-Azure OpenAI-compatible endpoints, or the Sleep
+> `--preferences` flag; use a source install from `main` for those features
+> until the next release.
+
 ## Training
 
 ```bash
 python scripts/train.py --config <config.yaml> [overrides...]
+# Installed equivalent:
+skillopt-train --config <config.yaml> [overrides...]
 ```
 
 ### Arguments
@@ -11,13 +19,15 @@ python scripts/train.py --config <config.yaml> [overrides...]
 | Argument | Description |
 |---|---|
 | `--config` | Path to YAML config file (required) |
-| `key=value` | Override any config parameter |
+| `--cfg-options key=value [...]` | Override structured config parameters |
 
 ### Examples
 
 ```bash
 # Basic training
-python scripts/train.py --config configs/searchqa/default.yaml
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --out_root outputs/searchqa_run
 
 # With overrides
 python scripts/train.py \
@@ -34,6 +44,8 @@ python scripts/train.py \
 
 ```bash
 python scripts/eval_only.py --config <config.yaml> --skill <skill.md>
+# Installed equivalent:
+skillopt-eval --config <config.yaml> --skill <skill.md>
 ```
 
 ### Arguments
@@ -42,7 +54,8 @@ python scripts/eval_only.py --config <config.yaml> --skill <skill.md>
 |---|---|
 | `--config` | Path to YAML config file (required) |
 | `--skill` | Path to skill document to evaluate (required) |
-| `--split` | Evaluation split: `test` (default), `valid`, `train` |
+| `--split` | `train`, `valid_seen`, `valid_unseen`, or `all` (default) |
+| `--cfg-options` | One or more `section.key=value` overrides |
 
 ### Examples
 
@@ -50,15 +63,64 @@ python scripts/eval_only.py --config <config.yaml> --skill <skill.md>
 # Evaluate best skill on test set
 python scripts/eval_only.py \
   --config configs/searchqa/default.yaml \
-  --skill outputs/searchqa/run_001/skills/best_skill.md
+  --skill outputs/searchqa_run/best_skill.md \
+  --split valid_unseen
 
 # Evaluate on validation set
 python scripts/eval_only.py \
   --config configs/searchqa/default.yaml \
-  --skill outputs/searchqa/run_001/skills/best_skill.md \
+  --skill outputs/searchqa_run/best_skill.md \
-  --split valid
+  --split valid_seen
 ```
 
+`--skill` consumes the artifact produced by training. Unless `--out_root` is
+set for evaluation, `eval_only.py` creates a separate timestamped
+`outputs/eval_<env>_<model>_<timestamp>/` directory and writes
+`eval_summary.json` there; it does not modify the training run directory.
+
+For the generic OpenAI-compatible research backend, select the role backends
+explicitly:
+
+```bash
+python scripts/train.py \
+  --config configs/searchqa/default.yaml \
+  --cfg-options \
+    model.optimizer_backend=openai_compatible \
+    model.target_backend=openai_compatible \
+    model.optimizer=deepseek-chat \
+    model.target=deepseek-chat
+```
+
+## SkillOpt-Sleep
+
+```bash
+skillopt-sleep <action> [options]
+# Equivalent from a source checkout:
+python -m skillopt_sleep <action> [options]
+```
+
+Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and
+`unschedule`. Common options include:
+
+| Argument | Description |
+|---|---|
+| `--project PATH` | Project to evolve (default: current directory) |
+| `--scope invoked\|all` | Harvest this project or all projects |
+| `--source claude\|codex\|auto` | Transcript source |
+| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | Replay/optimizer backend |
+| `--model NAME` | Backend-specific model override |
+| `--preferences TEXT` | House rules supplied to reflection |
+| `--lookback-hours N` | Initial transcript lookback; `0` scans all history |
+| `--max-sessions N` / `--max-tasks N` | Bound the harvested workload |
+| `--target-skill-path PATH` | Explicit skill document to stage/adopt |
+| `--tasks-file PATH` | Replay a reviewed task JSON file instead of harvesting |
+| `--edit-budget N` | Maximum bounded edits for the night |
+| `--progress` / `--json` | Progress or machine-readable output |
+| `--auto-adopt` | Apply an accepted staged proposal automatically |
+
+Backend-specific setup for compatible endpoints is documented in
+[OpenAI-compatible endpoints for SkillOpt-Sleep](../sleep/openai-compatible-endpoints.md).
+
 ## WebUI
 
 ```bash
@@ -68,4 +130,8 @@ python -m skillopt_webui.app [--port PORT] [--share]
 | Argument | Default | Description |
 |---|---|---|
 | `--port` | 7860 | Port number |
+| `--host` | `0.0.0.0` | Server bind address |
 | `--share` | false | Create public Gradio link |
+
+The default host binds every network interface. Use `--host 127.0.0.1` when
+the dashboard should be reachable only from the local machine.

docs/reference/config.md

@@ -1,85 +1,174 @@
 # Configuration Reference
 
-Complete reference for all SkillOpt configuration parameters.
+SkillOpt loads structured YAML, resolves `_base_` inheritance, and flattens
+the result for the trainer. Shipped defaults live in
+`configs/_base_/default.yaml`; benchmark configs override them.
 
-## Model
+## Model and Backend Selection
+
+Use explicit optimizer and target backends when the two roles differ or when
+selecting the generic OpenAI-compatible backend.
+
+| Backend | Optimizer | Target |
+|---|:---:|:---:|
+| `openai_chat` | ✓ | ✓ |
+| `openai_compatible` | ✓ | ✓ |
+| `claude_chat` | ✓ | ✓ |
+| `qwen_chat` | ✓ | ✓ |
+| `minimax_chat` | ✓ | ✓ |
+| `codex_exec` | ✓ | ✓ |
+| `claude_code_exec` | — | ✓ |
+
+MiniMax currently has one shared deployment. `model.minimax_model` is applied
+when MiniMax is the target; mixed-backend runs cannot independently choose a
+MiniMax optimizer model and a different target model.
 
 | Parameter | Type | Default | Description |
 |---|---|---|---|
-| `model.backend` | str | `azure_openai` | Backend: `azure_openai` / `openai_chat` / `claude_code_exec` / `qwen` |
+| `model.backend` | str | `azure_openai` | Backward-compatible high-level run label |
-| `model.optimizer` | str | `gpt-5.5` | Optimizer model (for reflection & slow update) |
+| `model.optimizer` | str | `gpt-5.5` | Optimizer deployment/model |
-| `model.target` | str | `gpt-5.5` | Target model (for rollout execution) |
+| `model.target` | str | `gpt-5.5` | Target deployment/model |
-| `model.reasoning_effort` | str | `medium` | Reasoning effort level |
+| `model.optimizer_backend` | str | `openai_chat` | Optimizer client path; chat backends plus `codex_exec` |
-| `model.optimizer_backend` | str | `openai_chat` | Optimizer backend: `openai_chat` / `claude_chat` / `qwen_chat` / `minimax_chat` |
+| `model.target_backend` | str | `openai_chat` | Target client path; chat or exec backend |
-| `model.target_backend` | str | `openai_chat` | Target backend: chat backends plus execution harnesses |
+| `model.reasoning_effort` | str | `medium` | Shared reasoning effort |
-| `model.qwen_chat_base_url` | str | `http://localhost:8000/v1` | Shared Qwen/vLLM OpenAI-compatible endpoint |
+| `model.rewrite_reasoning_effort` | str | empty | Optional full-rewrite effort override |
-| `model.qwen_chat_enable_thinking` | bool | `false` | Shared Qwen thinking flag |
+| `model.rewrite_max_completion_tokens` | int | `64000` | Full-rewrite output cap |
-| `model.optimizer_qwen_chat_base_url` | str | — | Optimizer-specific Qwen/vLLM endpoint; overrides shared `qwen_chat_base_url` |
+
-| `model.target_qwen_chat_base_url` | str | — | Target-specific Qwen/vLLM endpoint; overrides shared `qwen_chat_base_url` |
+### Azure/OpenAI `openai_chat`
+
+| Parameter | Default | Description |
+|---|---|---|
+| `model.azure_openai_endpoint` | empty | Shared Azure resource URL or compatibility-mode base URL |
+| `model.azure_openai_api_version` | `2024-12-01-preview` | Azure API version |
+| `model.azure_openai_api_key` | empty | Key for `api_key` or compatibility auth |
+| `model.azure_openai_auth_mode` | empty | Config value; empty falls back to env, whose default is `azure_cli` |
+| `model.azure_openai_ad_scope` | Azure Cognitive Services scope | AAD token scope |
+| `model.azure_openai_managed_identity_client_id` | empty | Optional user-assigned identity client ID |
+
+Every shared key also has an `optimizer_azure_openai_*` and
+`target_azure_openai_*` form.
+
+### Claude `claude_chat`
+
+`claude_chat` launches an installed, authenticated Claude Code CLI with
+`claude -p`; it does not instantiate an Anthropic API client. The executable
+defaults to `claude` and can be overridden with `CLAUDE_CLI_BIN`.
+`ANTHROPIC_API_KEY` is one authentication option understood by the CLI.
+
+### Qwen, MiniMax, and Exec Backends
+
+| Parameter family | Description |
+|---|---|
+| `model.qwen_chat_*` | Shared `base_url`, `api_key`, `temperature`, `timeout_seconds`, `max_tokens`, and `enable_thinking` |
+| `model.optimizer_qwen_chat_*` / `model.target_qwen_chat_*` | Per-role Qwen overrides |
+| `model.minimax_*` | MiniMax `base_url`, `api_key`, shared `minimax_model`, `temperature`, `max_tokens`, and `enable_thinking`; `minimax_model` applies when MiniMax is the target |
+| `model.codex_exec_*` | Codex path, sandbox, profile, SDK mode, reasoning, network/search, and approval policy |
+| `model.claude_code_exec_*` | Claude path, profile, SDK mode, effort, and thinking-token cap |
 
 ## Training (`train`)
 
-| Parameter | Type | Default | DL Analogy | Description |
+| Parameter | Type | Default | Description |
-|---|---|---|---|---|
+|---|---|---|---|
-| `train.num_epochs` | int | 4 | Epochs | Number of training epochs |
+| `train.num_epochs` | int | `4` | Training epochs |
-| `train.batch_size` | int | 40 | Batch size | Tasks sampled per step |
+| `train.train_size` | int | `0` | `0` derives the size from the dataset split |
-| `train.accumulation` | int | 1 | Gradient accumulation | Accumulation rounds per step |
+| `train.steps_per_epoch` | int | derived | Runtime field recomputed from train size, batch size, and accumulation; configured values are overwritten |
-| `train.seed` | int | 42 | Random seed | Reproducibility seed |
+| `train.batch_size` | int | `40` | Tasks sampled per step |
+| `train.accumulation` | int | `1` | Accumulation rounds per step |
+| `train.seed` | int | `42` | Random seed |
 
 ## Gradient / Reflection (`gradient`)
 
 | Parameter | Type | Default | Description |
 |---|---|---|---|
-| `gradient.minibatch_size` | int | 8 | Reflect minibatch size |
+| `gradient.minibatch_size` | int | `8` | Reflect minibatch size |
-| `gradient.merge_batch_size` | int | 8 | Patch merge batch size |
+| `gradient.merge_batch_size` | int | `8` | Patch merge batch size |
-| `gradient.analyst_workers` | int | 16 | Parallel reflection workers |
+| `gradient.analyst_workers` | int | `16` | Parallel reflection workers |
-| `gradient.max_analyst_rounds` | int | 3 | Max rounds of analyst reflection |
+| `gradient.max_analyst_rounds` | int | `3` | Maximum analyst rounds |
-| `gradient.failure_only` | bool | `false` | Only reflect on failures |
+| `gradient.failure_only` | bool | `false` | Reflect only on failures |
 
 ## Optimizer (`optimizer`)
 
-| Parameter | Type | Default | DL Analogy | Description |
+| Parameter | Type | Default | Description |
-|---|---|---|---|---|
+|---|---|---|---|
-| `optimizer.learning_rate` | int | 4 | Learning rate | Max edit patches per step (edit budget) |
+| `optimizer.learning_rate` | int | `4` | Maximum edit patches per step |
-| `optimizer.min_learning_rate` | int | 2 | Min LR | Min edits for decay schedulers |
+| `optimizer.min_learning_rate` | int | `2` | Floor for decaying schedules |
-| `optimizer.lr_scheduler` | str | `cosine` | LR schedule | `constant` / `linear` / `cosine` / `autonomous` |
+| `optimizer.lr_scheduler` | str | `cosine` | `constant`, `linear`, `cosine`, or `autonomous` |
-| `optimizer.skill_update_mode` | str | `patch` | — | `patch` / `rewrite_from_suggestions` / `full_rewrite_minibatch` |
+| `optimizer.lr_control_mode` | str | `fixed` | `fixed`, `autonomous`, or `none` |
-| `optimizer.use_slow_update` | bool | `true` | Momentum | Epoch-boundary longitudinal comparison & guidance |
+| `optimizer.skill_update_mode` | str | `patch` | `patch`, `rewrite_from_suggestions`, or `full_rewrite_minibatch` |
-| `optimizer.slow_update_samples` | int | 20 | — | Samples for slow update evaluation |
+| `optimizer.use_slow_update` | bool | `true` | Epoch-boundary longitudinal update |
-| `optimizer.use_meta_skill` | bool | `true` | Meta-learning | Cross-epoch optimizer-side strategy memory |
+| `optimizer.slow_update_samples` | int | `20` | Longitudinal evaluation samples |
-| `optimizer.longitudinal_pair_policy` | str | `mixed` | — | `mixed` / `changed` / `unchanged` |
+| `optimizer.slow_update_gate_with_selection` | bool | `false` | Gate slow-update guidance on the selection split |
+| `optimizer.longitudinal_pair_policy` | str | `mixed` | `mixed`, `changed`, or `unchanged` |
+| `optimizer.use_meta_skill` | bool | `true` | Cross-epoch optimizer memory |
+| `optimizer.use_skill_aware_reflection` | bool | `false` | Enable skill-defect vs execution-lapse routing |
+| `optimizer.skill_aware_appendix_source` | str | `both` | `both` or `failure_only` |
+| `optimizer.skill_aware_consolidate_threshold` | int | `0` | Appendix compaction threshold; `0` disables it |
 
 ## Evaluation (`evaluation`)
 
 | Parameter | Type | Default | Description |
 |---|---|---|---|
-| `evaluation.use_gate` | bool | `true` | Enable validation gating (accept/reject updates) |
+| `evaluation.use_gate` | bool | `true` | Accept only improvements when enabled; `false` records validation but force-accepts each candidate |
-| `evaluation.eval_test` | bool | `true` | Run test evaluation after training |
+| `evaluation.gate_metric` | str | `hard` | `hard`, `soft`, or `mixed` |
+| `evaluation.gate_mixed_weight` | float | `0.5` | Soft-score weight for `mixed` |
+| `evaluation.use_semantic_density` | bool | `false` | Add the optional instruction-density bonus |
+| `evaluation.semantic_density_weight` | float | `0.05` | Density bonus weight |
+| `evaluation.leading_words` | list/str | built in | Optional custom high-influence words |
+| `evaluation.sel_env_num` | int | `0` | Selection size; `0` uses the full split |
+| `evaluation.test_env_num` | int | `0` | Test size; `0` uses the full split |
+| `evaluation.eval_test` | bool | `true` | Run final test evaluation |
 
 ## Environment (`env`)
 
 | Parameter | Type | Default | Description |
 |---|---|---|---|
-| `env.name` | str | — | Benchmark name (e.g., `searchqa`, `docvqa`) |
+| `env.name` | str | empty | Benchmark name |
-| `env.data_path` | str | — | Path to dataset |
+| `env.skill_init` | str | empty | Initial skill document |
-| `env.skill_init` | str | — | Path to initial seed skill (optional) |
 | `env.split_mode` | str | `ratio` | `ratio` or `split_dir` |
-| `env.split_ratio` | str | `2:1:7` | Train:val:test ratio |
+| `env.split_ratio` | str | benchmark/default | Train:validation:test ratio |
-| `env.exec_timeout` | int | 120 | Per-task timeout in seconds |
+| `env.split_seed` | int | `42` | Deterministic split seed |
-| `env.out_root` | str | — | Output directory |
+| `env.split_dir` | str | empty | Materialized train/val/test directory |
+| `env.data_path` | str | empty | Raw data path for ratio mode |
+| `env.split_output_dir` | str | empty | Optional materialized split output |
+| `env.exec_timeout` | int | `120` | Per-task timeout in seconds |
+| `env.out_root` | str | generated by the train/eval CLIs | Output directory |
 
-## Azure OpenAI Credentials
+Benchmark-specific `env` keys are passed through to the adapter.
+
+## Credential Environment Variables
+
+### Azure-family backend
 
 | Variable | Description |
 |---|---|
-| `AZURE_OPENAI_ENDPOINT` / `model.azure_openai_endpoint` | Azure resource endpoint |
+| `AZURE_OPENAI_ENDPOINT` | Shared Azure endpoint or compatibility base URL |
-| `AZURE_OPENAI_API_KEY` / `model.azure_openai_api_key` | Azure API key |
+| `AZURE_OPENAI_API_VERSION` | Azure API version |
-| `OPENAI_API_KEY` | OpenAI API key (for `openai_chat` backend) |
+| `AZURE_OPENAI_AUTH_MODE` | `api_key`, `azure_cli`, `managed_identity`, or `openai_compatible` |
-| `ANTHROPIC_API_KEY` | Anthropic API key (for `claude_code_exec` backend) |
+| `AZURE_OPENAI_API_KEY` | Key for `api_key` or `openai_compatible` mode |
-| `QWEN_CHAT_BASE_URL` | Shared local vLLM endpoint for `qwen_chat` |
+| `AZURE_OPENAI_AD_SCOPE` | Optional AAD scope |
-| `QWEN_CHAT_MODEL` | Shared served model name for `qwen_chat` |
+| `AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID` | Optional managed-identity client ID |
-| `QWEN_CHAT_API_KEY` | Optional API key for the shared Qwen endpoint |
+
-| `OPTIMIZER_QWEN_CHAT_BASE_URL` | Optimizer-specific local vLLM endpoint |
+Use `OPTIMIZER_AZURE_OPENAI_*` and `TARGET_AZURE_OPENAI_*` for role-specific
-| `OPTIMIZER_QWEN_CHAT_MODEL` | Optimizer-specific served model name |
+overrides.
-| `TARGET_QWEN_CHAT_BASE_URL` | Target-specific local vLLM endpoint |
+
-| `TARGET_QWEN_CHAT_MODEL` | Target-specific served model name |
+### Generic OpenAI-compatible backend
+
+| Variable suffix | Shared / per-role forms |
+|---|---|
+| `BASE_URL` | `OPENAI_COMPATIBLE_BASE_URL`, `OPTIMIZER_OPENAI_COMPATIBLE_BASE_URL`, `TARGET_OPENAI_COMPATIBLE_BASE_URL` |
+| `API_KEY` | Corresponding shared/optimizer/target `*_API_KEY` names |
+| `MODEL` | Corresponding shared/optimizer/target `*_MODEL` names |
+| `TEMPERATURE` | Corresponding shared/optimizer/target `*_TEMPERATURE` names |
+| `MAX_TOKENS` | Corresponding shared/optimizer/target `*_MAX_TOKENS` names |
+| `TIMEOUT_SECONDS` | Corresponding shared/optimizer/target `*_TIMEOUT_SECONDS` names |
+
+The train/eval entry points set deployments from YAML `model.optimizer` and
+`model.target` after backend initialization. For selected OpenAI-compatible or
+Qwen roles, those values override the corresponding `*_MODEL` environment
+variables; the environment model names mainly seed direct library use.
+
+Other backend families use the authenticated Claude CLI (`CLAUDE_CLI_BIN`;
+optionally `ANTHROPIC_API_KEY`), `QWEN_CHAT_*`, and `MINIMAX_*`.
+SkillOpt-Sleep's compatible endpoint uses `AZURE_OPENAI_*`, not the research
+backend's `OPENAI_COMPATIBLE_*`; see
+[the Sleep endpoint guide](../sleep/openai-compatible-endpoints.md).

docs/sleep/README.md

@@ -0,0 +1,132 @@
+# SkillOpt-Sleep 😴 — deployment-time companion (preview)
+
+**SkillOpt-Sleep** applies SkillOpt's discipline to your *own daily usage*. It gives a
+local coding agent a nightly **sleep cycle** that reviews your past sessions, replays
+your recurring tasks on your own API budget, and consolidates what it learns into
+**validated** long-term memory and skills — behind a held-out gate, staged for your
+review. It requires **no weight training** and adds no separate optimization loop to
+normal agent requests.
+
+> **Preview.** This is an early preview we are actively iterating on; interfaces and
+> defaults may change. The engine lives in the top-level [`skillopt_sleep/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt_sleep)
+> package with **zero dependency** on the paper's `skillopt/` code (the validation gate
+> is vendored).
+
+## How it works
+
+One "night":
+
+```
+harvest Claude Code / Codex transcripts → mine recurring tasks → replay offline
+   → consolidate (reflect → bounded edit → GATE on real held-out tasks)
+   → stage proposal → (you) adopt
+```
+
+It synthesizes **SkillOpt** (validation-gated bounded text edits), **Claude Dreams**
+(offline consolidation; review-then-adopt), and the **agent-sleep** idea (short-term
+experience → long-term competence).
+
+> **Data boundary.** Harvesting is local and read-only. The `mock` backend makes no
+> provider calls. A real backend, however, sends truncated excerpts from harvested
+> sessions and derived tasks to the provider you select for mining, replay, judging,
+> and reflection. Outbound prompts are not currently guaranteed to be secret-free;
+> review your transcript source and provider policy before running on sensitive
+> projects. For a reviewable workflow, harvest to a task file, inspect/redact it, mark
+> it `"reviewed": true`, and then replay that file with the real backend.
+
+## How to use it
+
+### Quickest path: the `skillopt-sleep` CLI (pip)
+
+```bash
+pip install skillopt        # installs the engine + the `skillopt-sleep` command
+skillopt-sleep dry-run      # harvest + mine + replay, report only; stages nothing
+skillopt-sleep run          # a full nightly cycle; the proposal is staged for review
+skillopt-sleep status       # show state + the latest staged proposal
+skillopt-sleep adopt        # apply the latest staged proposal
+skillopt-sleep schedule     # install a nightly cron entry for this project
+```
+
+> **Version note.** This page tracks `main`. PyPI 0.2.0 provides the base
+> commands above. Sleep handoff, non-Azure OpenAI-compatible endpoints, and
+> `--preferences` landed later and require a source install from `main` until
+> the next release.
+
+The per-agent integrations below still come from the repo; the CLI above is the
+standalone, pip-only way to run a cycle. Claude Code, Codex, Copilot, and Devin wrap
+the shared engine. OpenClaw is a separate reference adaptation and has its own setup.
+
+One engine, thin per-agent shells (see [`plugins/`](https://github.com/microsoft/SkillOpt/tree/main/plugins)):
+
+| Platform | Folder | Install |
+|---|---|---|
+| **Claude Code** | [`plugins/claude-code`](https://github.com/microsoft/SkillOpt/tree/main/plugins/claude-code) | `/plugin marketplace add ./plugins/claude-code` → `/skillopt-sleep` |
+| **Codex** | [`plugins/codex`](https://github.com/microsoft/SkillOpt/tree/main/plugins/codex) | `bash plugins/codex/install.sh` → `skillopt-sleep` skill |
+| **Copilot** | [`plugins/copilot`](https://github.com/microsoft/SkillOpt/tree/main/plugins/copilot) | register `plugins/copilot/mcp_server.py` as an MCP server |
+| **Devin** | [`plugins/devin`](https://github.com/microsoft/SkillOpt/tree/main/plugins/devin) | register `plugins/devin/mcp_server.py` as an MCP server |
+| **OpenClaw** | [`plugins/openclaw`](https://github.com/microsoft/SkillOpt/tree/main/plugins/openclaw) | adapt the reference wrapper and paths for your installation |
+
+To use DeepSeek, vLLM, Ollama, or another Chat Completions server, see
+**[OpenAI-compatible endpoints](openai-compatible-endpoints.md)**. That guide also
+documents the separate HTTPS-only boundary for Azure managed-identity credentials.
+
+Deterministic proof (no API key):
+`python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves`.
+
+### Opt-in: experience replay & dream rollouts
+
+Two consolidation mechanisms, both default **off** (behavior is unchanged unless you
+enable them). They strengthen the nightly update when your tasks have a clean
+correctness signal; the validation gate still governs what ships.
+
+| Config knob | Default | Effect |
+|---|---|---|
+| `dream_rollouts` | `1` | Run each task K times → learn from the good-vs-bad contrast (contrastive reflection). |
+| `recall_k` | `0` | Associative recall — pull the K most-similar past tasks (from a persisted archive) into tonight's dream. |
+| `dream_factor` | `0` | Add N lightweight synthetic variants of each task. |
+
+## Results
+
+> 📊 **More results & analysis — the gate-safety stress test, experience-replay
+> scaling, and the dream-diversity ablation — are in
+> [`docs/sleep/RESULTS.md`](RESULTS.md).** The highlights:
+
+**Controlled experiment recipe (not the shipping CLI defaults).** 5 nights × 10 new
+real "today" tasks per night; the full held-out **test** split is scored before night
+1 (baseline) and after night 5 (after); optimizer = GPT-5.5; single seed (42). The
+experiments use the shipped consolidation and gate components, while the nightly CLI
+and benchmark harnesses remain separate entry points. Numbers are absolute held-out
+accuracy; **Δ** = `after − baseline` in percentage points.
+
+**(a) End-to-end on real agents — [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1`.**
+Deficient seed skills go **0.00 → 1.00** on the held-out set with **both Claude Code
+and Codex** as the target agent (all 4 seeds, including a real tool-use loop).
+
+**(b) Experience replay scales the gain — SearchQA** (1,400-item held-out test,
+SQuAD exact-match; target = GPT-5.5; **validation-gated**):
+
+| Replay config (`dream_rollouts=5`) | Baseline → After | Δ (pts) |
+|---|---|---|
+| `recall_k=10` | 0.802 → 0.834 | +3.1 |
+| `recall_k=20` | 0.803 → 0.848 | **+4.5** |
+| full-history replay *(reference, not a shipping default)* | 0.796 → 0.851 | +5.6 |
+| `recall_k=10`, `dream_rollouts=8` *(more dreaming, same recall)* | 0.798 → 0.835 | +3.7 |
+
+The gain rises monotonically with how much relevant past experience is recalled. The
+same SearchQA cell **without** the gate (`recall_k=10`) is 0.808 → 0.839 (+3.1).
+
+**(c) Second benchmark — SpreadsheetBench** (280-item held-out test; the agent's
+generated openpyxl code is executed and compared cell-by-cell to a golden workbook;
+target = GPT-5.4-nano; gate-free + the output-contract guardrail): 0.279 → 0.314 (**+3.6**).
+
+**(d) Honest scope.** These gains hold where tasks recur and have a checkable
+correctness signal. On saturated or noisy benchmarks (e.g. a strong model already
+near ceiling) the effect is **flat within run-to-run noise** — single-seed baseline
+variance here is ±1–2 pts, so treat sub-~1.5 pt differences as noise. The validation
+gate keeps the worst case bounded; keep it **on** by default.
+
+## Learn more
+
+See the [SkillOpt documentation index](../index.md), the
+[CLI reference](../reference/cli.md), and the integration-specific READMEs under
+[`plugins/`](https://github.com/microsoft/SkillOpt/tree/main/plugins).

docs/sleep/RESULTS.md

@@ -0,0 +1,191 @@
+# SkillOpt-Sleep — results & analysis
+
+This is the evidence behind SkillOpt-Sleep: does a nightly, offline sleep cycle
+actually make a *deployed* agent better, and is it safe to run unattended? We
+answer with a controlled deployment-scale study built from the same shipped
+consolidation and gate components. Its multi-night benchmark recipe is an
+experiment configuration, not the default configuration of the nightly CLI.
+
+## Setup
+
+**Protocol (identical for every cell unless stated).** 5 nights; each night adds
+**10 new real "today" tasks**; the skill carries over and is refined night to
+night. The full held-out **test** split is scored before night 1 (*baseline*) and
+after night 5 (*after*); **Δ = after − baseline** in percentage points. Optimizer
+model = **GPT-5.5**; single seed (42). The measurements use the shipped replay,
+consolidation, and gate implementations. The nightly CLI and the checked-in
+benchmark convenience harnesses are separate entry points and do not all call one
+shared wrapper function.
+
+**Benchmarks** (real evaluators, not format heuristics):
+
+| Benchmark | Held-out test | Scoring |
+|---|---|---|
+| SearchQA | 1,400 items | SQuAD exact-match vs gold |
+| LiveMathematicianBench | 124 items | multiple-choice label (choices shuffled per item) |
+| SpreadsheetBench | 280 items | the agent's generated openpyxl code is **executed**, output workbook compared cell-by-cell to a golden file |
+
+**Targets:** GPT-5.5, GPT-5.4-mini, GPT-5.4-nano. **Modes:** validation-gated
+(default) and gate-free.
+
+---
+
+## 1. The headline — the validation gate is what makes nightly self-evolution *safe*
+
+Self-evolution is easy to build and easy to ruin: an optimizer that accepts its
+own "lessons" unconditionally can adopt a plausible-but-wrong rule and an obedient
+model will follow it off a cliff. We reproduced exactly that failure, then showed
+the gate prevents it.
+
+Stress case — **GPT-5.4-nano on SearchQA**, weak model on a single-sample (degraded)
+reflection signal, same nights, same candidate edits, gate **off** vs **on**:
+
+| | Night 0 → Night 5 | Δ |
+|---|---|---|
+| **no gate** | 0.554 → **0.026** | **−52.8** |
+| **with gate (default)** | 0.570 → 0.570 | 0.0 |
+
+Ungated, the optimizer learned "answer with the document-title string, verbatim";
+the model complied and accuracy collapsed night after night
+(0.554 → 0.490 → 0.325 → 0.031 → 0.034 → 0.026). The gated twin **rejected every one
+of those edits** and never lost a point. This single experiment is the core
+argument for SkillOpt-Sleep's design, and why the gate ships **on by default**.
+
+---
+
+## 2. Cross-model scaling — bigger gains where there's headroom
+
+The same protocol on a weaker target model (**GPT-5.4-nano**, optimizer = GPT-5.5)
+produces substantially larger gains — because the weaker model has more room to
+learn. This is the realistic "cheap deployed agent, strong overnight optimizer"
+scenario:
+
+| Config (SearchQA, nano, gated) | Baseline → After | Δ | Night-by-night |
+|---|---|---|---|
+| **cumulative replay, nights=5** | 0.560 → **0.679** | **+11.9** | 0.560 → 0.626 → 0.665 → 0.665 → 0.665 → 0.679 |
+| recall_k=20, nights=5 | 0.566 → 0.681 | +11.5 | 0.566 → 0.659 → 0.685 → 0.685 → 0.681 → 0.681 |
+| cumulative, nights=8 | 0.562 → 0.657 | +9.5 | saturates after night 5 |
+
+Both replay strategies (cumulative and recall) agree within 0.4 pt — the gain is
+robust across configurations.
+
+**Compared to GPT-5.5 on the same benchmark (SearchQA, gated):**
+
+| Target model | Best Δ | Baseline | Headroom |
+|---|---|---|---|
+| GPT-5.4-nano | **+11.9** | 0.560 | 44 pt |
+| GPT-5.5 | +6.0 | 0.798 | 20 pt |
+
+The story: **SkillOpt-Sleep helps most where there's the most to learn** — weaker
+deployed models benefit ~2× as much from the same nightly optimization. This is
+also the economical deployment pattern (cheap inference model + one strong
+overnight optimizer call).
+
+---
+
+## 3. Experience replay turns a one-time bump into a climb
+
+The plugin's two opt-in knobs (`recall_k`, `dream_rollouts`) are what produce the
+gains. On **SearchQA, GPT-5.5, gated** — the gain rises monotonically with how
+much relevant past experience is recalled:
+
+| Replay (`dream_rollouts=5`) | Baseline → After | Δ |
+|---|---|---|
+| `recall_k=10` | 0.802 → 0.834 | +3.1 |
+| `recall_k=20` | 0.803 → 0.848 | **+4.5** |
+| full-history (reference, not a default) | 0.796 → 0.851 | +5.6 |
+
+And the curve genuinely **climbs across nights** rather than jumping once and
+plateauing — full-history replay, gated, night by night:
+
+```
+0.798 → 0.814 → 0.854 → 0.854 → 0.854 → 0.858
+```
+
+The gate accepts a new, better skill as late as **night 5** (0.854 → 0.858).
+Replay-policy ablation (SearchQA, GPT-5.5):
+
+| Replay policy | Gate-free Δ | Gated Δ |
+|---|---|---|
+| none (tonight's tasks only) | +3.9 | +2.0 |
+| **recall k=10 (opt-in experiment)** | +5.1 | +4.4 |
+| cumulative (full history) | +4.8 | +6.0 |
+
+Recall captures most of cumulative's benefit at a fraction of the per-night cost.
+
+---
+
+## 4. Sensitivity around the experiment recipe
+
+We swept `dream_factor`, `rollouts`, `per_night`, and `nights` on the nano cell
+(SearchQA, gated) around the study recipe: `dream_factor=2`, `rollouts=5`,
+`per_night=10`, and `nights=5`. These are **experiment values**, not the shipping
+defaults (`dream_factor=0`, `dream_rollouts=1`, and `recall_k=0`):
+
+| Variant | Δ | vs experiment baseline (+11.9) |
+|---|---|---|
+| dream_factor=4 (baseline 2) | +8.8 | −3.1 |
+| rollouts=10 (baseline 5) | +9.5 | −2.4 |
+| per_night=15 (baseline 10) | +2.7 | −9.2 |
+| nights=8 (baseline 5) | +9.5 | −2.4 |
+
+Every tested direction away from that baseline reduced the measured gain in this
+cell. The result supports that particular study recipe; it does not establish a
+universal optimum. Shipping stays conservative, and users must opt in to additional
+dream rollouts or recall after considering task quality and provider cost.
+
+---
+
+## 5. Why these gains exist — the dream-diversity fix (and a rigor note)
+
+Reflection learns from the **contrast** between good and bad rollouts of the same
+task, which requires the K dream rollouts to be *independent samples*. An early
+version of the engine collapsed them to one cached sample, so contrastive
+reflection never fired. Fixing that, then adding recall, is what produces the
+gains in Sections 1–2. Measured across an 18-cell deployment sweep (3 benchmarks ×
+3 targets × 2 modes), under three engine configurations:
+
+| Engine configuration | mean Δ | worst-cell Δ | cells > +0.5 | cells < −0.5 |
+|---|---|---|---|---|
+| single-sample reflection (degraded) | −2.66 | **−52.8** | 7 / 18 | 5 / 18 |
+| diverse rollouts (K=5), no recall | +0.24 | −4.0 | 6 / 18 | 7 / 18 |
+| **diverse rollouts + recall (experiment recipe)** | **+0.53** | **−2.4** | 7 / 18 | 7 / 18 |
+
+The catastrophic −52.8 is removed **at its source** by diverse rollouts: the same
+gate-free nano-SearchQA cell goes 0.554 → **0.586 (+2.7)** with no gate at all once
+the dream is fixed. Recall then lifts the grid mean and tightens the worst case.
+This is **defense in depth, each layer measured**: diverse rollouts propose better
+edits, recall remembers relevant experience, and the gate catches whatever still
+slips through.
+
+---
+
+## 6. End-to-end on real agents
+
+On the public [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1`
+benchmark — designed for exactly this learnable-gap setting — deficient seed skills
+go **0.00 → 1.00** on the held-out set with **both Claude Code and Codex** as the
+target agent (all 4 seeds, including a real tool-use loop), and the two agents
+cross-verify each other's consolidated skills.
+
+---
+
+## 7. Honest scope & limitations
+
+- **Where it helps:** recurring tasks with a checkable correctness signal and real
+  headroom. That is the plugin's actual use case (your repeated daily tasks and
+  house rules the agent keeps missing).
+- **Where it's flat:** saturated tasks on strong models, or noisy tasks with a weak
+  learning signal — within run-to-run noise.
+- **Single seed.** Cells aggregate one seed per config; treat sub-~1.5 pt
+  differences as noise. Spot seed-robustness check on the one flagged cell
+  (nano SearchQA gated): seeds 42/43/44 give −1.9 / +3.6 / +4.7 (3-seed mean
+  **+2.1**), i.e. the tabled −1.9 is a pessimistic draw, not the typical outcome.
+- **Keep the gate on.** It is the difference between bounded downside (−2.4) and a
+  −52.8 collapse. Gate-free mode is for users who cannot hold out a validation set
+  and is additionally protected by the output-contract guardrail.
+
+---
+
+Back to the module overview: [`docs/sleep/README.md`](README.md) ·
+documentation index: [SkillOpt documentation](../index.md).

docs/sleep/examples/runner.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Reference launcher for running SkillOpt-Sleep against an OpenAI-compatible
+endpoint (DeepSeek shown here), plus an Antigravity `session-end` hook.
+
+This is a *sanitized example*, not a supported entry point. Adapt the paths and
+provider details to your environment. No API keys are hardcoded — the key is read
+from an .env file or the process environment.
+
+Usage:
+    python runner.py run           # run a full sleep cycle against DeepSeek
+    python runner.py dry-run       # harvest + replay, report only
+    python runner.py session-end   # Antigravity Stop-hook: append rollout evidence
+"""
+import os
+import re
+import sys
+import json
+import subprocess
+import datetime
+from pathlib import Path
+
+# --- Configure these for your environment -----------------------------------
+# Path to a file containing your provider key as `sk-...` (kept out of source).
+PROVIDER_ENV_FILE = Path(os.environ.get("SKILLOPT_PROVIDER_ENV_FILE", "provider.env"))
+# Endpoint + model for the OpenAI-compatible provider.
+PROVIDER_ENDPOINT = os.environ.get("SKILLOPT_PROVIDER_ENDPOINT", "https://api.deepseek.com")
+PROVIDER_MODEL = os.environ.get("SKILLOPT_PROVIDER_MODEL", "deepseek-v4-pro")
+# Project whose SKILL.md files the sleep cycle should evolve.
+PROJECT_DIR = os.environ.get("SKILLOPT_PROJECT_DIR", os.getcwd())
+# Where the session-end hook appends rollout evidence.
+ROLLOUT_LOG = Path(os.environ.get("SKILLOPT_ROLLOUT_LOG", "brain/rollout-evidence.jsonl"))
+# ----------------------------------------------------------------------------
+
+
+def load_provider_key(env: dict) -> None:
+    """Ensure DEEPSEEK_API_KEY is set, reading it from PROVIDER_ENV_FILE if needed."""
+    if env.get("DEEPSEEK_API_KEY"):
+        return
+    try:
+        text = PROVIDER_ENV_FILE.read_text(encoding="utf-8")
+    except OSError:
+        return
+    m = re.search(r"sk-[A-Za-z0-9]+", text)
+    if m:
+        env["DEEPSEEK_API_KEY"] = m.group(0)
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        print("Usage: runner.py [dry-run|run|status|adopt|session-end]")
+        sys.exit(1)
+
+    command = sys.argv[1]
+
+    # Antigravity Stop-hook: enrich future nights with task-outcome metadata.
+    if command == "session-end":
+        ROLLOUT_LOG.parent.mkdir(parents=True, exist_ok=True)
+        outcome = {
+            "timestamp": datetime.datetime.now().isoformat(),
+            "event": "SessionEnd",
+            "metadata": "Appended task outcome metadata",
+        }
+        with open(ROLLOUT_LOG, "a", encoding="utf-8") as f:
+            f.write(json.dumps(outcome) + "\n")
+        print("Rollout evidence metadata appended.")
+        return
+
+    env = os.environ.copy()
+    load_provider_key(env)
+
+    if env.get("DEEPSEEK_API_KEY"):
+        # OpenAI-compatible path — see docs/sleep/openai-compatible-endpoints.md
+        backend = "azure_openai"
+        env["PYTHONIOENCODING"] = "utf-8"
+        env["AZURE_OPENAI_AUTH_MODE"] = "openai_compatible"
+        env["AZURE_OPENAI_ENDPOINT"] = PROVIDER_ENDPOINT
+        env["AZURE_OPENAI_API_KEY"] = env["DEEPSEEK_API_KEY"]
+        # Provider-specific request fields are opt-in, never inferred from the
+        # model name. For DeepSeek reasoning models, enable the thinking channel:
+        env.setdefault("SKILLOPT_SLEEP_CHAT_EXTRA_BODY",
+                       json.dumps({"thinking": {"type": "enabled"}}))
+        env.setdefault("SKILLOPT_SLEEP_COMPAT_MAX_TOKENS", "8192")
+    else:
+        # OPTIONAL, UNVERIFIED fallback: route the `claude` CLI backend through a
+        # local Anthropic-compatible proxy (e.g. LiteLLM) to reach Gemini. There
+        # is no native Gemini backend; this path was not validated. See the doc.
+        backend = "claude"
+        if "ANTHROPIC_API_KEY" not in env and "GEMINI_API_KEY" in env:
+            env["ANTHROPIC_API_KEY"] = env["GEMINI_API_KEY"]
+        env.setdefault("ANTHROPIC_BASE_URL", "http://127.0.0.1:4000")
+
+    args = ["skillopt-sleep", command]
+    if command in ("run", "dry-run"):
+        args = ["skillopt-sleep", command, "--backend", backend,
+                "--model", PROVIDER_MODEL, "--project", PROJECT_DIR]
+
+    print(f"Running: {' '.join(args)}")
+    # Propagate the child's exit code so supervisors (watchdog.py, systemd,
+    # Task Scheduler) see a failed sleep run as a failure, not a success.
+    proc = subprocess.run(args, env=env, check=False)
+    sys.exit(proc.returncode)
+
+
+if __name__ == "__main__":
+    main()

docs/sleep/examples/watchdog.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Minimal supervisor that runs the SkillOpt-Sleep cycle on a fixed interval.
+
+Sanitized example (see docs/sleep/openai-compatible-endpoints.md). On Windows,
+register this under a Scheduled Task so it survives logout; on Linux/macOS a
+systemd timer or cron entry serves the same purpose and is usually preferable to
+a long-lived process.
+"""
+import os
+import sys
+import time
+import subprocess
+import datetime
+import traceback
+
+INTERVAL_SECONDS = int(os.environ.get("SKILLOPT_WATCHDOG_INTERVAL", str(4 * 3600)))
+RUNNER = os.environ.get("SKILLOPT_RUNNER", os.path.join(os.path.dirname(__file__), "runner.py"))
+LOG_FILE = os.environ.get("SKILLOPT_WATCHDOG_LOG", "brain/watchdog.log")
+
+
+def log(msg: str) -> None:
+    os.makedirs(os.path.dirname(LOG_FILE) or ".", exist_ok=True)
+    line = f"[{datetime.datetime.now().isoformat()}] {msg}"
+    with open(LOG_FILE, "a", encoding="utf-8") as f:
+        f.write(line + "\n")
+    print(line)
+
+
+def run_once() -> None:
+    log("Invoking skillopt-sleep run via runner.py...")
+    try:
+        result = subprocess.run([sys.executable, RUNNER, "run"],
+                                capture_output=True, text=True)
+        if result.returncode == 0:
+            log("Successfully completed run.")
+        else:
+            log(f"Run failed (exit {result.returncode}).")
+            log(f"STDERR: {result.stderr}")
+    except Exception as e:
+        log(f"Exception while running skillopt: {e}")
+        log(traceback.format_exc())
+
+
+def main() -> None:
+    log(f"Watchdog started. Interval: {INTERVAL_SECONDS}s.")
+    while True:
+        try:
+            run_once()
+        except Exception as e:
+            log(f"Unexpected error in watchdog loop: {e}")
+        log(f"Sleeping for {INTERVAL_SECONDS}s...")
+        time.sleep(INTERVAL_SECONDS)
+
+
+if __name__ == "__main__":
+    main()

docs/sleep/openai-compatible-endpoints.md

@@ -0,0 +1,173 @@
+# OpenAI-compatible endpoints for SkillOpt-Sleep (DeepSeek, local vLLM, …)
+
+This document describes the `azure_openai` backend in
+`skillopt_sleep/backend.py`, which can drive servers that implement the expected
+OpenAI-compatible Chat Completions request shape — for example DeepSeek's hosted
+API or a self-hosted vLLM/Ollama server — in addition to native Azure OpenAI
+deployments. The included runner is a sanitized unattended-launch example that
+was originally used alongside Antigravity; it is not an Antigravity transcript
+integration.
+
+> **Version requirement.** This capability landed after v0.2.0. Until the next
+> release, install SkillOpt from the latest `main`; the current PyPI 0.2.0
+> package does not provide this compatible-endpoint path.
+
+## What changed
+
+All changes are backward-compatible — the default managed-identity Azure path
+is unchanged:
+
+1. **CLI acceptance.** `skillopt-sleep run --backend azure_openai` is now an
+   accepted choice in `skillopt_sleep/__main__.py` (it was previously rejected
+   by argparse even though `get_backend()` understood the name).
+
+2. **Endpoint resolution honors `AZURE_OPENAI_ENDPOINT`.**
+   `AzureOpenAIBackend.__init__` resolves the endpoint as `explicit arg` →
+   `AZURE_OPENAI_ENDPOINT` env → the built-in `_AZURE_ENDPOINTS` table.
+   Previously a non-Azure endpoint could not be supplied at all.
+
+3. **`openai_compatible` auth mode.** When
+   `AZURE_OPENAI_AUTH_MODE=openai_compatible` (also accepts `compat`/`openai`),
+   `_get_client()` builds a plain `openai.OpenAI(base_url=…)` client with
+   `AZURE_OPENAI_API_KEY` instead of an `AzureOpenAI` client. This mirrors the
+   auth mode already supported by the sibling `skillopt/model/azure_openai.py`
+   module. (The `AzureOpenAI` client rewrites request URLs with Azure-only
+   `?api-version=…` query params and deployment path segments, which non-Azure
+   servers reject with `404 Resource not found` — the sleep cycle then scores
+   every rollout `0.0` with no diagnostic.)
+
+4. **Managed-identity credential guard.** The managed-identity path attaches an
+   Azure AD bearer token to every request. It therefore accepts only an **HTTPS**
+   endpoint whose hostname ends in `*.openai.azure.com` or
+   `*.cognitiveservices.azure.com`. An HTTP endpoint — even one with an
+   Azure-looking hostname — and any host outside those suffixes are rejected
+   before a credential-bearing client is created.
+
+5. **Provider-neutral request shape.** In compat mode the backend sends only the
+   standard OpenAI-compatible contract (`model`, `messages`, `max_tokens`).
+   Provider-specific request fields are **opt-in** via environment variables
+   (below) and are attached only in compat mode — nothing is inferred from
+   model-name substrings, and the native Azure request remains unchanged.
+
+6. **Reliable error state.** `_call()` records the last exception in
+   `self.last_call_error` (surfaced in `diagnostics.json`), clears it when a
+   retry recovers, and sets an explicit `"empty response on all N attempts"`
+   diagnostic when every attempt returns empty text.
+
+## Configuration reference
+
+SkillOpt-Sleep's `azure_openai` backend reads these environment variables
+(unprefixed only — the `OPTIMIZER_*`/`TARGET_*` dual-role variables belong to
+the separate `skillopt.model.azure_openai` module and are **not** used by the
+sleep cycle):
+
+| Variable | Meaning |
+|---|---|
+| `AZURE_OPENAI_AUTH_MODE` | `openai_compatible` (or `compat`/`openai`) selects the plain OpenAI client. Unset/other = Azure managed identity (default). |
+| `AZURE_OPENAI_ENDPOINT` | Base URL of the server, e.g. `https://api.deepseek.com`. Azure managed identity requires HTTPS plus an approved Azure hostname. |
+| `AZURE_OPENAI_API_KEY` | API key sent by the compat client to the configured base URL. |
+| `SKILLOPT_SLEEP_COMPAT_MAX_TOKENS` | Optional int (default `8192`): `max_tokens` sent in compat mode. |
+| `SKILLOPT_SLEEP_CHAT_EXTRA_BODY` | Optional JSON object passed as `extra_body` for provider-specific fields in compat mode only. It is ignored in native Azure mode. |
+
+## Data and transport boundaries
+
+- Harvesting reads local transcripts without modifying them, and the `mock`
+  backend makes no provider calls. A real backend sends **truncated transcript
+  excerpts and derived task content** to the selected provider for mining,
+  replay, judging, and reflection.
+- Outbound prompts are not currently guaranteed to be free of secrets. Review
+  the provider's data policy and avoid a third-party endpoint for sensitive
+  transcripts unless you have first inspected and redacted the task material.
+  One reviewable path is `skillopt-sleep harvest --output tasks.json`, followed
+  by a reviewed `--tasks-file` run.
+- Use HTTPS for every remote compatible provider. Plain HTTP is appropriate only
+  for an explicitly trusted loopback development server such as
+  `http://127.0.0.1:8000/v1`; the compat client sends its API key to the configured
+  URL.
+- Azure managed-identity credentials have the stricter invariant described
+  above: HTTPS **and** an approved Azure hostname are both mandatory.
+
+## How to use it
+
+```bash
+export AZURE_OPENAI_AUTH_MODE=openai_compatible
+export AZURE_OPENAI_ENDPOINT=https://api.deepseek.com   # DeepSeek base URL
+export AZURE_OPENAI_API_KEY=sk-...                       # your provider key
+
+# DeepSeek reasoning models: enable the thinking channel (opt-in, not inferred)
+export SKILLOPT_SLEEP_CHAT_EXTRA_BODY='{"thinking": {"type": "enabled"}}'
+export SKILLOPT_SLEEP_COMPAT_MAX_TOKENS=8192
+
+skillopt-sleep run \
+  --backend azure_openai \
+  --model deepseek-v4-pro \
+  --project /path/to/your/project
+```
+
+The same pattern works for a server that implements this Chat Completions
+contract: point `AZURE_OPENAI_ENDPOINT` at the provider-specific base URL, set a
+matching `--model`, and omit `SKILLOPT_SLEEP_CHAT_EXTRA_BODY` unless the provider
+needs extra request fields. Self-hosted vLLM and Ollama commonly use a `/v1` base
+path, for example `http://127.0.0.1:8000/v1` or
+`http://127.0.0.1:11434/v1`.
+
+`--project` selects the project/transcript scope and the project `CLAUDE.md`; it
+does **not** by itself select an arbitrary project `SKILL.md`. Pass
+`--target-skill-path path/to/SKILL.md` when a specific skill is the optimization
+target. Without that flag, SkillOpt-Sleep uses its configured managed skill.
+
+## Unattended runner example (originally used with Antigravity)
+
+The [`examples/`](https://github.com/microsoft/SkillOpt/tree/main/docs/sleep/examples) directory contains a sanitized reference for running
+the compatible backend unattended:
+
+- **`examples/runner.py`** — a thin launcher that loads a provider key from an
+  `.env` file, exports the variables above, invokes `skillopt-sleep run` with
+  the DeepSeek backend, and **exits with the child's return code** so
+  supervisors see failures as failures. Its `session-end` action writes a small
+  local rollout-evidence event as an example hook target.
+- **`examples/watchdog.py`** — a minimal supervisor loop that invokes the runner
+  on a fixed interval (e.g. every 4 hours) and logs non-zero exits as failures.
+  On Windows this is registered as a Scheduled Task so it survives logout; on
+  Linux/macOS a `systemd` timer or cron entry serves the same role.
+
+The current engine does **not** read `brain/rollout-evidence.jsonl`, and it does
+not harvest Antigravity transcripts. That hook output is illustrative metadata,
+not additional training evidence. A real run must use a supported Claude
+Code/Codex transcript source or a reviewed task file converted by the operator.
+
+### Contributor-reported validation
+
+The contributor reported the following results from a private Windows 11 setup
+driving the cycle against `deepseek-v4-pro` in `openai_compatible` mode. They are
+useful integration evidence, but the private session set is not a reproducible
+benchmark bundled with this repository:
+
+- A direct backend smoke test returns a live completion (no `404`,
+  `last_call_error` empty, client type `OpenAI`).
+- A full nightly cycle using the configured session source moved the held-out
+  validation gate from `0.250 → 1.000`, **accepting** a DeepSeek-authored
+  skill edit (`accept_new_best`). `diagnostics.json` for that night reports
+  `"backend": "azure_openai"` with a non-empty token count and an empty
+  `call_error` — i.e. a genuine optimization night, versus the prior all-`0.0`
+  nights that the endpoint bug produced.
+- A subsequent unattended night triggered by the watchdog completed the full
+  chain (watchdog → runner → `skillopt-sleep` → DeepSeek) and the gate correctly
+  **rejected** a non-improving proposal (`0.3 → 0.3`), confirming the validation
+  gate behaves normally on the new backend.
+
+Deterministic no-network coverage for the new behavior lives in
+`tests/test_azure_openai_compat.py` (CLI acceptance, client selection,
+endpoint/auth guard, request kwargs, retry error-state, empty-response
+diagnostics, and runner exit-code propagation).
+
+## Unsupported Gemini proxy branch in the example
+
+`examples/runner.py` still contains an illustrative branch that routes the
+**`claude` CLI backend** through a loopback Anthropic-compatible proxy such as
+[LiteLLM](https://github.com/BerriAI/litellm). It is not a native Gemini backend,
+has no validated model mapping in this example, and is not part of the supported
+path documented here. The sample currently enters that branch whenever no
+DeepSeek key is found, so a production adaptation should remove it or replace it
+with an explicit opt-in, a separately configured model, and a trusted isolated
+loopback proxy. Do not treat this branch as tested Gemini support.

docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md

@@ -0,0 +1,244 @@
+# SkillOpt Sleep — Claude Code self-evolving plugin (design)
+
+> **Historical design proposal.** This document records the June 2026 design
+> target and includes planned controls that are not part of the current nightly
+> CLI. It is not an installation or configuration reference. For implemented
+> behavior, flags, defaults, and data boundaries, use
+> [`docs/sleep/README.md`](../../sleep/README.md),
+> [`docs/reference/cli.md`](../../reference/cli.md), and
+> [`plugins/README.md`](https://github.com/microsoft/SkillOpt/blob/main/plugins/README.md).
+
+**Status:** approved-for-build (autonomous offline session, 2026-06-07)
+**Author:** generated for Yifan Yang, executed autonomously while user is asleep
+**Branch:** `feat/claude-code-sleep-plugin` (worktree `my_repo/SkillOpt-sleep`)
+
+---
+
+## 1. One-paragraph summary
+
+`skillopt-sleep` is a Claude Code plugin that gives a user's local Claude
+agent a nightly **sleep cycle**. While the user is offline, it (1) **harvests**
+the day's real Claude Code session transcripts from `~/.claude`, (2) **mines**
+them into discrete *task records* with checkable outcomes, (3) **replays /
+"dreams"** those tasks offline using the user's own API budget, and (4) runs
+the **SkillOpt optimizer loop** (reflect → bounded edit → held-out gate) to
+consolidate short-term experience into long-term **memory** (`CLAUDE.md`) and
+**skills** (`SKILL.md`). Only changes that pass a validation gate are kept, and
+every change is written to a **review staging area** the user approves before it
+touches live config — mirroring Claude Dream's "input store is never modified"
+safety contract. The result: an agent that measurably gets better at *this
+user's* recurring work, every night, with zero model-weight training.
+
+## 2. Why this is the right synthesis of the three ingredients
+
+| Ingredient | What we take from it | Where it lives in this design |
+|---|---|---|
+| **SkillOpt** (your paper/code) | Skill = trainable text state; bounded add/delete/replace edits under a textual learning rate; **held-out validation gate**; rejected-edit buffer; epoch-wise slow/meta update. | The `consolidate` stage *is* a single SkillOpt epoch, reusing `skillopt.optimizer.*` and `skillopt.evaluation.gate`. |
+| **Claude Dreams** | Async offline job: read a memory store + 1–100 session transcripts → emit a **new, separate** reorganized memory store (dedup / merge / resolve contradictions / surface insights). Input never mutated; output reviewed then adopted or discarded. | The `harvest` + `consolidate-memory` stages and the **staging/adopt** safety model are modeled directly on Dreams. |
+| **Agent Sleep paper** (2605.26099) | Agents need periodic offline consolidation: short-term experience buffer → synthetic replay/self-generated data → self-update; "sleep" turns episodes into durable competence. | The whole nightly schedule, the `replay` step, and the short-term→long-term framing. |
+
+The key novel claim this enables for the project (and a future paper section):
+**SkillOpt's validation-gated bounded-edit optimizer is the missing "safe
+update rule" for Dream-style memory consolidation.** Dreams reorganize memory
+but don't *prove* the reorganization helps; the Sleep paper consolidates but
+assumes weight updates. SkillOpt-Sleep consolidates **text** (memory + skills)
+and **gates each change on replayed task performance**, so nightly evolution is
+both weight-free and regression-protected.
+
+## 3. Goals / non-goals
+
+**Goals**
+1. A working Claude Code plugin: scheduled (nightly/cron) **and** user-triggered (`/sleep`).
+2. Look back over the user's real past prompts & trajectories from local `~/.claude` records.
+3. Offline "dream training": re-run mined tasks (mock-env or fresh retry) on the user's budget.
+4. Continuous evolution of **memory** (`CLAUDE.md`) and **skills** (`SKILL.md`) via the SkillOpt gate.
+5. A reproducible experiment that answers: *does the nightly loop actually improve a held-out score?*
+6. Safety: never silently overwrite user config; stage → user approves → adopt.
+
+**Non-goals (now)**
+- Codex version (explicitly deferred by user; architecture keeps it pluggable).
+- Anthropic managed Dreams API integration (we *emulate* Dreams locally; managed API is a future backend).
+- Model fine-tuning / weight updates (out of scope by design — text-only).
+- Fully unattended auto-adopt by default (opt-in; default is review-gated).
+
+## 4. The local data we read (verified on this machine)
+
+- **Prompt history:** `~/.claude/history.jsonl` — one JSON/line: `{display, pastedContents, timestamp, project}`. The cross-session list of every prompt the user typed, with project path + epoch-ms timestamp.
+- **Full transcripts:** `~/.claude/projects/<path-slug>/<sessionId>.jsonl` — one record/line. Record `type` ∈ {`user`,`assistant`,`mode`,`permission-mode`,`attachment`,`file-history-snapshot`,`last-prompt`,…}. User/assistant records carry `message` (role+content blocks), plus `cwd`, `gitBranch`, `timestamp`, `sessionId`, `version`, `userType`. ~215k transcripts present on this box.
+- **Deployment targets we may evolve:**
+  - Project memory: `<project>/CLAUDE.md` (and `~/.claude/CLAUDE.md` global).
+  - User skills: `~/.claude/skills/<name>/SKILL.md` (frontmatter: `name`, `description`, optional `allowed-tools`, `argument-hint`).
+  - Plugin skills under `~/.claude/plugins/...`.
+
+Everything stays **on-disk and local**; the only network calls are the LLM
+optimizer/replay calls the user already pays for.
+
+## 5. Architecture
+
+### 5.1 The nightly Sleep Cycle (stages)
+
+```
+            ┌────────────────────────── SLEEP CYCLE (one "night") ──────────────────────────┐
+            │                                                                                │
+ trigger →  │  1.HARVEST     2.MINE          3.REPLAY            4.CONSOLIDATE      5.STAGE   │ → wake report
+ (cron or   │  read ~/.claude scan sessions  re-run tasks        SkillOpt epoch:   write to   │
+  /sleep)   │  transcripts → → task records   offline (mock or    reflect→edit→     .skillopt-│
+            │  + history     w/ outcomes &    fresh retry) under  GATE on held-out  sleep/    │
+            │                checkable refs   current skill/mem    replay split      staging/ │
+            │                                                                          ↓      │
+            │                                              6.ADOPT (opt-in / user-approved)   │
+            └────────────────────────────────────────────────────────────────────────────────┘
+```
+
+**1. Harvest** (`harvest.py`)
+Read `history.jsonl` + per-project transcript JSONLs for a time window
+(default: since last sleep, fallback last 24–72h). Group by project (`cwd` /
+`project`). Emit normalized `SessionDigest` objects: ordered user prompts,
+assistant final texts, tool-call summary, files touched (from
+`file-history-snapshot`), git branch, errors seen, and **user-feedback signals**
+(e.g. "still broken", "that's wrong", "perfect", re-asks of the same thing).
+
+**2. Mine** (`mine.py`)
+Turn digests into `TaskRecord`s — the unit the optimizer trains on. A task is a
+self-contained intent (the user's request) plus an *outcome label* and, where
+possible, a **checkable reference**:
+- *Explicit success/failure* from feedback signals ("works now" after N retries → the early attempts are failures, the fix is the success exemplar).
+- *Self-consistency check*: re-derivable answers (math, lookups) get a reference; open-ended ones get an LLM-judge rubric instead.
+- Each TaskRecord: `{id, project, intent, context_excerpt, attempted_solution, outcome ∈ {success,fail,mixed}, reference_kind ∈ {exact, rubric, none}, reference, tags}`.
+Mining is itself an LLM call (the **miner**), prompt-tunable, with a deterministic regex/heuristic fallback for offline/no-key runs.
+
+**3. Replay / "Dream"** (`replay.py`)
+For mined tasks, re-run the intent **offline** under the *current* skill+memory
+to get a fresh trajectory & score. Two modes:
+- `mock` (default, safe): reconstruct a sandboxed prompt from the task's captured context (no live repo mutation, no network side effects) and run the target model. Deterministic, cheap, safe to run unattended.
+- `fresh` (opt-in): actually re-attempt in a throwaway git worktree of the project. Higher fidelity, heavier, never touches the user's working tree.
+Scoring: exact-match / substring for `exact` refs; LLM-judge (0–1) for `rubric` refs; this yields the `hard`/`soft` scores SkillOpt already expects.
+
+**4. Consolidate** (`consolidate.py`) — *this is one SkillOpt epoch*
+Reuse the existing optimizer pieces rather than reinventing:
+- `reflect`: partition replayed tasks into failure/success minibatches → propose add/delete/replace edits to **skill** and a parallel proposer for **memory** (`CLAUDE.md`). (Memory consolidation also does Dream-style dedup/merge/contradiction-resolution over existing `CLAUDE.md` lines.)
+- `aggregate` + `rank_and_select` under an **edit budget** (textual learning rate).
+- `apply_patch_with_report` → candidate skill / candidate memory.
+- **GATE** (`skillopt.evaluation.gate.evaluate_gate`): replay a *held-out* slice of tasks with the candidate; accept only if it strictly beats current. Rejected edits go to the rejected-edit buffer (negative feedback) exactly as in the paper.
+- A **slow/meta** pass across nights (not just within one night) carries durable, cross-session lessons — the literal "short-term experience → long-term knowledge" of the Sleep paper. Per-night state persists in `~/.skillopt-sleep/state.json`.
+
+**5. Stage** (`staging/`)
+Write `proposed_CLAUDE.md`, `proposed_SKILL.md`, a unified diff, and a
+`sleep_report.md` (what changed, why, gate deltas, token cost) into
+`<project>/.skillopt-sleep/staging/<date>/`. **Nothing live is modified.**
+
+**6. Adopt**
+`/sleep adopt` (or `auto_adopt: true` in config for power users) copies staged
+files over the live `CLAUDE.md` / `SKILL.md`, after a `git`-style backup. This
+is the only stage that mutates user-facing config, and it is explicit by default
+— the Dreams "review the output, then adopt or discard" contract.
+
+### 5.2 Components & boundaries (each independently testable)
+
+```
+skillopt/sleep/
+  __init__.py
+  types.py         # SessionDigest, TaskRecord, ReplayResult, SleepConfig, SleepReport (dataclasses)
+  harvest.py       # ~/.claude transcripts + history.jsonl  ->  list[SessionDigest]
+  mine.py          # list[SessionDigest]  ->  list[TaskRecord]   (LLM miner + heuristic fallback)
+  replay.py        # TaskRecord + skill + memory  ->  ReplayResult (hard/soft)   (mock | fresh)
+  consolidate.py   # ReplayResults -> candidate skill+memory -> GATE -> accepted artifacts
+  memory.py        # CLAUDE.md read/merge/dedup/diff (Dream-style) + protected-region markers
+  state.py         # ~/.skillopt-sleep/state.json: last_sleep, night counter, slow/meta memory
+  staging.py       # write/adopt staging dir, backups
+  cli.py           # `python -m skillopt.sleep {run|status|adopt|harvest|dry-run}`
+  config.py        # SleepConfig load/merge (defaults + ~/.skillopt-sleep/config.yaml)
+  optimizer_backend.py  # thin: route reflect/judge to a chosen backend; mock backend for tests
+
+skillopt-sleep-plugin/            # the Claude Code plugin surface
+  .claude-plugin/plugin.json
+  commands/sleep.md               # /sleep [run|status|adopt|dry-run]
+  commands/sleep-status.md
+  skills/skillopt-sleep/SKILL.md  # so Claude knows how to drive the engine
+  hooks/hooks.json                # optional: schedule + on-session-end harvest
+  scripts/*                       # shims that call `python -m skillopt.sleep ...`
+```
+
+**Reuse, don't fork:** `consolidate.py` calls into existing
+`skillopt.optimizer.clip.rank_and_select`, `skillopt.gradient.aggregate.merge_patches`,
+`skillopt.optimizer.skill.apply_patch_with_report`, and
+`skillopt.evaluation.gate.evaluate_gate`. The sleep layer is an **EnvAdapter-shaped
+shim** over the user's own life, not a new optimizer.
+
+### 5.3 Data flow (one task, end to end)
+
+```
+history.jsonl + <session>.jsonl
+   └─harvest→ SessionDigest{prompts, finals, tools, feedback}
+        └─mine→ TaskRecord{intent, attempted, outcome, reference}
+             └─replay(current skill+mem)→ ReplayResult{hard, soft, trajectory}
+                  └─reflect→ edits(skill), edits(memory)
+                       └─rank/clip(edit_budget)→ candidate
+                            └─GATE(replay held-out)→ accept? → staging/  → (adopt) live CLAUDE.md/SKILL.md
+```
+
+## 6. Scheduling & triggering
+
+- **Cron/scheduled:** documented `crontab` line + an optional Claude Code hook; default `0 3 * * *` (3am local; pick an off-:00 minute in practice). The engine is a plain CLI so it works under cron, systemd-timer, or the Claude Code scheduler.
+- **User-triggered:** `/sleep run` (full cycle), `/sleep dry-run` (harvest+mine+replay, no edits), `/sleep status`, `/sleep adopt`.
+- **On-session-end harvest (optional hook):** cheaply append the just-finished session to the night's buffer so the 3am run has fresh data without a full rescan.
+
+## 7. Safety model (hard requirements)
+
+1. **Never mutate live `CLAUDE.md`/`SKILL.md` except via explicit `adopt`** (or opt-in `auto_adopt`). Default = staged + reviewed (Dreams contract).
+2. **Backups:** every adopt snapshots the prior file to `staging/<date>/backup/`.
+3. **Read-only harvest:** transcripts are read, never written.
+4. **`fresh` replay runs only in throwaway worktrees**, never the user's checkout; no `rm -rf`, no force-push, network off unless `replay.network: true`.
+5. **Budget cap:** `max_tokens_per_night` + `max_tasks_per_night`; stop early when hit, log what was skipped (no silent truncation).
+6. **Secret hygiene:** redact obvious secrets from digests before they enter prompts (reuse `_redact_*` ideas from trainer).
+7. **PII/scope:** only harvest projects on an allowlist (default: the project the plugin is invoked in) or `projects: all` opt-in.
+
+## 8. Validation experiment — "does it actually improve?"
+
+A self-contained, **deterministic-by-default** experiment lives in
+`skillopt/sleep/experiments/` and is the acceptance test for the whole idea.
+
+**Setup:** a synthetic "user persona" (e.g. *researcher who keeps asking for
+arXiv-id extraction in a fixed format*, or *programmer who keeps mis-formatting
+git commit messages*). We ship 12–20 tiny tasks with **exact checkable
+references**, split into `replay` (train) and `holdout` (test).
+
+**Procedure:**
+1. Score the holdout with an **empty** skill+memory → `baseline`.
+2. Run `N` sleep nights (each: replay train slice → reflect → gated edit).
+3. Score holdout with the evolved skill+memory → `after`.
+4. Report `after − baseline`, accept/reject counts, edit count, tokens.
+
+**Two backends:**
+- `mock` (default, **no API key, fully deterministic**): a scripted optimizer that proposes the known-good rule on failure and a scripted judge. Proves the *plumbing* (harvest→mine→replay→gate→adopt) monotonically improves the score and the gate blocks regressions. This is the CI-able acceptance test.
+- `anthropic` (opt-in, uses `ANTHROPIC_API_KEY`): the real optimizer/judge, to demonstrate genuine lift on the persona tasks.
+
+**Success criteria:**
+- Mock: `after > baseline`, gate rejects an injected harmful edit, adopt+backup works, re-run is reproducible. (Hard gate in CI.)
+- Anthropic (when run): `after ≥ baseline` on holdout with ≥1 accepted, human-readable edit; documented in the wake-up report.
+
+## 9. Personas (the user's framing) → concrete recurring-task families
+
+- **Programmer:** commit-message conventions, repo-specific build/test commands, "always run X before Y", framework gotchas → consolidated into project `CLAUDE.md` + a `repo-workflow` skill.
+- **Researcher:** citation/format preferences, experiment-logging habits, paper-section style, dataset-path memory → `research-prefs` skill + memory.
+- **Finance/analyst:** report formatting, recurring data-pull recipes, terminology → `report-style` skill + memory.
+The engine is domain-agnostic; the persona only changes which tasks get mined.
+
+## 10. Phased delivery
+
+- **Phase 0 — scaffold + types + harvest** (read-only, no API). Provable on this box's real `~/.claude`.
+- **Phase 1 — mine + replay(mock) + consolidate + gate + staging**, with the **mock** optimizer backend and the deterministic experiment green. *(primary deliverable of the offline session)*
+- **Phase 2 — plugin surface** (`/sleep`, skill, hooks, plugin.json) wired to the CLI.
+- **Phase 3 — real Anthropic backend** for miner/reflect/judge + `fresh` replay in worktrees.
+- **Phase 4 — slow/meta cross-night memory**, adopt automation, multi-project, polish + docs.
+
+This session targets **Phase 0 + Phase 1 fully**, **Phase 2 scaffolded**, and the
+**deterministic experiment passing**, all committed (not pushed) for review.
+
+## 11. Open questions for the user (answer when awake)
+
+1. **Adopt policy:** keep default *review-gated*, or do you want `auto_adopt` for your own machine?
+2. **Scope:** harvest only the invoked project, or all projects in `~/.claude/projects`?
+3. **Real-API demo:** want me to spend live `ANTHROPIC_API_KEY` budget on the persona demo, or keep everything mock until you say go?
+4. **Skill target:** evolve a *new* dedicated `skillopt-sleep`-managed skill, or also edit your existing hand-written skills in `~/.claude/skills`?
+5. **Paper:** should this become a section/figure in the SkillOpt arXiv (Dream+Sleep framing as "deployment-time continual skill optimization")?

index.html

@@ -1778,6 +1778,8 @@
       <a href="#evolution">Evolution</a>
       <a href="#transfer">Transfer</a>
       <a href="#citation">Citation</a>
+      <a href="https://microsoft.github.io/SkillOpt/blog/">Blog</a>
+      <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/index.md" target="_blank" rel="noopener">Docs</a>
       <a href="https://github.com/microsoft/SkillOpt" target="_blank" rel="noopener">Code</a>
     </nav>
   </header>
@@ -1915,7 +1917,7 @@
           <h3>A skill is external state for an agent.</h3>
           <p>
             Instead of fine-tuning a model or hand-maintaining prompts, SkillOpt runs
-            the frozen agent on scored batches, asks a separate optimizer model to
+            the frozen agent on scored batches, asks an optimizer model to
             propose structured edits, and accepts a candidate only when validation
             performance improves.
           </p>
@@ -2416,21 +2418,18 @@
 
       <div class="bibtex-box">
         <button class="copy-btn" type="button" onclick="copyBibtex(this)">Copy</button>
-        <pre><code>@misc{yang2026skilloptexecutivestrategyselfevolving,
+        <pre><code>@article{yang2026skillopt,
-      title={SkillOpt: Executive Strategy for Self-Evolving Agent Skills}, 
+  title={Skillopt: Executive strategy for self-evolving agent skills},
-      author={Yifan Yang and Ziyang Gong and Weiquan Huang and Qihao Yang and Ziwei Zhou and Zisu Huang and Yan Li and Xuemei Gao and Qi Dai and Bei Liu and Kai Qiu and Yuqing Yang and Dongdong Chen and Xue Yang and Chong Luo},
+  author={Yang, Yifan and Gong, Ziyang and Huang, Weiquan and Yang, Qihao and Zhou, Ziwei and Huang, Zisu and Li, Yan and Gao, Xuemei and Dai, Qi and Liu, Bei and others},
-      year={2026},
+  journal={arXiv preprint arXiv:2605.23904},
-      eprint={2605.23904},
+  year={2026}
-      archivePrefix={arXiv},
-      primaryClass={cs.AI},
-      url={https://arxiv.org/abs/2605.23904}, 
 }</code></pre>
       </div>
     </section>
 
     <footer class="footer">
       <span>SkillOpt: Executive Strategy for Self-Evolving Agent Skills</span>
-      <span><a href="https://github.com/microsoft/SkillOpt" target="_blank" rel="noopener">Code</a> / <a href="#citation">Citation</a></span>
+      <span><a href="https://microsoft.github.io/SkillOpt/blog/">Blog</a> / <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/index.md" target="_blank" rel="noopener">Docs</a> / <a href="https://github.com/microsoft/SkillOpt" target="_blank" rel="noopener">Code</a> / <a href="#citation">Citation</a></span>
     </footer>
   </main>
   <script>

mkdocs.yml

@@ -39,6 +39,7 @@ theme:
 
 nav:
   - Home: index.md
+  - Technical Blog: https://microsoft.github.io/SkillOpt/blog/
   - Getting Started:
     - Installation: guide/installation.md
     - First Experiment: guide/first-experiment.md
@@ -47,6 +48,10 @@ nav:
     - Training Loop: guide/training-loop.md
     - Skill Document: guide/skill-document.md
     - Deep Learning Analogy: guide/dl-analogy.md
+  - SkillOpt-Sleep:
+    - Overview: sleep/README.md
+    - OpenAI-compatible Endpoints: sleep/openai-compatible-endpoints.md
+    - Results: sleep/RESULTS.md
   - Extension Guides:
     - Add a New Benchmark: guide/new-benchmark.md
     - Local Environment Smoke Tests: guide/local-env-smoke.md

plugins/README.md

@@ -0,0 +1,178 @@
+# SkillOpt-Sleep integrations
+
+**SkillOpt-Sleep** reviews recent agent sessions, mines recurring tasks, replays
+them, and proposes bounded updates to memory and skills. A held-out validation
+gate decides whether a proposal is worth staging, and nothing live changes until
+the user explicitly adopts it.
+
+The shared engine lives in [`skillopt_sleep/`](../skillopt_sleep) and has no
+runtime dependency on the paper's `skillopt/` experiment package.
+
+## Available integrations
+
+Four integrations wrap the shared `skillopt_sleep` CLI. OpenClaw is a separate
+reference adaptation with its own backend and setup assumptions.
+
+| Platform | Folder | Mechanism | Status |
+|---|---|---|---|
+| **Claude Code** | [`claude-code/`](claude-code) | marketplace plugin, commands, skill, and hooks | installable shared-engine integration |
+| **Codex** | [`codex/`](codex) | user-level skill and shared runner | installable shared-engine integration |
+| **GitHub Copilot** | [`copilot/`](copilot) | MCP server exposing seven `sleep_*` tools | shared-engine MCP integration |
+| **Devin** | [`devin/`](devin) | MCP server plus Devin transcript conversion | shared-engine MCP integration |
+| **OpenClaw** | [`openclaw/`](openclaw) | custom DeepSeek/Ollama wrapper | independent reference adaptation; review and adapt before use |
+
+## Install
+
+Clone the repository first unless an installed `skillopt-sleep` CLI is sufficient
+for your workflow.
+
+| Platform | Install | Then |
+|---|---|---|
+| **Claude Code** | from the repository root, `/plugin marketplace add ./plugins/claude-code`, then `/plugin install skillopt-sleep@skillopt-sleep` | `/skillopt-sleep status` |
+| **Codex** | `bash plugins/codex/install.sh` | ask Codex to use the `skillopt-sleep` skill |
+| **Copilot** | register `plugins/copilot/mcp_server.py` using its example MCP config | ask Copilot to run `sleep_status` |
+| **Devin** | register `plugins/devin/mcp_server.py` using its example MCP config | ask Devin to run `sleep_status` |
+| **OpenClaw** | follow and adapt [`openclaw/README.md`](openclaw/README.md) | validate paths, credentials, and tasks locally |
+
+Python 3.10 or newer is required. Real CLI backends also require the selected
+agent CLI to be installed and authenticated.
+
+The shared [`run-sleep.sh`](run-sleep.sh) supports both source checkouts and
+installed packages. If it cannot find the repository, it tries the
+`skillopt-sleep` executable on `PATH` (including `uv tool`/`pipx` installs), then
+an importable `skillopt_sleep` module. Install with `uv tool install skillopt` or
+`pip install skillopt` when using that fallback.
+
+> **Version note.** This integration reference tracks `main`. PyPI 0.2.0
+> supports the base Sleep CLI, while handoff, Sleep support for non-Azure
+> OpenAI-compatible endpoints, and `--preferences` require a source checkout
+> from `main` until the next release.
+
+## One sleep cycle
+
+```text
+harvest supported local sessions → mine recurring tasks → replay tasks
+  → reflect and propose bounded edits → validate on held-out real tasks
+  → stage proposal → (you) review and adopt
+```
+
+The default backend is `mock`: it makes no provider calls and is useful for
+checking plumbing. A real backend is required for model-driven mining and genuine
+optimization.
+
+## Data boundary
+
+- Harvesting is local and read-only. The `mock` backend has no model-provider
+  data path and no API spend.
+- A real backend sends truncated transcript excerpts and derived task content to
+  the provider selected for mining, replay, judging, and reflection.
+- Outbound prompts are not currently guaranteed to be free of secrets. Do not
+  use a third-party provider on sensitive transcripts without reviewing the data
+  source and the provider's retention policy.
+- For a reviewable workflow, export tasks first, inspect and redact the JSON, set
+  its top-level `"reviewed"` field to `true`, and then use the task file with a
+  real backend:
+
+  ```bash
+  python -m skillopt_sleep harvest --project "$(pwd)" --output reviewed-tasks.json
+  python -m skillopt_sleep dry-run --project "$(pwd)" --backend codex \
+    --tasks-file reviewed-tasks.json --progress
+  ```
+
+  Real backends reject task files that are still marked unreviewed.
+
+For the separate API-key and Azure managed-identity transport boundaries, see
+[OpenAI-compatible endpoints](../docs/sleep/openai-compatible-endpoints.md).
+
+## Supported CLI surface
+
+Actions:
+
+| Action | Behavior |
+|---|---|
+| `status` | show state and the latest staged proposal |
+| `dry-run` | harvest, mine, replay, and report; stage nothing |
+| `run` | run the full cycle and stage a proposal |
+| `adopt` | apply the latest staged proposal, with backups |
+| `harvest` | inspect or export mined tasks |
+| `schedule` / `unschedule` | install or remove the managed nightly cron entry |
+
+Common implemented flags include:
+
+| Flag | Default | Purpose |
+|---|---|---|
+| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | `mock` | select who performs model calls |
+| `--model NAME` | backend default | select a backend-specific model |
+| `--source claude\|codex\|auto` | `claude` | select the transcript source |
+| `--project PATH` | current directory | select the project and invoked harvest scope |
+| `--scope invoked\|all` | `invoked` | limit transcript harvesting |
+| `--target-skill-path PATH` | managed skill | select a specific `SKILL.md` to stage/adopt |
+| `--tasks-file PATH` | none | replay a reviewed task file instead of harvesting |
+| `--max-sessions N` / `--max-tasks N` | unset → `3 × tasks` / `40` tasks | bound harvested work; these are not hard token or wall-clock budgets |
+| `--edit-budget N` | `4` | cap bounded edits per cycle |
+| `--preferences "..."` | empty | add house rules to the reflection prior |
+| `--progress` | off | print phase progress to stderr |
+| `--auto-adopt` | off | adopt an accepted proposal without a separate command |
+| `--json` | off | emit machine-readable output where supported |
+
+The nightly CLI does **not** currently expose `--gate`, `--rollouts-k`,
+`--optimizer-model`, `--target-model`, `--budget-tokens`, or `--budget-minutes`.
+Do not pass experiment-harness flags to the main CLI.
+
+### Preferences
+
+`--preferences` is the main user-facing steering knob:
+
+```bash
+python -m skillopt_sleep run --backend codex --project "$(pwd)" \
+  --preferences "Prefer pytest. Keep commit subjects imperative and concise."
+```
+
+Preferences guide reflection but remain subject to the validation gate.
+
+### Advanced config
+
+The JSON/YAML config under `~/.skillopt-sleep/` supports additional engine keys,
+including `gate_mode`, `gate_metric`, `dream_rollouts`, `dream_factor`, `recall_k`,
+`evolve_memory`, and `evolve_skill`. These are config keys, not aliases for the
+unsupported CLI flags listed above. Shipping defaults are conservative:
+`gate_mode="on"`, `dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`.
+
+### Handoff backend
+
+`--backend handoff` keeps model subprocesses out of the engine. It writes pending
+model calls to `.skillopt-sleep-handoff/PROMPTS.md` and `pending.json`, exits with
+code 3, and resumes after answers are placed in `answers/<id>.md`:
+
+```bash
+python -m skillopt_sleep run --backend handoff --project "$(pwd)"
+# answer each prompt in a fresh context, then run the same command again
+```
+
+Answering held-out prompts from a context that has already seen their references
+contaminates the validation gate. Claude Code's `/skillopt-sleep-handoff` command
+automates the loop with isolated fresh-context subagents.
+
+## Validation
+
+The deterministic no-provider check exercises consolidation and the gate:
+
+```bash
+python -m skillopt_sleep.experiments.run_experiment \
+  --persona researcher --assert-improves
+```
+
+Real-model benchmark results and their limitations are documented in
+[`docs/sleep/RESULTS.md`](../docs/sleep/RESULTS.md). The benchmark recipes are not
+the shipping CLI defaults.
+
+## Safety summary
+
+- Session harvesting is read-only.
+- `mock` replay makes no provider calls.
+- `run` stages proposals; `adopt` is the normal live-change boundary.
+- Adoption backs up existing target files.
+- `--max-sessions` and `--max-tasks` bound work, but the main CLI does not yet
+  enforce a hard token or elapsed-time budget.
+- Treat real-backend transcript excerpts as data shared with the selected
+  provider.

plugins/claude-code/.claude-plugin/marketplace.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
+  "name": "skillopt-sleep",
+  "description": "SkillOpt-Sleep: give your local Claude agent a nightly sleep cycle that reviews past sessions and consolidates validated memory + skills.",
+  "owner": {
+    "name": "Yifan Yang",
+    "email": "yifanyang@microsoft.com"
+  },
+  "plugins": [
+    {
+      "name": "skillopt-sleep",
+      "description": "Nightly offline self-evolution: harvest your past Claude Code sessions, replay recurring tasks on your own API budget, and consolidate what the agent learns into validated CLAUDE.md memory and SKILL.md skills, behind a held-out gate, staged for your review. Synthesizes SkillOpt (validation-gated skill optimization), Claude Dreams (offline memory consolidation), and agent sleep/consolidation.",
+      "author": {
+        "name": "Yifan Yang"
+      },
+      "category": "productivity",
+      "source": {
+        "source": "git-subdir",
+        "url": "https://github.com/microsoft/SkillOpt.git",
+        "path": "plugins/claude-code",
+        "ref": "main"
+      },
+      "homepage": "https://github.com/microsoft/SkillOpt"
+    }
+  ]
+}

plugins/claude-code/.claude-plugin/plugin.json

@@ -0,0 +1,22 @@
+{
+  "name": "skillopt-sleep",
+  "description": "Give your local Claude agent a nightly 'sleep cycle': it reviews your past sessions offline, replays recurring tasks on your own API budget, and consolidates what it learns into validated memory (CLAUDE.md) and skills (SKILL.md) so it gets better the more you use it. Synthesizes SkillOpt (validation-gated skill optimization), Claude Dreams (offline memory consolidation), and agent sleep/consolidation.",
+  "version": "0.1.0",
+  "author": {
+    "name": "Yifan Yang",
+    "email": "yifanyang@microsoft.com"
+  },
+  "homepage": "https://github.com/microsoft/SkillOpt",
+  "repository": "https://github.com/microsoft/SkillOpt",
+  "license": "MIT",
+  "keywords": [
+    "skillopt",
+    "self-improvement",
+    "memory-consolidation",
+    "dreams",
+    "sleep",
+    "skills",
+    "continual-learning",
+    "offline-optimization"
+  ]
+}

plugins/claude-code/README.md

@@ -0,0 +1,187 @@
+# SkillOpt-Sleep (Claude Code plugin)
+
+> Give your local Claude agent a **sleep cycle**. Every night it reviews your
+> past sessions offline, replays your recurring tasks on your own API budget,
+> and consolidates what it learns into **validated** memory (`CLAUDE.md`) and
+> skills (`SKILL.md`). Your agent gets better the more you use it — no
+> model-weight training.
+
+SkillOpt-Sleep is the **deployment-time** companion to
+[SkillOpt](https://github.com/microsoft/SkillOpt). SkillOpt trains a skill
+offline on a benchmark; SkillOpt-Sleep applies the same discipline to *your own
+daily usage*: bounded text edits, accepted only through a held-out validation
+gate, with rejected candidates recorded in the cycle report for review.
+
+It synthesizes three ideas:
+
+| Idea | Contribution |
+|---|---|
+| **SkillOpt** | skill/memory = trainable text; bounded add/delete/replace edits; **held-out gate** keeps only changes that help. |
+| **Claude Dreams** | offline consolidation over past sessions; input never mutated; output **reviewed then adopted**. |
+| **Agent sleep** | periodic offline replay turns short-term episodes into long-term skill. |
+
+## What it does (one "night")
+
+```
+harvest ~/.claude transcripts → mine recurring tasks → replay offline
+   → consolidate (reflect → bounded edit → GATE) → stage proposal → (you) adopt
+```
+
+Nothing live is modified until **you** run `/skillopt-sleep adopt` (the Dreams "review,
+then adopt or discard" contract). Every adopt backs up the prior file first.
+
+## Install
+
+**Requirements:** Python ≥ 3.10. A real CLI backend additionally requires its
+corresponding `claude` or `codex` executable on `PATH` and authenticated.
+
+```bash
+# 1) get the code (the plugin ships inside the SkillOpt repo)
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+
+# 2) add the plugin to Claude Code as a local marketplace
+/plugin marketplace add ./plugins/claude-code
+/plugin install skillopt-sleep@skillopt-sleep
+
+# 3) verify
+/skillopt-sleep status
+```
+
+The plugin's bundled runner (`scripts/sleep.sh`) auto-selects a Python ≥ 3.10
+interpreter and calls the `skillopt_sleep` engine. A source checkout needs no
+`pip install`. If the marketplace cache does not contain a usable source tree,
+the shared runner falls back first to a `skillopt-sleep` executable on `PATH`
+(including `uv tool`/`pipx` installs), then to an importable Python module. Use
+`uv tool install skillopt` or `pip install skillopt` for that fallback.
+
+> **Version note.** This page tracks `main`. PyPI 0.2.0 provides the base Sleep
+> CLI, but handoff mode and `--preferences` require a source checkout from
+> `main` until the next release.
+
+## Quick start
+
+```bash
+# from inside any project you use with Claude Code:
+/skillopt-sleep dry-run     # preview what it would learn; no changes staged
+/skillopt-sleep run         # full cycle: stages a reviewed proposal (still no live edits)
+/skillopt-sleep status      # see history + the latest staged proposal
+/skillopt-sleep adopt       # apply the staged proposal to CLAUDE.md / SKILL.md (with backup)
+
+/skillopt-sleep-handoff run # same cycle, but THIS session answers the model calls
+                            # (no claude -p subprocess, no API key — subscription-friendly)
+```
+
+Or call the engine directly (Python ≥ 3.10):
+
+```bash
+python -m skillopt_sleep run --project "$(pwd)" --scope invoked --backend mock
+python -m skillopt_sleep run --project "$(pwd)" --backend claude   # real lift via Claude
+python -m skillopt_sleep run --project "$(pwd)" --backend codex    # real lift via Codex
+```
+
+Default backend is **`mock`** — deterministic, no API spend — so you can try the
+plumbing for free. Switch to `--backend claude` or `--backend codex` for
+model-driven mining and optimization on your own budget; an accepted gain is
+task- and model-dependent, not guaranteed.
+
+### Data boundary for real backends
+
+Harvesting `~/.claude` is local and read-only, and the `mock` backend makes no
+provider calls. A real backend sends truncated transcript excerpts and derived
+tasks to the selected provider for mining, replay, judging, and reflection.
+Outbound prompts are not currently guaranteed to be secret-free. Review your
+session data and provider policy before using a real backend on a sensitive
+project; the [shared integration guide](../README.md#data-boundary) describes a
+reviewed task-file workflow.
+
+### Handoff mode (session answers the model calls)
+
+`--backend handoff` runs the cycle without any model subprocess: the engine
+executes the deterministic stages and writes every model call it needs to
+`.skillopt-sleep-handoff/PROMPTS.md` + `pending.json` (exit code 3). You (or
+the `/skillopt-sleep-handoff` command, which automates the loop with isolated
+fresh-context subagents) write each raw answer to `answers/<id>.md` and re-run
+the same command; it resumes from the answers and either finishes or stages
+the next batch. Typically 3–6 rounds per night.
+
+```bash
+python -m skillopt_sleep run --backend handoff --project "$(pwd)"
+# ... answer .skillopt-sleep-handoff/PROMPTS.md into answers/<id>.md ...
+python -m skillopt_sleep run --backend handoff --project "$(pwd)"   # resume
+```
+
+Answer every prompt in a **fresh context** — a session that has already seen
+the mined tasks and their references would contaminate the held-out gate.
+Details: [the plugins README](../README.md#handoff-backend).
+
+## Does it actually improve? (real models, public benchmark)
+
+SkillOpt-Sleep is validated against [gbrain-evals](https://github.com/garrytan/gbrain-evals)'
+public `skillopt-v1` suite — the same benchmark gbrain scores its own skill
+optimizer against. We take a deliberately **deficient** skill and run one sleep
+night; held-out scoring is done by a local rule judge (no judge-API, no way to
+grade its own homework).
+
+| Backend | Seed | Held-out before → after | Nights |
+|---|---|---|---|
+| **Claude (Haiku 4.5)** | brief-writer | **0.00 → 1.00** | 1 |
+| **Codex** | brief-writer | **0.00 → 1.00** | 2 |
+
+Both took a brief-writer with no risks section / no confidence level and, within
+1–2 nights, proposed gated edits that lifted the held-out score to perfect —
+into the protected `LEARNED` block, nothing else touched. The Codex 2-night
+trace even shows the optimizer **diagnosing its own residual failure** and
+adding a meta-rule to fix it. See the recorded results and limitations in
+[`docs/sleep/RESULTS.md`](../../docs/sleep/RESULTS.md).
+
+Reproduce:
+
+```bash
+git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals
+python -m skillopt_sleep.experiments.run_gbrain --backend claude --model haiku \
+  --seeds brief-writer --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 \
+  --nights 1 --limit-replay 3 --limit-holdout 3
+python -m skillopt_sleep.experiments.run_gbrain --backend codex \
+  --seeds brief-writer --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 \
+  --nights 1 --limit-replay 3 --limit-holdout 3
+```
+
+## Deterministic proof (no API, no keys)
+
+```bash
+python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves
+python -m skillopt_sleep.experiments.run_experiment --persona programmer  --assert-improves
+```
+
+Each prints the held-out score rising from baseline toward 1.0 as the gate
+accepts the general rules your tasks need, and confirms the gate **rejects** an
+injected harmful edit. Context for the measured experiments is in
+[`docs/sleep/RESULTS.md`](../../docs/sleep/RESULTS.md).
+
+## Schedule it nightly
+
+```bash
+/skillopt-sleep schedule --hour 3 --minute 17
+/skillopt-sleep unschedule
+```
+
+The built-in scheduler creates a managed cron entry and logs under the project.
+The scheduled run stages proposals unless `--auto-adopt` is explicitly selected.
+
+## Safety
+
+- **Read-only** harvest of `~/.claude`. `mock` replay has no side effects.
+- Proposals are **staged**, never auto-applied (unless you opt in with `--auto-adopt`).
+- Every adopt writes a backup under the staging dir's `backup/`.
+- `--max-sessions` and `--max-tasks` bound work, but the main CLI does not enforce
+  a hard token or wall-clock budget.
+- Real backends share truncated session/task content with the selected provider;
+  do not assume outbound prompts have been fully redacted.
+
+## Status
+
+The engine, deterministic experiment, Claude/Codex CLI backends, handoff mode,
+and staged adoption flow are implemented. Advanced experiment-harness flags are
+not automatically available on the nightly CLI; see the
+[shared integration reference](../README.md#supported-cli-surface).

plugins/claude-code/commands/skillopt-sleep-handoff.md

@@ -0,0 +1,74 @@
+---
+description: Run the SkillOpt-Sleep cycle with the handoff backend — no API subprocess; this session answers the engine's model calls via prompt/answer files, in isolated fresh-context subagents
+argument-hint: "[run | dry-run] [--preferences \"...\"] (default: run)"
+allowed-tools: Bash, Read, Write, Task
+---
+
+# /skillopt-sleep-handoff — session-executed sleep cycle
+
+You are driving **SkillOpt-Sleep in handoff mode**: the Python engine runs
+every deterministic stage (harvest → mine → replay scoring → gate → stage)
+and outsources each model call (attempt / judge / reflect) to YOU via
+prompt files. No `claude -p` subprocess, no API key — the model work runs
+on this session's budget, but each prompt MUST be answered in a fresh,
+isolated context so the validation gate stays honest.
+
+## Requested action: $ARGUMENTS
+
+(If `$ARGUMENTS` is empty, treat it as `run`.)
+
+## The loop
+
+Repeat until the engine exits 0 (done) — at most 8 rounds:
+
+1. **Run the engine** via the bundled runner. Split `$ARGUMENTS` into the action
+   and remaining options, and preserve those options on every resumed round:
+
+   ```bash
+   "${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" <action> --backend handoff --project "$(pwd)" --scope invoked <remaining options>
+   ```
+
+   - exit 0 → the night is complete; go to "Finish" below.
+   - exit 3 → pending model calls; continue with step 2.
+   - anything else → stop and show the user the error output.
+
+2. **Read the batch**: `Read` `.skillopt-sleep-handoff/pending.json` in the
+   project. Each entry has `id`, `prompt`, `max_tokens`, `answer_file`.
+
+3. **Answer each prompt in ISOLATION** — this is the integrity rule:
+   - For each entry, launch a subagent (Task tool) whose ENTIRE input is
+     the `prompt` text verbatim. Add nothing: no summary of this session,
+     no mention of SkillOpt, no other prompts from the batch.
+   - Take the subagent's reply and `Write` the raw answer text (no
+     commentary, no code fences) to the entry's `answer_file`.
+   - NEVER answer from this session's own context — you have seen the
+     mined tasks and their references, so inline answers would contaminate
+     the held-out gate and fake the improvement score.
+
+4. **Re-run the same engine command** — it resumes from the answers
+   directory and either finishes or stages the next batch.
+
+## Finish
+
+- For `run`, if the engine prints a staging directory, `Read` its `report.md`
+  and show the user: held-out baseline → candidate score, the gate decision,
+  the proposed edits, and where the proposal is staged. If an accepted proposal
+  was staged, tell the user nothing live changed and offer
+  `/skillopt-sleep adopt`.
+- For `dry-run`, no staging directory or `report.md` is created; summarize the
+  final stdout instead.
+- The engine archives `.skillopt-sleep-handoff/` on a completed real run;
+  do not delete it yourself.
+
+## Safety reminders
+
+- **Never** edit `CLAUDE.md` or `SKILL.md` yourself — only `adopt` does
+  that, with a backup.
+- Mined tasks are pinned to `.skillopt-sleep-handoff/tasks.json` on round
+  one, so sessions created while answering prompts cannot shift the task
+  set. Do not edit that file.
+- If a batch looks like it contains secrets or content the user would not
+  want re-processed, stop and ask before answering.
+- Handoff files apply pattern-based secret redaction, but that is not a
+  guarantee that prompts are free of sensitive data. Treat the pending batch as
+  private user data and do not copy it into chat, logs, or commits.

plugins/claude-code/commands/skillopt-sleep.md

@@ -0,0 +1,82 @@
+---
+description: Run or manage the SkillOpt-Sleep self-evolution cycle (review past sessions, replay tasks through a selected backend, consolidate validated memory + skills, or schedule nightly runs)
+argument-hint: "[run | dry-run | status | adopt | harvest | schedule | unschedule] (default: status)"
+allowed-tools: Bash, Read
+---
+
+# /skillopt-sleep — SkillOpt-Sleep nightly self-evolution
+
+You are driving **SkillOpt-Sleep**: a tool that lets this user's Claude agent
+improve from past usage by reviewing sessions, replaying recurring tasks, and
+consolidating what it learns into **validated** memory (`CLAUDE.md`) and skills
+(`SKILL.md`). With the default gate enabled, a change is kept only if it improves
+a held-out replay score. Nothing live is modified until adoption unless the
+user explicitly requests `--auto-adopt`.
+
+## Requested action: $ARGUMENTS
+
+(If `$ARGUMENTS` is empty, treat it as `status`.)
+
+## How to run it
+
+The engine is the `skillopt_sleep` Python package in this repo. Split
+`$ARGUMENTS` into the first action token and its remaining options, then use the
+**plugin's bundled runner** so the right interpreter and repo are on the path.
+Preserve the user's remaining options (for example `--preferences`, `--backend`,
+or `--target-skill-path`) instead of silently dropping them:
+
+```bash
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" <action> --project "$(pwd)" --scope invoked <remaining options>
+```
+
+`<action>` is one of:
+
+| action       | what it does |
+|--------------|--------------|
+| `status`     | show how many nights have run + the latest staged proposal (READ-ONLY) |
+| `dry-run`    | harvest → mine → replay → report, but **stage nothing** (no-staging preview) |
+| `run`        | full cycle: **stage** a validation report and any accepted proposal; only explicit `--auto-adopt` may also update live files |
+| `adopt`      | apply the latest staged proposal to live `CLAUDE.md` / `SKILL.md` (backs up first) |
+| `harvest`    | debug: print the recurring tasks mined from recent sessions |
+| `schedule`   | install a nightly cron entry for this project (`--hour --minute`, off-:00 by default) |
+| `unschedule` | remove the nightly cron entry (`--all` to remove every managed entry) |
+
+Default backend is `mock` (deterministic, no API spend). To use real budget for
+model-driven optimization, add `--backend claude` or `--backend codex`. An
+accepted gain is evidence on this run's held-out tasks, not a guarantee of
+general improvement; results depend on the tasks, model, and checks. To steer
+what the optimizer writes, add `--preferences "<your house rules>"`.
+
+## Steps to follow
+
+1. **Run the requested action** via the bundled runner above. Capture stdout and
+   stderr.
+2. **For `run`:** if it prints a staging directory, `Read` its `report.md` and
+   show the user:
+   - held-out score: baseline → candidate (evidence on this run's held-out tasks)
+   - the gate decision (accept/reject) and the exact edits it proposes
+   - where the proposal is staged
+3. **For `dry-run`:** no staging directory or `report.md` is created. Summarize
+   the score, gate decision, and edits from stdout (or request `--json` when
+   machine-readable output is useful).
+4. **For `run` that produced an accepted proposal:** inspect whether stdout says
+   it was auto-adopted. If not, tell the user nothing live changed and offer
+   `/skillopt-sleep adopt`; if it was, report the updated paths explicitly.
+5. **For `adopt`:** confirm which live files were updated and that backups were
+   written under the staging dir's `backup/`.
+6. **Never** edit `CLAUDE.md` or `SKILL.md` yourself — let the engine's explicit
+   `adopt` or user-requested `--auto-adopt` path apply its manifest and backup
+   behavior. Respect the review gate.
+
+## Safety reminders
+
+- Harvest is **read-only** over `~/.claude`. Replay in `mock` mode runs no
+  shell side effects.
+- The cycle stages proposals by default; auto-adoption requires explicit opt-in.
+- A real backend sends truncated transcript excerpts and derived tasks to its
+  provider for mining, replay, judging, and reflection. Pattern-based redaction
+  is not a guarantee that outbound prompts are secret-free. For sensitive data,
+  use `mock` or first run `harvest --output <file>`, review/redact the file, set
+  `"reviewed": true`, and then pass it with `--tasks-file`.
+- `schedule` manages a cron entry when `crontab` is available; otherwise it
+  prints a line for manual installation.

plugins/claude-code/hooks/hooks.json

@@ -0,0 +1,16 @@
+{
+  "hooks": {
+    "SessionEnd": [
+      {
+        "matcher": "*",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/on-session-end.sh\"",
+            "async": true
+          }
+        ]
+      }
+    ]
+  }
+}

plugins/claude-code/hooks/on-session-end.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# SkillOpt-Sleep SessionEnd hook (async, best-effort, NON-BLOCKING).
+#
+# This does NOT run the optimizer. It only appends a tiny marker so the next
+# nightly cycle knows there is fresh activity to harvest, and (optionally)
+# nudges the user once that a sleep cycle is available. It must never fail the
+# session or spend API budget.
+set -uo pipefail
+
+PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+STATE_DIR="${HOME}/.skillopt-sleep"
+mkdir -p "$STATE_DIR" 2>/dev/null || exit 0
+
+# Record that a session just ended (cheap; used for "is there new data?").
+printf '%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${PWD}" \
+  >> "$STATE_DIR/session-end.log" 2>/dev/null || true
+
+exit 0

plugins/claude-code/scripts/install-cron.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Print (does NOT install) a crontab line that runs SkillOpt-Sleep nightly.
+# The user copies the line into `crontab -e` if they want it.
+set -euo pipefail
+
+PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+RUNNER="$PLUGIN_ROOT/scripts/sleep.sh"
+PROJECT="${1:-$(pwd)}"
+BACKEND="${2:-mock}"
+
+# 3:17am local — deliberately off the :00 mark so many users don't all hit the
+# API at once (and we leave room for jitter).
+MIN=17
+HOUR=3
+
+cat <<EOF
+# ── SkillOpt-Sleep nightly cycle ────────────────────────────────────────────
+# Review past sessions, replay tasks, stage validated memory/skill updates.
+# Runs at ${HOUR}:$(printf '%02d' $MIN) local every day. Output goes to the project's
+# .skillopt-sleep/ dir; nothing live is changed until you run '/skillopt-sleep adopt'
+# (unless you pass --auto-adopt below).
+#
+# Copy the next line into 'crontab -e':
+${MIN} ${HOUR} * * *  "${RUNNER}" run --project "${PROJECT}" --scope invoked --backend ${BACKEND} >> "${PROJECT}/.skillopt-sleep/cron.log" 2>&1
+#
+# For fully-autonomous adoption (power users), append: --auto-adopt
+# To use the authenticated Claude CLI for model-driven optimization, set
+# BACKEND=claude above.
+# ────────────────────────────────────────────────────────────────────────────
+EOF

plugins/claude-code/scripts/run-sleep.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# SkillOpt-Sleep shared runner — used by all platform plugins (Claude Code,
+# Codex, Copilot). Resolves the repo root (which contains the skillopt_sleep
+# package), picks a Python >= 3.10, and execs the engine CLI.
+#
+# Usage: run-sleep.sh <run|dry-run|status|adopt|harvest|...> [args...]
+set -euo pipefail
+
+# This script lives at <repo>/plugins/run-sleep.sh, so the repo root (which
+# holds skillopt_sleep/) is one level up. CLAUDE_PLUGIN_ROOT (if set by Claude
+# Code) points at the plugin dir; the engine is then two levels above it.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+if [ -d "$SCRIPT_DIR/../skillopt_sleep" ]; then
+  REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -d "$CLAUDE_PLUGIN_ROOT/../../skillopt_sleep" ]; then
+  REPO_ROOT="$(cd "$CLAUDE_PLUGIN_ROOT/../.." && pwd)"
+elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -d "$SKILLOPT_SLEEP_REPO/skillopt_sleep" ]; then
+  REPO_ROOT="$SKILLOPT_SLEEP_REPO"
+else
+  # last resort: search upward from CWD
+  d="$PWD"
+  while [ "$d" != "/" ]; do
+    [ -d "$d/skillopt_sleep" ] && { REPO_ROOT="$d"; break; }
+    d="$(dirname "$d")"
+  done
+fi
+if [ "$#" -eq 0 ]; then set -- status; fi
+
+if [ -n "${REPO_ROOT:-}" ]; then
+  # Source checkout: run from repo root so skillopt_sleep/ is importable.
+  PY=""
+  # Allow explicit Python override (useful on macOS with old system Python).
+  if [ -n "${SKILLOPT_SLEEP_PYTHON:-}" ]; then
+    PY="$SKILLOPT_SLEEP_PYTHON"
+  else
+    for cand in python3.12 python3.11 python3.10 python3; do
+      if command -v "$cand" >/dev/null 2>&1; then
+        ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)"
+        if [ "${ver:-0}" -ge 310 ]; then PY="$cand"; break; fi
+      fi
+    done
+  fi
+  if [ -z "$PY" ]; then
+    echo "[sleep] ERROR: need Python >= 3.10 (found none)." >&2
+    exit 1
+  fi
+  cd "$REPO_ROOT"
+  exec "$PY" -m skillopt_sleep "$@"
+fi
+
+# No source checkout found — fall back to an installed engine.
+
+# Fallback 1: skillopt-sleep CLI on PATH (uv tool install / pipx / pip install).
+# Checked before the import fallback because uv tool install / pipx isolate the
+# package from the system Python's import path, so `python -c "import
+# skillopt_sleep"` would fail even though the CLI is available.
+if command -v skillopt-sleep >/dev/null 2>&1; then
+  exec skillopt-sleep "$@"
+fi
+
+# Fallback 2: importable as a module (pip install into the active Python).
+# Pick a Python >= 3.10 and check importability.
+PY=""
+for cand in python3.12 python3.11 python3.10 python3; do
+  if command -v "$cand" >/dev/null 2>&1; then
+    ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)"
+    if [ "${ver:-0}" -ge 310 ] && "$cand" -c "import skillopt_sleep" >/dev/null 2>&1; then
+      PY="$cand"; break
+    fi
+  fi
+done
+if [ -n "$PY" ]; then
+  exec "$PY" -m skillopt_sleep "$@"
+fi
+
+echo "[sleep] ERROR: could not locate the skillopt_sleep package." >&2
+echo "[sleep] Install it with 'uv tool install skillopt' or 'pip install skillopt'," >&2
+echo "[sleep] or set SKILLOPT_SLEEP_REPO to a clone of the SkillOpt repo." >&2
+exit 1

plugins/claude-code/scripts/sleep.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Claude Code plugin runner — thin wrapper over the shared runner so all
+# platform plugins share one engine launcher.
+#
+# After marketplace install the plugin is isolated in a cache directory and
+# the repo-relative path no longer works.  We try four locations:
+#   1. Co-located run-sleep.sh (bundled copy — works in marketplace cache)
+#   2. Repo-relative ../../run-sleep.sh (dev checkout)
+#   3. CLAUDE_PLUGIN_ROOT/../run-sleep.sh (plugin env variable)
+#   4. SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh (explicit env)
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+SHARED=""
+if [ -f "$HERE/run-sleep.sh" ]; then
+  SHARED="$HERE/run-sleep.sh"
+elif [ -f "$(cd "$HERE/../.." 2>/dev/null && pwd)/run-sleep.sh" ]; then
+  SHARED="$(cd "$HERE/../.." && pwd)/run-sleep.sh"
+elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -f "$(cd "$CLAUDE_PLUGIN_ROOT/.." 2>/dev/null && pwd)/run-sleep.sh" ]; then
+  SHARED="$(cd "$CLAUDE_PLUGIN_ROOT/.." && pwd)/run-sleep.sh"
+elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -f "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" ]; then
+  SHARED="$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh"
+fi
+
+if [ -z "$SHARED" ]; then
+  echo "[sleep] ERROR: cannot locate run-sleep.sh." >&2
+  echo "[sleep] Set SKILLOPT_SLEEP_REPO to the SkillOpt repo root, or pip install skillopt." >&2
+  exit 1
+fi
+exec bash "$SHARED" "$@"

plugins/claude-code/skills/skillopt-sleep/SKILL.md

@@ -0,0 +1,155 @@
+---
+name: skillopt-sleep
+description: "Use when the user wants their Claude agent to self-improve from past usage, asks about a nightly/offline 'sleep' or 'dream' cycle, memory/skill consolidation, or says things like 'make my agent better the more I use it', 'review my past sessions', 'learn my preferences', 'consolidate what you learned', 'run the sleep cycle', or wants to schedule background self-optimization. Drives the skillopt_sleep engine: harvest past sessions -> mine recurring tasks -> replay through a selected backend -> consolidate validated CLAUDE.md/SKILL.md behind a held-out gate."
+---
+
+# SkillOpt-Sleep: usage-driven self-evolution for a local Claude agent
+
+SkillOpt-Sleep gives the user's agent a **sleep cycle**. On demand or on a
+nightly schedule, it reviews real past Claude Code sessions, re-runs recurring
+tasks through the selected backend, and consolidates what it
+learns into **memory** (`CLAUDE.md`) and **skills** (`SKILL.md`). With the
+default validation gate enabled, it keeps only changes that improve a held-out
+score. Live files change only through explicit adoption or a user-requested
+`--auto-adopt`. It aims to improve this user's recurring work, while making
+each accepted proposal measurable on the run's held-out tasks,
+with no model-weight training. It is the deployment-time analogue of training:
+short-term experience → long-term competence.
+
+It synthesizes three ideas:
+- **SkillOpt** — the skill/memory doc is trainable text; bounded add/delete/replace
+  edits; accepted only through a held-out gate; rejected edits are recorded in
+  the run report for review.
+- **Claude Dreams** — consolidation that reads past sessions and proposes changes
+  inside protected learned blocks; the input is never mutated, and output is
+  reviewed before adoption.
+- **Agent sleep** — periodic background replay turns episodes into durable skill.
+
+## When to use this skill
+
+Trigger when the user wants any of:
+- "make my agent learn from how I use it" / "get better the more I use it" / "remember my preferences across sessions"
+- a nightly/scheduled or on-demand **offline self-improvement / dream / sleep** run
+- to **review past sessions/trajectories** and distill recurring tasks
+- to **consolidate** feedback into `CLAUDE.md` or a managed skill
+- to **schedule** the cycle (cron) or **adopt** a staged proposal
+
+## The cycle (six stages)
+
+1. **Harvest** — read `~/.claude/projects/*/<session>.jsonl` + `~/.claude/history.jsonl` (READ-ONLY) → session digests.
+2. **Mine** — digests → `TaskRecord`s (recurring intents + outcome labels + checkable refs where possible).
+3. **Replay** — re-run tasks through the selected backend under the *current*
+   skill+memory → (hard, soft) scores.
+4. **Consolidate** — reflect on failures → propose bounded edits → **gate** on a held-out slice; with the default gate enabled, accept only if it strictly improves.
+5. **Stage** — write the accepted `proposed_CLAUDE.md` and/or
+   `proposed_SKILL.md`, plus `report.md`, `report.json`, `manifest.json`, and
+   `diagnostics.json` into `<project>/.skillopt-sleep/staging/<timestamp>/`.
+   **Nothing live changes.** A rejected run still has a report but no proposed
+   live-file replacement.
+6. **Adopt** — explicit (or opt-in auto): copy staged files over live ones, backing up first.
+
+## How to drive it
+
+Prefer the `/skillopt-sleep` command. Under the hood it calls the bundled runner:
+
+```bash
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" status                       # what's happened
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" dry-run --project "$(pwd)"    # no-staging preview
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" run --project "$(pwd)"        # full cycle, stages a proposal
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" adopt --project "$(pwd)"      # apply staged proposal (with backup)
+```
+
+- Default backend is `mock` (deterministic, **no API spend**) — good for trying the plumbing.
+- Add `--backend claude` or `--backend codex` to spend the user's real budget
+  for model-driven optimization. A held-out gain is run-specific evidence, not
+  a guarantee of broader improvement; results depend on the tasks, model, and
+  checks.
+- Scope defaults to the invoked project; `--scope all` harvests every Claude
+  project into the current run's configured targets.
+- A real backend sends truncated transcript/task content to its provider. See
+  the data-boundary rules below before using one with sensitive sessions.
+
+### Scheduling
+
+```bash
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" schedule --project "$(pwd)" --hour 3 --minute 17
+"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" unschedule --project "$(pwd)"
+```
+
+Installs a nightly cron entry. `unschedule --all` removes every managed entry.
+
+## Common CLI flags
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--project PATH` | cwd | Project directory to evolve |
+| `--scope all\|invoked` | invoked | Harvest scope |
+| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | mock | Backend (mock = no provider calls) |
+| `--model NAME` | backend default | Override the model used for replay |
+| `--source claude\|codex\|auto` | claude | Transcript source |
+| `--lookback-hours N` | 72 | Harvest window |
+| `--max-sessions N` | derived | Cap harvested sessions; defaults to 3 × max tasks (120 with current defaults) |
+| `--max-tasks N` | 40 | Cap mined tasks |
+| `--target-skill-path PATH` | `~/.claude/skills/skillopt-sleep-learned/SKILL.md` | Explicit SKILL.md to evolve |
+| `--tasks-file PATH` | — | Reviewed TaskRecord JSON (skip harvest) |
+| `--progress` | off | Print phase progress to stderr |
+| `--auto-adopt` | off | Auto-adopt if gate passes |
+| `--edit-budget N` | 4 | Max bounded edits per night |
+| `--preferences TEXT` | empty | Add house rules to the optimizer's reflection prior |
+| `--json` | off | Machine-readable JSON output |
+
+The CLI also has source/runtime path overrides (`--claude-home`, `--codex-home`,
+and `--codex-path`) and action-specific flags. Use
+`python -m skillopt_sleep <action> --help` as the authoritative surface.
+
+## Config keys (`~/.skillopt-sleep/config.json`)
+
+Beyond the CLI flags, advanced behavior is controlled via config:
+
+- **`preferences`** — free-text house rules injected into the optimizer's reflect step (e.g. "Always use async/await", "Answers in `\boxed{}`").
+- **`gate_mode`** — `on` (default, validation-gated) or `off` (greedy, accept all edits).
+- **`gate_metric`** — `hard`, `soft`, or `mixed` (default). Controls how the held-out gate scores.
+- **`dream_rollouts`** — >1 enables multi-rollout contrastive reflection per task.
+- **`recall_k`** — >0 recalls K similar past tasks into the dream (long-term memory).
+- **`evolve_memory`** / **`evolve_skill`** — independently toggle CLAUDE.md vs SKILL.md consolidation.
+
+## Memory consolidation
+
+The sleep cycle can consolidate both:
+- **SKILL.md** — the managed skill file (bounded edits: add/delete/replace)
+- **CLAUDE.md** — the project memory (same bounded edits)
+
+With the default gate enabled, both are evaluated by the same held-out score.
+Set `evolve_memory: false` to consolidate only skills, or `evolve_skill: false`
+for only memory.
+
+## Hard rules
+
+- **Never** hand-edit the user's `CLAUDE.md` / `SKILL.md` as part of this skill.
+  Let the engine's explicit `adopt` or user-requested `--auto-adopt` path apply
+  the staging manifest and back up existing live files first.
+- Harvest is read-only. `mock` replay has no side effects.
+- Real backends send truncated transcript excerpts and derived tasks to the
+  selected provider for mining, replay, judging, and reflection. The Claude
+  transcript path is not guaranteed to remove every secret before those calls.
+  Review provider policy and session contents first. For sensitive data, use
+  `mock` or run `harvest --output <file>`, inspect/redact the JSON, set
+  `"reviewed": true`, and replay it with `--tasks-file`; real backends refuse an
+  unreviewed task file.
+- Always show the user the **held-out baseline → candidate** score and the
+  exact proposed edits before suggesting adoption. Evidence before adoption.
+- If asked to demonstrate the mechanism without provider calls, run
+  `python -m skillopt_sleep.experiments.run_experiment --persona researcher --json`
+  — a deterministic synthetic demo of held-out lift and gate rejection. It
+  validates the mechanism, not effectiveness on the user's own tasks.
+
+## Validate / demo
+
+```bash
+# deterministic synthetic demo (no API): score rises and the gate blocks a regression
+python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves
+python -m skillopt_sleep.experiments.run_experiment --persona programmer  --assert-improves
+```
+
+See the [SkillOpt-Sleep documentation](https://github.com/microsoft/SkillOpt/tree/main/docs/sleep)
+for recorded results, limitations, and the supported integration surface.

plugins/codex/README.md

@@ -0,0 +1,115 @@
+# SkillOpt-Sleep — Codex integration
+
+Give your **Codex** agent a nightly **sleep cycle**: it reviews past sessions
+offline, replays your recurring tasks on your own Codex budget, and consolidates
+what it learns into validated memory + skills behind a held-out gate. Same engine
+as the Claude Code plugin (`skillopt_sleep`), wrapped for Codex.
+
+> **Verified on Codex:** on the public
+> [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1`
+> benchmark, a deliberately deficient skill goes **0.00 → 1.00** on a held-out
+> set with the Codex backend (incl. the tool-use seed via a real tool loop).
+> See the recorded results and limitations in
+> [`docs/sleep/RESULTS.md`](../../docs/sleep/RESULTS.md).
+
+## What Codex supports (and what we use)
+
+Codex (`@openai/codex`) extends via **`AGENTS.md`** instructions, **skills** at
+`~/.agents/skills/<name>/SKILL.md`, and plugins that can distribute skills.
+Custom prompts are deprecated in Codex, so this integration is skill-first: the
+installed `skillopt-sleep` skill contains the launch commands and operating
+rules. The shared runner remains a plain shell entrypoint that the skill calls.
+
+## Install
+
+On Linux/macOS:
+```bash
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+bash plugins/codex/install.sh          # installs the skill
+export SKILLOPT_SLEEP_REPO="$(pwd)"    # so the runner is found from anywhere
+```
+
+On Windows (PowerShell):
+```powershell
+git clone <repo-url> SkillOpt-Sleep
+cd SkillOpt-Sleep
+powershell -File plugins/codex/install.ps1
+[System.Environment]::SetEnvironmentVariable("SKILLOPT_SLEEP_REPO", "$(pwd)", "User")
+```
+
+If a previous install created `~/.codex/prompts/sleep.md`, the installer moves
+that deprecated prompt aside with a `.skillopt-legacy*.bak` suffix.
+
+Requires Python ≥ 3.10 and the `codex` CLI on PATH.
+
+## Use
+
+Mention `$skillopt-sleep` where Codex supports explicit skill mentions, or ask
+Codex in natural language:
+
+```text
+Use the skillopt-sleep skill to run status for this project.
+Use the skillopt-sleep skill to run a dry-run for this project.
+Use the skillopt-sleep skill to run the full cycle for this project with the Codex backend.
+Use the skillopt-sleep skill to adopt the latest staged proposal.
+```
+
+Or call the engine directly:
+
+```bash
+python -m skillopt_sleep dry-run --project "$(pwd)" --source codex --backend mock
+python -m skillopt_sleep run --project "$(pwd)" --source codex --backend codex \
+  --max-sessions 5 --max-tasks 3 --progress
+python -m skillopt_sleep run --project "$(pwd)" --source codex --backend codex \
+  --target-skill-path .agents/skills/example/SKILL.md \
+  --max-sessions 5 --max-tasks 3 --progress
+```
+
+`--source codex` reads Codex Desktop archived sessions from
+`~/.codex/archived_sessions`. Use `--codex-home /path/to/.codex` to point at a
+different Codex home, or `--source auto` to try Codex archives first and fall
+back to Claude Code transcripts. Default backend is `mock` (no API spend).
+`--backend codex` uses your Codex budget for model-driven optimization; an
+accepted gain is task-dependent, not guaranteed. Bound live runs
+with `--max-sessions` and `--max-tasks`; add `--progress` because Codex-backed
+mining, replay, and reflection can be slow and otherwise quiet. Use
+`--target-skill-path` to stage/adopt into a repo-scoped Codex skill such as
+`.agents/skills/<name>/SKILL.md`; target runs over-sample mined tasks and
+prefer tasks that match the target skill's path, headings, and content. The
+implemented main-CLI flags work the same across the shared integrations, and
+`--preferences "..."` is available for house rules. Advanced keys such as
+`gate_mode`, `dream_rollouts`, and `recall_k` belong in the Sleep config; the
+nightly CLI does not expose `--gate`, `--rollouts-k`, token/time-budget, or
+optimizer/target-split flags. See the
+[shared CLI reference](../README.md#supported-cli-surface).
+
+For privacy-sensitive projects, split the run into reviewable steps:
+
+```bash
+python -m skillopt_sleep harvest --project "$(pwd)" --source codex \
+  --target-skill-path .agents/skills/example/SKILL.md \
+  --max-sessions 5 --max-tasks 3 \
+  --output reviewed-tasks.json
+
+python -m skillopt_sleep dry-run --project "$(pwd)" --backend codex \
+  --tasks-file reviewed-tasks.json --progress --json
+```
+
+Inspect/redact the JSON and set `"reviewed": true` before using a real backend.
+`--tasks-file` skips archive harvest/mining and replays only the reviewed JSON
+tasks; real backends refuse task files still marked `"reviewed": false`.
+
+This review step matters even though the Codex transcript converter removes
+known secret-shaped strings: pattern-based redaction is not a guarantee. A real
+backend sends truncated transcript/task content to the selected provider, while
+`--backend mock` makes no provider calls.
+
+## Notes / status
+
+- Codex's `exec` runs shell, so the real-tool-loop replay (e.g. the
+  `tool_called: search` benchmark seed) works natively.
+- This integration no longer installs a `.codex/prompts` slash command. Skills
+  are the reusable Codex workflow surface; mention `skillopt-sleep` explicitly
+  or ask for a sleep/dream/offline self-improvement run and Codex can load the
+  skill.

plugins/codex/install.ps1

@@ -0,0 +1,47 @@
+# Install the SkillOpt-Sleep Codex integration as a user-level Codex skill on Windows.
+# Idempotent; prints what it does.
+
+$ErrorActionPreference = "Stop"
+
+$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
+$CodexHome = if ($env:CODEX_HOME) { $env:CODEX_HOME } else { Join-Path $env:USERPROFILE ".codex" }
+$AgentsSkills = Join-Path $env:USERPROFILE ".agents\skills"
+$LegacyPrompt = Join-Path $CodexHome "prompts\sleep.md"
+
+Write-Output "[install] repo: $RepoRoot"
+
+# 1) user-level skill
+$SkillDir = Join-Path $AgentsSkills "skillopt-sleep"
+if (-not (Test-Path $SkillDir)) {
+    New-Item -ItemType Directory -Path $SkillDir -Force | Out-Null
+}
+Copy-Item (Join-Path $RepoRoot "plugins\codex\skills\skillopt-sleep\SKILL.md") (Join-Path $SkillDir "SKILL.md") -Force
+Write-Output "[install] skill           -> $(Join-Path $SkillDir 'SKILL.md')"
+
+# 2) retire the old custom prompt entrypoint from previous installs
+if (Test-Path $LegacyPrompt) {
+    $Backup = "${LegacyPrompt}.skillopt-legacy.bak"
+    if (Test-Path $Backup) {
+        $DateStr = Get-Date -Format "yyyyMMddHHmmss"
+        $Backup = "${LegacyPrompt}.skillopt-legacy.${DateStr}.bak"
+    }
+    Move-Item $LegacyPrompt $Backup -Force
+    Write-Output "[install] legacy prompt  -> $Backup"
+}
+
+# 3) record the repo location so the runner is found from anywhere
+Write-Output "[install] add to your environment variables:"
+Write-Output "    [System.Environment]::SetEnvironmentVariable('SKILLOPT_SLEEP_REPO', '$RepoRoot', 'User')"
+Write-Output "    Or set it via System Properties."
+
+# 4) optional: append an AGENTS.md hint (only if the user opts in)
+Write-Output ""
+Write-Output "[install] Optional — add this to ~/.codex/AGENTS.md so Codex always knows the tool:"
+Write-Output ""
+Write-Output "  ## SkillOpt-Sleep"
+Write-Output "  Use the skillopt-sleep skill when I ask to run a sleep/dream/offline"
+Write-Output "  self-improvement cycle. The runner is:"
+Write-Output "  \`powershell -File `"$RepoRoot\plugins\run-sleep.ps1`" status --project `"\$(pwd)\`"\`."
+Write-Output ""
+Write-Output "Done. Try asking Codex:"
+Write-Output "  Use the skillopt-sleep skill to run status for this project."

plugins/codex/install.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Install the SkillOpt-Sleep Codex integration as a user-level Codex skill.
+# Idempotent; prints what it does.
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+CODEX_HOME="${CODEX_HOME:-$HOME/.codex}"
+AGENTS_SKILLS="${HOME}/.agents/skills"
+LEGACY_PROMPT="$CODEX_HOME/prompts/sleep.md"
+
+echo "[install] repo: $REPO_ROOT"
+
+# 1) user-level skill
+mkdir -p "$AGENTS_SKILLS/skillopt-sleep"
+cp "$REPO_ROOT/plugins/codex/skills/skillopt-sleep/SKILL.md" "$AGENTS_SKILLS/skillopt-sleep/SKILL.md"
+echo "[install] skill           -> $AGENTS_SKILLS/skillopt-sleep/SKILL.md"
+
+# 2) retire the old custom prompt entrypoint from previous installs
+if [ -f "$LEGACY_PROMPT" ]; then
+  backup="${LEGACY_PROMPT}.skillopt-legacy.bak"
+  if [ -e "$backup" ]; then
+    backup="${LEGACY_PROMPT}.skillopt-legacy.$(date +%Y%m%d%H%M%S).bak"
+  fi
+  mv "$LEGACY_PROMPT" "$backup"
+  echo "[install] legacy prompt  -> $backup"
+fi
+
+# 3) record the repo location so the runner is found from anywhere
+echo "[install] add to your shell profile:"
+echo "    export SKILLOPT_SLEEP_REPO=\"$REPO_ROOT\""
+
+# 4) optional: append an AGENTS.md hint (only if the user opts in)
+cat <<EOF
+
+[install] Optional — add this to ~/.codex/AGENTS.md so Codex always knows the tool:
+
+  ## SkillOpt-Sleep
+  Use the skillopt-sleep skill when I ask to run a sleep/dream/offline
+  self-improvement cycle. The runner is:
+  \`bash "$REPO_ROOT/plugins/run-sleep.sh" status --project "\$(pwd)"\`.
+
+Done. Try asking Codex:
+  Use the skillopt-sleep skill to run status for this project.
+EOF

plugins/codex/skills/skillopt-sleep/SKILL.md

@@ -0,0 +1,185 @@
+---
+name: skillopt-sleep
+description: "Use when the user wants Codex to self-improve from past usage, asks about a nightly/offline 'sleep' or 'dream' cycle, wants Codex to review past sessions, learn preferences, consolidate memory/skills, run dry-run/run/adopt/status for SkillOpt-Sleep, or schedule background self-optimization. Drives the skillopt_sleep engine: harvest past sessions -> mine recurring tasks -> replay through a selected backend -> consolidate validated memory + skills behind a held-out gate."
+---
+
+# SkillOpt-Sleep: usage-driven self-evolution for a local Codex agent
+
+SkillOpt-Sleep gives the user's Codex agent a sleep cycle. On demand or on a
+nightly schedule, it reviews past local sessions, re-runs recurring tasks
+through the selected backend, and proposes changes to a configured skill and to
+the project's `CLAUDE.md`. With the default validation gate enabled, it keeps
+only changes that improve a held-out score. Live files change only through
+explicit adoption or a user-requested `--auto-adopt`. There is no model-weight
+training.
+
+The current shared engine does **not** write `AGENTS.md`. For a Codex-visible
+result, always select a Codex skill explicitly with `--target-skill-path` (for
+example `.agents/skills/<name>/SKILL.md`). If project `CLAUDE.md` is not a
+desired secondary target, set `"evolve_memory": false` in
+`~/.skillopt-sleep/config.json` before running.
+
+## When to use
+
+Trigger when the user wants any of:
+
+- Codex to learn from past sessions or get better the more they use it;
+- a nightly/scheduled or on-demand sleep/dream/offline self-improvement run;
+- to review past sessions and distill recurring tasks;
+- to consolidate feedback into memory or managed skills;
+- to run `status`, `harvest`, `dry-run`, `run`, or `adopt` for SkillOpt-Sleep.
+
+## The cycle
+
+1. **Harvest** - read local session transcripts according to the engine
+   configuration and normalize them into session digests.
+2. **Mine** - turn digests into recurring `TaskRecord`s with outcomes and
+   checkable references where possible.
+3. **Replay** - re-run mined tasks through the selected backend under the
+   current skill and memory.
+4. **Consolidate** - reflect on failures and propose bounded edits.
+5. **Gate** - with the default gate enabled, accept edits only when the held-out
+   validation score improves.
+6. **Stage** - write the proposal under
+   `<project>/.skillopt-sleep/staging/<date>/`; nothing live changes.
+7. **Adopt** - explicitly, or through user-requested auto-adopt, copy staged
+   files over live files with backups for existing targets.
+
+## How to drive it
+
+Invoke the bundled runner via shell (Codex `exec` has shell access). The runner
+finds the engine and a Python >= 3.10 automatically.
+
+```bash
+# point at the repo if it isn't auto-detected from CWD:
+export SKILLOPT_SLEEP_REPO=/path/to/SkillOpt
+TARGET_SKILL=.agents/skills/example/SKILL.md
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" status --project "$(pwd)"
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" harvest --project "$(pwd)" \
+  --source codex --target-skill-path "$TARGET_SKILL"
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" dry-run --project "$(pwd)" \
+  --source codex --target-skill-path "$TARGET_SKILL" --backend mock
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" run --project "$(pwd)" \
+  --source codex --target-skill-path "$TARGET_SKILL" --backend codex \
+  --max-sessions 5 --max-tasks 3 --progress
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" adopt --project "$(pwd)"
+```
+
+On Windows (CMD / PowerShell):
+```cmd
+:: CMD
+set SKILLOPT_SLEEP_REPO=C:\path\to\SkillOpt-Sleep
+"%SKILLOPT_SLEEP_REPO%\plugins\run-sleep.cmd" status --project "%CD%"
+```
+```powershell
+# PowerShell
+$env:SKILLOPT_SLEEP_REPO = "C:\path\to\SkillOpt-Sleep"
+powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" status --project "$(pwd)"
+```
+
+Actions are `status`, `harvest`, `dry-run`, `run`, `adopt`, `schedule`, and `unschedule`.
+
+- Default backend is `mock`, which is deterministic and spends no API budget.
+- `--backend codex` uses the user's Codex budget for model-driven optimization.
+  An accepted held-out gain is run-specific evidence, not a guarantee of
+  broader improvement; results depend on the tasks, model, and checks.
+- `--source codex` reads Codex Desktop archived sessions from `~/.codex/archived_sessions`;
+  use `--codex-home /path/to/.codex` if the archive lives elsewhere.
+- `--target-skill-path` is required for a Codex skill target. Without it, the
+  shared default is a Claude-managed skill under `~/.claude/skills/`, not an
+  `.agents` skill.
+- Keep `dry-run --backend mock` as the first smoke check unless the user
+  explicitly asked for a real optimization run.
+
+### Scheduling
+
+```bash
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" schedule --project "$(pwd)" \
+  --backend codex --hour 3 --minute 17
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" unschedule --project "$(pwd)"
+```
+
+The scheduler persists the project, backend, time, and optional auto-adopt flag;
+it does not persist `--source` or `--target-skill-path` from this command. Before
+scheduling a Codex-targeted run, set `"transcript_source": "codex"` and an
+absolute `"target_skill_path"` in `~/.skillopt-sleep/config.json`. On systems
+without `crontab`, `schedule` prints a line for manual installation.
+`unschedule --all` removes every managed entry.
+
+### All backends
+
+- `--backend mock` — deterministic, no API spend (default)
+- `--backend claude` — uses the Claude CLI
+- `--backend codex` — uses the Codex CLI
+- `--backend copilot` — uses the GitHub Copilot CLI
+- `--backend handoff` — emits prompt/answer files for an interactive session
+- `--backend azure_openai` — uses the configured Azure OpenAI endpoint
+
+### Additional flags
+
+| Flag | Description |
+|------|-------------|
+| `--auto-adopt` | Auto-adopt if the gate passes (default: stage only) |
+| `--edit-budget N` | Max bounded edits per night (default: 4) |
+| `--lookback-hours N` | Harvest window in hours (default: 72) |
+| `--json` | Machine-readable JSON output |
+
+### Config keys (`~/.skillopt-sleep/config.json`)
+
+- **`preferences`** — free-text house rules for the optimizer
+- **`gate_mode`** — `on` (validation-gated, default) or `off` (greedy)
+- **`gate_metric`** — `hard` | `soft` | `mixed` (default)
+- **`dream_rollouts`** — >1 for multi-rollout contrastive reflection
+- **`recall_k`** — >0 recalls similar past tasks from the archive
+
+### Memory consolidation
+
+The shared sleep cycle consolidates project **memory** (`CLAUDE.md`) and the
+selected **skill** (`SKILL.md`) by default. It does not update `AGENTS.md`.
+Each target is independently toggleable through `evolve_memory` /
+`evolve_skill`, and both are gated by the same held-out validation score.
+
+## Steps
+
+1. Run the requested action; capture stdout.
+2. For `dry-run` and `run`, report the held-out baseline -> candidate score,
+   gate action, task count, session count, and exact proposed edits.
+3. If a staging directory is printed, read `report.md` before summarizing.
+4. `run` stages by default; if `--auto-adopt` was explicitly supplied, report
+   the paths it updated instead of claiming nothing changed.
+5. Offer adoption only after the user has reviewed a still-staged proposal.
+6. Never hand-edit the configured `CLAUDE.md` or target skill as a substitute
+   for the engine's adopt path; adoption is the safety boundary and backs up
+   existing targets first.
+
+## Hard rules
+
+- Harvest is read-only. Do not edit archived sessions or raw transcripts.
+- Codex transcript harvesting removes known secret-shaped strings, developer
+  instructions, and raw tool payloads, but pattern-based redaction is not a
+  guarantee. A real backend still sends truncated transcript/task content to
+  its provider. Review sensitive sessions and provider policy first; prefer a
+  reviewed `--tasks-file` workflow when the data boundary matters.
+- Keep raw secrets, credentials, private user data, and transcript contents out
+  of messages, logs, generated artifacts, and commits.
+- Show validation evidence before recommending adoption.
+- Treat generated edits as proposals, not as source of truth.
+- Do not rely on deprecated custom prompts or `/sleep` slash commands for this
+  Codex integration. This skill is the entrypoint.
+
+## Validate
+
+```bash
+python -m skillopt_sleep dry-run --project "$(pwd)" --source codex \
+  --target-skill-path .agents/skills/example/SKILL.md --backend mock --json
+python -m skillopt_sleep.experiments.run_gbrain --backend codex \
+  --seeds brief-writer --data-root /path/to/gbrain-evals/eval/data/skillopt-v1 \
+  --nights 2 --limit-replay 3 --limit-holdout 3
+```
+
+In the recorded `brief-writer` gbrain run, the deliberately deficient fixture
+went 0.00 -> 1.00 on that run's held-out set. Treat this as reproducible
+benchmark evidence for that configuration, not a guarantee for other skills,
+tasks, or models; see the
+[recorded results](https://github.com/microsoft/SkillOpt/blob/main/docs/sleep/RESULTS.md)
+for context and limitations.

plugins/copilot/README.md

@@ -0,0 +1,84 @@
+# SkillOpt-Sleep — GitHub Copilot integration
+
+Give **Copilot** (CLI or VS Code) a nightly **sleep cycle** via a tiny **MCP
+server** that exposes the `skillopt_sleep` engine as tools. MCP is GitHub's
+supported way to extend Copilot, so this works across Copilot CLI, VS Code, and
+other MCP clients with the same server.
+
+## What's here
+
+| File | Purpose |
+|---|---|
+| `mcp_server.py` | stdlib-only MCP (stdio) server exposing `sleep_*` tools |
+| `mcp-config.example.json` | drop-in MCP server config |
+| `copilot-instructions.snippet.md` | paste into `.github/copilot-instructions.md` |
+
+## Install
+
+Requires Python ≥ 3.10. No third-party packages — the server is pure stdlib.
+
+1. **Register the MCP server.** Add the server to your Copilot MCP config
+   (Copilot CLI: `~/.copilot/mcp-config.json`; VS Code: your MCP settings).
+   Use `mcp-config.example.json` as a template — set `SKILLOPT_SLEEP_REPO` to
+   this repo's path:
+
+   ```json
+   {
+     "mcpServers": {
+       "skillopt-sleep": {
+         "command": "python3",
+        "args": ["/abs/path/SkillOpt/plugins/copilot/mcp_server.py"],
+        "env": { "SKILLOPT_SLEEP_REPO": "/abs/path/SkillOpt" }
+       }
+     }
+   }
+   ```
+
+2. **(Optional) Tell Copilot about it.** Append
+   `copilot-instructions.snippet.md` to your repo's
+   `.github/copilot-instructions.md` so Copilot reaches for the tools when the
+   user asks to "run the sleep cycle".
+
+## Use
+
+Ask Copilot things like *"run the sleep cycle"*, *"what did the last sleep
+propose?"*, *"adopt the staged sleep proposal"*. The server exposes seven MCP
+tools: `sleep_status`, `sleep_dry_run`, `sleep_run`, `sleep_adopt`,
+`sleep_harvest`, `sleep_schedule`, and `sleep_unschedule`.
+
+Each tool takes optional `project`, `backend` (`mock`/`claude`/`codex`/`copilot`), and
+`scope` arguments. Default backend is `mock` (no API spend). The `copilot`
+backend drives the GitHub Copilot CLI (`copilot -p ... --output-format json`)
+and requires the `copilot` CLI to be installed and authenticated.
+
+Harvesting is local and read-only, and the default `mock` backend makes no
+provider calls. A real backend sends truncated transcript excerpts and derived
+tasks to the selected provider. Outbound prompts are not currently guaranteed
+to be secret-free; review sensitive data and provider policy first. See the
+[shared data-boundary guidance](../README.md#data-boundary).
+
+For speed, the `copilot` backend runs each call against an isolated
+`COPILOT_HOME` with built-in MCP servers and custom instructions disabled, so
+your user MCP servers (including this project's own) are not spawned per call
+(~5x faster). Override with `SKILLOPT_SLEEP_COPILOT_HOME=<dir>`, pick a model
+with `SKILLOPT_SLEEP_COPILOT_MODEL`, or set `SKILLOPT_SLEEP_COPILOT_FULL_ENV=1`
+to use your real Copilot environment instead.
+
+## Verify the server directly (no Copilot needed)
+
+```bash
+printf '%s\n' \
+  '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
+  '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \
+  | SKILLOPT_SLEEP_REPO="$(pwd)" python3 plugins/copilot/mcp_server.py
+```
+You should see the server info and all seven `sleep_*` tools.
+
+## Notes / status
+
+- MCP is the stable, official Copilot extension surface, so this is the most
+  portable shared-engine integration (one server → CLI + IDE).
+- The MCP schema exposes the main CLI's implemented controls, including task and
+  session caps, target-skill selection, scheduling, and staged adoption. It does
+  not add experiment-only gate, rollout, token/time-budget, or optimizer/target
+  split flags. See the [shared CLI reference](../README.md#supported-cli-surface).

plugins/copilot/copilot-instructions.snippet.md

@@ -0,0 +1,56 @@
+<!--
+Copy this block into your repo's .github/copilot-instructions.md so Copilot
+knows the SkillOpt-Sleep tools exist. (Copilot reads copilot-instructions.md
+automatically as ambient guidance.)
+-->
+
+## SkillOpt-Sleep (offline self-evolution)
+
+This project has SkillOpt-Sleep available via an MCP server (`skillopt-sleep`).
+It gives the agent a nightly "sleep cycle": it reviews past sessions, replays
+recurring tasks through a selected backend, and stages validation-gated changes
+to project `CLAUDE.md` and a configured `SKILL.md`.
+
+When the user asks to "run the sleep cycle", "review my past sessions", "learn
+my preferences", or "make the agent improve from past usage", use the MCP tools:
+
+- `sleep_status` — what's happened + the latest staged proposal
+- `sleep_dry_run` — no-staging preview; a real backend still makes provider calls
+- `sleep_run` — full cycle, stages a validation-gated proposal by default;
+  explicit `auto_adopt` may update live files
+- `sleep_adopt` — apply the staged proposal (backs up an existing live file first)
+- `sleep_harvest` — list mined recurring tasks
+- `sleep_schedule` — install a nightly cron entry (set `hour`/`minute`)
+- `sleep_unschedule` — remove the nightly cron entry
+
+### Key parameters (pass as MCP tool arguments)
+
+- `backend` — `mock` (default, no provider calls), `claude`, `codex`, or `copilot`
+- `source` — `claude`, `codex`, or `auto` (where to read transcripts)
+- `target_skill_path` — explicit SKILL.md to evolve; use this for a skill that
+  the current agent actually loads
+- `tasks_file` — reviewed TaskRecord JSON (skip harvest); real backends require
+  its metadata to contain `"reviewed": true`
+- `max_tasks` / `max_sessions` — cap workload
+- `auto_adopt` — auto-adopt if the gate passes
+- `json` — machine-readable output for programmatic use
+
+### Advanced config (`~/.skillopt-sleep/config.json`)
+
+- `preferences` — free-text house rules for the optimizer
+- `gate_mode` — `on` (default) or `off`; `dream_rollouts` — >1 for more signal
+- `evolve_memory` / `evolve_skill` — toggle which docs consolidate
+
+Always show the user the held-out baseline → candidate score and the proposed
+edits before suggesting `sleep_adopt`. Never hand-edit the user's memory/skill
+files; use `sleep_adopt` (or an explicitly requested `auto_adopt`) so the engine
+applies its staging manifest and backup behavior.
+
+Harvesting is local and read-only, and `backend: "mock"` makes no provider
+calls. A real backend sends truncated transcript excerpts and derived tasks to
+the selected provider; outbound prompts are not guaranteed to be secret-free.
+Review sensitive data and provider policy before selecting a real backend.
+
+`sleep_schedule` persists only the project, backend, time, and optional
+auto-adopt setting. Put a non-default transcript source or target skill in
+`~/.skillopt-sleep/config.json` before scheduling it.

plugins/copilot/mcp-config.example.json

@@ -0,0 +1,11 @@
+{
+  "mcpServers": {
+    "skillopt-sleep": {
+      "command": "python3",
+      "args": ["plugins/copilot/mcp_server.py"],
+      "env": {
+        "SKILLOPT_SLEEP_REPO": "${workspaceFolder}"
+      }
+    }
+  }
+}

plugins/copilot/mcp_server.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""SkillOpt-Sleep — minimal MCP server (stdio, stdlib-only).
+
+Exposes the sleep engine as MCP tools so any MCP-capable client (GitHub Copilot
+CLI / VS Code, Claude Desktop, etc.) can drive it. No third-party deps: speaks
+JSON-RPC 2.0 over stdio with just the handful of MCP methods clients need.
+
+Tools exposed:
+  - sleep_status   : how many nights have run + the latest staged proposal
+  - sleep_dry_run  : harvest+mine+replay, report only (no staging)
+  - sleep_run      : full cycle, stages a proposal (nothing live changes)
+  - sleep_adopt    : apply the latest staged proposal (with backup)
+  - sleep_harvest  : debug — list mined recurring tasks
+
+Each tool shells out to `python -m skillopt_sleep <action> ...` and returns its
+stdout. Configure your client to launch:  python plugins/copilot/mcp_server.py
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+
+REPO_ROOT = os.environ.get("SKILLOPT_SLEEP_REPO") or os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "..", "..")
+)
+PROTOCOL_VERSION = "2024-11-05"
+
+TOOLS = [
+    {"name": "sleep_status", "action": "status",
+     "description": "Show how many SkillOpt-Sleep nights have run and the latest staged proposal."},
+    {"name": "sleep_dry_run", "action": "dry-run",
+     "description": "Preview a sleep cycle (harvest+mine+replay) without staging anything."},
+    {"name": "sleep_run", "action": "run",
+     "description": "Run a full sleep cycle; stages a reviewed proposal. Nothing live changes until adopt."},
+    {"name": "sleep_adopt", "action": "adopt",
+     "description": "Apply the latest staged proposal to CLAUDE.md/SKILL.md (backs up first)."},
+    {"name": "sleep_harvest", "action": "harvest",
+     "description": "Debug: list the recurring tasks mined from recent sessions."},
+    {"name": "sleep_schedule", "action": "schedule",
+     "description": "Install a nightly cron entry to run the sleep cycle automatically."},
+    {"name": "sleep_unschedule", "action": "unschedule",
+     "description": "Remove the nightly cron entry for a project."},
+]
+_BY_NAME = {t["name"]: t for t in TOOLS}
+
+_TOOL_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "project": {"type": "string",
+                     "description": "Project dir to evolve (default: cwd)."},
+        "backend": {"type": "string", "enum": ["mock", "claude", "codex", "copilot"],
+                     "description": "mock = no API spend (default); claude/codex/copilot = real."},
+        "scope": {"type": "string", "enum": ["invoked", "all"],
+                  "description": "Harvest scope (default: invoked project only)."},
+        "source": {"type": "string", "enum": ["claude", "codex", "auto"],
+                   "description": "Transcript source (default: claude)."},
+        "model": {"type": "string",
+                  "description": "Backend-specific model override."},
+        "tasks_file": {"type": "string",
+                       "description": "Path to reviewed TaskRecord JSON (skips harvest)."},
+        "target_skill_path": {"type": "string",
+                              "description": "Explicit SKILL.md path to evolve/stage/adopt."},
+        "progress": {"type": "boolean",
+                     "description": "Print phase progress to stderr."},
+        "max_sessions": {"type": "integer",
+                         "description": "Cap harvested sessions per run."},
+        "max_tasks": {"type": "integer",
+                      "description": "Cap mined tasks per run."},
+        "lookback_hours": {"type": "integer",
+                           "description": "Harvest window in hours (default: 72)."},
+        "auto_adopt": {"type": "boolean",
+                       "description": "Auto-adopt if gate passes (default: false)."},
+        "json": {"type": "boolean",
+                 "description": "Return machine-readable JSON output."},
+        "edit_budget": {"type": "integer",
+                        "description": "Max bounded edits per night (default: 4)."},
+        "hour": {"type": "integer",
+                 "description": "Hour for schedule (0-23, default: 3)."},
+        "minute": {"type": "integer",
+                   "description": "Minute for schedule (0-59, default: 17)."},
+    },
+    "additionalProperties": False,
+}
+
+
+def _run_engine(action: str, args: dict) -> str:
+    py = sys.executable or "python3"
+    cmd = [py, "-m", "skillopt_sleep", action]
+    # String-valued flags
+    for flag, key in [
+        ("--project", "project"), ("--backend", "backend"),
+        ("--scope", "scope"), ("--source", "source"),
+        ("--model", "model"), ("--tasks-file", "tasks_file"),
+        ("--target-skill-path", "target_skill_path"),
+    ]:
+        val = args.get(key)
+        if val:
+            cmd += [flag, str(val)]
+    # Integer-valued flags
+    for flag, key in [
+        ("--max-sessions", "max_sessions"), ("--max-tasks", "max_tasks"),
+        ("--lookback-hours", "lookback_hours"), ("--edit-budget", "edit_budget"),
+        ("--hour", "hour"), ("--minute", "minute"),
+    ]:
+        val = args.get(key)
+        if val is not None:
+            cmd += [flag, str(int(val))]
+    # Boolean flags
+    for flag, key in [
+        ("--progress", "progress"), ("--auto-adopt", "auto_adopt"),
+        ("--json", "json"),
+    ]:
+        if args.get(key):
+            cmd.append(flag)
+    try:
+        proc = subprocess.run(cmd, cwd=REPO_ROOT, capture_output=True, text=True, timeout=3600)
+    except Exception as e:
+        return f"[error] failed to run engine: {e}"
+    out = (proc.stdout or "").strip()
+    err = (proc.stderr or "").strip()
+    return out + (("\n[stderr]\n" + err) if err else "")
+
+
+def _result(id_, result):
+    return {"jsonrpc": "2.0", "id": id_, "result": result}
+
+
+def _error(id_, code, message):
+    return {"jsonrpc": "2.0", "id": id_, "error": {"code": code, "message": message}}
+
+
+def handle(req: dict):
+    method = req.get("method")
+    id_ = req.get("id")
+    if method == "initialize":
+        return _result(id_, {
+            "protocolVersion": PROTOCOL_VERSION,
+            "capabilities": {"tools": {}},
+            "serverInfo": {"name": "skillopt-sleep", "version": "0.1.0"},
+        })
+    if method in ("notifications/initialized", "initialized"):
+        return None  # notification, no response
+    if method == "tools/list":
+        return _result(id_, {"tools": [
+            {"name": t["name"], "description": t["description"], "inputSchema": _TOOL_SCHEMA}
+            for t in TOOLS
+        ]})
+    if method == "tools/call":
+        params = req.get("params") or {}
+        name = params.get("name")
+        tool = _BY_NAME.get(name)
+        if not tool:
+            return _error(id_, -32602, f"unknown tool: {name}")
+        text = _run_engine(tool["action"], params.get("arguments") or {})
+        return _result(id_, {"content": [{"type": "text", "text": text}]})
+    if method == "ping":
+        return _result(id_, {})
+    return _error(id_, -32601, f"method not found: {method}")
+
+
+def main() -> int:
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            req = json.loads(line)
+        except Exception:
+            continue
+        resp = handle(req)
+        if resp is not None:
+            sys.stdout.write(json.dumps(resp) + "\n")
+            sys.stdout.flush()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

plugins/copilot/skillopt/README.md

@@ -0,0 +1,107 @@
+# SkillOpt — GitHub Copilot integration
+
+Give **Copilot** (CLI or VS Code) direct access to the **SkillOpt** research
+engine via a tiny **MCP server**. MCP is GitHub's supported way to extend
+Copilot, so this works across Copilot CLI, VS Code, and other MCP clients with
+the same server.
+
+SkillOpt is **validation-gated, text-space skill optimization**: it reflects on
+rollouts, makes bounded edits to a skill, and keeps a change only if it improves
+a held-out validation set. This plugin exposes the repo's training and eval
+entry points (`scripts/train.py`, `scripts/eval_only.py`) as Copilot tools.
+
+> This is the companion to the **SkillOpt-Sleep** plugin (`../mcp_server.py`,
+> `sleep_*` tools). Sleep evolves a *local coding agent* from your past
+> sessions; this server drives the *research* training/eval loops on the
+> benchmark configs in [`../../../configs`](../../../configs).
+
+## What's here
+
+| File | Purpose |
+|---|---|
+| `mcp_server.py` | stdlib-only MCP (stdio) server exposing `skillopt_*` tools |
+| `mcp-config.example.json` | drop-in MCP server config |
+| `copilot-instructions.snippet.md` | paste into `.github/copilot-instructions.md` |
+
+## Install
+
+Requires Python ≥ 3.10. The MCP server itself is pure stdlib, but the tools it
+launches need SkillOpt's runtime deps — install the package first:
+
+```bash
+pip install -e .   # or: pip install -r requirements.txt
+```
+
+1. **Register the MCP server.** Add the server to your Copilot MCP config
+   (Copilot CLI: `~/.copilot/mcp-config.json`; VS Code: your MCP settings).
+   Use `mcp-config.example.json` as a template — set `SKILLOPT_REPO` to this
+   repo's path:
+
+   ```json
+   {
+     "mcpServers": {
+       "skillopt": {
+         "command": "python3",
+         "args": ["/abs/path/SkillOpt/plugins/copilot/skillopt/mcp_server.py"],
+         "env": { "SKILLOPT_REPO": "/abs/path/SkillOpt" }
+       }
+     }
+   }
+   ```
+
+2. **(Optional) Tell Copilot about it.** Append
+   `copilot-instructions.snippet.md` to your repo's
+   `.github/copilot-instructions.md` so Copilot reaches for the tools when the
+   user asks to "optimize a skill" or "train on a benchmark".
+
+## Use
+
+Ask Copilot things like *"what configs can I run?"*, *"optimize the searchqa
+skill"*, or *"evaluate this skill on the dataset"*. Copilot calls the MCP tools:
+`skillopt_list_configs`, `skillopt_train`, `skillopt_eval`.
+
+| Tool | Required args | Notes |
+|---|---|---|
+| `skillopt_list_configs` | — | Lists `configs/**/*.yaml` you can pass as `config`. |
+| `skillopt_train` | `config` | Runs a reflective optimization loop. Long-running; spends budget. |
+| `skillopt_eval` | `config`, `skill` | Evaluates one skill markdown file; no training. |
+
+Common optional args (both train and eval): `env`, `backend`,
+`optimizer_model`, `target_model`, `out_root`, `cfg_options` (space-separated
+`section.key=value` YAML overrides), and `extra_args` (raw passthrough flags
+for the underlying script). `skillopt_train` also accepts `num_epochs`,
+`batch_size`, `seed`, and `use_gate`. `use_gate` defaults to `true`; setting it
+to `false` still records validation scores but force-accepts every candidate,
+which changes the optimization semantics.
+
+The MCP schema's `backend` argument follows the underlying script's
+`--backend` choices. Role-specific and generic OpenAI-compatible backends can
+be selected through config or `extra_args` (for example,
+`--optimizer_backend openai_compatible --target_backend openai_compatible`);
+see the repository's [backend guide](../../../docs/guide/new-backend.md) for
+the required environment variables.
+
+Runs can be very long. The server's subprocess timeout defaults to 6 hours;
+override it with the `SKILLOPT_RUN_TIMEOUT` environment variable (seconds).
+
+## Verify the server directly (no Copilot needed)
+
+```bash
+printf '%s\n' \
+  '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
+  '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \
+  '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"skillopt_list_configs","arguments":{}}}' \
+  | SKILLOPT_REPO="$(pwd)" python3 plugins/copilot/skillopt/mcp_server.py
+```
+
+You should see the server info, the three `skillopt_*` tools, and the list of
+benchmark configs.
+
+## Notes / status
+
+- MCP is the stable, official Copilot extension surface, so this is portable
+  across Copilot CLI and IDE from one server.
+- `skillopt_list_configs` is filesystem-only and safe to call anytime;
+  `skillopt_train` / `skillopt_eval` shell out to the repo scripts and require
+  the SkillOpt runtime deps (and, for real backends, model credentials — see
+  [`../../../.env.example`](../../../.env.example)).

plugins/copilot/skillopt/copilot-instructions.snippet.md

@@ -0,0 +1,34 @@
+<!--
+Copy this block into your repo's .github/copilot-instructions.md so Copilot
+knows the SkillOpt research-engine tools exist. (Copilot reads
+copilot-instructions.md automatically as ambient guidance.)
+-->
+
+## SkillOpt (research skill-optimization engine)
+
+This repo exposes the core **SkillOpt** training/eval engine via an MCP server
+(`skillopt`). SkillOpt is validation-gated, text-space skill optimization: it
+reflects on rollouts, makes bounded edits to a skill, and keeps a change only
+if it improves a held-out validation set.
+
+When the user asks to "optimize a skill", "train on <benchmark>", "run
+SkillOpt", "evaluate this skill", or "what configs can I run", use the MCP
+tools:
+
+- `skillopt_list_configs` — list the benchmark YAML configs you can pass as `config`
+- `skillopt_train` — run a reflective skill-optimization loop on a config (long-running; spends API/compute budget)
+- `skillopt_eval` — evaluate a single skill markdown file on a dataset (no training)
+
+Guidance:
+- Always run `skillopt_list_configs` first if you don't already know a valid `config` path.
+- `skillopt_train` and `skillopt_eval` are long-running and consume the user's
+  model backend/budget — confirm the `config`, `backend`, and model choices
+  with the user before launching. Surface the held-out gate result for training,
+  or the evaluation score and output directory for eval-only runs.
+- For one-off YAML overrides use dotted `cfg_options` for structured configs
+  (e.g. `train.seed=123 train.batch_size=40`);
+  for any other underlying flag use `extra_args`.
+
+This is distinct from the **SkillOpt-Sleep** MCP server (`skillopt-sleep`,
+`sleep_*` tools), which evolves a local coding agent from past sessions rather
+than running the research benchmarks.

plugins/copilot/skillopt/mcp-config.example.json

@@ -0,0 +1,11 @@
+{
+  "mcpServers": {
+    "skillopt": {
+      "command": "python3",
+      "args": ["plugins/copilot/skillopt/mcp_server.py"],
+      "env": {
+        "SKILLOPT_REPO": "${workspaceFolder}"
+      }
+    }
+  }
+}

plugins/copilot/skillopt/mcp_server.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+"""SkillOpt (research engine) — minimal MCP server (stdio, stdlib-only).
+
+Exposes the core SkillOpt skill-optimization engine as MCP tools so any
+MCP-capable client (GitHub Copilot CLI / VS Code, Claude Desktop, etc.) can
+drive it. No third-party deps: speaks JSON-RPC 2.0 over stdio with just the
+handful of MCP methods clients need.
+
+This is the companion to the SkillOpt-Sleep MCP server (``../mcp_server.py``).
+Where Sleep evolves a *local agent* from past sessions, this server drives the
+*research* training/eval loops from this repo (``scripts/train.py`` /
+``scripts/eval_only.py``) against the benchmark configs in ``configs/``.
+
+Tools exposed:
+  - skillopt_list_configs : discover the benchmark YAML configs you can use
+  - skillopt_train        : run a reflective skill-optimization (training) loop
+  - skillopt_eval         : evaluate a single skill on a dataset (no training)
+
+``skillopt_train`` and ``skillopt_eval`` shell out to the repo's entry-point
+scripts and stream back their stdout/stderr. Configure your client to launch:
+  python plugins/copilot/skillopt/mcp_server.py
+"""
+from __future__ import annotations
+
+import glob
+import json
+import os
+import subprocess
+import sys
+
+# Repo root: three levels up from plugins/copilot/skillopt/mcp_server.py
+REPO_ROOT = os.environ.get("SKILLOPT_REPO") or os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "..", "..", "..")
+)
+PROTOCOL_VERSION = "2024-11-05"
+
+# Training/eval runs are long; give the engine plenty of headroom.
+RUN_TIMEOUT_SECONDS = int(os.environ.get("SKILLOPT_RUN_TIMEOUT", "21600"))  # 6h
+
+
+def _list_configs() -> str:
+    """List the benchmark configs available under configs/ (filesystem only)."""
+    pattern = os.path.join(REPO_ROOT, "configs", "**", "*.yaml")
+    paths = sorted(glob.glob(pattern, recursive=True))
+    if not paths:
+        return f"[no configs found under {os.path.join(REPO_ROOT, 'configs')}]"
+    rels = [os.path.relpath(p, REPO_ROOT).replace(os.sep, "/") for p in paths]
+    lines = ["Available SkillOpt configs (pass as `config`):", ""]
+    lines += [f"  - {r}" for r in rels]
+    return "\n".join(lines)
+
+
+def _run_script(script_rel: str, args: dict, *, required: tuple[str, ...] = ()) -> str:
+    """Shell out to a repo entry-point script, mapping args -> --flags."""
+    for key in required:
+        if not args.get(key):
+            return f"[error] missing required argument: {key}"
+
+    py = sys.executable or "python3"
+    cmd = [py, os.path.join("scripts", script_rel)]
+
+    # Ordered flags that the train/eval scripts accept directly.
+    flag_args = (
+        "config", "skill", "split", "env", "backend",
+        "optimizer_model", "target_model", "out_root",
+        "num_epochs", "batch_size", "seed", "use_gate",
+    )
+    for key in flag_args:
+        val = args.get(key)
+        if val is None or val == "":
+            continue
+        cmd += [f"--{key}", str(val)]
+
+    # cfg-options: arbitrary KEY=VALUE YAML overrides (nargs="+").
+    cfg_options = args.get("cfg_options")
+    if cfg_options:
+        if isinstance(cfg_options, str):
+            cfg_options = cfg_options.split()
+        cmd += ["--cfg-options", *[str(x) for x in cfg_options]]
+
+    # extra_args: raw passthrough for any other train/eval flag.
+    extra = args.get("extra_args")
+    if extra:
+        if isinstance(extra, str):
+            extra = extra.split()
+        cmd += [str(x) for x in extra]
+
+    try:
+        proc = subprocess.run(
+            cmd, cwd=REPO_ROOT, capture_output=True, text=True,
+            timeout=RUN_TIMEOUT_SECONDS,
+        )
+    except subprocess.TimeoutExpired:
+        return f"[error] run exceeded {RUN_TIMEOUT_SECONDS}s timeout: {' '.join(cmd)}"
+    except Exception as e:  # noqa: BLE001
+        return f"[error] failed to run script: {e}"
+    out = (proc.stdout or "").strip()
+    err = (proc.stderr or "").strip()
+    body = out + (("\n[stderr]\n" + err) if err else "")
+    return body or f"[done] exit code {proc.returncode}, no output"
+
+
+TOOLS = [
+    {
+        "name": "skillopt_list_configs",
+        "description": "List the benchmark YAML configs under configs/ that can be passed as `config` to train/eval.",
+    },
+    {
+        "name": "skillopt_train",
+        "description": "Run a SkillOpt reflective skill-optimization (training) loop on a benchmark config. Long-running; uses your model backend/budget.",
+    },
+    {
+        "name": "skillopt_eval",
+        "description": "Evaluate a single skill markdown file on a dataset without training (scripts/eval_only.py).",
+    },
+]
+_BY_NAME = {t["name"]: t for t in TOOLS}
+
+_NO_ARGS_SCHEMA = {"type": "object", "properties": {}, "additionalProperties": False}
+
+_COMMON_PROPS = {
+    "config": {"type": "string",
+               "description": "Path to a benchmark YAML config (e.g. configs/searchqa/default.yaml). See skillopt_list_configs."},
+    "env": {"type": "string", "description": "Override the environment/adapter name (e.g. searchqa, alfworld)."},
+    "backend": {"type": "string", "description": "Model backend (e.g. azure_openai, claude, codex, qwen, minimax)."},
+    "optimizer_model": {"type": "string", "description": "Model used for reflection/skill rewriting (the optimizer)."},
+    "target_model": {"type": "string", "description": "Model used to execute tasks (the target)."},
+    "out_root": {"type": "string", "description": "Output directory root for run artifacts."},
+    "cfg_options": {"type": "string", "description": "Space-separated YAML overrides, e.g. 'seed=123 batch_size=40'."},
+    "extra_args": {"type": "string", "description": "Raw passthrough flags for the underlying script, e.g. '--workers 8 --max_turns 30'."},
+}
+
+_TRAIN_SCHEMA = {
+    "type": "object",
+    "properties": {
+        **_COMMON_PROPS,
+        "num_epochs": {"type": "integer", "description": "Number of optimization epochs."},
+        "batch_size": {"type": "integer", "description": "Tasks per optimization step."},
+        "seed": {"type": "integer", "description": "Random seed."},
+        "use_gate": {"type": "string", "enum": ["true", "false"],
+                     "description": "Whether to keep the held-out validation gate on (default on)."},
+    },
+    "required": ["config"],
+    "additionalProperties": False,
+}
+
+_EVAL_SCHEMA = {
+    "type": "object",
+    "properties": {
+        **_COMMON_PROPS,
+        "skill": {"type": "string", "description": "Path to the skill markdown file to evaluate."},
+        "split": {"type": "string", "description": "Dataset split to evaluate (default: all)."},
+    },
+    "required": ["config", "skill"],
+    "additionalProperties": False,
+}
+
+_SCHEMA_BY_NAME = {
+    "skillopt_list_configs": _NO_ARGS_SCHEMA,
+    "skillopt_train": _TRAIN_SCHEMA,
+    "skillopt_eval": _EVAL_SCHEMA,
+}
+
+
+def _result(id_, result):
+    return {"jsonrpc": "2.0", "id": id_, "result": result}
+
+
+def _error(id_, code, message):
+    return {"jsonrpc": "2.0", "id": id_, "error": {"code": code, "message": message}}
+
+
+def _dispatch(name: str, args: dict) -> str:
+    if name == "skillopt_list_configs":
+        return _list_configs()
+    if name == "skillopt_train":
+        return _run_script("train.py", args, required=("config",))
+    if name == "skillopt_eval":
+        return _run_script("eval_only.py", args, required=("config", "skill"))
+    return f"[error] unknown tool: {name}"
+
+
+def handle(req: dict):
+    method = req.get("method")
+    id_ = req.get("id")
+    if method == "initialize":
+        return _result(id_, {
+            "protocolVersion": PROTOCOL_VERSION,
+            "capabilities": {"tools": {}},
+            "serverInfo": {"name": "skillopt", "version": "0.1.0"},
+        })
+    if method in ("notifications/initialized", "initialized"):
+        return None  # notification, no response
+    if method == "tools/list":
+        return _result(id_, {"tools": [
+            {"name": t["name"], "description": t["description"],
+             "inputSchema": _SCHEMA_BY_NAME[t["name"]]}
+            for t in TOOLS
+        ]})
+    if method == "tools/call":
+        params = req.get("params") or {}
+        name = params.get("name")
+        if name not in _BY_NAME:
+            return _error(id_, -32602, f"unknown tool: {name}")
+        text = _dispatch(name, params.get("arguments") or {})
+        return _result(id_, {"content": [{"type": "text", "text": text}]})
+    if method == "ping":
+        return _result(id_, {})
+    return _error(id_, -32601, f"method not found: {method}")
+
+
+def main() -> int:
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            req = json.loads(line)
+        except Exception:
+            continue
+        resp = handle(req)
+        if resp is not None:
+            sys.stdout.write(json.dumps(resp) + "\n")
+            sys.stdout.flush()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

plugins/devin/README.md

@@ -0,0 +1,81 @@
+# SkillOpt-Sleep — Devin integration
+
+Give **Devin** (Cognition) a nightly **sleep cycle** via a tiny **MCP server**
+that exposes the `skillopt_sleep` engine as tools. MCP is Devin's supported way
+to add custom tooling, so this works in Devin's CLI and IDE.
+
+Devin doesn't write transcripts in the format the engine consumes, so this
+plugin adds a **Devin-specific harvester** that converts every locally available
+source into the Claude Code-compatible JSONL the engine reads.
+
+## What's here
+
+| File | Purpose |
+|---|---|
+| `mcp_server.py` | stdlib-only MCP (stdio) server exposing `sleep_*` tools |
+| `harvest_devin.py` | converts Devin ATIF-v1.7 transcripts + agentmemory + `.devin/skills` into JSONL, with `taskKey` + outcome envelopes |
+| `judge.py` | reference judge for the deferred/judge branch of the validation gate |
+| `mcp-config.example.json` | drop-in MCP server config |
+| `devin-rules.snippet.md` | paste into `.devin/rules/skillopt-sleep.md` |
+
+## What it harvests
+
+| Source | Where |
+|---|---|
+| Devin transcripts (ATIF-v1.7) | `~/.local/share/devin/cli/transcripts/*.json` |
+| agentmemory | `~/.agentmemory/standalone.json` |
+| Skill files | `.devin/skills/*/SKILL.md` |
+
+Workspaces are auto-detected from `~/.config/Devin/User/workspaceStorage/*/workspace.json`.
+After `sleep_adopt`, the evolved skill is synced to `.devin/skills/skillopt-sleep-learned/SKILL.md`.
+
+## Install
+
+Requires Python ≥ 3.10. No third-party packages — the server is pure stdlib.
+
+1. **Register the MCP server.** Use `mcp-config.example.json` as a template; set
+   `args` to the absolute path of this `mcp_server.py`. The engine is found
+   automatically (this plugin lives inside the SkillOpt repo). Or via the Devin
+   CLI:
+
+   ```bash
+   devin mcp add skillopt-sleep \
+     --env "SKILLOPT_DEVIN_CLAUDE_HOME=$HOME/.skillopt-sleep-devin" \
+     -- python3 /abs/path/to/SkillOpt/plugins/devin/mcp_server.py
+   ```
+
+2. **(Optional)** copy `devin-rules.snippet.md` to `.devin/rules/skillopt-sleep.md`
+   so Devin proactively offers the tools.
+
+3. Ask Devin: *"run the sleep cycle"*, *"what did the last sleep propose?"*, *"adopt it"*.
+
+## Tools
+
+| Tool | What it does |
+|---|---|
+| `sleep_status` | nights run so far + latest staged proposal |
+| `sleep_dry_run` | preview cycle — no staging; a real backend still makes provider calls |
+| `sleep_run` | full cycle; stages a proposal for review |
+| `sleep_adopt` | apply the staged proposal; syncs skill to the workspace |
+| `sleep_harvest` | debug: list the recurring tasks mined |
+| `sleep_schedule` | install a nightly cron entry (`--hour` / `--minute`) |
+| `sleep_unschedule` | remove the nightly cron entry |
+
+Default backend is `mock` (no API spend); the `claude`, `codex`, and `copilot`
+backends use the corresponding authenticated CLI and budget. The `handoff`
+backend runs the cycle with no model subprocess or API key — the engine writes
+pending model calls to `.skillopt-sleep-handoff/PROMPTS.md` + `pending.json`
+(exit code 3) and resumes after answers are placed in `answers/<id>.md`; re-run
+`sleep_run` with the same arguments to resume. The seven tools call the same
+`python -m skillopt_sleep` actions as the other shared-engine integrations.
+
+## Data boundary
+
+The Devin harvester reads local ATIF transcripts, agentmemory, and skill files
+and converts them into the engine's session format. The `mock` backend keeps
+that workflow local. A real backend sends truncated excerpts and derived tasks
+to the selected provider for mining, replay, judging, and reflection. The
+conversion step is not a guarantee that outbound prompts contain no secrets;
+review sensitive sources and provider policy before enabling a real backend.
+See the [shared data-boundary guidance](../README.md#data-boundary) and
+[implemented CLI reference](../README.md#supported-cli-surface).

plugins/devin/devin-rules.snippet.md

@@ -0,0 +1,41 @@
+# SkillOpt-Sleep (Devin)
+
+You have access to a nightly self-evolution cycle via the `skillopt-sleep` MCP
+server. Use these tools to improve your long-term skills over time:
+
+- **`sleep_status`** — refresh the converted local cache, then show how many
+  nights have run and the latest staged proposal
+- **`sleep_dry_run`** — refresh the converted local cache and preview a cycle
+  without engine staging/adoption; a real backend still makes provider calls
+- **`sleep_run`** — run a full cycle; stages a proposal by default, while an
+  explicit `auto_adopt` may also update live files
+- **`sleep_adopt`** — apply the staged proposal, then sync the managed skill to
+  `.devin/skills/skillopt-sleep-learned/SKILL.md` when `project` is the Devin
+  workspace and that workspace already contains a `.devin/` directory
+- **`sleep_harvest`** — debug: list the recurring tasks mined from recent sessions
+- **`sleep_schedule`** / **`sleep_unschedule`** — low-level shared-engine cron
+  controls; the current scheduled command does not run Devin's conversion step,
+  so do not use it as an unattended Devin-harvest workflow
+
+When a user asks about the sleep cycle or skill evolution, prefer calling these
+tools over explaining the concept.
+
+Always pass the absolute Devin workspace as `project`, especially for
+`sleep_adopt`. Default backend is `mock` (no provider calls). The `claude`,
+`codex`, and `copilot` backend values use the corresponding installed and
+authenticated CLI; they do not require this plugin to implement a separate
+API-key flow. The `handoff` backend runs the cycle with no model subprocess
+or API key — the engine writes pending model calls to
+`.skillopt-sleep-handoff/` and exits; answer each prompt in a fresh context
+and re-run `sleep_run` to resume (typically 3–6 rounds).
+
+The Devin conversion and mock workflow stay local. A real backend sends
+truncated transcript excerpts and derived tasks to the selected provider for
+mining, replay, judging, and reflection; conversion is not a guarantee that
+outbound prompts contain no secrets. Review local sources and provider policy
+before selecting a real backend.
+
+For a reviewed task file, pass `tasks_file`; before using it with a real backend,
+inspect/redact it and ensure its metadata contains `"reviewed": true`.
+
+Place this file at `.devin/rules/skillopt-sleep.md` in your workspace.

plugins/devin/fixtures/devin_sample.json

@@ -0,0 +1,21 @@
+{
+  "schema_version": "ATIF-v1.7",
+  "session_id": "demo-001",
+  "steps": [
+    {
+      "source": "user",
+      "message": "Fix the failing NullPointerException in OrderService.persist() in the dutch-kis project",
+      "timestamp": "2026-06-20T10:00:00Z"
+    },
+    {
+      "source": "agent",
+      "message": "The repository call returns an Optional that is being unwrapped with .get(). I'll switch to orElseThrow(NotFoundException::new) so the missing-row case is handled.",
+      "timestamp": "2026-06-20T10:00:05Z"
+    },
+    {
+      "source": "agent",
+      "message": "Applied the fix and ran the suite: rtk mvn test -Dtest=OrderServiceTest -> BUILD SUCCESS, 142 passed, 0 failed.",
+      "timestamp": "2026-06-20T10:01:00Z"
+    }
+  ]
+}

plugins/devin/harvest_devin.py

@@ -0,0 +1,533 @@
+#!/usr/bin/env python3
+"""Convert Devin IDE local data into Claude Code-format JSONL transcripts.
+
+Devin (Cognition) does not persist agent conversation transcripts to disk in a
+format the sleep engine understands.  This script bridges that gap by synthesising
+JSONL files from every locally available source:
+
+  1. **Devin transcripts** (~/.local/share/devin/cli/transcripts/*.json)
+     Native ATIF-v1.7 format — source:"user" / source:"agent" messages
+     converted directly to user/assistant JSONL turns.
+
+  2. **agentmemory** (~/.agentmemory/standalone.json)
+     Memories saved by the `agentmemory` MCP server — each memory's title
+     becomes a synthetic user prompt; its content becomes the assistant reply.
+
+  3. **Skill files** (.devin/skills/*/SKILL.md)
+     Each skill description is converted to a session where the user asked
+     "use the <skill> skill" and the assistant described how to apply it.
+
+Output layout (mirrors ~/.claude/projects/<slug>/<sessionId>.jsonl):
+    <out_dir>/projects/<slug>/<session_id>.jsonl
+
+Workspace auto-detection order:
+  1. ``SKILLOPT_DEVIN_WORKSPACES`` env var — colon-separated abs paths
+  2. Devin registry: ``~/.config/Devin/User/workspaceStorage/*/workspace.json``
+  4. Working directory fallback
+
+Usage (standalone):
+    python harvest_devin.py [--out-dir PATH] [--workspaces PATH ...]
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from urllib.parse import unquote, urlparse
+
+# ── cross-platform path resolution (Linux + Windows + macOS) ──────────────────
+#
+# Devin is a VS Code-family app, so its user-data dir moves with the OS:
+# Linux ~/.config/<App>, Windows %APPDATA%\<App>, macOS
+# ~/Library/Application Support/<App>.  Resolve all candidates and let callers
+# keep whichever actually exists.
+
+def _app_data_roots(app: str) -> List[str]:
+    """User-data dir candidates for a VS Code-family app, current OS first."""
+    home = os.path.expanduser("~")
+    roots: List[str] = []
+    if os.name == "nt":
+        appdata = os.environ.get("APPDATA") or os.path.join(home, "AppData", "Roaming")
+        roots.append(os.path.join(appdata, app))
+    elif sys.platform == "darwin":
+        roots.append(os.path.join(home, "Library", "Application Support", app))
+    # XDG / Linux (also a sensible fallback everywhere)
+    xdg = os.environ.get("XDG_CONFIG_HOME") or os.path.join(home, ".config")
+    roots.append(os.path.join(xdg, app))
+    # de-dupe, preserve order
+    return list(dict.fromkeys(roots))
+
+
+def _devin_transcript_candidates() -> List[str]:
+    """Where the Devin CLI may store ATIF transcripts, per OS."""
+    home = os.path.expanduser("~")
+    cands: List[str] = []
+    if os.name == "nt":
+        for base in (os.environ.get("LOCALAPPDATA"), os.environ.get("APPDATA")):
+            if base:
+                cands.append(os.path.join(base, "devin", "cli", "transcripts"))
+    elif sys.platform == "darwin":
+        cands.append(os.path.join(home, "Library", "Application Support",
+                                  "devin", "cli", "transcripts"))
+    cands.append(os.path.join(home, ".local", "share", "devin", "cli", "transcripts"))
+    return list(dict.fromkeys(cands))
+
+
+def _first_existing(paths: List[str]) -> str:
+    """First path that exists, else the first candidate (for nice messaging)."""
+    for p in paths:
+        if os.path.exists(p):
+            return p
+    return paths[0] if paths else ""
+
+
+def _uri_to_path(folder: str) -> str:
+    """Convert a VS Code ``file://`` workspace URI to a local path, cross-platform.
+
+    Linux:   file:///home/u/proj      -> /home/u/proj
+    Windows: file:///c%3A/Users/u/p   -> c:/Users/u/p
+    """
+    if not folder.startswith("file://"):
+        return folder
+    path = unquote(urlparse(folder).path)
+    # Windows drive paths come through as '/C:/...' — strip the leading slash.
+    if os.name == "nt" and re.match(r"^/[A-Za-z]:", path):
+        path = path[1:]
+    return path
+
+# ── workspace auto-detection ─────────────────────────────────────────────────
+
+def _workspaces_from_registry(storage_root: str) -> List[tuple]:
+    """Read VS Code-style workspaceStorage to get (mtime, path) pairs."""
+    results: List[tuple] = []
+    if not os.path.isdir(storage_root):
+        return results
+    for entry in os.scandir(storage_root):
+        ws_json = os.path.join(entry.path, "workspace.json")
+        if not os.path.isfile(ws_json):
+            continue
+        try:
+            with open(ws_json, encoding="utf-8") as f:
+                data = json.load(f)
+            folder = _uri_to_path(data.get("folder", ""))
+            if folder and os.path.isdir(folder):
+                results.append((os.path.getmtime(ws_json), folder))
+        except Exception:
+            continue
+    return results
+
+
+def _detect_workspaces() -> List[str]:
+    """Return known workspace paths (Devin registry), newest first."""
+    env_val = os.environ.get("SKILLOPT_DEVIN_WORKSPACES", "")
+    if env_val:
+        # os.pathsep so Windows 'C:\a;C:\b' splits correctly (not on the drive colon)
+        return [p for p in env_val.split(os.pathsep) if p and os.path.isdir(p)]
+
+    registries: List[str] = [
+        os.path.join(r, "User", "workspaceStorage")
+        for r in _app_data_roots("Devin")
+    ]
+
+    seen: set = set()
+    results: List[tuple] = []
+    for registry in registries:
+        for mtime, folder in _workspaces_from_registry(registry):
+            if folder not in seen:
+                seen.add(folder)
+                results.append((mtime, folder))
+    results.sort(reverse=True)
+    paths = [p for _, p in results]
+    return paths if paths else [os.getcwd()]
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+def _slug(path: str) -> str:
+    """SHA-256 of abs-path, first 16 hex chars — matches Claude Code's scheme."""
+    return hashlib.sha256(os.path.abspath(path).encode()).hexdigest()[:16]
+
+
+def _iso(epoch_ms: Optional[float] = None) -> str:
+    dt = (datetime.fromtimestamp(epoch_ms / 1000.0, tz=timezone.utc)
+          if epoch_ms is not None else datetime.now(tz=timezone.utc))
+    return dt.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+
+def _write_session(
+    out_dir: str, project: str, session_id: str,
+    user_prompts: List[str], assistant_replies: List[str],
+    timestamp_base_ms: float,
+    task_key: Optional[str] = None,
+) -> None:
+    slug = _slug(project)
+    session_dir = os.path.join(out_dir, "projects", slug)
+    os.makedirs(session_dir, exist_ok=True)
+    out_path = os.path.join(session_dir, f"{session_id}.jsonl")
+    ts = timestamp_base_ms
+    with open(out_path, "w", encoding="utf-8") as f:
+        for user_text, asst_text in zip(user_prompts, assistant_replies):
+            user_rec = {
+                "type": "user",
+                "message": {"role": "user", "content": user_text},
+                "cwd": project,
+                "timestamp": _iso(ts),
+                "sessionId": session_id,
+                "version": "1.0",
+            }
+            if task_key:
+                # grouping key so the miner can collapse repeats into one recurring task
+                user_rec["taskKey"] = task_key
+            f.write(json.dumps(user_rec, ensure_ascii=False) + "\n")
+            # space the reply >=5s after the prompt so a single-turn session
+            # isn't misclassified as a <3s headless replay and dropped by the
+            # engine's harvest filter (skillopt_sleep Issue #62).
+            ts += 5000
+            f.write(json.dumps({
+                "type": "assistant",
+                "message": {"role": "assistant", "content": asst_text},
+                "timestamp": _iso(ts),
+                "sessionId": session_id,
+                "version": "1.0",
+            }, ensure_ascii=False) + "\n")
+            ts += 2000
+
+
+def _append_history(out_dir: str, display: str, project: str, timestamp_ms: float) -> None:
+    record = {"display": display, "timestamp": timestamp_ms, "project": project}
+    with open(os.path.join(out_dir, "history.jsonl"), "a", encoding="utf-8") as f:
+        f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+def _infer_project(text: str, workspaces: List[str]) -> str:
+    for ws in workspaces:
+        if os.path.basename(ws.rstrip("/")).lower() in text.lower():
+            return ws
+    return workspaces[0] if workspaces else os.getcwd()
+
+# ── task identity + outcome extraction (fuel for the validation gate) ─────────
+#
+# SkillOpt's gate only works "where tasks recur and have a checkable correctness
+# signal."  These helpers add the two things a raw transcript lacks:
+#   * a stable taskKey so repeats collapse into one recurring task, and
+#   * an outcome envelope (success + verifier + re-runnable reference) so the
+#     held-out replay has something to score against.
+
+_LANG_HINTS = [
+    ("java",   r"(java|spring|maven|\bmvn\b|gradle|\.java\b|lombok)"),
+    ("python", r"(python|pytest|\bpip\b|\.py\b|django|flask)"),
+    ("ts",     r"(typescript|\.tsx?\b|\bnpm\b|jest|node)"),
+    ("js",     r"(javascript|\.jsx?\b)"),
+    ("sql",    r"(\bsql\b|select\s|mariadb|mysql|postgres|\.sql\b)"),
+    ("go",     r"(golang|\bgo test\b|\.go\b)"),
+    ("rust",   r"(rust|cargo|\.rs\b)"),
+]
+_INTENT_HINTS = [
+    ("fix",       r"(fix|bug|error|fail|npe|exception|broken|crash)"),
+    ("implement", r"(implement|add|create|build|introduce|support)"),
+    ("refactor",  r"(refactor|clean ?up|rename|extract|simplify)"),
+    ("test",      r"(test|coverage|assert)"),
+    ("review",    r"(review|audit|inspect)"),
+    ("optimize",  r"(optimi[sz]e|perf|speed up|slow)"),
+    ("explain",   r"(explain|understand|what does|how does)"),
+]
+_STOPWORDS = {"please", "this", "that", "with", "from", "into", "should",
+              "would", "code", "using", "the", "have"}
+
+
+def _normalize_task_key(text: str, project: str) -> str:
+    """Stable '<lang>:<intent>:<target>' grouping key for a task."""
+    low = text.lower()
+    lang = next((n for n, pat in _LANG_HINTS if re.search(pat, low)), "general")
+    intent = next((n for n, pat in _INTENT_HINTS if re.search(pat, low)), "task")
+    # target: prefer a CamelCase identifier, then a filename, then first real word
+    m = re.search(r"\b([A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+)\b", text)  # CamelCase
+    if not m:
+        m = re.search(r"\b([\w-]+\.\w+)\b", text)                     # filename.ext
+    if m:
+        target = m.group(1)
+    else:
+        # first content word that isn't a stopword or an intent verb (e.g. "implement")
+        target = next((w for w in re.findall(r"[a-zA-Z]{4,}", low)
+                       if w not in _STOPWORDS
+                       and not any(re.search(pat, w) for _, pat in _INTENT_HINTS)),
+                      "general")
+    target = re.sub(r"[^a-zA-Z0-9]+", "-", target).strip("-").lower()[:40] or "general"
+    return f"{lang}:{intent}:{target}"
+
+
+_PASS_PAT = re.compile(
+    r"(build success|all tests? pass(?:ed)?|\b\d+ passed\b|\b0 failed\b|"
+    r"tests? pass(?:ed)?|✓|no errors)", re.IGNORECASE)
+_FAIL_PAT = re.compile(
+    r"(build failure|tests? failed|\b[1-9]\d* failed\b|error:|traceback|"
+    r"assertion ?error)", re.IGNORECASE)  # note: "0 failed" must NOT match
+_CMD_PAT = re.compile(
+    r"((?:rtk\s+)?(?:mvn|gradle|pytest|npm(?:\s+run)?\s+test|yarn\s+test|"
+    r"go\s+test|cargo\s+test)[^\n`]*)", re.IGNORECASE)
+
+
+def _detect_outcome(messages: List[str]) -> Optional[Dict[str, Any]]:
+    """Best-effort checkable signal from agent messages. None ⇒ no hard signal."""
+    blob = "\n".join(m for m in messages if m)
+    pass_hit, fail_hit = _PASS_PAT.search(blob), _FAIL_PAT.search(blob)
+    if not pass_hit and not fail_hit:
+        return None
+    verifier = "tests" if re.search(r"test|pytest", blob, re.IGNORECASE) else "build"
+    out: Dict[str, Any] = {
+        "success": bool(pass_hit) and not fail_hit,
+        "verifier": verifier,
+        "evidence": (pass_hit or fail_hit).group(0).strip(),
+    }
+    cmd = _CMD_PAT.search(blob)
+    if cmd:
+        # keep only the command itself, dropping any "-> result" / ": output" tail
+        repro = re.split(r"\s*(?:->|→|:|,)\s*", cmd.group(1))[0].strip()
+        out["reference"] = {"repro": repro}
+    return out
+
+
+def _build_rubric(user_prompt: str) -> List[str]:
+    """Derive checkable criteria from the task so a judge has something to score."""
+    crit: List[str] = []
+    ids = re.findall(r"\b([A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+|[\w-]+\.\w+)\b", user_prompt)
+    for i in dict.fromkeys(ids):           # dedupe, preserve order
+        crit.append(f"Addresses {i}")
+    intent = _normalize_task_key(user_prompt, "").split(":")[1]
+    crit.append({
+        "fix":       "Resolves the reported defect without introducing new errors",
+        "implement": "Implements the requested behavior end to end",
+        "refactor":  "Preserves behavior while improving structure",
+        "test":      "Adds or fixes tests that actually exercise the change",
+        "optimize":  "Improves performance without changing results",
+    }.get(intent, "Satisfies the user's stated request"))
+    crit.append("Response is concrete and actionable, not a restatement of the task")
+    return crit[:5]
+
+
+def _judge_rubric_fallback(user_prompt: str) -> Dict[str, Any]:
+    """When no hard signal exists, attach a rubric and mark the task for judge
+    scoring. success=None tells the gate to defer/judge rather than trust it.
+    The actual scoring is done by judge.py (or the engine) at replay time."""
+    return {
+        "success": None,
+        "verifier": "judge",
+        "rubric": _build_rubric(user_prompt or ""),
+    }
+
+
+def _write_outcome(out_dir: str, session_id: str, task_key: str, project: str,
+                   ts_ms: float, outcome: Dict[str, Any]) -> None:
+    rec = {"type": "outcome", "sessionId": session_id, "taskKey": task_key,
+           "project": project, "timestamp": _iso(ts_ms), **outcome}
+    with open(os.path.join(out_dir, "outcomes.jsonl"), "a", encoding="utf-8") as f:
+        f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+
+# ── source 1: Devin ATIF-v1.7 transcripts ────────────────────────────────────
+
+def harvest_devin_transcripts(
+    transcripts_dir: str, out_dir: str, workspaces: List[str]
+) -> int:
+    """Convert Devin CLI ATIF-v1.7 transcripts to Claude Code JSONL."""
+    if not os.path.isdir(transcripts_dir):
+        return 0
+    written = 0
+    for entry in os.scandir(transcripts_dir):
+        if not entry.name.endswith(".json"):
+            continue
+        try:
+            with open(entry.path, encoding="utf-8") as f:
+                data = json.load(f)
+        except Exception:
+            continue
+        if data.get("schema_version", "").startswith("ATIF"):
+            pass  # Devin native format
+        else:
+            continue
+        session_id = data.get("session_id") or entry.name[:-5]
+        steps = data.get("steps") or []
+        user_prompts: List[str] = []
+        agent_replies: List[str] = []
+        project = ""
+        ts_base: Optional[float] = None
+        for step in steps:
+            src = step.get("source", "")
+            msg = str(step.get("message") or "").strip()
+            if not msg or src == "system":
+                continue
+            if src == "user":
+                user_prompts.append(msg)
+                if not project:
+                    project = _infer_project(msg, workspaces)
+            elif src == "agent":
+                agent_replies.append(msg)
+            if ts_base is None:
+                raw_ts = step.get("timestamp", "")
+                if raw_ts:
+                    try:
+                        from datetime import datetime as _dt
+                        ts_base = _dt.fromisoformat(
+                            raw_ts.replace("Z", "+00:00")
+                        ).timestamp() * 1000
+                    except Exception:
+                        pass
+        if not user_prompts:
+            continue
+        if not project:
+            project = workspaces[0] if workspaces else os.getcwd()
+        if ts_base is None:
+            ts_base = datetime.now(tz=timezone.utc).timestamp() * 1000
+        # Identity + outcome: what makes this trajectory replayable & gradeable.
+        task_key = _normalize_task_key(user_prompts[0], project)
+        outcome = _detect_outcome(agent_replies) or _judge_rubric_fallback(user_prompts[0])
+        # Pair turns; pad shorter list
+        n = max(len(user_prompts), len(agent_replies))
+        user_prompts += [""] * (n - len(user_prompts))
+        agent_replies += [""] * (n - len(agent_replies))
+        sid = f"devin_{session_id}"
+        _write_session(
+            out_dir, project, sid,
+            user_prompts=[p for p in user_prompts if p],
+            assistant_replies=[r if r else "[no reply recorded]" for r, p in
+                               zip(agent_replies, user_prompts) if p],
+            timestamp_base_ms=ts_base,
+            task_key=task_key,
+        )
+        _write_outcome(out_dir, sid, task_key, project, ts_base, outcome)
+        _append_history(
+            out_dir,
+            display=(user_prompts[0] or session_id)[:120],
+            project=project,
+            timestamp_ms=ts_base,
+        )
+        written += 1
+    return written
+
+
+# ── source 2: agentmemory ─────────────────────────────────────────────────────
+
+def harvest_agentmemory(agentmemory_path: str, out_dir: str,
+                        workspaces: List[str]) -> int:
+    if not os.path.isfile(agentmemory_path):
+        return 0
+    with open(agentmemory_path, encoding="utf-8") as f:
+        data = json.load(f)
+    memories: Dict[str, Any] = data.get("mem:memories", {})
+    written = 0
+    base_ts = datetime.now(tz=timezone.utc).timestamp() * 1000 - len(memories) * 60_000
+    for i, (mem_id, mem) in enumerate(memories.items()):
+        title = str(mem.get("title", "")).strip()
+        content = str(mem.get("content", "")).strip()
+        if not title or not content:
+            continue
+        project = _infer_project(title + " " + content, workspaces)
+        ts = base_ts + i * 60_000
+        _write_session(out_dir, project, mem_id,
+                       user_prompts=[title],
+                       assistant_replies=[content],
+                       timestamp_base_ms=ts)
+        _append_history(out_dir, display=title[:120], project=project, timestamp_ms=ts)
+        written += 1
+    return written
+
+# ── source 3: skill files (.devin/skills) ─────────────────────────────────────
+
+def harvest_skills(workspaces: List[str], out_dir: str) -> int:
+    written = 0
+    seen_ids: set = set()
+    for ws in workspaces:
+        skills_root = os.path.join(ws, ".devin", "skills")
+        if not os.path.isdir(skills_root):
+            continue
+        for skill_dir in os.scandir(skills_root):
+            if not skill_dir.is_dir():
+                continue
+            skill_md = os.path.join(skill_dir.path, "SKILL.md")
+            if not os.path.isfile(skill_md):
+                continue
+            sid = f"skill_{skill_dir.name}"
+            if sid in seen_ids:
+                continue
+            seen_ids.add(sid)
+            with open(skill_md, encoding="utf-8") as f:
+                raw = f.read()
+            body = re.sub(r"^---.*?---\s*", "", raw, flags=re.DOTALL).strip()
+            if not body:
+                continue
+            first_line = body.split("\n")[0].lstrip("# ").strip()
+            user_ask = f"Please use the {skill_dir.name} skill: {first_line}"
+            ts = datetime.now(tz=timezone.utc).timestamp() * 1000 - 3_600_000
+            _write_session(out_dir, ws, sid,
+                           user_prompts=[user_ask],
+                           assistant_replies=[body[:1200]],
+                           timestamp_base_ms=ts)
+            _append_history(out_dir, display=user_ask[:120], project=ws, timestamp_ms=ts)
+            written += 1
+    return written
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Generate SkillOpt-Sleep transcripts from Devin local data"
+    )
+    parser.add_argument(
+        "--out-dir",
+        default=os.path.expanduser("~/.skillopt-sleep-devin"),
+        help="Output claude_home dir (default: ~/.skillopt-sleep-devin)",
+    )
+    parser.add_argument(
+        "--agentmemory",
+        default=os.path.expanduser("~/.agentmemory/standalone.json"),
+        help="Path to agentmemory standalone.json",
+    )
+    parser.add_argument(
+        "--devin-transcripts",
+        default=_first_existing(_devin_transcript_candidates()),
+        help="Devin CLI ATIF transcripts directory (default: per-OS auto-detect)",
+    )
+    parser.add_argument(
+        "--workspaces", nargs="*",
+        help="Workspace paths (default: auto-detect from Devin registry)",
+    )
+    parser.add_argument("--quiet", action="store_true")
+    args = parser.parse_args(argv)
+
+    out_dir = os.path.expanduser(args.out_dir)
+    os.makedirs(out_dir, exist_ok=True)
+    os.makedirs(os.path.join(out_dir, "projects"), exist_ok=True)
+
+    workspaces = args.workspaces or _detect_workspaces()
+    workspaces = [ws for ws in workspaces if os.path.isdir(ws)]
+    if not workspaces:
+        workspaces = [os.getcwd()]
+
+    total = 0
+    devin_transcripts = os.path.expanduser(args.devin_transcripts)
+    n = harvest_devin_transcripts(devin_transcripts, out_dir, workspaces)
+    if not args.quiet:
+        print(f"[harvest_devin] devin        : {n} sessions")
+    total += n
+
+    n = harvest_agentmemory(args.agentmemory, out_dir, workspaces)
+    if not args.quiet:
+        print(f"[harvest_devin] agentmemory  : {n} sessions")
+    total += n
+
+    n = harvest_skills(workspaces, out_dir)
+    if not args.quiet:
+        print(f"[harvest_devin] skill files  : {n} sessions")
+    total += n
+
+    if not args.quiet:
+        print(f"[harvest_devin] total        : {total} synthetic sessions → {out_dir}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

plugins/devin/judge.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Reference judge for SkillOpt-Sleep — score a candidate reply against a rubric.
+
+Tasks harvested without a hard test/build signal get ``verifier: "judge"`` and a
+``rubric`` (see ``_build_rubric`` in harvest_devin.py).  This module is the
+scorer the validation gate calls for those tasks: given the rubric and a
+candidate reply produced during replay, it returns a score in ``[0, 1]``.  The
+gate accepts a skill edit only if the *new* skill scores strictly higher on the
+held-out tasks.
+
+It is self-contained on purpose — in a full deployment the SkillOpt engine owns
+replay+scoring, but having a runnable reference here lets you sanity-check the
+judge path without the engine.
+
+Backends (select via ``SKILLOPT_JUDGE``):
+  * ``heuristic`` (default) — keyword-coverage, offline, no API key, deterministic.
+  * ``claude``              — LLM judge via the Anthropic API (needs ANTHROPIC_API_KEY).
+
+Usage:
+    python judge.py --rubric rubric.json --reply reply.txt
+    echo "<reply>" | python judge.py --rubric-inline '["Addresses OrderService", ...]'
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from typing import List
+
+_STOPWORDS = {"addresses", "resolves", "implements", "without", "introducing",
+              "behavior", "request", "response", "concrete", "actionable", "not",
+              "the", "and", "that", "with", "stated", "reported", "actually",
+              "preserves", "improving", "structure", "requested", "satisfies"}
+
+# Cheap, fast model is the right default for a judge.
+_JUDGE_MODEL = os.environ.get("SKILLOPT_JUDGE_MODEL", "claude-haiku-4-5-20251001")
+
+
+def _content_words(text: str) -> List[str]:
+    return [w for w in re.findall(r"[A-Za-z][A-Za-z0-9_.\-]{3,}", text.lower())
+            if w not in _STOPWORDS]
+
+
+def heuristic_score(reply: str, rubric: List[str]) -> float:
+    """Fraction of rubric criteria whose key content words appear in the reply.
+
+    Crude but deterministic: each criterion is 'met' if at least one of its
+    content words shows up in the candidate reply. Good enough to smoke-test the
+    gate wiring; swap in the claude backend for real judging.
+    """
+    if not rubric:
+        return 0.0
+    low = reply.lower()
+    met = 0
+    for criterion in rubric:
+        words = _content_words(criterion)
+        if not words:                       # nothing to check → treat as met
+            met += 1
+            continue
+        if any(w in low for w in words):
+            met += 1
+    return round(met / len(rubric), 3)
+
+
+def claude_score(reply: str, rubric: List[str]) -> float:
+    """LLM judge via the Anthropic API. Returns a 0..1 score.
+
+    Stdlib-only (urllib) so this file stays dependency-free. Falls back to the
+    heuristic if the key is missing or the call fails, so the gate never hard-errors.
+    """
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if not api_key:
+        print("[judge] ANTHROPIC_API_KEY unset — using heuristic", file=sys.stderr)
+        return heuristic_score(reply, rubric)
+    import urllib.request
+
+    rubric_block = "\n".join(f"- {c}" for c in rubric)
+    prompt = (
+        "You are scoring an AI agent's reply against a rubric. For each criterion, "
+        "decide if the reply satisfies it. Respond with ONLY a number between 0 and "
+        "1 — the fraction of criteria satisfied.\n\n"
+        f"Rubric:\n{rubric_block}\n\nReply:\n{reply}\n\nScore:"
+    )
+    body = json.dumps({
+        "model": _JUDGE_MODEL,
+        "max_tokens": 8,
+        "messages": [{"role": "user", "content": prompt}],
+    }).encode()
+    req = urllib.request.Request(
+        "https://api.anthropic.com/v1/messages", data=body,
+        headers={"content-type": "application/json", "x-api-key": api_key,
+                 "anthropic-version": "2023-06-01"},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            data = json.load(resp)
+        text = "".join(b.get("text", "") for b in data.get("content", []))
+        m = re.search(r"[01](?:\.\d+)?", text)
+        return max(0.0, min(1.0, float(m.group(0)))) if m else heuristic_score(reply, rubric)
+    except Exception as exc:                 # network/auth/parse — degrade gracefully
+        print(f"[judge] claude backend failed ({exc}) — using heuristic", file=sys.stderr)
+        return heuristic_score(reply, rubric)
+
+
+def score(reply: str, rubric: List[str]) -> float:
+    backend = os.environ.get("SKILLOPT_JUDGE", "heuristic")
+    return claude_score(reply, rubric) if backend == "claude" else heuristic_score(reply, rubric)
+
+
+def main(argv=None) -> int:
+    p = argparse.ArgumentParser(description="Score a reply against a rubric (0..1)")
+    g = p.add_mutually_exclusive_group(required=True)
+    g.add_argument("--rubric", help="Path to a JSON file containing a list of criteria")
+    g.add_argument("--rubric-inline", help="Inline JSON list of criteria")
+    p.add_argument("--reply", help="Path to the reply text (default: stdin)")
+    args = p.parse_args(argv)
+
+    rubric = (json.load(open(args.rubric, encoding="utf-8")) if args.rubric
+              else json.loads(args.rubric_inline))
+    reply = (open(args.reply, encoding="utf-8").read() if args.reply
+             else sys.stdin.read())
+    print(score(reply, rubric))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

plugins/devin/mcp-config.example.json

@@ -0,0 +1,11 @@
+{
+  "mcpServers": {
+    "skillopt-sleep": {
+      "command": "python3",
+      "args": ["/abs/path/to/SkillOpt/plugins/devin/mcp_server.py"],
+      "env": {
+        "SKILLOPT_DEVIN_CLAUDE_HOME": "~/.skillopt-sleep-devin"
+      }
+    }
+  }
+}

plugins/devin/mcp_server.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+"""SkillOpt-Sleep — Devin MCP server (stdio, stdlib-only).
+
+Exposes the sleep engine as MCP tools so Devin (Cognition) can drive it. No
+third-party deps: speaks JSON-RPC 2.0 over stdio with just the handful of MCP
+methods clients need. Same `sleep_*` interface and engine flags as
+`plugins/copilot`, plus a Devin-specific harvest step.
+
+Before each data-reading action this server runs `harvest_devin.py` to convert
+locally available Devin data (ATIF-v1.7 transcripts, agentmemory memories, and
+.devin skill files) into the Claude Code-compatible JSONL the engine consumes,
+writing it under SKILLOPT_DEVIN_CLAUDE_HOME and pointing the engine there with
+`--claude-home`. After `sleep_adopt` the evolved skill is synced back into the
+workspace's `.devin/skills/`.
+
+Tools: sleep_status, sleep_dry_run, sleep_run, sleep_adopt, sleep_harvest,
+sleep_schedule, sleep_unschedule. Each shells out to
+`python -m skillopt_sleep <action> ...`. Configure Devin to launch:
+  python plugins/devin/mcp_server.py
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+
+# expanduser wraps the whole value so a "~/..." env var is expanded too (not
+# just a default) — otherwise a literal ~ dir gets created.
+REPO_ROOT = os.path.expanduser(
+    os.environ.get("SKILLOPT_SLEEP_REPO")
+    or os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+)
+PLUGIN_DIR = os.path.dirname(os.path.abspath(__file__))
+CLAUDE_HOME = os.path.expanduser(
+    os.environ.get("SKILLOPT_DEVIN_CLAUDE_HOME", "~/.skillopt-sleep-devin")
+)
+MANAGED_SKILL_NAME = os.environ.get("SKILLOPT_MANAGED_SKILL", "skillopt-sleep-learned")
+PROTOCOL_VERSION = "2024-11-05"
+
+TOOLS = [
+    {"name": "sleep_status", "action": "status",
+     "description": "Show how many SkillOpt-Sleep nights have run and the latest staged proposal."},
+    {"name": "sleep_dry_run", "action": "dry-run",
+     "description": "Preview a sleep cycle (harvest+mine+replay) without staging anything."},
+    {"name": "sleep_run", "action": "run",
+     "description": "Run a full sleep cycle; stages a reviewed proposal. Nothing live changes until adopt."},
+    {"name": "sleep_adopt", "action": "adopt",
+     "description": "Apply the latest staged proposal to the managed SKILL.md and sync it into .devin/skills/."},
+    {"name": "sleep_harvest", "action": "harvest",
+     "description": "Debug: list the recurring tasks mined from recent Devin sessions."},
+    {"name": "sleep_schedule", "action": "schedule",
+     "description": "Install a nightly cron entry to run the sleep cycle automatically."},
+    {"name": "sleep_unschedule", "action": "unschedule",
+     "description": "Remove the nightly cron entry for a project."},
+]
+_BY_NAME = {t["name"]: t for t in TOOLS}
+
+_TOOL_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "project": {"type": "string",
+                    "description": "Project dir to evolve (default: cwd)."},
+        "backend": {"type": "string", "enum": ["mock", "claude", "codex", "copilot", "handoff"],
+                    "description": "mock = no API spend (default); claude/codex/copilot = real; handoff = session answers prompts, no API subprocess."},
+        "scope": {"type": "string", "enum": ["invoked", "all"],
+                  "description": "Harvest scope (default: invoked project only)."},
+        "source": {"type": "string", "enum": ["claude", "codex", "auto"],
+                   "description": "Transcript source (default: claude)."},
+        "model": {"type": "string",
+                  "description": "Backend-specific model override."},
+        "tasks_file": {"type": "string",
+                       "description": "Path to reviewed TaskRecord JSON (skips harvest)."},
+        "target_skill_path": {"type": "string",
+                              "description": "Explicit SKILL.md path to evolve/stage/adopt."},
+        "progress": {"type": "boolean",
+                     "description": "Print phase progress to stderr."},
+        "max_sessions": {"type": "integer",
+                         "description": "Cap harvested sessions per run."},
+        "max_tasks": {"type": "integer",
+                      "description": "Cap mined tasks per run."},
+        "lookback_hours": {"type": "integer",
+                           "description": "Harvest window in hours (default: 72)."},
+        "auto_adopt": {"type": "boolean",
+                       "description": "Auto-adopt if gate passes (default: false)."},
+        "json": {"type": "boolean",
+                 "description": "Return machine-readable JSON output."},
+        "edit_budget": {"type": "integer",
+                        "description": "Max bounded edits per night (default: 4)."},
+        "hour": {"type": "integer",
+                 "description": "Hour for schedule (0-23, default: 3)."},
+        "minute": {"type": "integer",
+                   "description": "Minute for schedule (0-59, default: 17)."},
+    },
+    "additionalProperties": False,
+}
+
+# actions that read harvested Devin data (schedule/unschedule/adopt don't)
+_HARVEST_ACTIONS = {"status", "dry-run", "run", "harvest"}
+
+
+def _run_harvest() -> str:
+    """Convert local Devin data into the JSONL the engine reads, under CLAUDE_HOME."""
+    harvester = os.path.join(PLUGIN_DIR, "harvest_devin.py")
+    env = dict(os.environ)
+    env["PYTHONPATH"] = REPO_ROOT + os.pathsep + env.get("PYTHONPATH", "")
+    try:
+        proc = subprocess.run(
+            [sys.executable, harvester, "--out-dir", CLAUDE_HOME],
+            capture_output=True, text=True, timeout=60, env=env,
+        )
+        out = (proc.stdout or "").strip()
+        err = (proc.stderr or "").strip()
+        return out + (("\n[harvest stderr]\n" + err) if err else "")
+    except Exception as exc:
+        return f"[harvest_devin] warning: {exc}"
+
+
+def _sync_skill(project: str) -> str:
+    """After adopt, copy the evolved skill into the workspace's .devin/skills/."""
+    src = os.path.join(CLAUDE_HOME, "skills", MANAGED_SKILL_NAME, "SKILL.md")
+    if not (os.path.isfile(src) and project and os.path.isdir(project)):
+        return ""
+    dot_root = os.path.join(project, ".devin")
+    if not os.path.isdir(dot_root):
+        return ""
+    dst_dir = os.path.join(dot_root, "skills", MANAGED_SKILL_NAME)
+    os.makedirs(dst_dir, exist_ok=True)
+    dst = os.path.join(dst_dir, "SKILL.md")
+    shutil.copy2(src, dst)
+    return f"\n[sleep] synced evolved skill → {dst}"
+
+
+def _run_engine(action: str, args: dict) -> str:
+    harvest_out = _run_harvest() if action in _HARVEST_ACTIONS else ""
+
+    py = sys.executable or "python3"
+    cmd = [py, "-m", "skillopt_sleep", action, "--claude-home", CLAUDE_HOME]
+    # Devin transcripts are converted to the Claude format, so default source=claude
+    if not args.get("source"):
+        cmd += ["--source", "claude"]
+    # String-valued flags
+    for flag, key in [
+        ("--project", "project"), ("--backend", "backend"),
+        ("--scope", "scope"), ("--source", "source"),
+        ("--model", "model"), ("--tasks-file", "tasks_file"),
+        ("--target-skill-path", "target_skill_path"),
+    ]:
+        val = args.get(key)
+        if val:
+            cmd += [flag, str(val)]
+    # Integer-valued flags
+    for flag, key in [
+        ("--max-sessions", "max_sessions"), ("--max-tasks", "max_tasks"),
+        ("--lookback-hours", "lookback_hours"), ("--edit-budget", "edit_budget"),
+        ("--hour", "hour"), ("--minute", "minute"),
+    ]:
+        val = args.get(key)
+        if val is not None:
+            cmd += [flag, str(int(val))]
+    # Boolean flags
+    for flag, key in [
+        ("--progress", "progress"), ("--auto-adopt", "auto_adopt"), ("--json", "json"),
+    ]:
+        if args.get(key):
+            cmd.append(flag)
+
+    env = dict(os.environ)
+    env["PYTHONPATH"] = REPO_ROOT + os.pathsep + env.get("PYTHONPATH", "")
+    try:
+        proc = subprocess.run(cmd, cwd=REPO_ROOT, capture_output=True,
+                              text=True, timeout=3600, env=env)
+    except Exception as e:
+        return f"[harvest]\n{harvest_out}\n[error] failed to run engine: {e}"
+    out = (proc.stdout or "").strip()
+    err = (proc.stderr or "").strip()
+    result = (f"[harvest]\n{harvest_out}\n\n" if harvest_out else "") + f"[engine]\n{out}"
+    if err:
+        result += f"\n[stderr]\n{err}"
+    if action == "adopt":
+        result += _sync_skill(args.get("project") or os.getcwd())
+    return result
+
+
+def _result(id_, result):
+    return {"jsonrpc": "2.0", "id": id_, "result": result}
+
+
+def _error(id_, code, message):
+    return {"jsonrpc": "2.0", "id": id_, "error": {"code": code, "message": message}}
+
+
+def handle(req: dict):
+    method = req.get("method")
+    id_ = req.get("id")
+    if method == "initialize":
+        return _result(id_, {
+            "protocolVersion": PROTOCOL_VERSION,
+            "capabilities": {"tools": {}},
+            "serverInfo": {"name": "skillopt-sleep", "version": "0.1.0"},
+        })
+    if method in ("notifications/initialized", "initialized"):
+        return None
+    if method == "tools/list":
+        return _result(id_, {"tools": [
+            {"name": t["name"], "description": t["description"], "inputSchema": _TOOL_SCHEMA}
+            for t in TOOLS
+        ]})
+    if method == "tools/call":
+        params = req.get("params") or {}
+        name = params.get("name")
+        tool = _BY_NAME.get(name)
+        if not tool:
+            return _error(id_, -32602, f"unknown tool: {name}")
+        text = _run_engine(tool["action"], params.get("arguments") or {})
+        return _result(id_, {"content": [{"type": "text", "text": text}]})
+    if method == "ping":
+        return _result(id_, {})
+    return _error(id_, -32601, f"method not found: {method}")
+
+
+def main() -> int:
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            req = json.loads(line)
+        except Exception:
+            continue
+        resp = handle(req)
+        if resp is not None:
+            sys.stdout.write(json.dumps(resp) + "\n")
+            sys.stdout.flush()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

plugins/openclaw/README.md

@@ -0,0 +1,115 @@
+# OpenClaw reference adaptation for SkillOpt-Sleep
+
+This directory is a contributed reference for connecting
+[SkillOpt-Sleep](https://github.com/microsoft/SkillOpt) to
+[OpenClaw](https://github.com/openclaw/openclaw) with a custom DeepSeek/Ollama
+backend.
+
+> **Reference status.** This is not one of the shared, plug-and-play
+> `skillopt_sleep` wrappers. Several scripts and the sample config contain
+> environment-specific absolute paths and assumptions from the original setup,
+> and the contributed wrapper has unresolved Python 3.10 syntax and backend
+> factory-signature gaps. The current checkout is not directly runnable; treat
+> it as porting source material, not an installation.
+
+## Included components
+
+| File | Purpose |
+|---|---|
+| `run_sleep.py` | custom cycle entry point |
+| `skillopt_sleep_openclaw.py` | DeepSeek Chat Completions backend plus local Ollama embeddings |
+| `run_sleep_cron.sh` | category-oriented cron wrapper |
+| `slash_sleep.py` | experimental `/sleep` command helper |
+| `config.json` | example engine configuration |
+| `SKILL.md` | OpenClaw skill manifest |
+| `tests/*.json` | example task sets for research, DevOps, and wiki workflows |
+
+The adaptation imports the shared engine but registers its own backend and
+maintains its own wrapper behavior. Changes to the shared CLI documentation do
+not automatically make every option available through these custom scripts.
+
+## Intended cycle
+
+```text
+harvest supported session data or load a task file
+  → replay with the current skill
+  → propose bounded edits
+  → validate the candidate on held-out tasks
+  → stage a proposal for operator review
+```
+
+The intended safety boundary is manual adoption: review the generated report and
+staged files before changing a live skill.
+
+## Adapt before use
+
+1. Clone SkillOpt into a location you control:
+
+   ```bash
+   git clone https://github.com/microsoft/SkillOpt.git
+   cd SkillOpt/plugins/openclaw
+   ```
+
+2. Inspect and replace the sample absolute paths in `run_sleep.py`,
+   `slash_sleep.py`, `run_sleep_cron.sh`, and `config.json`. Confirm the engine
+   checkout, OpenClaw workspace, state directory, skill directory, and task-file
+   paths all point to isolated test locations.
+
+3. Review `config.json`. In particular, do not assume that values such as
+   `max_tokens_per_night` or `replay_mode` are enforced by this custom wrapper
+   merely because they appear in the example config.
+
+4. Supply credentials through your normal secret-management mechanism. Do not
+   commit a DeepSeek key or place it in a world-readable file.
+
+5. Resolve every known porting gap listed in [`SKILL.md`](SKILL.md), add
+   isolated tests for your adapted backend, and verify that `--help` imports
+   cleanly on Python 3.10+. Only then start with a dry run and one reviewed
+   task file. The target command should be shaped like:
+
+   ```bash
+   cd /path/to/SkillOpt/plugins/openclaw
+   python3 run_sleep.py --config /path/to/reviewed-config.json \
+     --tasks tests/research-cron-tasks.json --dry-run
+   ```
+
+6. Inspect the report, paths, network destinations, and proposed edits before
+   considering a non-dry run or scheduling.
+
+## Data boundary
+
+The custom `openclaw-deepseek` backend sends task, skill, response, rubric, and
+reflection content to the configured DeepSeek endpoint. Its embedding helper can
+send truncated text to the configured local Ollama service. Do not assume these
+outbound prompts have been fully redacted; inspect transcript/task inputs and the
+provider's retention policy before using real data.
+
+Use HTTPS for a remote DeepSeek-compatible endpoint. Keep any plaintext Ollama
+endpoint on a trusted loopback interface. For a network-free engine smoke test,
+use the shared SkillOpt-Sleep CLI with `--backend mock` rather than assuming this
+custom wrapper is isolated.
+
+## Scheduling
+
+`run_sleep_cron.sh` and the scheduling helpers are examples, not portable
+installers. Adapt their paths, create log directories, verify their environment,
+and run the exact command manually before adding a cron entry. Scheduled runs
+must preserve the same manual-adoption and credential boundaries as interactive
+runs.
+
+## Validation scope
+
+The bundled JSON files are example held-out task sets, not a universal OpenClaw
+benchmark. Provider cost and quality depend on the selected model, task content,
+number of calls, and pricing at run time; this reference does not promise a fixed
+nightly cost. Validate the adapted workflow in an isolated workspace before
+using it on live skills.
+
+For the supported shared-engine CLI and its current flags, see the
+[integration reference](../README.md#supported-cli-surface). For measured
+SkillOpt-Sleep results and limitations, see
+[`docs/sleep/RESULTS.md`](../../docs/sleep/RESULTS.md).
+
+## License
+
+MIT, consistent with SkillOpt core.

plugins/openclaw/SKILL.md

@@ -0,0 +1,103 @@
+---
+name: skillopt-sleep
+description: Reference-only OpenClaw adaptation of SkillOpt-Sleep. Use it to study or port the contributed DeepSeek wrapper, not as a ready-to-run installation.
+---
+
+# SkillOpt-Sleep OpenClaw reference adaptation
+
+This directory is a contributed **reference**, not a supported, plug-and-play
+OpenClaw integration. It illustrates one way to connect the shared
+`skillopt_sleep` cycle to a custom DeepSeek Chat Completions backend and a set of
+environment-specific task fixtures.
+
+Do not run or schedule the files unchanged. Several scripts and the sample
+configuration preserve assumptions from the contributor's original machine,
+and parts of the wrapper have not yet been ported to the current shared-engine
+interfaces. Start with the directory's [README.md](README.md), which is the
+authoritative status and adaptation guide.
+
+## What is included
+
+- `skillopt_sleep_openclaw.py` — a contributed DeepSeek backend prototype. It
+  also contains an Ollama embedding helper, but that helper is not wired into
+  the current shared sleep cycle.
+- `run_sleep.py` — a custom cycle wrapper with environment-specific paths and a
+  backend-registration shim.
+- `slash_sleep.py` — an experimental command helper written for an older
+  staging-manifest shape.
+- `run_sleep_cron.sh` — a machine-specific category runner, not a portable cron
+  installer.
+- `config.json` — a sample configuration, not a set of guaranteed or enforced
+  runtime limits.
+- `tests/*.json` — example task fixtures from one environment, not a universal
+  OpenClaw benchmark.
+
+## Known porting gaps
+
+Before treating this as an integration, a maintainer must at least:
+
+1. Replace every absolute workspace, repository, state, skill, log, and task
+   path with explicit user configuration.
+2. Update the custom backend factory to the current `get_backend` call contract,
+   including the project directory, and update its backend methods and edit
+   records to the current protocol.
+3. Replace the experimental adoption logic with the current staging manifest
+   and `skillopt_sleep.staging.adopt` behavior. Current staging artifacts use
+   `proposed_SKILL.md` / `proposed_CLAUDE.md`, `manifest.json`, and report files;
+   they do not expose the old `manifest.proposed_skill` field.
+4. Decide how real OpenClaw transcripts are converted into a supported session
+   format. Pointing `claude_home` at an arbitrary agent directory does not by
+   itself make its files Claude Code-compatible JSONL.
+5. Build scheduling around the adapted wrapper. The shared scheduler launches
+   the shared CLI; it does not automatically preserve this custom backend or
+   its category task-file flow.
+6. Add isolated end-to-end tests for dry-run, accepted/rejected gates, staging,
+   adoption and backup, credential failure, and scheduled execution.
+
+Until those gaps are resolved, use the supported shared
+`python -m skillopt_sleep` CLI with `--backend mock` to test SkillOpt-Sleep itself,
+and treat this directory only as source material for a future OpenClaw port.
+
+## Shared-engine features are not wrapper features
+
+At this revision the supported shared CLI backends are `mock`, `claude`,
+`codex`, `copilot`, `handoff`, and `azure_openai`; the
+[plugin integration reference](../README.md#supported-cli-surface) is the
+authoritative list. The shared engine can consolidate a selected skill and
+project `CLAUDE.md` memory (controlled by `evolve_skill` and `evolve_memory`),
+and its `schedule` / `unschedule` actions manage shared-engine cron entries.
+Those capabilities do **not** make the custom OpenClaw wrapper portable: the
+shared scheduler will not invoke the prototype backend or its category
+fixtures. Use the shared documentation for those features, not this reference
+SKILL.
+
+## Data and credential boundary
+
+The prototype DeepSeek backend sends task, skill, memory, response, rubric, and
+reflection content to its configured Chat Completions endpoint. Its source also
+contains a helper that can send text to an Ollama service if a future port wires
+that helper into the cycle. Neither path should be assumed to remove every
+secret or private detail.
+
+Before any port is tested with real data:
+
+- use isolated, synthetic or explicitly reviewed task files;
+- replace sample business names, personal references, URLs, and machine paths;
+- load credentials through the operator's secret-management mechanism;
+- verify TLS and retention policy for every remote endpoint; and
+- inspect all staged artifacts before adoption.
+
+The bundled fixtures are examples only. Their scores and any old cost estimates
+do not establish effectiveness, safety, or a stable nightly price for another
+OpenClaw deployment.
+
+## Further information
+
+- [OpenClaw README](README.md) — current reference status and adaptation checklist
+- [plugin integration reference](../README.md) — supported shared-engine CLI
+  surface and data boundary
+- [SkillOpt-Sleep documentation](../../docs/sleep/README.md) — concepts,
+  results, and limitations
+
+Contributions that turn this reference into a portable integration should add
+tests and update all three documents together.

plugins/openclaw/config.json

@@ -0,0 +1,30 @@
+{
+  "_comment": "OpenClaw adaptation of skillopt-sleep. Edit and run via run_sleep.py",
+
+  "claude_home": "/home/ethanclaw/.openclaw/agents",
+  "invoked_project": "/home/ethanclaw/.openclaw/workspace",
+  "projects": "invoked",
+  "lookback_hours": 168,
+
+  "max_tasks_per_night": 12,
+  "max_tokens_per_night": 800000,
+  "holdout_fraction": 0.34,
+  "val_fraction": 0.34,
+  "test_fraction": 0.0,
+
+  "backend": "openclaw-deepseek",
+  "model": "deepseek-v4-pro",
+  "gate_mode": "on",
+  "edit_budget": 3,
+  "gate_metric": "mixed",
+  "gate_mixed_weight": 0.5,
+  "replay_mode": "fresh",
+  "evolve_memory": true,
+  "evolve_skill": true,
+  "llm_mine": false,
+
+  "auto_adopt": false,
+  "managed_skill_name": "skillopt-sleep-learned",
+  "redact_secrets": true,
+  "seed": 42
+}

plugins/openclaw/run_sleep.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""run_sleep.py — OpenClaw entry point for SkillOpt-Sleep.
+
+Runs one nightly sleep cycle:
+  1. harvest recent session transcripts
+  2. mine recurring task patterns
+  3. replay tasks with current skill (baseline) + candidate skill (with proposed edit)
+  4. gate candidate vs baseline on held-out accuracy
+  5. stage the proposal in ~/.skillopt-sleep/staging/<night>/
+  6. leave adoption to Ethan (auto_adopt=false)
+
+Usage:
+  python3 run_sleep.py                  # one cycle, default config
+  python3 run_sleep.py --dry-run        # compute report only, no staging
+  python3 run_sleep.py --tasks path.json  # use a pre-built task file
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+# Ensure the skillopt_sleep package is importable (it lives in the cloned repo)
+REPO = Path("/home/ethanclaw/.openclaw/workspace/SkillOpt")
+sys.path.insert(0, str(REPO))
+
+# Register our backend before importing cycle
+from skillopt_sleep_openclaw import OpenClawDeepSeekBackend
+import skillopt_sleep.backend as _b
+_b._BACKENDS = getattr(_b, "_BACKENDS", {})
+_b._BACKENDS["openclaw-deepseek"] = OpenClawDeepSeekBackend
+
+# Patch get_backend to know about our backend
+_orig_get_backend = _b.get_backend
+
+def get_backend(name, model="", codex_path=""):
+    if name == "openclaw-deepseek":
+        return OpenClawDeepSeekBackend(model=model or "deepseek-v4-pro")
+    return _orig_get_backend(name, model=model, codex_path=codex_path)
+
+_b.get_backend = get_backend
+
+from skillopt_sleep.cycle import run_sleep_cycle
+from skillopt_sleep.config import load_config
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="OpenClaw SkillOpt-Sleep nightly cycle")
+    ap.add_argument("--dry-run", action="store_true", help="Compute but don't stage")
+    ap.add_argument("--config", default="/home/ethanclaw/.openclaw/workspace/skills/skillopt-sleep/config.json")
+    ap.add_argument("--tasks", default=None, help="Path to pre-built tasks JSON")
+    ap.add_argument("--verbose", action="store_true")
+    args = ap.parse_args()
+
+    # Load config from file then override with our defaults
+    overrides = {}
+    if os.path.exists(args.config):
+        with open(args.config) as f:
+            overrides.update(json.load(f))
+    overrides.pop("_comment", None)
+
+    cfg = load_config(**overrides)
+
+    seed_tasks = None
+    if args.tasks:
+        from skillopt_sleep.types import TaskRecord
+        with open(args.tasks) as f:
+            raw = json.load(f)
+        # Translate our test-set fields → TaskRecord fields
+        seed_tasks = []
+        for t in raw:
+            seed_tasks.append(TaskRecord(
+                id=t['id'],
+                project=t.get('project', 'openclaw'),
+                intent=t.get('intent') or t.get('prompt', ''),
+                context_excerpt=t.get('context_excerpt', ''),
+                attempted_solution=t.get('attempted_solution', ''),
+                outcome=t.get('outcome', 'unknown'),
+                reference_kind=t.get('reference_kind', 'rubric'),
+                reference=t.get('reference', ''),
+                judge=t.get('judge', {}),
+                tags=t.get('tags', []),
+                source_sessions=t.get('source_sessions', []),
+                split=t.get('split', 'train'),
+            ))
+
+    print(f"[skillopt-sleep] starting cycle...")
+    print(f"  backend: {cfg.get('backend')}")
+    print(f"  project: {cfg.get('invoked_project')}")
+    print(f"  max tasks: {cfg.get('max_tasks_per_night')}")
+    print(f"  edit budget: {cfg.get('edit_budget')}")
+    print(f"  dry_run: {args.dry_run}")
+
+    outcome = run_sleep_cycle(cfg, seed_tasks=seed_tasks, dry_run=args.dry_run)
+
+    r = outcome.report
+    print(f"\n=== Report — night {r.night} ===")
+    print(f"  sessions harvested: {r.n_sessions}")
+    print(f"  tasks mined: {r.n_tasks}  (replayed: {r.n_replayed})")
+    print(f"  baseline: {r.baseline_score:.3f}  ->  candidate: {r.candidate_score:.3f}")
+    print(f"  gate: {r.gate_action}  accepted={r.accepted}")
+    print(f"  tokens: {r.tokens_used}")
+    if r.edits:
+        print(f"  applied edits ({len(r.edits)}):")
+        for e in r.edits:
+            print(f"    [{e.target}/{e.op}] {e.content[:80]}...")
+    if r.rejected_edits:
+        print(f"  rejected edits ({len(r.rejected_edits)}) — kept as negative feedback")
+    if r.notes:
+        for n in r.notes:
+            print(f"  note: {n}")
+    if outcome.staging_dir:
+        print(f"\n  STAGED at: {outcome.staging_dir}")
+        print(f"  Review with: ls {outcome.staging_dir}")
+
+    return 0 if r.accepted or r.candidate_score >= r.baseline_score else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

plugins/openclaw/run_sleep_cron.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# run_sleep_cron.sh — wrapper for cron-driven nightly sleep cycle
+#
+# Usage: bash run_sleep_cron.sh [category1 category2 ...]
+#   No args: run on all categories in tests/
+#   With args: run only on listed categories (research-cron, devops, wiki)
+#
+# Cron (3am MYT daily):
+#   0 3 * * * cd /home/ethanclaw/.openclaw/workspace/skills/skillopt-sleep && bash run_sleep_cron.sh >> ~/.skillopt-sleep/nightly.log 2>&1
+
+set -euo pipefail
+
+SKILL_DIR="/home/ethanclaw/.openclaw/workspace/skills/skillopt-sleep"
+TESTS_DIR="$SKILL_DIR/tests"
+LOG_DIR="$HOME/.skillopt-sleep/logs"
+mkdir -p "$LOG_DIR"
+
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+LOG_FILE="$LOG_DIR/night-$TIMESTAMP.log"
+
+# category → test file map
+declare -A CATEGORIES=(
+    ["research-cron"]="research-cron-tasks.json"
+    ["devops"]="devops-tasks.json"
+    ["wiki"]="wiki-tasks.json"
+)
+
+# Determine which categories to run
+if [ $# -eq 0 ]; then
+    CATS=("research-cron" "devops" "wiki")
+else
+    CATS=("$@")
+fi
+
+{
+    echo "=========================================="
+    echo "SkillOpt-Sleep nightly — $TIMESTAMP"
+    echo "Categories: ${CATS[*]}"
+    echo "=========================================="
+} | tee -a "$LOG_FILE"
+
+# Pre-flight: check DeepSeek API key
+if ! grep -q "DEEPSEEK_API_KEY=" "$HOME/.openclaw/.env" 2>/dev/null; then
+    echo "ERROR: DEEPSEEK_API_KEY not found in ~/.openclaw/.env" | tee -a "$LOG_FILE"
+    exit 1
+fi
+
+EXIT_CODE=0
+for cat in "${CATS[@]}"; do
+    tasks_file="$TESTS_DIR/${CATEGORIES[$cat]:-}"
+    if [ ! -f "$tasks_file" ]; then
+        echo "SKIP: $cat (no tasks file: $tasks_file)" | tee -a "$LOG_FILE"
+        continue
+    fi
+
+    echo "" | tee -a "$LOG_FILE"
+    echo "--- [$cat] starting cycle ---" | tee -a "$LOG_FILE"
+
+    cd "$SKILL_DIR"
+    if python3 run_sleep.py --tasks "$tasks_file" 2>&1 | tee -a "$LOG_FILE"; then
+        echo "--- [$cat] OK ---" | tee -a "$LOG_FILE"
+    else
+        EC=$?
+        echo "--- [$cat] FAILED (exit $EC) ---" | tee -a "$LOG_FILE"
+        EXIT_CODE=$EC
+    fi
+done
+
+{
+    echo ""
+    echo "=========================================="
+    echo "Done. Exit: $EXIT_CODE"
+    echo "=========================================="
+} | tee -a "$LOG_FILE"
+
+exit $EXIT_CODE

plugins/openclaw/skillopt_sleep_openclaw.py

@@ -0,0 +1,275 @@
+"""OpenClaw backend for SkillOpt-Sleep.
+
+Adapts the skillopt_sleep Backend protocol to our DeepSeek + Ollama stack:
+  - attempt/judge/reflect  ->  DeepSeek V4 Pro (or Flash for cost)
+  - embeddings              ->  Ollama nomic-embed-text (already configured)
+
+This backend NEVER mutates live state. It only returns text + EditRecord
+proposals that the gate stages for human review.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+from typing import Any, Dict, List, Optional, Tuple
+
+from skillopt_sleep.backend import Backend, _normalize, exact_score
+from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord
+
+
+# ── DeepSeek + Ollama OpenAI-compatible API client (curl-based, no extra deps) ──
+
+
+def _chat(messages: List[Dict[str, str]], *, model: str, temperature: float = 0.2, max_tokens: int = 1500) -> str:
+    """Call DeepSeek V4 Pro via curl + jq. No extra Python deps needed."""
+    import json as _json
+    import urllib.request
+
+    api_key = os.environ.get("DEEPSEEK_API_KEY", "")
+    if not api_key:
+        # try loading from .env
+        env_path = os.path.expanduser("~/.openclaw/.env")
+        if os.path.exists(env_path):
+            with open(env_path) as f:
+                for line in f:
+                    if line.startswith("DEEPSEEK_API_KEY="):
+                        api_key = line.split("=", 1)[1].strip()
+                        break
+
+    base = os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1")
+
+    payload = {
+        "model": model,
+        "messages": messages,
+        "temperature": temperature,
+        "max_tokens": max_tokens,
+        "stream": False,
+    }
+    req = urllib.request.Request(
+        f"{base}/chat/completions",
+        data=_json.dumps(payload).encode("utf-8"),
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=180) as resp:
+            data = _json.loads(resp.read().decode("utf-8"))
+            return data["choices"][0]["message"]["content"]
+    except Exception as e:
+        return f"[BACKEND_ERROR] {type(e).__name__}: {str(e)[:200]}"
+
+
+def _embed(text: str) -> List[float]:
+    """Call Ollama for embeddings. Uses the configured nomic-embed-text model."""
+    import json as _json
+    import urllib.request
+
+    try:
+        req = urllib.request.Request(
+            "http://127.0.0.1:11434/api/embeddings",
+            data=_json.dumps({"model": "nomic-embed-text:latest", "prompt": text[:2000]}).encode("utf-8"),
+            headers={"Content-Type": "application/json"},
+        )
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            data = _json.loads(resp.read().decode("utf-8"))
+            return data.get("embedding", [])
+    except Exception:
+        return []
+
+
+# ── Backend implementation ────────────────────────────────────────────────────
+
+
+class OpenClawDeepSeekBackend(Backend):
+    """Use DeepSeek V4 Pro for attempt/judge/reflect, Ollama for embeddings.
+
+    - "model" passed to constructor = optimizer model (default: deepseek-v4-pro)
+    - "judge_model" = judge model (default: deepseek-v4-pro for quality)
+    - "cheap_model" = budget-fallback (deepseek-v4-flash)
+    """
+
+    name = "openclaw-deepseek"
+
+    def __init__(
+        self,
+        model: str = "deepseek-v4-pro",
+        judge_model: str = "deepseek-v4-pro",
+        cheap_model: str = "deepseek-v4-flash",
+    ):
+        self._model = model
+        self._judge_model = judge_model
+        self._cheap_model = cheap_model
+        self._tokens = 0  # rough estimate
+
+    def tokens_used(self) -> int:
+        return self._tokens
+
+    # ── 1. attempt: produce a response given the task + skill + memory ──
+    def attempt(self, task: TaskRecord, skill: str, memory: str) -> str:
+        sys = (
+            "You are an OpenClaw agent (Kobe ecosystem). Use the skill and memory below to complete the task. "
+            "If the task asks for a structured output, follow the rubric exactly. "
+            "Be concise. No preamble, no explanation unless the task asks for it."
+        )
+        usr = f"""## SKILL
+{skill or '(no skill yet)'}
+
+## MEMORY
+{memory or '(no memory yet)'}
+
+## TASK
+{task.intent}
+
+## CONTEXT (if any)
+{task.context_excerpt or '(none)'}
+
+## RESPONSE
+"""
+        out = _chat(
+            [{"role": "system", "content": sys}, {"role": "user", "content": usr}],
+            model=self._model,
+            temperature=0.2,
+        )
+        self._tokens += len(usr) // 4 + 200
+        return out
+
+    # ── 2. judge: score the response ──
+    def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]:
+        # Hard score: exact-match against task.reference (if available)
+        hard = exact_score(task.reference or "", response)
+
+        # Soft score: LLM judge against rubric (reference if reference_kind=='rubric')
+        rubric_text = task.reference if task.reference_kind == "rubric" else ""
+        if rubric_text:
+            judge_prompt = f"""You are a strict grader. Score the response 0.0-1.0 against the rubric.
+
+## TASK
+{task.intent}
+
+## REFERENCE
+{task.reference or '(none)'}
+
+## RUBRIC
+{rubric_text}
+
+## RESPONSE
+{response[:3000]}
+
+## INSTRUCTIONS
+Return ONLY a single float 0.0-1.0 on one line. No explanation. No markdown.
+"""
+            try:
+                j_out = _chat(
+                    [{"role": "user", "content": judge_prompt}],
+                    model=self._judge_model,
+                    temperature=0.0,
+                    max_tokens=20,
+                ).strip()
+                soft = float(re.search(r"[\d.]+", j_out.splitlines()[0]).group())
+                soft = max(0.0, min(1.0, soft))
+            except Exception:
+                soft = hard
+            self._tokens += 600
+        else:
+            soft = hard
+
+        rationale = f"hard={hard:.2f} soft={soft:.2f}"
+        return hard, soft, rationale
+
+    # ── 3. reflect: produce bounded EditRecord proposals ──
+    def reflect(
+        self,
+        failures: List[Tuple[TaskRecord, ReplayResult]],
+        successes: List[Tuple[TaskRecord, ReplayResult]],
+        skill: str,
+        memory: str,
+        *,
+        edit_budget: int,
+        evolve_skill: bool,
+        evolve_memory: bool,
+    ) -> List[EditRecord]:
+        # Compact digest of failures + successes
+        fail_digest = "\n".join(
+            f"- TASK: {t.intent[:200]}\n  RESPONSE: {r.response[:300]}\n  WHY FAIL: {r.judge_rationale or r.fail_reason or 'unknown'}\n  REFERENCE: {t.reference[:200]}"
+            for t, r in failures[:5]
+        ) or "(none)"
+        succ_digest = "\n".join(
+            f"- TASK: {t.intent[:150]} -> OK ({r.judge_rationale or 'high score'})"
+            for t, r in successes[:3]
+        ) or "(none)"
+
+        rubric_text = ""
+        if failures:
+            rubric_text = f"\n\n## REFERENCE ANSWERS\n{chr(10).join(f'Q: {t.intent[:120]}\\nA: {t.reference}' for t, _ in failures[:3] if t.reference)}"
+
+        sys = (
+            "You are SkillOpt-Sleep's bounded-edit optimizer. Your job is to propose 1-4 MINIMAL text edits to a skill or memory document "
+            "that, if applied, would help future agents do better on the failed tasks. "
+            "NEVER propose adding new sections wholesale. NEVER delete entire sections. "
+            "Edit primitives: ADD (append a step/rule at end), DELETE (remove a specific line by exact match), REPLACE (swap a specific line for another by exact match). "
+            "If you cannot identify a clear, minimal improvement, return an empty list."
+        )
+        usr = f"""## CURRENT SKILL
+{skill or '(empty)'}
+
+## CURRENT MEMORY
+{memory or '(empty)'}
+
+## FAILED TASKS
+{fail_digest}
+
+## SUCCESSFUL TASKS
+{succ_digest}
+{rubric_text}
+
+## CONSTRAINTS
+- max {edit_budget} edits total
+- edits go to {"skill + memory" if (evolve_skill and evolve_memory) else ("skill" if evolve_skill else "memory")}
+- if evolve_skill=False, target="memory" only; if evolve_memory=False, target="skill" only
+- target must be "skill" or "memory"
+
+## OUTPUT FORMAT (JSON, no markdown)
+{{"edits": [{{"op": "ADD"|"DELETE"|"REPLACE", "target": "skill"|"memory", "content": "the text to add or replace with", "old_text": "for REPLACE/DELETE, the exact line to find", "rationale": "one short sentence why"}}]}}
+"""
+        out = _chat(
+            [{"role": "system", "content": sys}, {"role": "user", "content": usr}],
+            model=self._model,
+            temperature=0.4,
+            max_tokens=2000,
+        )
+        self._tokens += len(usr) // 3 + 1500
+
+        # parse
+        try:
+            # strip markdown fences if any
+            cleaned = out.strip()
+            if cleaned.startswith("```"):
+                cleaned = re.sub(r"^```[a-z]*\n?", "", cleaned)
+                cleaned = re.sub(r"\n?```$", "", cleaned)
+            data = json.loads(cleaned)
+            edits: List[EditRecord] = []
+            for e in data.get("edits", [])[:edit_budget]:
+                if e.get("op") not in ("ADD", "DELETE", "REPLACE"):
+                    continue
+                target = e.get("target", "skill")
+                if target not in ("skill", "memory"):
+                    continue
+                if not evolve_skill and target == "skill":
+                    continue
+                if not evolve_memory and target == "memory":
+                    continue
+                edits.append(EditRecord(
+                    op=e["op"],
+                    target=target,
+                    content=e.get("content", ""),
+                    old_text=e.get("old_text", ""),
+                    rationale=e.get("rationale", ""),
+                ))
+            return edits
+        except Exception as e:
+            # log + return empty list (no edit is better than a bad edit)
+            return []

plugins/openclaw/slash_sleep.py

@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+"""slash_sleep.py — OpenClaw slash command equivalent of SkillOpt's /sleep.
+
+Use from the main session as a /sleep command:
+  /sleep status    — show current state + last 5 nights
+  /sleep run       — trigger one cycle (all categories) right now
+  /sleep run research-cron  — one cycle, single category
+  /sleep adopt [night]      — adopt the most recent (or specified) staged proposal
+  /sleep reject [night]     — discard the most recent (or specified) staging dir
+  /sleep dry-run   — report-only cycle
+  /sleep cost      — estimate per-night cost for current config
+
+This script is a thin shell over run_sleep.py. It can be invoked either
+manually from the main session or by an OpenClaw command handler.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+from datetime import datetime
+
+SKILL_DIR = Path("/home/ethanclaw/.openclaw/workspace/skills/skillopt-sleep")
+STATE_DIR = Path(os.path.expanduser("~/.skillopt-sleep"))  # default
+STAGING_ROOT = STATE_DIR
+
+def _resolve_state_dir():
+    """Find the actual state dir.
+
+    Priority: scan in order:
+      1. ~/.skillopt-sleep/                 (default)
+      2. /home/ethanclaw/.openclaw/workspace/.skillopt-sleep/  (when staging is there)
+      3. /home/ethanclaw/.openclaw/.skillopt-sleep/            (parent of overridden claude_home)
+    Pick the first one that has a state.json OR staging dir.
+    """
+    candidates = [
+        Path(os.path.expanduser("~/.skillopt-sleep")),
+        Path("/home/ethanclaw/.openclaw/workspace/.skillopt-sleep"),
+        Path("/home/ethanclaw/.openclaw/.skillopt-sleep"),
+    ]
+    # Prefer the one with state.json
+    for c in candidates:
+        if (c / "state.json").exists():
+            return c
+    # Then the one with staging
+    for c in candidates:
+        if (c / "staging").exists():
+            return c
+    return candidates[0]
+
+TESTS_DIR = SKILL_DIR / "tests"
+
+
+def status() -> int:
+    state_dir = _resolve_state_dir()
+    state_file = state_dir / "state.json"
+    staging_dir = state_dir / "staging"
+    print(f"=== SkillOpt-Sleep status ===")
+    print(f"  state dir: {state_dir}")
+    print(f"  staging dir: {staging_dir}")
+    if staging_dir.exists():
+        stages = sorted(staging_dir.iterdir(), key=lambda p: p.stat().st_mtime, reverse=True)
+        print(f"  staging entries: {len(stages)}")
+        for s in stages[:3]:
+            print(f"    {s.name}")
+    if not state_file.exists():
+        print("  no state.json — run a cycle first (state is written at end of each non-dry-run)")
+        return 0
+
+    with open(state_file) as f:
+        state = json.load(f)
+
+    nights = state.get("history") or state.get("nights", [])
+    print(f"  total nights: {len(nights)}")
+    print(f"  accepted: {sum(1 for n in nights if n.get('accepted'))}")
+    print(f"  rejected: {sum(1 for n in nights if not n.get('accepted'))}")
+    if nights:
+        last = nights[-1]
+        print(f"  last night: {last.get('night')}")
+        print(f"    accepted: {last.get('accepted')}")
+        print(f"    baseline: {last.get('baseline'):.3f}  ->  candidate: {last.get('candidate'):.3f}")
+        print(f"    staging: {last.get('staging') or '(none)'}")
+    return 0
+
+
+def run_category(category: str, *, dry_run: bool = False) -> int:
+    cat_to_file = {
+        "research-cron": "research-cron-tasks.json",
+        "devops": "devops-tasks.json",
+        "wiki": "wiki-tasks.json",
+    }
+    tasks_file = TESTS_DIR / cat_to_file.get(category, f"{category}-tasks.json")
+    if not tasks_file.exists():
+        print(f"ERROR: no tasks file for category '{category}': {tasks_file}")
+        return 1
+
+    cmd = [sys.executable, str(SKILL_DIR / "run_sleep.py")]
+    if dry_run:
+        cmd.append("--dry-run")
+    cmd.extend(["--tasks", str(tasks_file)])
+
+    print(f"=== /sleep run {category}{' (dry-run)' if dry_run else ''} ===")
+    print(f"  cmd: {' '.join(cmd)}")
+    rc = os.system(" ".join(f'"{c}"' for c in cmd))
+    return rc
+
+
+def run_all(*, dry_run: bool = False) -> int:
+    rc = 0
+    for cat in ("research-cron", "devops", "wiki"):
+        r = run_category(cat, dry_run=dry_run)
+        if r != 0:
+            rc = r
+    return rc
+
+
+def adopt(night: str = None) -> int:
+    state_dir = _resolve_state_dir()
+    state_file = state_dir / "state.json"
+    if not state_file.exists():
+        print("ERROR: no state to adopt from")
+        return 1
+    with open(state_file) as f:
+        state = json.load(f)
+    nights = state.get("history") or state.get("nights", [])
+    if not nights:
+        print("ERROR: no nights recorded")
+        return 1
+
+    target = None
+    if night:
+        target = next((n for n in nights if str(n.get("night")) == night), None)
+        if not target:
+            print(f"ERROR: night '{night}' not found")
+            return 1
+    else:
+        # most recent accepted
+        candidates = [n for n in nights if n.get("accepted") and n.get("staging")]
+        if not candidates:
+            print("ERROR: no accepted nights with staging to adopt")
+            return 1
+        target = candidates[-1]
+
+    staging = target["staging"]
+    if not os.path.isdir(staging):
+        print(f"ERROR: staging dir missing: {staging}")
+        return 1
+
+    print(f"=== /sleep adopt night {target['night']} ===")
+    print(f"  staging: {staging}")
+    print(f"  baseline: {target.get('baseline'):.3f}  candidate: {target.get('candidate'):.3f}")
+
+    # Read proposed skill from staging
+    manifest = Path(staging) / "manifest.json"
+    if manifest.exists():
+        with open(manifest) as f:
+            m = json.load(f)
+        proposed = m.get("proposed_skill")
+        if proposed and Path(proposed).exists():
+            live = STATE_DIR / "live_skill.md"
+            backup = STATE_DIR / f"live_skill.md.bak-{target['night']}"
+            if live.exists():
+                shutil.copy2(live, backup)
+                print(f"  backed up current live skill → {backup}")
+            shutil.copy2(proposed, live)
+            print(f"  adopted proposed skill → {live}")
+            print()
+            print("✅ Adoption complete. Next cycle will use the new skill.")
+            return 0
+
+    print("ERROR: no proposed_skill in manifest")
+    return 1
+
+
+def reject(night: str = None) -> int:
+    state_dir = _resolve_state_dir()
+    state_file = state_dir / "state.json"
+    if not state_file.exists():
+        print("ERROR: no state")
+        return 1
+    with open(state_file) as f:
+        state = json.load(f)
+    nights = state.get("history") or state.get("nights", [])
+    target = None
+    if night:
+        target = next((n for n in nights if str(n.get("night")) == night), None)
+    else:
+        candidates = [n for n in reversed(nights) if n.get("staging")]
+        target = candidates[0] if candidates else None
+
+    if not target or not target.get("staging"):
+        print("ERROR: nothing to reject")
+        return 1
+
+    staging = target["staging"]
+    if os.path.isdir(staging):
+        shutil.rmtree(staging)
+        print(f"🗑️  Removed staging: {staging}")
+    # remove from state
+    state["history"] = [n for n in nights if n.get("night") != target["night"]]
+    with open(state_file, "w") as f:
+        json.dump(state, f, indent=2)
+    print("✅ Rejected. State updated.")
+    return 0
+
+
+def schedule_cmd(hour: int, minute: int) -> int:
+    """Install a nightly cron entry via the shared SkillOpt-Sleep scheduler.
+
+    Note: this schedules the shared engine (``python -m skillopt_sleep run``),
+    not the OpenClaw-specific ``run_sleep.py``.  Use ``run_sleep_cron.sh`` if
+    you need the OpenClaw-native backend and category task files instead.
+    """
+    try:
+        from skillopt_sleep.scheduler import schedule
+    except ImportError:
+        print("ERROR: skillopt_sleep.scheduler not available — is SkillOpt-Sleep installed?")
+        return 1
+    project = str(SKILL_DIR)
+    ok, msg = schedule(project, hour=hour, minute=minute)
+    print(msg)
+    return 0 if ok else 1
+
+
+def unschedule_cmd(all_projects: bool) -> int:
+    """Remove cron entry via the shared SkillOpt-Sleep scheduler."""
+    try:
+        from skillopt_sleep.scheduler import unschedule
+    except ImportError:
+        print("ERROR: skillopt_sleep.scheduler not available — is SkillOpt-Sleep installed?")
+        return 1
+    project = str(SKILL_DIR)
+    ok, msg = unschedule(project, all_projects=all_projects)
+    print(msg)
+    return 0 if ok else 1
+
+
+def cost() -> int:
+    """Estimate per-night cost based on the actual measurement from Phase 2.
+
+    From the real dry-run: 5 devops tasks used 14,427 tokens total.
+    That is ~2,885 tokens per task (all 3 phases combined).
+    """
+    cfg_path = SKILL_DIR / "config.json"
+    cfg = {}
+    if cfg_path.exists():
+        cfg = json.loads(cfg_path.read_text())
+    cfg.pop("_comment", None)
+
+    max_tasks = cfg.get("max_tasks_per_night", 12)
+    model = cfg.get("model", "deepseek-v4-pro")
+    # DeepSeek V4 pricing
+    if "pro" in model:
+        cost_in = 0.435  # per 1M
+        cost_out = 0.87
+    elif "flash" in model:
+        cost_in = 0.14
+        cost_out = 0.28
+    else:
+        cost_in, cost_out = 0.5, 1.0
+
+    # Measured: ~2,900 tokens per task, 30% output / 70% input
+    toks_per_task = 2900
+    input_toks = int(toks_per_task * 0.7)
+    output_toks = int(toks_per_task * 0.3)
+
+    cost_in_total = (input_toks * max_tasks / 1_000_000) * cost_in
+    cost_out_total = (output_toks * max_tasks / 1_000_000) * cost_out
+    cost = cost_in_total + cost_out_total
+
+    print(f"=== Cost estimate (per actual measurement) ===")
+    print(f"  model: {model}")
+    print(f"  max tasks/night: {max_tasks}")
+    print(f"  ~tokens/night: {toks_per_task * max_tasks:,}")
+    print(f"  cost/night: ${cost:.3f}")
+    print(f"  cost/month (30 nights): ${cost*30:.2f}")
+    print(f"  cost/year (365 nights): ${cost*365:.2f}")
+    return 0
+
+
+def main():
+    ap = argparse.ArgumentParser(description="OpenClaw /sleep command")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    sub.add_parser("status", help="show state + last 5 nights")
+    p_run = sub.add_parser("run", help="trigger one cycle")
+    p_run.add_argument("category", nargs="?", default=None,
+                        choices=["research-cron", "devops", "wiki", None])
+    p_run.add_argument("--dry-run", action="store_true")
+    sub.add_parser("dry-run", help="report-only cycle (all categories)")
+    p_adopt = sub.add_parser("adopt", help="adopt most recent accepted staging")
+    p_adopt.add_argument("night", nargs="?", default=None)
+    p_reject = sub.add_parser("reject", help="discard most recent staging")
+    p_reject.add_argument("night", nargs="?", default=None)
+    sub.add_parser("cost", help="estimate cost")
+    p_schedule = sub.add_parser("schedule", help="install nightly cron entry")
+    p_schedule.add_argument("--hour", type=int, default=3, help="hour (0-23)")
+    p_schedule.add_argument("--minute", type=int, default=17, help="minute (0-59)")
+    p_unschedule = sub.add_parser("unschedule", help="remove cron entry")
+    p_unschedule.add_argument("--all", dest="all_projects", action="store_true",
+                               help="remove entries for all projects")
+
+    args = ap.parse_args()
+
+    if args.cmd == "status":
+        return status()
+    if args.cmd == "run":
+        if args.category:
+            return run_category(args.category, dry_run=args.dry_run)
+        return run_all(dry_run=args.dry_run)
+    if args.cmd == "dry-run":
+        return run_all(dry_run=True)
+    if args.cmd == "adopt":
+        return adopt(args.night)
+    if args.cmd == "reject":
+        return reject(args.night)
+    if args.cmd == "cost":
+        return cost()
+    if args.cmd == "schedule":
+        return schedule_cmd(args.hour, args.minute)
+    if args.cmd == "unschedule":
+        return unschedule_cmd(args.all_projects)
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

plugins/openclaw/tests/devops-tasks.json

@@ -0,0 +1,87 @@
+[
+  {
+    "id": "do-01",
+    "reference": "[STATUS] devops-agent | Site Uptime \u2192 geoxylia.com OK (200) | 14/06 22:30 MYT",
+    "rubric": "Score 1.0 if output matches the exact format [STATUS] devops-agent | Site Uptime \u2192 geoxylia.com OK (200) | DD/MM HH:MM MYT, with a real current time. Score 0.5 if format is close but missing one field. Score 0.0 if wrong format or hallucinated values.",
+    "project": "devops-infrastructure-check",
+    "intent": "Site Uptime check. Run: `curl -o /dev/null -s -w '%{http_code}' https://geoxylia.com`. Interpret the result 200, and report in our standard format: 'STATUS | TASK \u2192 RESULT | TIME'. If not 200, escalate.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "devops-infrastructure-check"
+    ],
+    "source_sessions": [],
+    "split": "val"
+  },
+  {
+    "id": "do-02",
+    "reference": "Backup complete. Files: 87, Size: 1.2G, Last: 2026-06-14 22:00:00 MYT",
+    "rubric": "Score 1.0 if output includes the exact 'Backup complete. Files: N, Size: X, Last: timestamp' structure with plausible values. Score 0.5 if structure is close but one field missing. Score 0.0 if hallucinated or wrong structure.",
+    "project": "devops-infrastructure-check",
+    "intent": "Daily Memory Backup. Confirm this ran successfully by checking: `ls -t ~/backups/memory/memory-backup-*.tar.gz | head -3`. Report the file count, total size, and most recent backup time. Use format: 'Backup complete. Files: [N], Size: [X], Last: [timestamp]'.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "devops-infrastructure-check"
+    ],
+    "source_sessions": [],
+    "split": "val"
+  },
+  {
+    "id": "do-03",
+    "reference": "1) Vercel CSP missing frame-ancestors: MEDIUM. Allows clickjacking if anyone embeds our pages; not exploitable for our content, but best-practice gap.\n2) OpenClaw plaintext API keys: LOW. The config is chmod 600, loopback-only, not in git. Standard OpenClaw behavior. Rotating would add zero real security given current exposure.",
+    "rubric": "Score 1.0 if both are classified correctly (MEDIUM and LOW respectively) and justifications are accurate (not panicky, not dismissive). Score 0.5 if classifications are wrong by one tier or justifications are weak. Score 0.0 if both over-classified as CRITICAL or both wrong.",
+    "project": "devops-infrastructure-check",
+    "intent": "Security Check daily run. Two findings: 1) Vercel CSP header missing 'frame-ancestors' directive, 2) OpenClaw config has 3 plaintext API keys. Classify each as: CRITICAL / HIGH / MEDIUM / LOW / INFO. Justify each in 1 sentence.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "devops-infrastructure-check"
+    ],
+    "source_sessions": [],
+    "split": "train"
+  },
+  {
+    "id": "do-04",
+    "reference": "[INCIDENT] supabase.audit_results: anon role has no RLS policy \u2014 anyone with the URL can read all audit results. Fix: add policy 'audit_results_select_own' granting SELECT WHERE user_id = auth.uid(). Severity: HIGH (data exposure). Estimated 2-min fix.",
+    "rubric": "Score 1.0 if: (a) severity correctly identified as HIGH, (b) fix is a real RLS policy (not just 'enable RLS' since it's already enabled), (c) under 50 words, (d) Telegram-friendly format. Score 0.5 if severity right but fix is generic. Score 0.0 if missing severity or wrong fix.",
+    "project": "devops-infrastructure-check",
+    "intent": "Incident Check. The Supabase RLS check returned: 'table public.audit_results: rls enabled but policy missing for anon role'. Interpret severity, propose fix, and format as a Telegram alert (max 50 words).",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "devops-infrastructure-check"
+    ],
+    "source_sessions": [],
+    "split": "val"
+  },
+  {
+    "id": "do-05",
+    "reference": "\ud83d\udee1\ufe0f Week security digest:\n\n\u2022 0 critical incidents, 1 high resolved (Supabase RLS policy added)\n\u2022 22 plaintext secrets: expected OpenClaw behavior, no action\n\u2022 1 medium open: Vercel CSP frame-ancestors, schedule for next sprint\n\nTrend: stable. No regressions vs last week.",
+    "rubric": "Score 1.0 if all 3 priority tiers mentioned with correct counts, ends with a trend statement, Telegram-friendly. Score 0.5 if structure is right but one tier wrong. Score 0.0 if missing a tier or wrong format.",
+    "project": "devops-infrastructure-check",
+    "intent": "Weekly security digest. Synthesize this week's findings: 22 plaintext secrets in openclaw.json (expected), 0 critical incidents, 1 high (Supabase RLS), 1 medium (CSP frame-ancestors), 0 low. Output a 3-bullet Telegram status.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "devops-infrastructure-check"
+    ],
+    "source_sessions": [],
+    "split": "train"
+  }
+]

plugins/openclaw/tests/research-cron-tasks.json

@@ -0,0 +1,87 @@
+[
+  {
+    "id": "rc-01",
+    "reference": "COMPETITOR MOVES: Otterly adds Perplexity tracker, joining Profound and LLMRefs in multi-platform citations.\nBACKLINK OPPORTUNITIES: 3 SEO directories (G2, Capterra, GetApp) have not been claimed.\nAGENCY BLUEPRINT: Top 2 agency sites bundle GEO audit + content refresh as $3K/mo tier.\nACTION ITEMS: Build Perplexity citation test into GeoXylia audit; claim G2 listing by Friday.",
+    "rubric": "Score 1.0 if all 4 section headings present in correct order, each with a substantive (not generic) 1-sentence content. Score 0.5 if headings present but content is generic. Score 0.0 if any heading missing or order wrong.",
+    "project": "research-cron-output",
+    "intent": "Weekly Competitive Deep Dive for GeoXylia. The competitor otterly.ai just added a Perplexity citation tracker. Produce the report header (top section) in our standard format: COMPETITOR MOVES, BACKLINK OPPORTUNITIES, AGENCY BLUEPRINT, ACTION ITEMS. Keep it to 4 lines, one per section heading with a 1-sentence placeholder.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "research-cron-output"
+    ],
+    "source_sessions": [],
+    "split": "train"
+  },
+  {
+    "id": "rc-02",
+    "reference": "1. 'ai seo audit tool': 420 imp, pos 8.2, on page 1 \u2014 needs CTR lift (snippet/schema).\n2. 'geo audit tool': 230 imp, pos 12.5, page 2 \u2014 target blog post could push to page 1.\n3. 'llm optimization': 85 imp, pos 18.3, deep page-2 \u2014 fresh content with answer capsule could compete.",
+    "rubric": "Score 1.0 if the response correctly identifies 'ai seo audit tool', 'geo audit tool', and 'llm optimization' as the top 3 (NOT 'best free seo audit' which is already converting well, NOT 'free audit tool' which has too few impressions). Each must have correct impression count, position, and a substantive rationale. Score 0.5 if correct 3 keywords but rationale is weak. Score 0.0 if wrong keywords selected.",
+    "project": "research-cron-output",
+    "intent": "GSC keyword opportunity scan. From this snippet of GSC data, identify the top 3 keyword opportunities (high impressions, low CTR, position 5-15):\n\n1. 'ai seo audit tool' \u2014 420 imp, 12 clicks, pos 8.2\n2. 'best free seo audit' \u2014 1100 imp, 95 clicks, pos 4.1\n3. 'geo audit tool' \u2014 230 imp, 4 clicks, pos 12.5\n4. 'llm optimization' \u2014 85 imp, 1 click, pos 18.3\n5. 'free audit tool' \u2014 50 imp, 0 clicks, pos 22.0\n\nOutput: one line per opportunity, format 'KEYWORD: impressions, position, why-it-matters (1 short clause)'.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "research-cron-output"
+    ],
+    "source_sessions": [],
+    "split": "train"
+  },
+  {
+    "id": "rc-03",
+    "reference": "Google AI Overviews now show source links more prominently + author bylines. For GeoXylia: this favors pages with clear authorship (add author schema to blog posts). Action: this week, add author + E-E-A-T schema markup to top 10 blog posts. Source: Google Search Central blog.",
+    "rubric": "Score 1.0 if: (a) under 60 words, (b) names the change, (c) gives GeoXylia-specific implication, (d) gives a concrete action item, (e) cites the source. Score 0.5 if missing 1-2 of these. Score 0.0 if over 60 words or missing 3+.",
+    "project": "research-cron-output",
+    "intent": "Daily Industry News scan. The Google Search Central blog just announced: 'AI Overviews now showing source links more prominently, with author bylines for E-E-A-T-heavy content.' Write a 1-paragraph Telegram alert (max 60 words) for Ethan. Include: 1) what changed, 2) what it means for GeoXylia, 3) any action item.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "research-cron-output"
+    ],
+    "source_sessions": [],
+    "split": "val"
+  },
+  {
+    "id": "rc-04",
+    "reference": "Hi [Name], I saw seo-skill.com's resources page is one of the most-respected SEO learning hubs in the industry \u2014 your 2026 algorithm breakdown was spot-on. We just published a free 2026 AI SEO Audit comparison that your readers would find genuinely useful (no paywall, no signup). It covers the 8 leading AI-audit tools with hands-on screenshots and a clear feature matrix. GeoXylia is the only fully-free option in the comparison, so it's a natural fit for a 'tools to know' section. Mind if I share the link for inclusion?",
+    "rubric": "Score 1.0 if exactly 4 sentences, all four functional pieces present (compliment / mention resource / audience benefit / GeoXylia one-liner), conversational tone, no aggressive sales language. Score 0.5 if 3 of 4 pieces present or tone is too salesy. Score 0.0 if more than 5 sentences or missing 2+ pieces.",
+    "project": "research-cron-output",
+    "intent": "Backlink Outreach draft for the blog post 'Free AI SEO Audit Tool: 2026 Comparison'. The prospect is seo-skill.com (a popular SEO training site with a 'resources' page). Write a 4-sentence outreach email: 1) compliment, 2) mention our resource, 3) explain audience benefit, 4) one-line about GeoXylia.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "research-cron-output"
+    ],
+    "source_sessions": [],
+    "split": "train"
+  },
+  {
+    "id": "rc-05",
+    "reference": "1) DO MORE: AI citation / LLM-mention topics \u2014 the 0.9% CTR at position 9.4 means we're visible but need richer answer capsules to lift CTR. Target 2x posts/week on this cluster.\n2) PAUSE: Pure schema-markup how-tos \u2014 'Schema Markup for SEO' has 0 clicks at position 41, the audience isn't searching this way. Rework as 'How to appear in AI answers' framing.\n3) TEST: 'Perplexity vs ChatGPT citation rates for [niche]' \u2014 unexplored angle, could capture comparison-intent traffic.",
+    "rubric": "Score 1.0 if all 3 are specific (not generic), cite actual data from the prompt, and contain a clear actionable change. Score 0.5 if 2 of 3 are specific. Score 0.0 if generic advice or no data citations.",
+    "project": "research-cron-output",
+    "intent": "Performance \u2192 Strategy feedback loop. Last week's top blog post was 'AI Citation Audit: Does Your Site Appear in ChatGPT?' with 4,200 impressions and 38 clicks (CTR 0.9%, position 9.4). The bottom post was 'Schema Markup for SEO: A 2026 Guide' with 110 impressions and 0 clicks (CTR 0%, position 41). Write 3 specific strategy adjustments: 1) what to do more of, 2) what to pause, 3) what new topic to test.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "research-cron-output"
+    ],
+    "source_sessions": [],
+    "split": "val"
+  }
+]

plugins/openclaw/tests/wiki-tasks.json

@@ -0,0 +1,70 @@
+[
+  {
+    "id": "wk-01",
+    "reference": "1. What GEO is and isn't (define vs SEO/AEO, dispel the 'just add FAQ' myth)\n2. The 3 citation mechanisms LLMs use (RAG, fine-tuning, in-context; weight each)\n3. The 2026 citation data (real statistics from Profound/Otterly/Peec; what % of queries get citations)\n4. The action framework (a 5-step audit-and-fix process, concrete)\n5. Measurement (which metrics actually predict citation lift; vanity vs real)",
+    "rubric": "Score 1.0 if 5 sections, in a logical order, each with a substantive (not generic) purpose, and the section content is GEO-specific (not generic SEO). Score 0.5 if 5 sections but 1-2 are generic. Score 0.0 if wrong number of sections or wrong order.",
+    "project": "wiki-canonical-guide",
+    "intent": "Wiki canonical guide: 'GEO 2026 Standards'. Audience: a mid-level SEO specialist who has heard of GEO but not done it. Tone: technical, evidence-driven, no fluff. Length target: 1500-2200 words. Outline the 5 sections that should appear in order. For each, give a 1-sentence sub-purpose.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "wiki-canonical-guide"
+    ],
+    "source_sessions": [],
+    "split": "val"
+  },
+  {
+    "id": "wk-02",
+    "reference": "Yes, add inbound links. (1) geo-2026-standards.md \u2192 '## Action Framework' section, anchor: 'platform-specific citation rules' \u2014 natural since GEO standards reference ChatGPT/Perplexity behavior. (2) seo-2026-standards.md \u2192 '## AI Overviews' section, anchor: 'AI platform citations' \u2014 links to the mechanism guide. (3) content-strategy.md \u2192 '## Content Types' section, anchor: 'per-platform citation' \u2014 content strategy needs to know which platform favors which content.",
+    "rubric": "Score 1.0 if all 3 inbound links proposed with specific section + natural anchor text, demonstrating the link solves a real navigational gap (not just SEO-link-building). Score 0.5 if 2 of 3 are well-placed. Score 0.0 if generic anchors like 'click here' or no specific sections named.",
+    "project": "wiki-canonical-guide",
+    "intent": "Cross-link audit. The wiki page 'ai-platform-citation-guide.md' has 4 outbound links to other wiki pages, but no inbound links from: 'geo-2026-standards.md', 'seo-2026-standards.md', 'content-strategy.md'. Should we add inbound links? In which page should each inbound link go, and what anchor text would be natural?",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "wiki-canonical-guide"
+    ],
+    "source_sessions": [],
+    "split": "val"
+  },
+  {
+    "id": "wk-03",
+    "reference": "Priorities:\n1. Refresh 'geo-glossary.md' (last update 2026-04-12, 63 days) \u2014 add new terms like RAG, in-context citation, agentic SEO.\n2. Refresh 'competitor-pricing.md' (last update 2026-05-01, 44 days) \u2014 Profound raised enterprise tier.\n3. No structural fixes needed.\n\nTelegram: 'Wiki lint: 2 stale pages flagged (geo-glossary 63d, competitor-pricing 44d). No broken links. Both need refresh this week.'",
+    "rubric": "Score 1.0 if both stale pages correctly identified with specific (not generic) refresh notes, and Telegram summary is under 40 words with the right action. Score 0.5 if stale pages identified but refresh notes are vague. Score 0.0 if missing stale pages or Telegram over 40 words.",
+    "project": "wiki-canonical-guide",
+    "intent": "Wiki lint report. Today's scan: 14 wiki pages, 2 with 'Updated' dates > 30 days old ('geo-glossary.md' and 'competitor-pricing.md'), 0 broken internal links, 0 missing YAML frontmatter. Output: 1) prioritized action list, 2) Telegram summary (max 40 words).",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "wiki-canonical-guide"
+    ],
+    "source_sessions": [],
+    "split": "train"
+  },
+  {
+    "id": "wk-04",
+    "reference": "Index rebuilt: 14 wiki pages registered in _index.md (was 12 \u2014 added competitor-pricing-rev2 and citations-q2-2026).\nQuestion for Ethan: should 'competitor-pricing.md' and 'competitor-pricing-rev2.md' be merged? They're 78% similar in content.",
+    "rubric": "Score 1.0 if both sentences are accurate (count matches, names are plausible) and the question identifies a real consolidation opportunity (not a fabricated one). Score 0.5 if structure is right but content vague. Score 0.0 if wrong format or no question.",
+    "project": "wiki-canonical-guide",
+    "intent": "Index rebuild check. Run `python3 ~/agent-shared/scripts/update-index.py` (assume it works). After the run, the new wiki/_index.md should list all 14 pages. Generate a 2-sentence confirmation message + 1 question for Ethan to verify.",
+    "context_excerpt": "",
+    "attempted_solution": "",
+    "outcome": "unknown",
+    "reference_kind": "rubric",
+    "judge": {},
+    "tags": [
+      "wiki-canonical-guide"
+    ],
+    "source_sessions": [],
+    "split": "train"
+  }
+]

plugins/run-sleep.cmd

@@ -0,0 +1,122 @@
+@echo off
+setlocal enabledelayedexpansion
+
+:: Resolve REPO_ROOT
+set "SCRIPT_DIR=%~dp0"
+:: Strip trailing backslash
+set "SCRIPT_DIR=%SCRIPT_DIR:~0,-1%"
+
+set "REPO_ROOT="
+if exist "%SCRIPT_DIR%\..\skillopt_sleep" (
+    cd /d "%SCRIPT_DIR%\.."
+    set "REPO_ROOT=%CD%"
+    goto root_resolved
+)
+if not "%CLAUDE_PLUGIN_ROOT%"=="" if exist "%CLAUDE_PLUGIN_ROOT%\..\..\skillopt_sleep" (
+    cd /d "%CLAUDE_PLUGIN_ROOT%\..\.."
+    set "REPO_ROOT=%CD%"
+    goto root_resolved
+)
+if not "%SKILLOPT_SLEEP_REPO%"=="" if exist "%SKILLOPT_SLEEP_REPO%\skillopt_sleep" (
+    set "REPO_ROOT=%SKILLOPT_SLEEP_REPO%"
+    goto root_resolved
+)
+
+:: Search upward from current directory
+set "d=%CD%"
+:loop
+if exist "!d!\skillopt_sleep" (
+    set "REPO_ROOT=!d!"
+    goto root_resolved
+)
+for %%I in ("!d!") do set "parent=%%~dpI"
+:: Strip trailing backslash from parent if it's not root
+if "!parent!"=="!d!" goto root_resolved
+set "parent=!parent:~0,-1!"
+if "!parent!"=="" goto root_resolved
+set "d=!parent!"
+goto loop
+
+:root_resolved
+
+if "%REPO_ROOT%"=="" goto fallback_mode
+
+:: ── Source Checkout Mode ───────────────────────────────────────────
+set "PY="
+if not "%SKILLOPT_SLEEP_PYTHON%"=="" (
+    set "PY=%SKILLOPT_SLEEP_PYTHON%"
+    goto py_found
+)
+
+for %%p in (python3.exe python.exe py.exe) do (
+    where %%p >nul 2>nul
+    if !errorlevel! equ 0 (
+        %%p -c "import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)" >nul 2>nul
+        if !errorlevel! equ 0 (
+            set "PY=%%p"
+            goto py_found
+        )
+    )
+)
+
+:py_found
+if "%PY%"=="" (
+    echo [sleep] ERROR: need Python >= 3.10 (found none). >&2
+    exit /b 1
+)
+
+cd /d "%REPO_ROOT%"
+if "%~1" == "" (
+    "%PY%" -m skillopt_sleep status
+) else (
+    "%PY%" -m skillopt_sleep %*
+)
+exit /b !errorlevel!
+
+
+:: ── Fallback Mode (No Source Checkout) ─────────────────────────────
+:fallback_mode
+
+:: Fallback 1: skillopt-sleep CLI on PATH (uv tool install / pipx / pip install).
+where skillopt-sleep >nul 2>nul
+if !errorlevel! neq 0 goto try_fallback_2
+
+if "%~1" == "" (
+    skillopt-sleep status
+) else (
+    skillopt-sleep %*
+)
+exit /b !errorlevel!
+
+:try_fallback_2
+:: Fallback 2: importable as a module (pip install into the active Python).
+set "PY="
+for %%p in (python3.exe python.exe py.exe) do (
+    where %%p >nul 2>nul
+    if !errorlevel! equ 0 (
+        %%p -c "import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)" >nul 2>nul
+        if !errorlevel! equ 0 (
+            %%p -c "import skillopt_sleep" >nul 2>nul
+            if !errorlevel! equ 0 (
+                set "PY=%%p"
+                goto py_import_found
+            )
+        )
+    )
+)
+:py_import_found
+
+if "%PY%" == "" goto not_found
+
+if "%~1" == "" (
+    "%PY%" -m skillopt_sleep status
+) else (
+    "%PY%" -m skillopt_sleep %*
+)
+exit /b !errorlevel!
+
+:not_found
+echo [sleep] ERROR: could not locate the skillopt_sleep package. >&2
+echo [sleep] Install it with 'uv tool install skillopt' or 'pip install skillopt', >&2
+echo [sleep] or set SKILLOPT_SLEEP_REPO to a clone of the SkillOpt repo. >&2
+exit /b 1

plugins/run-sleep.ps1

@@ -0,0 +1,94 @@
+# SkillOpt-Sleep shared runner for Windows PowerShell
+# Resolves the repo root, picks Python >= 3.10, and runs the engine CLI.
+#
+# Usage: .\run-sleep.ps1 [status|run|dry-run|adopt|...] [args...]
+
+$ErrorActionPreference = "Stop"
+
+$ScriptDir = $PSScriptRoot
+
+$RepoRoot = $null
+if (Test-Path (Join-Path $ScriptDir "..\skillopt_sleep")) {
+    $RepoRoot = Resolve-Path (Join-Path $ScriptDir "..")
+} elseif ($env:CLAUDE_PLUGIN_ROOT -and (Test-Path (Join-Path $env:CLAUDE_PLUGIN_ROOT "..\..\skillopt_sleep"))) {
+    $RepoRoot = Resolve-Path (Join-Path $env:CLAUDE_PLUGIN_ROOT "..\..")
+} elseif ($env:SKILLOPT_SLEEP_REPO -and (Test-Path (Join-Path $env:SKILLOPT_SLEEP_REPO "skillopt_sleep"))) {
+    $RepoRoot = Resolve-Path $env:SKILLOPT_SLEEP_REPO
+} else {
+    # search upward from current location
+    $d = Get-Item .
+    while ($d -and $d.FullName -ne $d.Root.FullName) {
+        if (Test-Path (Join-Path $d.FullName "skillopt_sleep")) {
+            $RepoRoot = $d.FullName
+            break
+        }
+        $d = Split-Path -Parent $d.FullName -ErrorAction SilentlyContinue | Get-Item -ErrorAction SilentlyContinue
+    }
+}
+
+$argsList = @($args)
+if ($argsList.Count -eq 0) {
+    $argsList = @("status")
+}
+
+if ($RepoRoot) {
+    $Py = ""
+    if ($env:SKILLOPT_SLEEP_PYTHON) {
+        $Py = $env:SKILLOPT_SLEEP_PYTHON
+    } else {
+        foreach ($cand in @("python3", "python", "py")) {
+            $cmd = Get-Command $cand -ErrorAction SilentlyContinue
+            if ($cmd) {
+                $ver = & $cand -c "import sys; print('%d%d' % sys.version_info[:2])" 2>$null
+                if ($ver -and [int]$ver -ge 310) {
+                    $Py = $cand
+                    break
+                }
+            }
+        }
+    }
+
+    if (-not $Py) {
+        [Console]::Error.WriteLine("[sleep] ERROR: need Python >= 3.10 (found none).")
+        exit 1
+    }
+
+    Set-Location $RepoRoot
+    & $Py -m skillopt_sleep $argsList
+    exit $LASTEXITCODE
+}
+
+# No source checkout found — fall back to an installed engine.
+
+# Fallback 1: skillopt-sleep CLI on PATH (uv tool install / pipx / pip install).
+$cliCmd = Get-Command "skillopt-sleep" -ErrorAction SilentlyContinue
+if ($cliCmd) {
+    & skillopt-sleep $argsList
+    exit $LASTEXITCODE
+}
+
+# Fallback 2: importable as a module (pip install into the active Python).
+$Py = ""
+foreach ($cand in @("python3", "python", "py")) {
+    $cmd = Get-Command $cand -ErrorAction SilentlyContinue
+    if ($cmd) {
+        $ver = & $cand -c "import sys; print('%d%d' % sys.version_info[:2])" 2>$null
+        if ($ver -and [int]$ver -ge 310) {
+            $hasModule = & $cand -c "import skillopt_sleep" 2>$null
+            if ($LASTEXITCODE -eq 0) {
+                $Py = $cand
+                break
+            }
+        }
+    }
+}
+
+if ($Py) {
+    & $Py -m skillopt_sleep $argsList
+    exit $LASTEXITCODE
+}
+
+[Console]::Error.WriteLine("[sleep] ERROR: could not locate the skillopt_sleep package.")
+[Console]::Error.WriteLine("[sleep] Install it with 'uv tool install skillopt' or 'pip install skillopt',")
+[Console]::Error.WriteLine("[sleep] or set SKILLOPT_SLEEP_REPO to a clone of the SkillOpt repo.")
+exit 1

plugins/run-sleep.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# SkillOpt-Sleep shared runner — used by all platform plugins (Claude Code,
+# Codex, Copilot). Resolves the repo root (which contains the skillopt_sleep
+# package), picks a Python >= 3.10, and execs the engine CLI.
+#
+# Usage: run-sleep.sh <run|dry-run|status|adopt|harvest|...> [args...]
+set -euo pipefail
+
+# This script lives at <repo>/plugins/run-sleep.sh, so the repo root (which
+# holds skillopt_sleep/) is one level up. CLAUDE_PLUGIN_ROOT (if set by Claude
+# Code) points at the plugin dir; the engine is then two levels above it.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+if [ -d "$SCRIPT_DIR/../skillopt_sleep" ]; then
+  REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -d "$CLAUDE_PLUGIN_ROOT/../../skillopt_sleep" ]; then
+  REPO_ROOT="$(cd "$CLAUDE_PLUGIN_ROOT/../.." && pwd)"
+elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -d "$SKILLOPT_SLEEP_REPO/skillopt_sleep" ]; then
+  REPO_ROOT="$SKILLOPT_SLEEP_REPO"
+else
+  # last resort: search upward from CWD
+  d="$PWD"
+  while [ "$d" != "/" ]; do
+    [ -d "$d/skillopt_sleep" ] && { REPO_ROOT="$d"; break; }
+    d="$(dirname "$d")"
+  done
+fi
+if [ "$#" -eq 0 ]; then set -- status; fi
+
+if [ -n "${REPO_ROOT:-}" ]; then
+  # Source checkout: run from repo root so skillopt_sleep/ is importable.
+  PY=""
+  # Allow explicit Python override (useful on macOS with old system Python).
+  if [ -n "${SKILLOPT_SLEEP_PYTHON:-}" ]; then
+    PY="$SKILLOPT_SLEEP_PYTHON"
+  else
+    for cand in python3.12 python3.11 python3.10 python3; do
+      if command -v "$cand" >/dev/null 2>&1; then
+        ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)"
+        if [ "${ver:-0}" -ge 310 ]; then PY="$cand"; break; fi
+      fi
+    done
+  fi
+  if [ -z "$PY" ]; then
+    echo "[sleep] ERROR: need Python >= 3.10 (found none)." >&2
+    exit 1
+  fi
+  cd "$REPO_ROOT"
+  exec "$PY" -m skillopt_sleep "$@"
+fi
+
+# No source checkout found — fall back to an installed engine.
+
+# Fallback 1: skillopt-sleep CLI on PATH (uv tool install / pipx / pip install).
+# Checked before the import fallback because uv tool install / pipx isolate the
+# package from the system Python's import path, so `python -c "import
+# skillopt_sleep"` would fail even though the CLI is available.
+if command -v skillopt-sleep >/dev/null 2>&1; then
+  exec skillopt-sleep "$@"
+fi
+
+# Fallback 2: importable as a module (pip install into the active Python).
+# Pick a Python >= 3.10 and check importability.
+PY=""
+for cand in python3.12 python3.11 python3.10 python3; do
+  if command -v "$cand" >/dev/null 2>&1; then
+    ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)"
+    if [ "${ver:-0}" -ge 310 ] && "$cand" -c "import skillopt_sleep" >/dev/null 2>&1; then
+      PY="$cand"; break
+    fi
+  fi
+done
+if [ -n "$PY" ]; then
+  exec "$PY" -m skillopt_sleep "$@"
+fi
+
+echo "[sleep] ERROR: could not locate the skillopt_sleep package." >&2
+echo "[sleep] Install it with 'uv tool install skillopt' or 'pip install skillopt'," >&2
+echo "[sleep] or set SKILLOPT_SLEEP_REPO to a clone of the SkillOpt repo." >&2
+exit 1

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "skillopt"
-version = "0.1.0"
+version = "0.2.0"
 description = "SkillOpt: Agentic Skill Optimization via Reflective Training Loops"
 readme = "README.md"
 license = {text = "MIT"}
@@ -37,9 +37,11 @@ dependencies = [
 # Benchmark-specific dependencies
 alfworld = ["alfworld>=0.4.0", "gymnasium>=0.29.0"]
 # Claude model backend
-claude = ["claude-agent-sdk>=0.1.0"]
+claude = ["claude-agent-sdk>=0.1.0", "json_repair>=0.61.0"]
 # Qwen local model backend (via vLLM)
-qwen = ["vllm>=0.4.0"]
+qwen = ["vllm>=0.4.0", "json_repair>=0.61.0"]
+# SearchQA data materialization
+searchqa = ["datasets>=2.18.0"]
 # Documentation site
 docs = ["mkdocs-material>=9.5.0", "mkdocstrings[python]>=0.24.0"]
 # WebUI dashboard
@@ -51,11 +53,13 @@ all = [
     "alfworld>=0.4.0",
     "gymnasium>=0.29.0",
     "claude-agent-sdk>=0.1.0",
+    "json_repair>=0.61.0",
 ]
 
 [project.scripts]
 skillopt-train = "scripts.train:main"
 skillopt-eval = "scripts.eval_only:main"
+skillopt-sleep = "skillopt_sleep.__main__:main"
 
 [project.urls]
 Homepage = "https://github.com/microsoft/SkillOpt"
@@ -64,7 +68,13 @@ Repository = "https://github.com/microsoft/SkillOpt"
 Issues = "https://github.com/microsoft/SkillOpt/issues"
 
 [tool.setuptools.packages.find]
-include = ["skillopt*", "scripts*"]
+# skillopt*      = the research package
+# skillopt_sleep = the open-source Sleep tool (decoupled, zero research dep)
+# skillopt_webui = the Gradio dashboard (installed via the `webui` extra)
+include = ["skillopt", "skillopt.*", "skillopt_sleep", "skillopt_sleep.*", "skillopt_webui", "skillopt_webui.*", "scripts*"]
+
+[tool.setuptools.package-data]
+"*" = ["*.md"]
 
 [tool.ruff]
 line-length = 120

requirements.txt

@@ -17,6 +17,12 @@ httpx>=0.27.0
 # ── Optional: Qwen local model (via vLLM) ────────
 # vllm>=0.4.0
 
+# ── Optional: tolerant JSON repair for free-form output from non-OpenAI
+#    backends (Claude/Qwen). Without it extract_json() falls back safely and
+#    drops a malformed analyst edit instead of repairing it. Installed by the
+#    `claude`, `qwen`, and `all` extras in pyproject.toml.
+# json_repair>=0.61.0
+
 # ── Optional: WebUI dashboard ────────────────────
 # gradio>=4.0.0
 

scripts/eval_only.py

@@ -28,6 +28,8 @@ from skillopt.model import (
     configure_azure_openai,
     configure_claude_code_exec,
     configure_codex_exec,
+    configure_qwen_chat,
+    configure_minimax_chat,
     set_reasoning_effort,
     set_target_backend,
     set_target_deployment,
@@ -137,7 +139,7 @@ def parse_args() -> argparse.Namespace:
     # Legacy flat overrides
     p.add_argument("--env", type=str)
     p.add_argument("--backend", type=str,
-                   choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec"])
+                   choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "minimax", "minimax_chat"])
     p.add_argument("--optimizer_model", type=str)
     p.add_argument("--target_model", type=str)
     p.add_argument("--optimizer_backend", type=str)
@@ -179,6 +181,12 @@ def parse_args() -> argparse.Namespace:
     p.add_argument("--claude_code_exec_use_sdk", type=str)
     p.add_argument("--claude_code_exec_effort", type=str)
     p.add_argument("--claude_code_exec_max_thinking_tokens", type=int)
+    p.add_argument("--minimax_base_url", type=str)
+    p.add_argument("--minimax_api_key", type=str)
+    p.add_argument("--minimax_model", type=str)
+    p.add_argument("--minimax_temperature", type=float)
+    p.add_argument("--minimax_max_tokens", type=int)
+    p.add_argument("--minimax_enable_thinking", type=_BOOL)
     p.add_argument("--out_root", type=str)
     p.add_argument("--data_path", type=str)
     p.add_argument("--split_mode", type=str,
@@ -254,6 +262,12 @@ def main() -> None:
                 "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk",
                 "claude_code_exec_effort": "model.claude_code_exec_effort",
                 "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens",
+                "minimax_base_url": "model.minimax_base_url",
+                "minimax_api_key": "model.minimax_api_key",
+                "minimax_model": "model.minimax_model",
+                "minimax_temperature": "model.minimax_temperature",
+                "minimax_max_tokens": "model.minimax_max_tokens",
+                "minimax_enable_thinking": "model.minimax_enable_thinking",
                 "seed": "train.seed",
                 "test_env_num": "evaluation.test_env_num",
                 "env": "env.name",
@@ -306,11 +320,16 @@ def main() -> None:
             cfg.setdefault("optimizer_backend", "claude_chat")
             cfg.setdefault("target_backend", "claude_chat")
         elif backend in {"codex", "codex_exec"}:
-            cfg.setdefault("optimizer_backend", "openai_chat")
+            if not _has_model_override("model.optimizer_backend", "optimizer_backend"):
-            cfg.setdefault("target_backend", "codex_exec")
+                cfg["optimizer_backend"] = "codex_exec"
+            if not _has_model_override("model.target_backend", "target_backend"):
+                cfg["target_backend"] = "codex_exec"
         elif backend == "claude_code_exec":
             cfg.setdefault("optimizer_backend", "openai_chat")
             cfg.setdefault("target_backend", "claude_code_exec")
+        elif backend in {"minimax", "minimax_chat"}:
+            cfg.setdefault("optimizer_backend", "openai_chat")
+            cfg.setdefault("target_backend", "minimax_chat")
         else:
             cfg.setdefault("optimizer_backend", "openai_chat")
             cfg.setdefault("target_backend", "openai_chat")
@@ -336,6 +355,15 @@ def main() -> None:
             and not _has_model_override("model.target", "target_model")
         ):
             cfg["target_model"] = default_model_for_backend("claude_chat")
+    if cfg.get("target_backend") == "minimax_chat":
+        if (
+            str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+            and not _has_model_override("model.target", "target_model")
+        ):
+            cfg["target_model"] = (
+                cfg.get("minimax_model")
+                or default_model_for_backend("minimax_chat")
+            )
 
     if not cfg.get("out_root"):
         env = cfg.get("env", "unknown")
@@ -401,6 +429,30 @@ def main() -> None:
         effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")),
         max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384),
     )
+    configure_qwen_chat(
+        base_url=cfg.get("qwen_chat_base_url") or None,
+        api_key=cfg.get("qwen_chat_api_key") or None,
+        temperature=cfg.get("qwen_chat_temperature"),
+        timeout_seconds=cfg.get("qwen_chat_timeout_seconds"),
+        max_tokens=cfg.get("qwen_chat_max_tokens"),
+        enable_thinking=cfg.get("qwen_chat_enable_thinking"),
+        target_base_url=cfg.get("target_qwen_chat_base_url") or None,
+        target_api_key=cfg.get("target_qwen_chat_api_key") or None,
+        target_temperature=cfg.get("target_qwen_chat_temperature"),
+        target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"),
+        target_max_tokens=cfg.get("target_qwen_chat_max_tokens"),
+        target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"),
+    )
+    configure_minimax_chat(
+        base_url=cfg.get("minimax_base_url") or None,
+        api_key=cfg.get("minimax_api_key") or None,
+        temperature=cfg.get("minimax_temperature"),
+        max_tokens=cfg.get("minimax_max_tokens"),
+        enable_thinking=cfg.get("minimax_enable_thinking"),
+    )
+    minimax_model_cfg = cfg.get("minimax_model")
+    if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat":
+        set_target_deployment(str(minimax_model_cfg))
     set_reasoning_effort(cfg.get("reasoning_effort", "") or None)
 
     # Build adapter

scripts/materialize_searchqa.py

@@ -0,0 +1,170 @@
+"""Materialize runnable SearchQA splits from the released ID manifest."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+SPLITS = ("train", "val", "test")
+REQUIRED_FIELDS = ("question", "context", "answers")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--manifest-dir",
+        type=Path,
+        default=PROJECT_ROOT / "data" / "searchqa_id_split",
+        help="Directory containing train/val/test ID manifests.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=PROJECT_ROOT / "data" / "searchqa_split",
+        help="Directory to write runnable train/val/test splits.",
+    )
+    parser.add_argument(
+        "--dataset",
+        default="lucadiliello/searchqa",
+        help="Hugging Face dataset repository to load.",
+    )
+    return parser.parse_args()
+
+
+def load_manifest_ids(manifest_dir: Path) -> dict[str, list[str]]:
+    split_ids = {}
+    for split in SPLITS:
+        path = manifest_dir / split / "items.json"
+        with path.open(encoding="utf-8") as file:
+            items = json.load(file)
+        split_ids[split] = [str(item["id"]) for item in items]
+    return split_ids
+
+
+def _reject_duplicate_manifest_ids(split_ids: Mapping[str, Iterable[str]]) -> None:
+    seen: dict[str, str] = {}
+    duplicates: dict[str, list[str]] = {}
+    for split, ids in split_ids.items():
+        for item_id in ids:
+            if item_id in seen:
+                duplicates.setdefault(item_id, [seen[item_id]]).append(split)
+            else:
+                seen[item_id] = split
+
+    if duplicates:
+        preview = ", ".join(
+            f"{item_id} ({'/'.join(splits)})"
+            for item_id, splits in sorted(duplicates.items())[:5]
+        )
+        raise ValueError(
+            "SearchQA split manifest contains duplicate IDs across splits. "
+            f"First IDs: {preview}"
+        )
+
+
+def _iter_dataset_rows(dataset: Mapping[str, Iterable[dict]]) -> Iterable[dict]:
+    for source_split in dataset.values():
+        yield from source_split
+
+
+def _normalize_row(row: dict) -> dict:
+    try:
+        key = str(row["key"])
+    except KeyError as exc:
+        raise ValueError("SearchQA source row is missing required field: key") from exc
+
+    missing = [field for field in REQUIRED_FIELDS if field not in row]
+    if missing:
+        raise ValueError(f"SearchQA source row {key!r} is missing required fields: {', '.join(missing)}")
+
+    return {
+        "id": key,
+        "question": row["question"],
+        "context": row["context"],
+        "answers": row["answers"],
+    }
+
+
+def materialize_searchqa_splits(
+    manifest_dir: Path,
+    output_dir: Path,
+    dataset: Mapping[str, Iterable[dict]],
+    *,
+    dataset_name: str,
+) -> dict[str, int]:
+    """Write runnable SearchQA train/val/test splits from a source dataset."""
+    manifest_dir = manifest_dir.resolve()
+    output_dir = output_dir.resolve()
+    split_ids = load_manifest_ids(manifest_dir)
+    _reject_duplicate_manifest_ids(split_ids)
+    wanted_ids = {item_id for ids in split_ids.values() for item_id in ids}
+
+    selected: dict[str, dict] = {}
+    duplicate_ids: set[str] = set()
+    for row in _iter_dataset_rows(dataset):
+        key = str(row.get("key", ""))
+        if key not in wanted_ids:
+            continue
+        if key in selected:
+            duplicate_ids.add(key)
+            continue
+        selected[key] = _normalize_row(row)
+
+    if duplicate_ids:
+        preview = ", ".join(sorted(duplicate_ids)[:5])
+        raise ValueError(f"SearchQA source dataset contains duplicate manifest IDs. First IDs: {preview}")
+
+    missing = sorted(wanted_ids - selected.keys())
+    if missing:
+        preview = ", ".join(missing[:5])
+        raise RuntimeError(f"SearchQA source dataset is missing {len(missing)} manifest IDs. First IDs: {preview}")
+
+    counts = {}
+    for split, ids in split_ids.items():
+        items = [selected[item_id] for item_id in ids]
+        split_dir = output_dir / split
+        split_dir.mkdir(parents=True, exist_ok=True)
+        with (split_dir / "items.json").open("w", encoding="utf-8") as file:
+            json.dump(items, file, ensure_ascii=False, indent=2)
+        counts[split] = len(items)
+
+    manifest = {
+        "source_manifest_dir": str(manifest_dir),
+        "source_dataset": dataset_name,
+        "counts": counts,
+        "item_fields": ["id", *REQUIRED_FIELDS],
+    }
+    with (output_dir / "split_manifest.json").open("w", encoding="utf-8") as file:
+        json.dump(manifest, file, ensure_ascii=False, indent=2)
+
+    return counts
+
+
+def main() -> None:
+    args = parse_args()
+    try:
+        from datasets import load_dataset
+    except ImportError as exc:
+        raise SystemExit(
+            "Missing dependency 'datasets'. Install it with:\n"
+            "  python -m pip install 'skillopt[searchqa]'\n"
+            "or:\n"
+            "  python -m pip install datasets"
+        ) from exc
+
+    print(f"Loading {args.dataset}...")
+    dataset = load_dataset(args.dataset)
+    counts = materialize_searchqa_splits(
+        args.manifest_dir,
+        args.output_dir,
+        dataset,
+        dataset_name=args.dataset,
+    )
+    print(f"Wrote SearchQA splits to {args.output_dir.resolve()}: {counts}")
+
+
+if __name__ == "__main__":
+    main()

scripts/train.py

@@ -245,6 +245,10 @@ def parse_args() -> argparse.Namespace:
     p.add_argument("--longitudinal_pair_policy", type=str,
                    choices=["mixed", "changed", "unchanged"])
     p.add_argument("--use_meta_skill", type=_BOOL)
+    p.add_argument("--use_skill_aware_reflection", type=_BOOL)
+    p.add_argument("--skill_aware_appendix_source", type=str,
+                   choices=["both", "failure_only"])
+    p.add_argument("--skill_aware_consolidate_threshold", type=int)
     p.add_argument("--data_path", type=str)
     p.add_argument("--split_mode", type=str,
                    choices=["ratio", "split_dir"])
@@ -360,6 +364,9 @@ _LEGACY_TO_STRUCTURED: dict[str, str] = {
     "slow_update_samples": "optimizer.slow_update_samples",
     "longitudinal_pair_policy": "optimizer.longitudinal_pair_policy",
     "use_meta_skill": "optimizer.use_meta_skill",
+    "use_skill_aware_reflection": "optimizer.use_skill_aware_reflection",
+    "skill_aware_appendix_source": "optimizer.skill_aware_appendix_source",
+    "skill_aware_consolidate_threshold": "optimizer.skill_aware_consolidate_threshold",
     "use_gate": "evaluation.use_gate",
     "sel_env_num": "evaluation.sel_env_num",
     "test_env_num": "evaluation.test_env_num",
@@ -431,8 +438,10 @@ def load_config(args: argparse.Namespace) -> dict:
             flat.setdefault("optimizer_backend", "claude_chat")
             flat.setdefault("target_backend", "claude_chat")
         elif backend in {"codex", "codex_exec"}:
-            flat.setdefault("optimizer_backend", "openai_chat")
+            if not _has_model_override("model.optimizer_backend", "optimizer_backend"):
-            flat.setdefault("target_backend", "codex_exec")
+                flat["optimizer_backend"] = "codex_exec"
+            if not _has_model_override("model.target_backend", "target_backend"):
+                flat["target_backend"] = "codex_exec"
         elif backend == "claude_code_exec":
             flat.setdefault("optimizer_backend", "openai_chat")
             flat.setdefault("target_backend", "claude_code_exec")
@@ -527,6 +536,7 @@ def main() -> None:
     print(f"  minibatch_size: {cfg.get('minibatch_size')}")
     print(f"  seed:           {cfg.get('seed')}")
     print(f"  meta_skill:     {cfg.get('use_meta_skill', False)}")
+    print(f"  skill_aware_reflection: {cfg.get('use_skill_aware_reflection', False)}")
     print(f"  slow_update:    {cfg.get('use_slow_update', False)}")
     print(f"  out_root:       {cfg.get('out_root')}")
     print(f"{'='*60}\n")

skillopt.html

@@ -1778,6 +1778,8 @@
       <a href="#evolution">Evolution</a>
       <a href="#transfer">Transfer</a>
       <a href="#citation">Citation</a>
+      <a href="https://microsoft.github.io/SkillOpt/blog/">Blog</a>
+      <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/index.md" target="_blank" rel="noopener">Docs</a>
       <a href="https://github.com/microsoft/SkillOpt" target="_blank" rel="noopener">Code</a>
     </nav>
   </header>
@@ -1915,7 +1917,7 @@
           <h3>A skill is external state for an agent.</h3>
           <p>
             Instead of fine-tuning a model or hand-maintaining prompts, SkillOpt runs
-            the frozen agent on scored batches, asks a separate optimizer model to
+            the frozen agent on scored batches, asks an optimizer model to
             propose structured edits, and accepts a candidate only when validation
             performance improves.
           </p>
@@ -2416,21 +2418,18 @@
 
       <div class="bibtex-box">
         <button class="copy-btn" type="button" onclick="copyBibtex(this)">Copy</button>
-        <pre><code>@misc{yang2026skilloptexecutivestrategyselfevolving,
+        <pre><code>@article{yang2026skillopt,
-      title={SkillOpt: Executive Strategy for Self-Evolving Agent Skills}, 
+  title={Skillopt: Executive strategy for self-evolving agent skills},
-      author={Yifan Yang and Ziyang Gong and Weiquan Huang and Qihao Yang and Ziwei Zhou and Zisu Huang and Yan Li and Xuemei Gao and Qi Dai and Bei Liu and Kai Qiu and Yuqing Yang and Dongdong Chen and Xue Yang and Chong Luo},
+  author={Yang, Yifan and Gong, Ziyang and Huang, Weiquan and Yang, Qihao and Zhou, Ziwei and Huang, Zisu and Li, Yan and Gao, Xuemei and Dai, Qi and Liu, Bei and others},
-      year={2026},
+  journal={arXiv preprint arXiv:2605.23904},
-      eprint={2605.23904},
+  year={2026}
-      archivePrefix={arXiv},
-      primaryClass={cs.AI},
-      url={https://arxiv.org/abs/2605.23904}, 
 }</code></pre>
       </div>
     </section>
 
     <footer class="footer">
       <span>SkillOpt: Executive Strategy for Self-Evolving Agent Skills</span>
-      <span><a href="https://github.com/microsoft/SkillOpt" target="_blank" rel="noopener">Code</a> / <a href="#citation">Citation</a></span>
+      <span><a href="https://microsoft.github.io/SkillOpt/blog/">Blog</a> / <a href="https://github.com/microsoft/SkillOpt/blob/main/docs/index.md" target="_blank" rel="noopener">Docs</a> / <a href="https://github.com/microsoft/SkillOpt" target="_blank" rel="noopener">Code</a> / <a href="#citation">Citation</a></span>
     </footer>
   </main>
   <script>

skillopt/__init__.py

@@ -12,7 +12,7 @@ Pipeline stages:
   6. Evaluate  — validate candidate skill, accept/reject
 """
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 from skillopt.types import (  # noqa: F401
     BatchSpec,

skillopt/config.py

@@ -119,9 +119,15 @@ _FLATTEN_MAP: dict[str, str] = {
     "optimizer.slow_update_gate_with_selection": "slow_update_gate_with_selection",
     "optimizer.longitudinal_pair_policy": "longitudinal_pair_policy",
     "optimizer.use_meta_skill": "use_meta_skill",
+    "optimizer.use_skill_aware_reflection": "use_skill_aware_reflection",
+    "optimizer.skill_aware_appendix_source": "skill_aware_appendix_source",
+    "optimizer.skill_aware_consolidate_threshold": "skill_aware_consolidate_threshold",
     "evaluation.use_gate": "use_gate",
     "evaluation.gate_metric": "gate_metric",
     "evaluation.gate_mixed_weight": "gate_mixed_weight",
+    "evaluation.use_semantic_density": "use_semantic_density",
+    "evaluation.semantic_density_weight": "semantic_density_weight",
+    "evaluation.leading_words": "leading_words",
     "evaluation.sel_env_num": "sel_env_num",
     "evaluation.test_env_num": "test_env_num",
     "evaluation.eval_test": "eval_test",
@@ -155,7 +161,7 @@ def _load_yaml(path: str, _visited: set[str] | None = None) -> dict:
         raise ValueError(f"Circular _base_ inheritance: {abs_path}")
     _visited.add(abs_path)
 
-    with open(abs_path) as f:
+    with open(abs_path, encoding="utf-8") as f:
         cfg = yaml.safe_load(f) or {}
 
     base_ref = cfg.pop("_base_", None)
@@ -189,13 +195,6 @@ def flatten_config(cfg: dict) -> dict:
 
     flat: dict[str, Any] = {}
 
-    evaluation_section = cfg.get("evaluation", {})
-    if isinstance(evaluation_section, dict) and evaluation_section.get("use_gate") is False:
-        raise ValueError(
-            "Gate validation is mandatory in this branch. Remove "
-            "`evaluation.use_gate: false` from the config."
-        )
-
     # Apply the explicit mapping
     for dotted, flat_key in _FLATTEN_MAP.items():
         section, key = dotted.split(".", 1)

skillopt/engine/trainer.py

@@ -24,7 +24,7 @@ from collections import defaultdict
 
 from skillopt.datasets.base import BatchSpec
 from skillopt.envs.base import EnvAdapter
-from skillopt.evaluation.gate import evaluate_gate, select_gate_score
+from skillopt.evaluation.gate import GateResult, evaluate_gate, select_gate_score
 from skillopt.gradient.aggregate import merge_patches
 from skillopt.optimizer.meta_skill import run_meta_skill
 from skillopt.optimizer.clip import rank_and_select
@@ -32,6 +32,17 @@ from skillopt.optimizer.lr_autonomous import decide_autonomous_learning_rate
 from skillopt.optimizer.rewrite import rewrite_skill_from_suggestions
 from skillopt.optimizer.scheduler import build_scheduler
 from skillopt.optimizer.skill import apply_patch_with_report
+from skillopt.optimizer.appendix import (
+    append_to_appendix_field,
+    extract_appendix_notes as extract_appendix_notes_from_skill,
+    inject_empty_appendix_field,
+    _strip_all_appendix_fields,
+)
+from skillopt.optimizer.skill_aware import (
+    configure_skill_aware_reflection,
+    consolidate_appendix_notes,
+    extract_appendix_notes as extract_appendix_notes_from_result,
+)
 from skillopt.optimizer.slow_update import (
     build_comparison_pairs,
     extract_slow_update_field,
@@ -48,6 +59,7 @@ from skillopt.optimizer.update_modes import (
     short_item_summary,
 )
 from skillopt.model import (
+    chat_optimizer,
     configure_azure_openai,
     configure_claude_code_exec,
     configure_codex_exec,
@@ -64,6 +76,74 @@ from skillopt.model import (
 from skillopt.utils import compute_score, skill_hash
 
 
+# ── Skill-aware reflection: appendix flush ───────────────────────────────────
+
+def _flush_skill_aware_appendix(
+    current_skill: str,
+    all_raw_patches: list,
+    step_rec: dict,
+    step_dir: str,
+    cfg: dict,
+) -> str:
+    """Append this step's EXECUTION_LAPSE notes into the protected appendix.
+
+    Returns the (possibly) updated skill. Must be called on BOTH the normal
+    update path and the skip branches: a lapse-only step yields no body
+    patches by design (analysts return ``edits: []`` carriers), so the skip
+    paths would otherwise silently drop every note of the step.
+    """
+    step_appendix_notes: list[str] = []
+    for rp in all_raw_patches:
+        if isinstance(rp, dict):
+            step_appendix_notes.extend(extract_appendix_notes_from_result(rp))
+    if not step_appendix_notes:
+        return current_skill
+
+    before_notes = extract_appendix_notes_from_skill(current_skill)
+    current_skill = append_to_appendix_field(
+        current_skill, step_appendix_notes,
+    )
+    after_notes = extract_appendix_notes_from_skill(current_skill)
+    n_added = len(after_notes) - len(before_notes)
+    step_rec["n_execution_lapse_notes"] = len(step_appendix_notes)
+    step_rec["n_appendix_notes_added"] = n_added
+    step_rec["n_appendix_notes_total"] = len(after_notes)
+    with open(os.path.join(step_dir, "appendix_notes.json"), "w") as f:
+        json.dump(
+            {
+                "step_notes": step_appendix_notes,
+                "appendix_after": after_notes,
+            },
+            f, indent=2, ensure_ascii=False,
+        )
+    print(
+        f"    [skill-aware] +{n_added} appendix note(s) "
+        f"(total {len(after_notes)}) from {len(step_appendix_notes)} lapse signal(s)"
+    )
+    # Threshold-gated LLM consolidation (paper Eq.11): when the
+    # appendix grows past N notes, compact it with one optimizer
+    # call (dedupe / merge / shorten). 0 disables it. Any failure
+    # leaves the appendix unchanged.
+    consolidate_threshold = int(
+        cfg.get("skill_aware_consolidate_threshold", 0) or 0
+    )
+    if consolidate_threshold > 0 and len(after_notes) > consolidate_threshold:
+        compacted = consolidate_appendix_notes(
+            after_notes, chat_fn=chat_optimizer,
+        )
+        if compacted and len(compacted) < len(after_notes):
+            current_skill = append_to_appendix_field(
+                _strip_all_appendix_fields(current_skill), compacted,
+            )
+            step_rec["n_appendix_notes_consolidated"] = len(compacted)
+            step_rec["n_appendix_notes_total"] = len(compacted)
+            print(
+                f"    [skill-aware] consolidated appendix "
+                f"{len(after_notes)} -> {len(compacted)} notes"
+            )
+    return current_skill
+
+
 # ── Patch normalization ───────────────────────────────────────────────────────
 
 def _normalise_patches(
@@ -467,7 +547,7 @@ def _format_step_buffer(buffer: list[dict]) -> str:
 
         # Failure patterns
         for p in entry.get("failure_patterns", []):
-            ids = ", ".join(p["task_ids"][:3])
+            ids = ", ".join(p["task_ids"])
             parts.append(f'  - "{p["pattern"]}" (×{p["count"]}, tasks: {ids})')
 
         # Rejected edits (only present on reject)
@@ -484,7 +564,7 @@ def _format_step_buffer(buffer: list[dict]) -> str:
                     content = e.get("content", "")
                     target = e.get("target", "")
                     if target:
-                        parts.append(f'    {i}. [{op}] target="{target[:80]}" → "{content}"')
+                        parts.append(f'    {i}. [{op}] target="{target}" → "{content}"')
                     else:
                         parts.append(f'    {i}. [{op}] "{content}"')
                 else:
@@ -594,8 +674,10 @@ class ReflACTTrainer:
                 optimizer_backend = optimizer_backend or "claude_chat"
                 target_backend = target_backend or "claude_chat"
             elif backend in {"codex", "codex_exec"}:
-                optimizer_backend = optimizer_backend or "openai_chat"
+                if optimizer_backend in (None, "", "openai_chat"):
-                target_backend = target_backend or "codex_exec"
+                    optimizer_backend = "codex_exec"
+                if target_backend in (None, "", "openai_chat"):
+                    target_backend = "codex_exec"
             elif backend == "claude_code_exec":
                 optimizer_backend = optimizer_backend or "openai_chat"
                 target_backend = target_backend or "claude_code_exec"
@@ -838,6 +920,16 @@ class ReflACTTrainer:
 
         _save_skill(out_root, 0, skill_init)
 
+        use_skill_aware = cfg.get("use_skill_aware_reflection", False)
+        # Publish the toggle process-wide so run_minibatch_reflect resolves it
+        # from config for EVERY env adapter — no per-benchmark wiring needed.
+        configure_skill_aware_reflection(
+            use_skill_aware,
+            cfg.get("skill_aware_appendix_source", "both"),
+        )
+        if use_skill_aware:
+            current_skill = inject_empty_appendix_field(current_skill)
+
         def _persist_runtime_state(last_completed_step: int) -> None:
             _save_runtime_state(
                 out_root,
@@ -863,11 +955,10 @@ class ReflACTTrainer:
                 sel_cache[sh] = (rec["selection_hard"], rec["selection_soft"])
 
         # ── Baseline evaluation on selection set ─────────────────────────
-        if cfg.get("use_gate") is False:
+        # `use_gate=False` keeps validation running (selection rollout +
-            raise ValueError(
+        # scoring are unconditional below) but force-accepts every candidate
-                "Gate validation is mandatory in this branch. Remove "
+        # instead of gating it; final skill is chosen manually afterwards.
-                "`evaluation.use_gate=false` from the config."
+        use_gate = cfg.get("use_gate", True) is not False
-            )
         gate_metric = str(cfg.get("gate_metric", "hard")).strip().lower()
         if gate_metric not in {"hard", "soft", "mixed"}:
             raise ValueError(
@@ -875,6 +966,15 @@ class ReflACTTrainer:
                 f"got {gate_metric!r}"
             )
         gate_mixed_weight = float(cfg.get("gate_mixed_weight", 0.5))
+        use_semantic_density = bool(cfg.get("use_semantic_density", False))
+        semantic_density_weight = float(cfg.get("semantic_density_weight", 0.05))
+        leading_words_raw = cfg.get("leading_words", None)
+        leading_words = None
+        if leading_words_raw is not None:
+            if isinstance(leading_words_raw, str):
+                leading_words = [w.strip() for w in leading_words_raw.split(",") if w.strip()]
+            else:
+                leading_words = list(leading_words_raw)
         if not 0.0 <= gate_mixed_weight <= 1.0:
             raise ValueError(
                 f"evaluation.gate_mixed_weight must be in [0, 1], "
@@ -887,6 +987,8 @@ class ReflACTTrainer:
                 if gate_metric == "mixed"
                 else ""
             )
+            + ("" if use_gate
+               else "  (DISABLED → validation runs, candidates force-accepted)")
         )
         slow_gate_with_selection = bool(
             cfg.get("slow_update_gate_with_selection", False)
@@ -912,6 +1014,10 @@ class ReflACTTrainer:
             baseline_hard, baseline_soft = compute_score(baseline_results)
             current_score = select_gate_score(
                 baseline_hard, baseline_soft, gate_metric, gate_mixed_weight,
+                skill_content=skill_init,
+                use_semantic_density=use_semantic_density,
+                semantic_density_weight=semantic_density_weight,
+                leading_words=leading_words,
             )
             best_score = current_score
             sh = skill_hash(skill_init)
@@ -1104,6 +1210,13 @@ class ReflACTTrainer:
 
                 # ── No patches? Skip ─────────────────────────────────────
                 if not all_failure_patches and not all_success_patches:
+                    # Skill-aware: a lapse-only step has no body patches but
+                    # may still carry appendix notes — flush them BEFORE
+                    # skipping, or they would be silently dropped.
+                    if use_skill_aware:
+                        current_skill = _flush_skill_aware_appendix(
+                            current_skill, all_raw_patches, step_rec, step_dir, cfg,
+                        )
                     step_rec["action"] = "skip_no_patches"
                     step_rec["current_score"] = current_score
                     step_rec["best_score"] = best_score
@@ -1292,6 +1405,12 @@ class ReflACTTrainer:
                     is_full_rewrite_minibatch_mode(update_mode)
                     and rewrite_result is None
                 ):
+                    # Skill-aware: flush appendix notes before skipping (see
+                    # the skip_no_patches branch above).
+                    if use_skill_aware:
+                        current_skill = _flush_skill_aware_appendix(
+                            current_skill, all_raw_patches, step_rec, step_dir, cfg,
+                        )
                     step_rec["action"] = "skip_no_rewrite"
                     step_rec["current_score"] = current_score
                     step_rec["best_score"] = best_score
@@ -1346,9 +1465,37 @@ class ReflACTTrainer:
                     cand_soft=cand_soft,
                     metric=gate_metric,
                     mixed_weight=gate_mixed_weight,
-                )
+                    use_semantic_density=use_semantic_density,
+                    semantic_density_weight=semantic_density_weight,
+                    leading_words=leading_words,
+                ) if use_gate else None
                 cand_gate_score = select_gate_score(
                     cand_hard, cand_soft, gate_metric, gate_mixed_weight,
+                    skill_content=candidate_skill,
+                    use_semantic_density=use_semantic_density,
+                    semantic_density_weight=semantic_density_weight,
+                    leading_words=leading_words,
+                )
+                if not use_gate:
+                    # Validation ran (scores recorded above) but the gate is
+                    # disabled: force-accept the candidate as the new current
+                    # skill. Best-so-far is still tracked for convenience; the
+                    # final skill is selected manually from the trajectory.
+                    if cand_gate_score > best_score:
+                        fa_best_skill = candidate_skill
+                        fa_best_score = cand_gate_score
+                        fa_best_step = global_step
+                    else:
+                        fa_best_skill = best_skill
+                        fa_best_score = best_score
+                        fa_best_step = best_step
+                    gate = GateResult(
+                        action="force_accept",
+                        current_skill=candidate_skill,
+                        current_score=cand_gate_score,
+                        best_skill=fa_best_skill,
+                        best_score=fa_best_score,
+                        best_step=fa_best_step,
                     )
                 step_rec["gate_metric"] = gate_metric
                 step_rec["candidate_gate_score"] = cand_gate_score
@@ -1360,11 +1507,18 @@ class ReflACTTrainer:
                 best_skill = gate.best_skill
                 best_score = gate.best_score
                 best_step = gate.best_step
-                if gate.action in {"accept", "accept_new_best"}:
+                if gate.action in {"accept", "accept_new_best", "force_accept"}:
                     current_origin = f"step_{global_step:04d}"
-                if gate.action == "accept_new_best":
+                if gate.action == "accept_new_best" or (
+                    gate.action == "force_accept" and best_step == global_step
+                ):
                     best_origin = current_origin
 
+                if use_skill_aware:
+                    current_skill = _flush_skill_aware_appendix(
+                        current_skill, all_raw_patches, step_rec, step_dir, cfg,
+                    )
+
                 if gate_metric == "hard":
                     score_label = f"hard={cand_hard:.4f}"
                 elif gate_metric == "soft":
@@ -1384,6 +1538,11 @@ class ReflACTTrainer:
                         f"    [6/6 EVALUATE] ACCEPT "
                         f"{score_label} > current={prev_current:.4f}"
                     )
+                elif gate.action == "force_accept":
+                    print(
+                        f"    [6/6 EVALUATE] FORCE-ACCEPT (gate disabled) "
+                        f"{score_label}"
+                    )
                 else:
                     print(
                         f"    [6/6 EVALUATE] REJECT "
@@ -1514,13 +1673,13 @@ class ReflACTTrainer:
                         elif action in {
                             "accept", "accept_new_best", "force_accept",
                         }:
-                            # Force-accept mode: re-apply to both current & best.
+                            # Force-accept mode: re-apply guidance to
+                            # current_skill only. best_skill must remain a
+                            # faithful snapshot of the val-best step and must
+                            # NOT receive force-injected slow-update content.
                             current_skill = replace_slow_update_field(
                                 current_skill, slow_saved["slow_update_content"],
                             )
-                            best_skill = replace_slow_update_field(
-                                best_skill, slow_saved["slow_update_content"],
-                            )
                 elif epoch == 1:
                     # Epoch 1: inject empty placeholder
                     os.makedirs(slow_dir, exist_ok=True)
@@ -1528,7 +1687,7 @@ class ReflACTTrainer:
                     current_origin = f"slow_update_placeholder_epoch_{epoch:02d}"
                     _save_skill(out_root, global_step, current_skill)
                     with open(os.path.join(out_root, "best_skill.md"), "w") as f:
-                        f.write(best_skill if best_score > current_score else current_skill)
+                        f.write(best_skill)
                     with open(slow_done_path, "w") as f:
                         json.dump({"action": "inject_placeholder", "epoch": epoch}, f, indent=2)
                     _persist_runtime_state(global_step)
@@ -1707,6 +1866,9 @@ class ReflACTTrainer:
                                 cand_soft=slow_sel_soft,
                                 metric=gate_metric,
                                 mixed_weight=gate_mixed_weight,
+                                use_semantic_density=use_semantic_density,
+                                semantic_density_weight=semantic_density_weight,
+                                leading_words=leading_words,
                             )
                             slow_result["selection_hard"] = slow_sel_hard
                             slow_result["selection_soft"] = slow_sel_soft
@@ -1749,16 +1911,15 @@ class ReflACTTrainer:
                         else:
                             # ── Force-accept mode (default) ──────────────────
                             # The epoch-level longitudinal guidance is injected
-                            # into both current_skill and best_skill
+                            # into current_skill ONLY, so training continues
-                            # unconditionally — it must not be gated by
+                            # with the accumulated slow memory. best_skill is
-                            # step-level selection scores.
+                            # left untouched: it must remain a faithful snapshot
+                            # of the val-best step (which may be a pre-slow step
+                            # such as S_0 carrying no slow_update field at all).
                             slow_content = slow_result["slow_update_content"]
                             current_skill = replace_slow_update_field(
                                 current_skill, slow_content,
                             )
-                            best_skill = replace_slow_update_field(
-                                best_skill, slow_content,
-                            )
                             # Update caches so downstream steps use the
                             # slow-update-injected skill for hashing.
                             slow_candidate_hash = skill_hash(current_skill)
@@ -1769,7 +1930,7 @@ class ReflACTTrainer:
 
                             print(
                                 f"    [slow update] force-injected into "
-                                f"current & best "
+                                f"current only "
                                 f"({len(slow_content)} chars), "
                                 f"{slow_time}s"
                             )
@@ -1922,10 +2083,74 @@ class ReflACTTrainer:
         baseline_test_soft = None
         test_hard = None
         test_soft = None
+        final_test_hard = None
+        final_test_soft = None
+        final_selection_hard = None
+        final_selection_soft = None
 
         if cfg["eval_test"]:
             task_types = adapter.get_task_types()
 
+            # ── Final skill validation (valid_seen) + best promotion ─────
+            # The final (last) skill may carry an epoch-end slow_update that
+            # was force-injected WITHOUT a val pass (use_gate=false or
+            # slow_update_gate_with_selection=false), so it never competed for
+            # best. Run one real val on the final skill; if its gate score
+            # beats the incumbent best, PROMOTE it to best so that best is the
+            # true val-argmax over all skills (including the final slow_update).
+            # When final == best, reuse the existing val score (no rollout).
+            try:
+                if skill_hash(current_skill) == skill_hash(best_skill):
+                    final_selection_hard, final_selection_soft = best_score, None
+                    print(
+                        "\n  [final skill == best skill] "
+                        f"final_selection_hard={best_score:.4f} (reused)"
+                    )
+                else:
+                    fval_env, fval_n = _build_eval_env(
+                        split="valid_seen",
+                        env_num=cfg["sel_env_num"],
+                        seed=seed,
+                    )
+                    fval_dir = os.path.join(out_root, "final_selection_eval")
+                    fval_results = adapter.rollout(fval_env, current_skill, fval_dir)
+                    final_selection_hard, final_selection_soft = compute_score(fval_results)
+                    final_gate_score = select_gate_score(
+                        final_selection_hard, final_selection_soft,
+                        gate_metric, gate_mixed_weight,
+                        skill_content=current_skill,
+                        use_semantic_density=use_semantic_density,
+                        semantic_density_weight=semantic_density_weight,
+                        leading_words=leading_words,
+                    )
+                    print(
+                        f"\n  [final skill val] items={fval_n} "
+                        f"final_selection_hard={final_selection_hard:.4f} "
+                        f"gate={final_gate_score:.4f} "
+                        f"(best={best_score:.4f})"
+                    )
+                    if final_gate_score > best_score:
+                        # Promote: the final (slow-updated) skill is val-better
+                        # than the incumbent best. Make it the new best so the
+                        # subsequent BEST-skill test rollout evaluates it and
+                        # best/final test scores coincide.
+                        print(
+                            f"  [promote] final {final_gate_score:.4f} > "
+                            f"best {best_score:.4f} → final becomes new best "
+                            f"(step {global_step}, origin {current_origin})"
+                        )
+                        best_skill = current_skill
+                        best_score = final_gate_score
+                        best_step = global_step
+                        best_origin = current_origin
+                        with open(os.path.join(out_root, "best_skill.md"), "w") as f:
+                            f.write(best_skill)
+                        _persist_runtime_state(global_step)
+            except Exception as _e:  # noqa: BLE001
+                final_selection_hard = None
+                final_selection_soft = None
+                print(f"\n  [final skill val FAILED: {_e!r}]")
+
             # Baseline: S_0 on test set (valid_unseen)
             print(f"\n{'='*60}")
             print("  BASELINE TEST — evaluate initial skill on Test set (valid_unseen)")
@@ -1937,6 +2162,7 @@ class ReflACTTrainer:
             )
             print(f"  Test items: {test_n}")
             baseline_test_dir = os.path.join(out_root, "test_eval_baseline")
+            os.makedirs(baseline_test_dir, exist_ok=True)
             baseline_test_results = adapter.rollout(test_env, skill_init, baseline_test_dir)
             baseline_test_hard, baseline_test_soft = compute_score(baseline_test_results)
             baseline_buckets = _compute_task_type_buckets(baseline_test_results, task_types)
@@ -1971,6 +2197,7 @@ class ReflACTTrainer:
             )
             print(f"  Test items: {test_n2}")
             test_dir = os.path.join(out_root, "test_eval")
+            os.makedirs(test_dir, exist_ok=True)
             test_results = adapter.rollout(test_env2, best_skill, test_dir)
             test_hard, test_soft = compute_score(test_results)
             best_buckets = _compute_task_type_buckets(test_results, task_types)
@@ -1994,13 +2221,88 @@ class ReflACTTrainer:
                     f, indent=2, ensure_ascii=False,
                 )
 
+            # Final skill (last skill in trajectory) on test set.
+            # Distinct from best_skill: with use_gate=False every candidate is
+            # force-accepted so the final skill is whatever the last step
+            # produced; with use_gate=True it is the last accepted skill, which
+            # may differ from the best-on-val skill. We always evaluate it so
+            # every run reports baseline / best-on-val / final on test.
+            # Guarded so a failure here never prevents summary.json from being
+            # written (the orchestrator's post-hoc safety net fills it in).
+            try:
+                if skill_hash(current_skill) == skill_hash(best_skill):
+                    # Final == best: reuse results, skip a redundant rollout.
+                    final_test_hard, final_test_soft = test_hard, test_soft
+                    final_test_dir = os.path.join(out_root, "test_eval_final")
+                    os.makedirs(final_test_dir, exist_ok=True)
+                    with open(os.path.join(final_test_dir, "summary.json"), "w") as f:
+                        json.dump(
+                            {
+                                k: {
+                                    "total": b["total"],
+                                    "hard_acc": b["hard"] / max(b["total"], 1),
+                                }
+                                for k, b in best_buckets.items()
+                            },
+                            f, indent=2, ensure_ascii=False,
+                        )
+                    print(
+                        "\n  [final skill == best skill] "
+                        f"final_test_hard={final_test_hard:.4f} (reused)"
+                    )
+                else:
+                    print(f"\n{'='*60}")
+                    print("  FINAL SKILL TEST — evaluate last skill on Test set (valid_unseen)")
+                    print(f"{'='*60}")
+                    test_env3, test_n3 = _build_eval_env(
+                        split="valid_unseen",
+                        env_num=cfg["test_env_num"],
+                        seed=seed,
+                    )
+                    print(f"  Test items: {test_n3}")
+                    final_test_dir = os.path.join(out_root, "test_eval_final")
+                    os.makedirs(final_test_dir, exist_ok=True)
+                    final_test_results = adapter.rollout(test_env3, current_skill, final_test_dir)
+                    final_test_hard, final_test_soft = compute_score(final_test_results)
+                    final_buckets = _compute_task_type_buckets(final_test_results, task_types)
+                    print("\n  === Final Skill Test Results ===")
+                    for task_type in task_types + ["overall"]:
+                        b = final_buckets.get(task_type, {"total": 0, "hard": 0})
+                        t = max(b["total"], 1)
+                        print(
+                            f"    {task_type:<40s}: "
+                            f"hard={b['hard']}/{b['total']}={b['hard']/t:.4f}"
+                        )
+                    with open(os.path.join(final_test_dir, "summary.json"), "w") as f:
+                        json.dump(
+                            {
+                                k: {
+                                    "total": b["total"],
+                                    "hard_acc": b["hard"] / max(b["total"], 1),
+                                }
+                                for k, b in final_buckets.items()
+                            },
+                            f, indent=2, ensure_ascii=False,
+                        )
+            except Exception as _e:  # noqa: BLE001
+                final_test_hard = None
+                final_test_soft = None
+                print(f"\n  [final skill test FAILED: {_e!r}] "
+                      "— will be filled by post-hoc eval")
+
             # Comparison
             delta_hard = (test_hard or 0) - (baseline_test_hard or 0)
-            print(f"\n  === Improvement (best vs baseline) ===")
+            print(f"\n  === Improvement vs baseline (init S_0) ===")
             print(
-                f"    hard: {baseline_test_hard:.4f} -> {test_hard:.4f}  "
+                f"    [2] best-on-val hard: {baseline_test_hard:.4f} -> {test_hard:.4f}  "
                 f"(delta={delta_hard:+.4f})"
             )
+            if final_test_hard is not None:
+                final_delta_hard = (final_test_hard or 0) - (baseline_test_hard or 0)
+                print(
+                    f"    [3] final/last  hard: {baseline_test_hard:.4f} -> {final_test_hard:.4f}  "
+                    f"(delta={final_delta_hard:+.4f})"
+                )
 
         # ── Global summary ───────────────────────────────────────────────
         total_wall = time.time() - t_loop_start
@@ -2032,6 +2334,8 @@ class ReflACTTrainer:
                 skill_hash(skill_init), (None, None),
             )[0],
             "best_selection_hard": best_score,
+            "final_selection_hard": final_selection_hard,
+            "final_selection_soft": final_selection_soft,
             "best_step": best_step,
             "current_origin": current_origin,
             "best_origin": best_origin,
@@ -2044,11 +2348,18 @@ class ReflACTTrainer:
             "baseline_test_soft": baseline_test_soft,
             "test_hard": test_hard,
             "test_soft": test_soft,
+            "final_test_hard": final_test_hard,
+            "final_test_soft": final_test_soft,
             "test_delta_hard": (
                 (test_hard or 0) - (baseline_test_hard or 0)
                 if test_hard is not None
                 else None
             ),
+            "final_test_delta_hard": (
+                (final_test_hard or 0) - (baseline_test_hard or 0)
+                if final_test_hard is not None
+                else None
+            ),
             "total_wall_time_s": round(total_wall, 1),
             "token_summary": token_summary,
         }
@@ -2069,8 +2380,22 @@ class ReflACTTrainer:
                     f"    epoch {es['epoch']}: accept={es['accepts']} reject={es['rejects']} "
                     f"best={es['best_score_at_epoch_end']:.4f}"
                 )
+        if baseline_test_hard is not None:
+            print("\n  === TEST scores (3 skills, split=valid_unseen) ===")
+            print(
+                f"    [1] init/baseline (S_0)          : "
+                f"test_hard={baseline_test_hard:.4f}"
+            )
         if test_hard is not None:
-            print(f"  test_hard={test_hard:.4f} test_soft={test_soft:.4f}")
+            print(
+                f"    [2] best-on-val (step {best_step})".ljust(37)
+                + f": test_hard={test_hard:.4f} test_soft={test_soft:.4f}"
+            )
+        if final_test_hard is not None:
+            print(
+                f"    [3] final/last skill             : "
+                f"test_hard={final_test_hard:.4f} test_soft={final_test_soft:.4f}"
+            )
         if token_summary.get("_total"):
             t = token_summary["_total"]
             print(

skillopt/envs/_template/README.md

@@ -5,8 +5,8 @@ This directory provides scaffold files for adding a new benchmark to SkillOpt.
 ## Files
 
 - `env_template.py` — Environment adapter template (subclasses
-  `EnvAdapter`; implements the 5 abstract methods so the file is
+  `EnvAdapter`; implements the 4 abstract methods so the file is
-  instantiable out of the box).
+  instantiable out of the box — `reflect` is inherited).
 - `loader_template.py` — Data loader template (subclasses
   `SplitDataLoader`; implements `load_split_items` for `.json`/`.jsonl`).
 - `config_template.yaml` — Config file template.
@@ -21,20 +21,23 @@ This directory provides scaffold files for adding a new benchmark to SkillOpt.
    ```bash
    cd skillopt/envs/your_benchmark
    mv env_template.py    adapter.py
-   mv loader_template.py loader.py
+   mv loader_template.py dataloader.py
    ```
    …and inside each file rename the classes
    (`TemplateBenchmarkEnv → YourBenchmarkAdapter`,
    `TemplateBenchmarkLoader → YourBenchmarkLoader`)
    and fix the cross-import in `adapter.py`.
 3. **Implement the TODO blocks** inside `adapter.py:rollout` and the
-   `_normalize_item` helper in `loader.py`. If you want real reflection,
+   `_normalize_item` helper in `dataloader.py`. In addition to returning
-   uncomment the `run_minibatch_reflect` block in `adapter.py:reflect`.
+   `id`/`hard`/`soft`, persist each non-empty trajectory at
-4. **Register** the adapter — add a `try / except ImportError` block in
+   `<out_dir>/predictions/<id>/conversation.json`; the inherited `reflect`
-   `scripts/train.py`'s `_register_builtins()` mapping the registry key
+   method reads those files. Override `reflect` only for custom reflection
-   to your `YourBenchmarkAdapter` class. There is no
+   logic.
-   `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`; the live
+4. **Register** the adapter — add matching `try / except ImportError` blocks
-   registry is `_ENV_REGISTRY` in `scripts/train.py`.
+   to `_register_builtins()` in both `scripts/train.py` and
+   `scripts/eval_only.py`, mapping the registry key to your
+   `YourBenchmarkAdapter` class. There is no `BENCHMARK_REGISTRY` dict in
+   `skillopt/envs/__init__.py`; each CLI keeps its own lazy `_ENV_REGISTRY`.
 5. **Create the config** at `configs/your_benchmark/default.yaml`
    (start from `config_template.yaml`). `_base_` is a **string path**,
    not a list.

skillopt/envs/_template/config_template.yaml

@@ -50,6 +50,6 @@ evaluation:
 # ── Model ────────────────────────────────────────
 # Override only what differs from the inherited defaults.
 model:
-  optimizer_backend: openai_chat  # openai_chat | claude_chat | qwen_chat | minimax_chat
+  optimizer_backend: openai_chat  # openai_chat | claude_chat | qwen_chat | minimax_chat | codex_exec
-  target_backend: openai_chat     # … plus codex_exec / claude_code_exec for target only
+  target_backend: openai_chat     # chat backends plus codex_exec / claude_code_exec
   reasoning_effort: medium

skillopt/envs/_template/env_template.py

@@ -14,13 +14,9 @@ For a fully worked example see ``skillopt/envs/officeqa/``.
 """
 from __future__ import annotations
 
-import os
-
 from skillopt.datasets.base import BatchSpec
 from skillopt.envs.base import EnvAdapter
 from skillopt.envs._template.loader_template import TemplateBenchmarkLoader
-# When you wire in real reflection, also import:
-# from skillopt.gradient.reflect import run_minibatch_reflect
 
 
 class TemplateBenchmarkEnv(EnvAdapter):
@@ -131,53 +127,12 @@ class TemplateBenchmarkEnv(EnvAdapter):
             )
         return results
 
-    # ── Reflect: turn rollout results into patch dicts ─────────────────
+    # ── Reflect (inherited) ─────────────────────────────────────────────
-
+    #
-    def reflect(
+    # ``reflect`` is inherited from ``EnvAdapter``: the default delegates to
-        self,
+    # ``skillopt.gradient.reflect.run_minibatch_reflect`` using your
-        results: list[dict],
+    # ``analyst_error_*`` / ``analyst_success_*`` prompts. You do NOT need to
-        skill_content: str,
+    # implement it — override only if your benchmark needs custom reflection.
-        out_dir: str,
-        **kwargs,
-    ) -> list[dict | None]:
-        """
-        Turn rollouts into a list of raw patch dicts (or None to drop).
-
-        Each non-None dict MUST have:
-          - "patch":       {"edits": [...]}     a Patch.to_dict() payload
-          - "source_type": "failure" | "success"
-
-        Most benchmarks delegate to
-        :func:`skillopt.gradient.reflect.run_minibatch_reflect` which
-        will call the optimizer model with the
-        ``analyst_error_*`` / ``analyst_success_*`` prompts. To enable it,
-        uncomment the import above and call:
-
-            from skillopt.gradient.reflect import run_minibatch_reflect
-            return run_minibatch_reflect(
-                results=results,
-                skill_content=skill_content,
-                prediction_dir=kwargs.get(
-                    "prediction_dir", os.path.join(out_dir, "predictions")
-                ),
-                patches_dir=kwargs.get(
-                    "patches_dir", os.path.join(out_dir, "patches")
-                ),
-                workers=self.analyst_workers,
-                failure_only=self.failure_only,
-                minibatch_size=self.minibatch_size,
-                edit_budget=self.edit_budget,
-                random_seed=kwargs.get("random_seed"),
-                error_system=self.get_error_minibatch_prompt(),
-                success_system=self.get_success_minibatch_prompt(),
-                step_buffer_context=kwargs.get("step_buffer_context", ""),
-                update_mode=getattr(self, "_cfg", {}).get(
-                    "skill_update_mode", "patch"
-                ),
-            )
-        """
-        # Template default: produce no patches (no-op trainer step).
-        return [None for _ in results]
 
     # ── Stratification hint ────────────────────────────────────────────
 

skillopt/envs/alfworld/adapter.py

@@ -17,7 +17,6 @@ from skillopt.envs.alfworld.rollout import (
     run_alfworld_batch,
     TASKS,
 )
-from skillopt.gradient.reflect import run_minibatch_reflect
 from skillopt.utils import compute_score
 
 
@@ -425,35 +424,5 @@ class ALFWorldAdapter(EnvAdapter):
             all_results.extend(chunk_results)
         return all_results
 
-    def reflect(
-        self,
-        results: list[dict],
-        skill_content: str,
-        out_dir: str,
-        **kwargs,
-    ) -> list[dict | None]:
-        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
-        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
-        random_seed = kwargs.get("random_seed")
-        step_buffer_context = kwargs.get("step_buffer_context", "")
-        meta_skill_context = kwargs.get("meta_skill_context", "")
-
-        return run_minibatch_reflect(
-            results=results,
-            skill_content=skill_content,
-            prediction_dir=prediction_dir,
-            patches_dir=patches_dir,
-            workers=self.analyst_workers,
-            failure_only=self.failure_only,
-            minibatch_size=self.minibatch_size,
-            edit_budget=self.edit_budget,
-            random_seed=random_seed,
-            error_system=self.get_error_minibatch_prompt(),
-            success_system=self.get_success_minibatch_prompt(),
-            step_buffer_context=step_buffer_context,
-            meta_skill_context=meta_skill_context,
-        )
-
-
     def get_task_types(self) -> list[str]:
         return list(TASKS)

skillopt/envs/alfworld/rollout.py

@@ -7,12 +7,10 @@ Provides:
 """
 from __future__ import annotations
 
+import concurrent.futures
 import json
 import os
 import re
-import sys
-import concurrent.futures
-import numpy as np
 
 from skillopt.model import chat_target
 
@@ -65,6 +63,25 @@ def _append_diagnostic_instruction(prompt: str, diagnostic_instruction: str) ->
     return f"{prompt}\n\n## Training Readout\n{diagnostic_instruction.strip()}\n"
 
 
+def _resolve_alfworld_gamefile(gamefile: str) -> str:
+    path = os.path.expanduser(os.path.expandvars(str(gamefile)))
+    if os.path.isabs(path):
+        return path
+
+    data_root = os.environ.get("ALFWORLD_DATA", "").strip()
+    if not data_root:
+        return path
+
+    root = os.path.expanduser(os.path.expandvars(data_root))
+    return os.path.abspath(os.path.join(root, path))
+
+
+def _resolve_alfworld_gamefiles(gamefiles: list[str] | None) -> list[str] | None:
+    if gamefiles is None:
+        return None
+    return [_resolve_alfworld_gamefile(gamefile) for gamefile in gamefiles]
+
+
 # ── Environment builder ──────────────────────────────────────────────────────
 
 
@@ -86,9 +103,10 @@ def build_alfworld_env(
     Returns:
         env_manager: AlfWorldEnvironmentManager instance
     """
-    from omegaconf import OmegaConf
     from functools import partial
 
+    from omegaconf import OmegaConf
+
     from skillopt.envs.alfworld.vendor.alfworld_envs import build_alfworld_envs
     from skillopt.envs.alfworld.vendor.alfworld_projection import alfworld_projection
     from skillopt.envs.alfworld.vendor.env_manager import AlfWorldEnvironmentManager
@@ -97,6 +115,7 @@ def build_alfworld_env(
 
     alf_config_path = os.path.join(HERE, "vendor", "config_tw.yaml")
     env_kwargs = {"eval_dataset": eval_dataset}
+    resolved_gamefiles = _resolve_alfworld_gamefiles(specific_gamefiles)
 
     envs = build_alfworld_envs(
         alf_config_path,
@@ -106,7 +125,7 @@ def build_alfworld_env(
         is_train=is_train,
         env_kwargs=env_kwargs,
         resources_per_worker=None,
-        gamefiles=specific_gamefiles,
+        gamefiles=resolved_gamefiles,
     )
 
     config = OmegaConf.create(
@@ -222,7 +241,7 @@ def run_alfworld_batch(
                 if _extract_action(response) is None:
                     return idx, "<think>missing action tag</think><action>look</action>"
                 return idx, response
-            except Exception as e:
+            except Exception:
                 return idx, "<think>error</think><action>look</action>"
 
         executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_api_workers)

skillopt/envs/base.py

@@ -231,7 +231,6 @@ class EnvAdapter(ABC):
             (float 0-1). May include env-specific fields.
         """
 
-    @abstractmethod
     def reflect(
         self,
         results: list[dict],
@@ -241,15 +240,36 @@ class EnvAdapter(ABC):
     ) -> list[dict | None]:
         """Analyze rollout results and produce patches.
 
+        Default implementation: delegate to the shared minibatch reflect
+        stage. Every built-in benchmark uses this unchanged — override only
+        if your environment needs custom reflection logic.
+
         Each returned dict conforms to :class:`~skillopt.types.RawPatch`:
         ``"patch"`` (with ``"edits"`` list) + ``"source_type"``
-        (``"failure"`` or ``"success"``).
+        (``"failure"`` or ``"success"``); ``None`` entries are filtered out.
-
-        Returns
-        -------
-        list[dict | None]
-            Raw analyst outputs; ``None`` entries are filtered out.
         """
+        from skillopt.gradient.reflect import run_minibatch_reflect
+
+        return run_minibatch_reflect(
+            results=results,
+            skill_content=skill_content,
+            prediction_dir=kwargs.get(
+                "prediction_dir", os.path.join(out_dir, "predictions")
+            ),
+            patches_dir=kwargs.get(
+                "patches_dir", os.path.join(out_dir, "patches")
+            ),
+            workers=self.analyst_workers,
+            failure_only=self.failure_only,
+            minibatch_size=self.minibatch_size,
+            edit_budget=self.edit_budget,
+            random_seed=kwargs.get("random_seed"),
+            error_system=self.get_error_minibatch_prompt(),
+            success_system=self.get_success_minibatch_prompt(),
+            step_buffer_context=kwargs.get("step_buffer_context", ""),
+            meta_skill_context=kwargs.get("meta_skill_context", ""),
+            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
+        )
 
     @abstractmethod
     def get_task_types(self) -> list[str]:

skillopt/envs/docvqa/adapter.py

@@ -1,12 +1,9 @@
 from __future__ import annotations
 
-import os
-
 from skillopt.datasets.base import BatchSpec
 from skillopt.envs.base import EnvAdapter
 from skillopt.envs.docvqa.dataloader import DocVQADataLoader
 from skillopt.envs.docvqa.rollout import run_batch
-from skillopt.gradient.reflect import run_minibatch_reflect
 
 
 class DocVQAAdapter(EnvAdapter):
@@ -84,28 +81,6 @@ class DocVQAAdapter(EnvAdapter):
             task_timeout=self.exec_timeout,
         )
 
-    def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
-        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
-        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
-        random_seed = kwargs.get("random_seed")
-        step_buffer_context = kwargs.get("step_buffer_context", "")
-        return run_minibatch_reflect(
-            results=results,
-            skill_content=skill_content,
-            prediction_dir=prediction_dir,
-            patches_dir=patches_dir,
-            workers=self.analyst_workers,
-            failure_only=self.failure_only,
-            minibatch_size=self.minibatch_size,
-            edit_budget=self.edit_budget,
-            random_seed=random_seed,
-            error_system=self.get_error_minibatch_prompt(),
-            success_system=self.get_success_minibatch_prompt(),
-            step_buffer_context=step_buffer_context,
-            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
-        )
-
-
     def get_task_types(self) -> list[str]:
         seen: list[str] = []
         for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:

skillopt/envs/livemathematicianbench/adapter.py

@@ -2,10 +2,8 @@
 from __future__ import annotations
 
 import json
-import os
 
 from skillopt.datasets.base import BatchSpec
-from skillopt.gradient.reflect import run_minibatch_reflect
 from skillopt.envs.base import EnvAdapter
 from skillopt.envs.livemathematicianbench.dataloader import LiveMathematicianBenchDataLoader
 from skillopt.envs.livemathematicianbench.rollout import run_batch
@@ -127,36 +125,5 @@ class LiveMathematicianBenchAdapter(EnvAdapter):
             task_timeout=self.exec_timeout,
         )
 
-    def reflect(
-        self,
-        results: list[dict],
-        skill_content: str,
-        out_dir: str,
-        **kwargs,
-    ) -> list[dict | None]:
-        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
-        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
-        random_seed = kwargs.get("random_seed")
-        step_buffer_context = kwargs.get("step_buffer_context", "")
-        meta_skill_context = kwargs.get("meta_skill_context", "")
-
-        return run_minibatch_reflect(
-            results=results,
-            skill_content=skill_content,
-            prediction_dir=prediction_dir,
-            patches_dir=patches_dir,
-            workers=self.analyst_workers,
-            failure_only=self.failure_only,
-            minibatch_size=self.minibatch_size,
-            edit_budget=self.edit_budget,
-            random_seed=random_seed,
-            error_system=self.get_error_minibatch_prompt(),
-            success_system=self.get_success_minibatch_prompt(),
-            step_buffer_context=step_buffer_context,
-            meta_skill_context=meta_skill_context,
-            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
-        )
-
-
     def get_task_types(self) -> list[str]:
         return self.dataloader.get_task_types()

skillopt/envs/officeqa/adapter.py

@@ -6,7 +6,6 @@ from skillopt.datasets.base import BatchSpec
 from skillopt.envs.base import EnvAdapter
 from skillopt.envs.officeqa.dataloader import OfficeQADataLoader
 from skillopt.envs.officeqa.rollout import run_batch
-from skillopt.gradient.reflect import run_minibatch_reflect
 
 
 class OfficeQAAdapter(EnvAdapter):
@@ -104,28 +103,6 @@ class OfficeQAAdapter(EnvAdapter):
             diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
         )
 
-    def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
-        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
-        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
-        random_seed = kwargs.get("random_seed")
-        step_buffer_context = kwargs.get("step_buffer_context", "")
-        return run_minibatch_reflect(
-            results=results,
-            skill_content=skill_content,
-            prediction_dir=prediction_dir,
-            patches_dir=patches_dir,
-            workers=self.analyst_workers,
-            failure_only=self.failure_only,
-            minibatch_size=self.minibatch_size,
-            edit_budget=self.edit_budget,
-            random_seed=random_seed,
-            error_system=self.get_error_minibatch_prompt(),
-            success_system=self.get_success_minibatch_prompt(),
-            step_buffer_context=step_buffer_context,
-            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
-        )
-
-
     def get_task_types(self) -> list[str]:
         seen: list[str] = []
         for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:

skillopt/envs/searchqa/adapter.py

@@ -2,13 +2,11 @@
 from __future__ import annotations
 
 import json
-import os
 
 from skillopt.datasets.base import BatchSpec
 from skillopt.envs.base import EnvAdapter
 from skillopt.envs.searchqa.dataloader import SearchQADataLoader
 from skillopt.envs.searchqa.rollout import run_batch
-from skillopt.gradient.reflect import run_minibatch_reflect
 from skillopt.model import get_target_backend
 
 
@@ -94,36 +92,5 @@ class SearchQAAdapter(EnvAdapter):
             task_timeout=self.exec_timeout,
         )
 
-    def reflect(
-        self,
-        results: list[dict],
-        skill_content: str,
-        out_dir: str,
-        **kwargs,
-    ) -> list[dict | None]:
-        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
-        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
-        random_seed = kwargs.get("random_seed")
-        step_buffer_context = kwargs.get("step_buffer_context", "")
-        meta_skill_context = kwargs.get("meta_skill_context", "")
-
-        return run_minibatch_reflect(
-            results=results,
-            skill_content=skill_content,
-            prediction_dir=prediction_dir,
-            patches_dir=patches_dir,
-            workers=self.analyst_workers,
-            failure_only=self.failure_only,
-            minibatch_size=self.minibatch_size,
-            edit_budget=self.edit_budget,
-            random_seed=random_seed,
-            error_system=self.get_error_minibatch_prompt(),
-            success_system=self.get_success_minibatch_prompt(),
-            step_buffer_context=step_buffer_context,
-            meta_skill_context=meta_skill_context,
-            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
-        )
-
-
     def get_task_types(self) -> list[str]:
         return ["qa"]

skillopt/envs/searchqa/rollout.py

@@ -13,20 +13,31 @@ from __future__ import annotations
 import json
 import os
 import time
-import traceback
+from collections import Counter
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 
-from skillopt.model import chat_target, get_target_backend, is_target_exec_backend
+from skillopt.envs.searchqa.evaluator import evaluate
+from skillopt.model import chat_target, is_target_exec_backend
 from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec
 from skillopt.prompts import load_prompt
-from skillopt.envs.searchqa.evaluator import evaluate
-
 
 # ── Prompt templates ─────────────────────────────────────────────────────────
 
 _MAX_CONTEXT_CHARS = 6000
 
 
+def _raise_on_systemic_failure(results: list[dict]) -> None:
+    """Abort when all rollout rows failed before any agent response."""
+    if not results or not all(row.get("agent_ok") is False for row in results):
+        return
+    reasons = Counter(str(row.get("fail_reason") or "unknown error") for row in results)
+    common_reason, count = reasons.most_common(1)[0]
+    raise RuntimeError(
+        f"SearchQA rollout failed for all {len(results)} items before an agent "
+        f"response ({count}x): {common_reason}"
+    )
+
+
 def _truncate_context(context: str, max_chars: int = _MAX_CONTEXT_CHARS) -> str:
     """Truncate context at [DOC] boundaries to stay within budget."""
     if len(context) <= max_chars:
@@ -379,6 +390,7 @@ def run_batch(
 
     pending = [it for it in items if str(it["id"]) not in done_ids]
     if not pending:
+        _raise_on_systemic_failure(existing)
         return existing
 
     total = len(existing) + len(pending)
@@ -478,4 +490,5 @@ def run_batch(
         finally:
             ex.shutdown(wait=False, cancel_futures=True)
 
+    _raise_on_systemic_failure(results)
     return results

skillopt/envs/spreadsheetbench/adapter.py

@@ -16,7 +16,6 @@ from skillopt.envs.spreadsheetbench.rollout import (
     run_spreadsheet_batch,
     run_spreadsheet_batch_codegen,
 )
-from skillopt.gradient.reflect import run_minibatch_reflect
 from skillopt.model import get_target_backend, is_target_exec_backend
 
 
@@ -156,37 +155,5 @@ class SpreadsheetBenchAdapter(EnvAdapter):
 
         return results
 
-    def reflect(
-        self,
-        results: list[dict],
-        skill_content: str,
-        out_dir: str,
-        **kwargs,
-    ) -> list[dict | None]:
-        """Analyze rollout results and produce patches (minibatch mode)."""
-        prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
-        patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
-        random_seed = kwargs.get("random_seed")
-        step_buffer_context = kwargs.get("step_buffer_context", "")
-        meta_skill_context = kwargs.get("meta_skill_context", "")
-
-        return run_minibatch_reflect(
-            results=results,
-            skill_content=skill_content,
-            prediction_dir=prediction_dir,
-            patches_dir=patches_dir,
-            workers=self.analyst_workers,
-            failure_only=self.failure_only,
-            minibatch_size=self.minibatch_size,
-            edit_budget=self.edit_budget,
-            random_seed=random_seed,
-            error_system=self.get_error_minibatch_prompt(),
-            success_system=self.get_success_minibatch_prompt(),
-            step_buffer_context=step_buffer_context,
-            meta_skill_context=meta_skill_context,
-            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
-        )
-
-
     def get_task_types(self) -> list[str]:
         return list(TASK_TYPES)

skillopt/envs/spreadsheetbench/codegen_agent.py

@@ -54,8 +54,8 @@ def _build_eval_feedback(verify_report: str) -> str:
     output and whether each cell is correct or wrong.
     """
     import re
-    lines = ["Your code executed successfully but produced incorrect results.",
+    wrong_lines = []
-             "The following cells have wrong values:"]
+    n_correct = 0
     for raw_line in verify_report.splitlines():
         raw_line = raw_line.strip()
         if not raw_line:
@@ -68,9 +68,14 @@ def _build_eval_feedback(verify_report: str) -> str:
         if m:
             cell, got_val, mark = m.groups()
             if mark == "✗":
-                lines.append(f"  {cell}: your output = {got_val} (WRONG)")
+                wrong_lines.append(f"  {cell}: your output = {got_val} (WRONG)")
             else:
-                lines.append(f"  {cell}: correct ✓")
+                n_correct += 1
+    lines = ["Your code executed successfully but produced incorrect results.",
+             "The following cells have wrong values:"]
+    lines.extend(wrong_lines)
+    if n_correct:
+        lines.append(f"  ({n_correct} other cells are correct.)")
     lines.append(
         "\nPlease analyze the spreadsheet data more carefully and fix the code. "
         "Return a complete corrected Python script inside a ```python``` block."

skillopt/envs/spreadsheetbench/rollout.py

@@ -26,7 +26,9 @@ from concurrent.futures import (
 import openpyxl
 
 from skillopt.envs.spreadsheetbench.react_agent import run_react
-from skillopt.envs.spreadsheetbench.evaluator import evaluate, _generate_cell_names
+from skillopt.envs.spreadsheetbench.evaluator import (
+    evaluate, _generate_cell_names, _compare_cell_value,
+)
 from skillopt.envs.spreadsheetbench.executor import run_generated_code
 
 
@@ -87,6 +89,21 @@ def _find_test_cases(task_dir: str) -> list[tuple[str, str, str]]:
 
 # ── Auto-verify helper ──────────────────────────────────────────────────────
 
+# The official SpreadsheetBench evaluator never serialises cells to text — it
+# compares in memory and returns only a pass/fail bool. The per-cell report
+# below is a repo-local training aid (fed back to the model on retry and saved
+# into the trajectory for reflection). On most tasks the answer range is a
+# handful of cells, so the full report is tiny. But a few tasks have answer
+# ranges spanning tens of thousands of cells (e.g. 80-42 =
+# 'Consolidate_ALL'!A2:L8000 ≈ 96k cells); dumping every cell explodes the
+# report to several MB, floods the model's context and bloats conversation
+# files. We therefore apply the same head+tail character truncation the rest of
+# the codebase uses for oversized trajectory text (cf. reflect.py / slow_update.py
+# `text[:half] + "...[truncated]...\n" + text[-half:]`): keep the first and last
+# `_MAX_REPORT_CHARS // 2` chars so both the leading and trailing wrong cells
+# stay visible. Small reports are unchanged.
+_MAX_REPORT_CHARS = 12000      # head+tail char budget (~6000 head + 6000 tail)
+
 
 def _auto_verify_output(
     pred_path: str,
@@ -97,7 +114,8 @@ def _auto_verify_output(
 
     Returns a human-readable verification report that can be appended to the
     trajectory so the error analyst can see exactly what went wrong (e.g.
-    ``cell A1: got=None, expected=420``).
+    ``cell A1: got=None, expected=420``). Oversized reports are head+tail
+    truncated to `_MAX_REPORT_CHARS` chars, matching the rest of the codebase.
     """
     if not os.path.exists(pred_path):
         return "Verification: output file does not exist."
@@ -129,11 +147,28 @@ def _auto_verify_output(
                 lines.append(f"  Sheet '{sheet_name}' NOT FOUND in output.")
                 continue
 
+            n_empty_correct = 0   # empty-on-both correct cells collapsed to a count
             for cn in cell_names:
                 gv = ws_gold[cn].value if ws_gold else "N/A"
                 pv = ws_pred[cn].value
-                match = "✓" if repr(gv) == repr(pv) else "✗"
+                # Use the official cell comparator so this report's ✓/✗ agrees
+                # with the real scorer (evaluate). repr() equality would wrongly
+                # flag e.g. 5 vs 5.0 or None vs "" as mismatches and mislead the
+                # model into "fixing" cells that already pass scoring.
+                ok_cell = ws_gold is not None and _compare_cell_value(gv, pv)
+                # Collapse only cells that are correct AND empty on both sides
+                # (got=None, expected=None ✓): pure noise. Every other cell —
+                # including non-empty correct cells — is listed in full; the
+                # final head+tail char cap keeps the report bounded.
+                if ok_cell and gv in (None, "") and pv in (None, ""):
+                    n_empty_correct += 1
+                    continue
+                match = "✓" if ok_cell else "✗"
                 lines.append(f"  {sheet_name}!{cn}: got={pv!r}, expected={gv!r} {match}")
+            if n_empty_correct:
+                lines.append(
+                    f"  (+{n_empty_correct} empty cells correct, omitted)"
+                )
 
         # Also check if any cells in the output contain formula strings
         formula_cells = []
@@ -159,7 +194,17 @@ def _auto_verify_output(
         wb_pred.close()
         wb_gold.close()
 
-    return "\n".join(lines)
+    report = "\n".join(lines)
+    # Head+tail truncation, matching reflect.py / slow_update.py: keep the first
+    # and last half so both leading and trailing wrong cells remain visible.
+    if len(report) > _MAX_REPORT_CHARS:
+        half = _MAX_REPORT_CHARS // 2
+        report = (
+            report[:half]
+            + f"\n  ...[verification report truncated, {len(report)} chars total]...\n"
+            + report[-half:]
+        )
+    return report
 
 
 # ── Per-task worker ──────────────────────────────────────────────────────────