Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Shunsuke
|
98d0430bee | ||
|
Yifan Yang
|
eef4805b25 | ||
|
Yifan Yang
|
86bad36ffe | ||
|
Shunsuke
|
54e4b3eafb |
@@ -9,7 +9,7 @@
|
||||
---
|
||||
|
||||
## News 🔥🔥🔥
|
||||
- **[2026-06-08]** 😴 **SkillOpt-Sleep is here — plugins for Claude Code, Codex, and Copilot.** Give your local coding agent a nightly *sleep cycle*: it reviews your past sessions offline, replays your recurring tasks, and consolidates validated long-term memory + skills behind a held-out gate, so it gets better the more you use it. Validated on the public [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` benchmark with **real Claude and Codex** (deficient skills 0.00 → 1.00 on held-out, all 4 seeds). It's an **open-source tool decoupled from the paper code**. See [`plugins/`](plugins/) and the [SkillOpt-Sleep section](#-skillopt-sleep--the-deployment-time-companion) below.
|
||||
- **[2026-06-14]** 😴 **SkillOpt-Sleep (preview).** A nightly *sleep cycle* for local coding agents (Claude Code / Codex / Copilot): review past sessions offline, replay recurring tasks, and consolidate validated skills behind a held-out gate. This is an early **preview** — open-source and decoupled from the paper code — that we'll keep iterating on. See [`plugins/`](plugins/) and the [section below](#-skillopt-sleep--the-deployment-time-companion).
|
||||
- **[2026-06-03]** 🎉 **[gbrain](https://github.com/garrytan/gbrain), [gbrain-evals](https://github.com/garrytan/gbrain-evals/blob/main/docs/benchmarks/2026-06-03-skillopt.md), and [darwin-skill](https://github.com/alchaincyf/darwin-skill) have all integrated SkillOpt.**
|
||||
- **[2026-06-02]** 🎉 **SkillOpt [v0.1.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.1.0) is now available on [PyPI](https://pypi.org/project/skillopt/)!** Install with `pip install skillopt`. This initial release includes the full training loop (rollout → reflect → aggregate → select → update → evaluate), multi-backend support (OpenAI / Azure / Claude / Qwen / MiniMax), six built-in benchmarks, and WebUI dashboard.
|
||||
|
||||
@@ -55,6 +55,9 @@ https://github.com/user-attachments/assets/eb12d3bc-371c-467f-904d-91b61f339ed7
|
||||
|
||||
## 😴 SkillOpt-Sleep — the deployment-time companion
|
||||
|
||||
> **Preview.** SkillOpt-Sleep is an early preview that we are actively iterating
|
||||
> on; interfaces and defaults may change. Feedback and issues are welcome.
|
||||
|
||||
SkillOpt (above) trains a skill offline on a benchmark. **SkillOpt-Sleep**
|
||||
applies the same discipline to *your own daily usage*: it gives a local coding
|
||||
agent a nightly **sleep cycle** that reviews your past sessions, replays your
|
||||
@@ -76,8 +79,8 @@ harvest session transcripts → mine recurring tasks → replay offline
|
||||
|
||||
| Platform | Folder | Install |
|
||||
|---|---|---|
|
||||
| **Claude Code** | [`plugins/claude-code`](plugins/claude-code) | `/plugin marketplace add ./plugins/claude-code` → `/sleep` |
|
||||
| **Codex** | [`plugins/codex`](plugins/codex) | `bash plugins/codex/install.sh` → `/sleep` |
|
||||
| **Claude Code** | [`plugins/claude-code`](plugins/claude-code) | `/plugin marketplace add ./plugins/claude-code` → `/skillopt-sleep` |
|
||||
| **Codex** | [`plugins/codex`](plugins/codex) | `bash plugins/codex/install.sh` → `/skillopt-sleep` |
|
||||
| **Copilot** | [`plugins/copilot`](plugins/copilot) | register `plugins/copilot/mcp_server.py` as an MCP server |
|
||||
|
||||
**Validated on real models.** On the public
|
||||
|
||||
@@ -36,7 +36,7 @@ touch skillopt/envs/docfaithful/__init__.py
|
||||
|
||||
## Step 2 — Implement the data loader
|
||||
|
||||
`skillopt/envs/docfaithful/loader.py`:
|
||||
`skillopt/envs/docfaithful/dataloader.py`:
|
||||
|
||||
```python
|
||||
from __future__ import annotations
|
||||
@@ -161,13 +161,10 @@ Two design points worth flagging:
|
||||
```python
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs.docfaithful.loader import DocFaithfulDataLoader
|
||||
from skillopt.envs.docfaithful.dataloader import DocFaithfulDataLoader
|
||||
from skillopt.envs.docfaithful.rollout import run_batch
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
|
||||
|
||||
class DocFaithfulAdapter(EnvAdapter):
|
||||
@@ -234,7 +231,7 @@ class DocFaithfulAdapter(EnvAdapter):
|
||||
)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
# ── The two real action methods ─────────────────────────────────────
|
||||
# ── The rollout method (reflect is inherited) ───────────────────────
|
||||
|
||||
def rollout(self, env_manager, skill_content: str,
|
||||
out_dir: str, **kwargs) -> list[dict]:
|
||||
@@ -247,27 +244,9 @@ class DocFaithfulAdapter(EnvAdapter):
|
||||
max_completion_tokens=self.max_completion_tokens,
|
||||
)
|
||||
|
||||
def reflect(self, results: list[dict], skill_content: str,
|
||||
out_dir: str, **kwargs) -> list[dict | None]:
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=kwargs.get(
|
||||
"prediction_dir", os.path.join(out_dir, "predictions")
|
||||
),
|
||||
patches_dir=kwargs.get(
|
||||
"patches_dir", os.path.join(out_dir, "patches")
|
||||
),
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=kwargs.get("random_seed"),
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=kwargs.get("step_buffer_context", ""),
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
# reflect() is inherited from EnvAdapter — it delegates to
|
||||
# run_minibatch_reflect with your analyst_error_* / analyst_success_*
|
||||
# prompts. Override it only if you need custom reflection logic.
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
seen: list[str] = []
|
||||
@@ -373,9 +352,8 @@ If you get `ValueError: Unknown environment 'docfaithful'. Available: [...]`,
|
||||
you forgot Step 5.
|
||||
|
||||
If you get `TypeError: Can't instantiate abstract class DocFaithfulAdapter`,
|
||||
you forgot to implement one of the five abstract methods on `EnvAdapter`:
|
||||
`build_train_env`, `build_eval_env`, `rollout`, `reflect`,
|
||||
`get_task_types`.
|
||||
you forgot to implement one of the four abstract methods on `EnvAdapter`:
|
||||
`build_train_env`, `build_eval_env`, `rollout`, `get_task_types`.
|
||||
|
||||
## Tips
|
||||
|
||||
|
||||
+188
-49
@@ -1,74 +1,213 @@
|
||||
# SkillOpt-Sleep — plugins for Claude Code, Codex, and Copilot
|
||||
|
||||
One engine, three thin shells. **SkillOpt-Sleep** gives a local coding agent a
|
||||
nightly **sleep cycle**: it reviews your past sessions offline, replays your
|
||||
recurring tasks on your own API budget, and consolidates what it learns into
|
||||
**validated** long-term memory and skills — behind a held-out gate, staged for
|
||||
your review. Your agent gets better the more you use it, with no model-weight
|
||||
training.
|
||||
**Your coding agent forgets everything between sessions. SkillOpt-Sleep fixes
|
||||
that.** While you sleep, it reviews what you did today, notices the rules you
|
||||
keep repeating ("always add a LIMIT", "answers in `\boxed{}`", "cite the
|
||||
source"), and writes them into your agent's long-term memory and skills — but
|
||||
only the rules that actually make it score better on *your own* past tasks. You
|
||||
wake up to an agent that's better at *your* work, and you approve every change
|
||||
before it sticks.
|
||||
|
||||
It synthesizes three ideas: **SkillOpt** (validation-gated bounded text
|
||||
optimization — the research in this repo), **Claude Dreams** (offline memory
|
||||
consolidation; input never mutated; review-then-adopt), and the **agent sleep**
|
||||
literature (short-term experience → long-term competence).
|
||||
One engine, three thin shells. It synthesizes **SkillOpt** (validation-gated
|
||||
bounded text optimization — the research in this repo), **Claude Dreams**
|
||||
(offline consolidation; input never mutated; review-then-adopt), and the **agent
|
||||
sleep** idea (short-term experience → long-term competence).
|
||||
|
||||
> **This is an open-source tool, decoupled from the research code.** The engine
|
||||
> lives in the top-level [`skillopt_sleep/`](../skillopt_sleep) package and has
|
||||
> **zero dependency** on the paper's `skillopt/` experiment package (the
|
||||
> validation gate is vendored). You can ship/use it without the research stack.
|
||||
> **Open-source tool, decoupled from the research.** The engine lives in the
|
||||
> top-level [`skillopt_sleep/`](../skillopt_sleep) package with **zero
|
||||
> dependency** on the paper's `skillopt/` experiment code (the validation gate is
|
||||
> vendored). Use it without the research stack.
|
||||
|
||||
## The three integrations
|
||||
---
|
||||
|
||||
| Platform | Folder | Mechanism | Status |
|
||||
|---|---|---|---|
|
||||
| **Claude Code** | [`claude-code/`](claude-code) | `.claude-plugin` + `/sleep` command + skill + hooks | full, installable |
|
||||
| **Codex** | [`codex/`](codex) | `~/.codex/prompts/sleep.md` + `~/.agents/skills` + `AGENTS.md` | full |
|
||||
| **Copilot** | [`copilot/`](copilot) | MCP server (`sleep_*` tools) + `copilot-instructions` | full (MCP) |
|
||||
## Install (pick your agent)
|
||||
|
||||
All three call the **same** [`plugins/run-sleep.sh`](run-sleep.sh) → `python -m
|
||||
skillopt_sleep`, so behaviour is identical everywhere. Per-platform setup is in
|
||||
each folder's README.
|
||||
| Platform | Install | Then |
|
||||
|---|---|---|
|
||||
| **Claude Code** | `/plugin marketplace add microsoft/SkillOpt` → `/plugin install skillopt-sleep` | `/skillopt-sleep status` |
|
||||
| **Codex** | `git clone` → `bash plugins/codex/install.sh` | `/skillopt-sleep status` |
|
||||
| **Copilot** | `git clone` → register `plugins/copilot/mcp_server.py` as an MCP server | ask "run the sleep cycle" |
|
||||
|
||||
## Quick start (Claude Code)
|
||||
Requirements: Python ≥ 3.10 and the agent's CLI on PATH. All three call the same
|
||||
[`run-sleep.sh`](run-sleep.sh) → `python -m skillopt_sleep`, so behaviour is
|
||||
identical everywhere. Default backend is `mock` (no API spend); `--backend
|
||||
claude|codex` uses your own budget.
|
||||
|
||||
---
|
||||
|
||||
## How it works: one "night", in plain terms
|
||||
|
||||
```
|
||||
harvest your past sessions → mine the tasks you keep doing → replay them offline
|
||||
→ reflect on failures → propose a few rule edits → KEEP only edits that raise
|
||||
your held-out score → stage a proposal → (you) review & adopt
|
||||
```
|
||||
|
||||
Nothing live changes until you `adopt`; every adopt backs up the prior file.
|
||||
|
||||
### The split that keeps it honest: dream-train / real-val / real-test
|
||||
|
||||
This is the heart of the design, borrowed from the SkillOpt paper's
|
||||
train/selection/test protocol:
|
||||
|
||||
| Split | Where it comes from | What it's for |
|
||||
|---|---|---|
|
||||
| **train** | your real tasks **+ optional "dreamed" variants** | what the optimizer *learns from*. Over-dreaming here is fine — it's imagination. |
|
||||
| **val** (selection) | **your real tasks only**, held out | the **gate**: an edit is kept only if it raises this score. Stops overfitting. |
|
||||
| **test** | **your real tasks only**, held out, never seen during optimization | the **final score** we report. Kept as close to your real usage as possible. |
|
||||
|
||||
So you can **dream up extra training examples** to learn a rule robustly, while
|
||||
the rule is still **judged on real, unseen tasks**. A `dream` task can *never*
|
||||
land in val or test — that invariant is unit-tested.
|
||||
|
||||
---
|
||||
|
||||
## What each feature does **for you** (with examples)
|
||||
|
||||
Every control below works on all three platforms (pass it after the action,
|
||||
e.g. `/skillopt-sleep run --rollouts-k 3`).
|
||||
|
||||
### `--preferences "..."` — tell it your house rules
|
||||
|
||||
The single most useful knob. Free text that steers what the optimizer writes,
|
||||
as a prior. Use it to encode the conventions you're tired of repeating.
|
||||
|
||||
```bash
|
||||
git clone <repo-url> && cd SkillOpt-Sleep
|
||||
# Claude Code:
|
||||
/plugin marketplace add ./plugins/claude-code
|
||||
/plugin install skillopt-sleep@skillopt-sleep
|
||||
/sleep status
|
||||
# A backend engineer:
|
||||
/skillopt-sleep run --preferences "Always use async/await, never callbacks. \
|
||||
Prefer pytest over unittest. Commit subjects in imperative mood under 50 chars."
|
||||
|
||||
# A data analyst:
|
||||
/skillopt-sleep run --preferences "Every SQL query must end with LIMIT 1000 unless \
|
||||
I say otherwise. Money in USD with 2 decimals. Prefer CTEs over nested subqueries."
|
||||
|
||||
# A researcher:
|
||||
/skillopt-sleep run --preferences "Cite sources as [Author, Year]. Math answers in \
|
||||
\\boxed{}. Keep explanations under 150 words unless I ask for depth."
|
||||
```
|
||||
Codex: `bash plugins/codex/install.sh`.
|
||||
Copilot: register `plugins/copilot/mcp_server.py` as an MCP server.
|
||||
*What it does for you:* the next morning your agent already follows these
|
||||
without you re-typing them, and the rules are validated against your real tasks
|
||||
(if a "preference" actually hurts your held-out score, the gate drops it).
|
||||
|
||||
## What one "night" does
|
||||
### `--gate on|off` — strict vs. greedy
|
||||
|
||||
- `on` (default): an edit is kept **only if it raises your held-out score**.
|
||||
Safe — blocks plausible-but-wrong rules and reward-hacking.
|
||||
- `off`: greedy — keep edits without the strict check (still reports whether
|
||||
quality moved).
|
||||
|
||||
*What it does for you:* leave it `on` for trust. Flip it `off` when you're
|
||||
exploring and want to see everything the optimizer proposes.
|
||||
|
||||
### `--rollouts-k K` — learn from contrast, not just failure
|
||||
|
||||
Re-runs each task `K` times and learns from the difference between the **good**
|
||||
and **bad** attempts, not just a single failure.
|
||||
|
||||
```bash
|
||||
/skillopt-sleep run --rollouts-k 3
|
||||
```
|
||||
harvest ~/.claude (or session) transcripts → mine recurring tasks → replay offline
|
||||
→ consolidate (reflect → bounded edit → GATE on real held-out tasks)
|
||||
→ stage proposal → (you) adopt
|
||||
*What it does for you:* a much stronger signal. If your agent gets a task right 1
|
||||
time in 3, the optimizer figures out *what the winning attempt did* and makes it
|
||||
reliable.
|
||||
|
||||
### `--optimizer-model` / `--target-model` — optimize cheap, deploy anywhere
|
||||
|
||||
Use a strong model to *write* the rules and a cheap model to *run* your tasks.
|
||||
The learned skill then helps the cheap model — or any model.
|
||||
|
||||
```bash
|
||||
/skillopt-sleep run --optimizer-model sonnet --target-model haiku
|
||||
```
|
||||
*What it does for you:* spend a little on a smart optimizer overnight; your
|
||||
everyday cheap/fast agent inherits the upgrade. (Verified: a skill optimized on
|
||||
one model lifts a different one — cross-model and even cross-runtime
|
||||
Codex↔Claude.)
|
||||
|
||||
Nothing live changes until you adopt; every adopt backs up first.
|
||||
### `--budget-tokens N` / `--budget-minutes M` — cap the spend
|
||||
|
||||
## Controls (work on all platforms)
|
||||
You decide how much the nightly "dreaming" costs; it auto-plans how many nights
|
||||
× how many rollouts fit.
|
||||
|
||||
`--gate on|off` · `--rollouts-k K` (multi-rollout contrastive reflection) ·
|
||||
`--budget-tokens/--budget-minutes` · `--preferences "..."` · separate
|
||||
optimizer/target models (`--optimizer-model` / `--target-model`) · slow-update
|
||||
long-term memory. Full guide:
|
||||
[`../docs/sleep/CONTROLLABLE_DREAMING.md`](../docs/sleep/CONTROLLABLE_DREAMING.md).
|
||||
```bash
|
||||
/skillopt-sleep run --backend claude --budget-tokens 60000
|
||||
```
|
||||
*What it does for you:* predictable cost. It stops cleanly when the budget is hit
|
||||
and tells you what it skipped.
|
||||
|
||||
### multi-objective (accuracy ↑, tokens ↓, latency ↓)
|
||||
|
||||
The reward can weight not just correctness but **cost and speed**, so a skill can
|
||||
learn to be cheaper and faster, not only more accurate. *What it does for you:*
|
||||
"answer directly instead of opening five files" becomes a learned habit.
|
||||
|
||||
### `schedule` / `unschedule` — set it and forget it
|
||||
|
||||
Built-in nightly scheduling (no manual cron):
|
||||
|
||||
```bash
|
||||
/skillopt-sleep schedule --hour 3 --minute 17 # runs every night for this project
|
||||
/skillopt-sleep unschedule # stop it
|
||||
```
|
||||
*What it does for you:* it just gets better while you sleep. The nightly run only
|
||||
*stages* a proposal — adopting is still your call (or add `--auto-adopt` when you
|
||||
schedule, if you trust it).
|
||||
|
||||
---
|
||||
|
||||
## Full action / flag reference
|
||||
|
||||
| Action | Does |
|
||||
|---|---|
|
||||
| `status` | nights so far + the latest staged proposal (read-only) |
|
||||
| `dry-run` | harvest→mine→replay→report; **stages nothing** |
|
||||
| `run` | full cycle; **stages** a proposal; nothing live changes |
|
||||
| `adopt` | apply the staged proposal to `CLAUDE.md`/`SKILL.md` (backs up first) |
|
||||
| `harvest` | debug: print the recurring tasks it mined |
|
||||
| `schedule` / `unschedule` | install/remove the nightly cron entry |
|
||||
|
||||
| Flag | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `--backend mock\|claude\|codex` | `mock` | who runs/optimizes (mock = free) |
|
||||
| `--preferences "..."` | – | your house rules, as a prior |
|
||||
| `--gate on\|off` | `on` | strict held-out gate vs. greedy |
|
||||
| `--rollouts-k K` | `1` | multi-rollout contrastive reflection |
|
||||
| `--optimizer-model` / `--target-model` | – | split the optimizer from the target |
|
||||
| `--budget-tokens` / `--budget-minutes` | – | cap the nightly spend |
|
||||
| `--scope invoked\|all` | `invoked` | this project only, or all projects |
|
||||
| `--auto-adopt` | off | apply without manual review (power users) |
|
||||
|
||||
Deep dive: [`../docs/sleep/CONTROLLABLE_DREAMING.md`](../docs/sleep/CONTROLLABLE_DREAMING.md).
|
||||
|
||||
---
|
||||
|
||||
## Does it actually work?
|
||||
|
||||
Validated on the public
|
||||
[gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` benchmark
|
||||
with **real models on both Claude and Codex**: deficient skills go **0.00 →
|
||||
1.00** on held-out sets (all 4 seeds incl. a real tool-use loop), cross-model
|
||||
transfer is positive, and the gate blocks regressions. Full results:
|
||||
[`../docs/sleep/FINAL_REPORT.md`](../docs/sleep/FINAL_REPORT.md).
|
||||
Yes — measured with **real models on both Claude and Codex**, scored on held-out
|
||||
tasks the optimizer never trained on:
|
||||
|
||||
Deterministic proof (no API key):
|
||||
- **gbrain-evals `skillopt-v1`** (the public suite gbrain scores SkillOpt on):
|
||||
deficient skills go **0.00 → 1.00** on all 4 seeds, including a real tool-use
|
||||
loop; cross-model transfer is positive; the gate blocks regressions.
|
||||
→ [`../docs/sleep/FINAL_REPORT.md`](../docs/sleep/FINAL_REPORT.md)
|
||||
- **Academic daily-cases** (math / spreadsheet / search-QA, the paper's 4:1:5
|
||||
split with dream-augmented train): see
|
||||
[`../docs/sleep/daily_cases_results.md`](../docs/sleep/daily_cases_results.md).
|
||||
- **Fresh load-test** (a "SQL must always include LIMIT" analyst, built from
|
||||
scratch): held-out **0.00 → 1.00** on both backends.
|
||||
→ [`../docs/sleep/plugin_load_test.md`](../docs/sleep/plugin_load_test.md)
|
||||
|
||||
Try the deterministic proof yourself (no API key, no spend):
|
||||
```bash
|
||||
python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves
|
||||
```
|
||||
It prints the held-out score rising to 1.0 as the gate accepts the right rules,
|
||||
and confirms the gate **rejects** an injected harmful edit.
|
||||
|
||||
---
|
||||
|
||||
## Safety
|
||||
|
||||
- **Read-only** harvest of your sessions. `mock` replay has no side effects.
|
||||
- Proposals are **staged**, never auto-applied (unless you opt in with `--auto-adopt`).
|
||||
- Every adopt writes a backup. Per-night token/time budget caps. Secrets redacted.
|
||||
|
||||
@@ -27,7 +27,7 @@ harvest ~/.claude transcripts → mine recurring tasks → replay offline
|
||||
→ consolidate (reflect → bounded edit → GATE) → stage proposal → (you) adopt
|
||||
```
|
||||
|
||||
Nothing live is modified until **you** run `/sleep adopt` (the Dreams "review,
|
||||
Nothing live is modified until **you** run `/skillopt-sleep adopt` (the Dreams "review,
|
||||
then adopt or discard" contract). Every adopt backs up the prior file first.
|
||||
|
||||
## Install
|
||||
@@ -44,7 +44,7 @@ cd SkillOpt
|
||||
/plugin install skillopt-sleep@skillopt-sleep
|
||||
|
||||
# 3) verify
|
||||
/sleep status
|
||||
/skillopt-sleep status
|
||||
```
|
||||
|
||||
The plugin's bundled runner (`scripts/sleep.sh`) auto-selects a Python ≥ 3.10
|
||||
@@ -56,10 +56,10 @@ they shell out to the CLIs you already have.
|
||||
|
||||
```bash
|
||||
# from inside any project you use with Claude Code:
|
||||
/sleep dry-run # safe preview: what it would learn, no changes staged
|
||||
/sleep run # full cycle: stages a reviewed proposal (still no live edits)
|
||||
/sleep status # see history + the latest staged proposal
|
||||
/sleep adopt # apply the staged proposal to CLAUDE.md / SKILL.md (with backup)
|
||||
/skillopt-sleep dry-run # safe preview: what it would learn, no changes staged
|
||||
/skillopt-sleep run # full cycle: stages a reviewed proposal (still no live edits)
|
||||
/skillopt-sleep status # see history + the latest staged proposal
|
||||
/skillopt-sleep adopt # apply the staged proposal to CLAUDE.md / SKILL.md (with backup)
|
||||
```
|
||||
|
||||
Or call the engine directly (Python ≥ 3.10):
|
||||
|
||||
+16
-13
@@ -1,10 +1,10 @@
|
||||
---
|
||||
description: Run or manage the SkillOpt-Sleep self-evolution cycle (review past sessions, replay tasks offline, consolidate validated memory + skills)
|
||||
argument-hint: "[run | dry-run | status | adopt | harvest] (default: status)"
|
||||
description: Run or manage the SkillOpt-Sleep self-evolution cycle (review past sessions, replay tasks offline, consolidate validated memory + skills; can also schedule nightly runs)
|
||||
argument-hint: "[run | dry-run | status | adopt | harvest | schedule | unschedule] (default: status)"
|
||||
allowed-tools: Bash, Read
|
||||
---
|
||||
|
||||
# /sleep — SkillOpt-Sleep nightly self-evolution
|
||||
# /skillopt-sleep — SkillOpt-Sleep nightly self-evolution
|
||||
|
||||
You are driving **SkillOpt-Sleep**: a tool that lets this user's Claude agent
|
||||
improve offline by reviewing past sessions, replaying recurring tasks, and
|
||||
@@ -27,16 +27,19 @@ The engine is the `skillopt_sleep` Python package in this repo. Use the
|
||||
|
||||
`<action>` is one of:
|
||||
|
||||
| action | what it does |
|
||||
|-----------|--------------|
|
||||
| `status` | show how many nights have run + the latest staged proposal (READ-ONLY) |
|
||||
| `dry-run` | harvest → mine → replay → report, but **stage nothing** (safe preview) |
|
||||
| `run` | full cycle: also **stage** a reviewed proposal (still does NOT touch live files) |
|
||||
| `adopt` | apply the latest staged proposal to live `CLAUDE.md` / `SKILL.md` (backs up first) |
|
||||
| `harvest` | debug: print the recurring tasks mined from recent sessions |
|
||||
| action | what it does |
|
||||
|--------------|--------------|
|
||||
| `status` | show how many nights have run + the latest staged proposal (READ-ONLY) |
|
||||
| `dry-run` | harvest → mine → replay → report, but **stage nothing** (safe preview) |
|
||||
| `run` | full cycle: also **stage** a reviewed proposal (still does NOT touch live files) |
|
||||
| `adopt` | apply the latest staged proposal to live `CLAUDE.md` / `SKILL.md` (backs up first) |
|
||||
| `harvest` | debug: print the recurring tasks mined from recent sessions |
|
||||
| `schedule` | install a nightly cron entry for this project (`--hour --minute`, off-:00 by default) |
|
||||
| `unschedule` | remove the nightly cron entry (`--all` to remove every managed entry) |
|
||||
|
||||
Default backend is `mock` (deterministic, no API spend). To use real Anthropic
|
||||
budget for genuine improvement, add `--backend anthropic`.
|
||||
Default backend is `mock` (deterministic, no API spend). To use real budget for
|
||||
genuine improvement, add `--backend claude` or `--backend codex`. To steer what
|
||||
the optimizer writes, add `--preferences "<your house rules>"`.
|
||||
|
||||
## Steps to follow
|
||||
|
||||
@@ -47,7 +50,7 @@ budget for genuine improvement, add `--backend anthropic`.
|
||||
- the gate decision (accept/reject) and the exact edits it proposes
|
||||
- where the proposal is staged
|
||||
3. **For `run` that produced an accepted proposal:** tell the user the diff is
|
||||
staged and that **nothing live changed yet**. Offer to run `/sleep adopt`.
|
||||
staged and that **nothing live changed yet**. Offer to run `/skillopt-sleep adopt`.
|
||||
4. **For `adopt`:** confirm which live files were updated and that backups were
|
||||
written under the staging dir's `backup/`.
|
||||
5. **Never** edit `CLAUDE.md` or `SKILL.md` yourself — only the `adopt` action
|
||||
@@ -17,7 +17,7 @@ cat <<EOF
|
||||
# ── SkillOpt-Sleep nightly cycle ────────────────────────────────────────────
|
||||
# Review past sessions, replay tasks, stage validated memory/skill updates.
|
||||
# Runs at ${HOUR}:$(printf '%02d' $MIN) local every day. Output goes to the project's
|
||||
# .skillopt-sleep/ dir; nothing live is changed until you run '/sleep adopt'
|
||||
# .skillopt-sleep/ dir; nothing live is changed until you run '/skillopt-sleep adopt'
|
||||
# (unless you pass --auto-adopt below).
|
||||
#
|
||||
# Copy the next line into 'crontab -e':
|
||||
|
||||
@@ -41,7 +41,7 @@ Trigger when the user wants any of:
|
||||
|
||||
## How to drive it
|
||||
|
||||
Prefer the `/sleep` command. Under the hood it calls the bundled runner:
|
||||
Prefer the `/skillopt-sleep` command. Under the hood it calls the bundled runner:
|
||||
|
||||
```bash
|
||||
"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" status # what's happened
|
||||
|
||||
@@ -23,7 +23,7 @@ three, plus a shared runner.
|
||||
```bash
|
||||
git clone <repo-url> SkillOpt-Sleep
|
||||
cd SkillOpt-Sleep
|
||||
bash plugins/codex/install.sh # installs the /sleep prompt + skill
|
||||
bash plugins/codex/install.sh # installs the /skillopt-sleep prompt + skill
|
||||
export SKILLOPT_SLEEP_REPO="$(pwd)" # so the runner is found from anywhere
|
||||
```
|
||||
|
||||
@@ -32,10 +32,10 @@ Requires Python ≥ 3.10 and the `codex` CLI on PATH.
|
||||
## Use
|
||||
|
||||
```text
|
||||
/sleep status # what's happened
|
||||
/sleep dry-run # safe preview, stages nothing
|
||||
/sleep run # full cycle, stages a reviewed proposal (no live edits)
|
||||
/sleep adopt # apply the staged proposal (with backup)
|
||||
/skillopt-sleep status # what's happened
|
||||
/skillopt-sleep dry-run # safe preview, stages nothing
|
||||
/skillopt-sleep run # full cycle, stages a reviewed proposal (no live edits)
|
||||
/skillopt-sleep adopt # apply the staged proposal (with backup)
|
||||
```
|
||||
|
||||
Or call the engine directly:
|
||||
|
||||
@@ -9,10 +9,10 @@ AGENTS_SKILLS="${HOME}/.agents/skills"
|
||||
|
||||
echo "[install] repo: $REPO_ROOT"
|
||||
|
||||
# 1) custom /sleep prompt
|
||||
# 1) custom /skillopt-sleep prompt
|
||||
mkdir -p "$CODEX_HOME/prompts"
|
||||
cp "$REPO_ROOT/plugins/codex/prompts/sleep.md" "$CODEX_HOME/prompts/sleep.md"
|
||||
echo "[install] /sleep prompt -> $CODEX_HOME/prompts/sleep.md"
|
||||
cp "$REPO_ROOT/plugins/codex/prompts/skillopt-sleep.md" "$CODEX_HOME/prompts/skillopt-sleep.md"
|
||||
echo "[install] /skillopt-sleep prompt -> $CODEX_HOME/prompts/skillopt-sleep.md"
|
||||
|
||||
# 2) user-level skill
|
||||
mkdir -p "$AGENTS_SKILLS/skillopt-sleep"
|
||||
@@ -30,7 +30,7 @@ cat <<EOF
|
||||
|
||||
## SkillOpt-Sleep
|
||||
An offline self-improvement cycle is available. To run it:
|
||||
\`bash "$REPO_ROOT/plugins/run-sleep.sh" status\`. Use \`/sleep\` for the guided flow.
|
||||
\`bash "$REPO_ROOT/plugins/run-sleep.sh" status\`. Use \`/skillopt-sleep\` for the guided flow.
|
||||
|
||||
Done. Try: /sleep status
|
||||
Done. Try: /skillopt-sleep status
|
||||
EOF
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# /sleep — SkillOpt-Sleep for Codex
|
||||
# /skillopt-sleep — SkillOpt-Sleep for Codex
|
||||
#
|
||||
# Custom prompt: copy this file to ~/.codex/prompts/sleep.md and invoke with
|
||||
# `/sleep` in the Codex CLI. ($ARGUMENTS is the text after /sleep.)
|
||||
# Custom prompt: copy this file to ~/.codex/prompts/skillopt-sleep.md and invoke with
|
||||
# `/skillopt-sleep` in the Codex CLI. ($ARGUMENTS is the text after /skillopt-sleep.)
|
||||
|
||||
Run the SkillOpt-Sleep offline self-evolution cycle. Action: $ARGUMENTS
|
||||
(empty → "status").
|
||||
@@ -34,7 +34,7 @@ for real improvement on the user's own Codex budget (default `mock` = no spend).
|
||||
2. For `run`/`dry-run`: read the staged `report.md` it prints and show the user
|
||||
the held-out baseline → candidate score and the exact proposed edits.
|
||||
3. `run` only **stages** a proposal under `<project>/.skillopt-sleep/staging/`;
|
||||
nothing live changes until `adopt`. Offer `/sleep adopt`.
|
||||
nothing live changes until `adopt`. Offer `/skillopt-sleep adopt`.
|
||||
4. Never hand-edit the user's `AGENTS.md` / skills yourself — only `adopt` does,
|
||||
and it backs up first.
|
||||
|
||||
|
||||
@@ -5,8 +5,8 @@ This directory provides scaffold files for adding a new benchmark to SkillOpt.
|
||||
## Files
|
||||
|
||||
- `env_template.py` — Environment adapter template (subclasses
|
||||
`EnvAdapter`; implements the 5 abstract methods so the file is
|
||||
instantiable out of the box).
|
||||
`EnvAdapter`; implements the 4 abstract methods so the file is
|
||||
instantiable out of the box — `reflect` is inherited).
|
||||
- `loader_template.py` — Data loader template (subclasses
|
||||
`SplitDataLoader`; implements `load_split_items` for `.json`/`.jsonl`).
|
||||
- `config_template.yaml` — Config file template.
|
||||
@@ -21,15 +21,15 @@ This directory provides scaffold files for adding a new benchmark to SkillOpt.
|
||||
```bash
|
||||
cd skillopt/envs/your_benchmark
|
||||
mv env_template.py adapter.py
|
||||
mv loader_template.py loader.py
|
||||
mv loader_template.py dataloader.py
|
||||
```
|
||||
…and inside each file rename the classes
|
||||
(`TemplateBenchmarkEnv → YourBenchmarkAdapter`,
|
||||
`TemplateBenchmarkLoader → YourBenchmarkLoader`)
|
||||
and fix the cross-import in `adapter.py`.
|
||||
3. **Implement the TODO blocks** inside `adapter.py:rollout` and the
|
||||
`_normalize_item` helper in `loader.py`. If you want real reflection,
|
||||
uncomment the `run_minibatch_reflect` block in `adapter.py:reflect`.
|
||||
`_normalize_item` helper in `dataloader.py`. (`reflect` is inherited from
|
||||
`EnvAdapter`; override it only for custom reflection logic.)
|
||||
4. **Register** the adapter — add a `try / except ImportError` block in
|
||||
`scripts/train.py`'s `_register_builtins()` mapping the registry key
|
||||
to your `YourBenchmarkAdapter` class. There is no
|
||||
|
||||
@@ -14,13 +14,9 @@ For a fully worked example see ``skillopt/envs/officeqa/``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs._template.loader_template import TemplateBenchmarkLoader
|
||||
# When you wire in real reflection, also import:
|
||||
# from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
|
||||
|
||||
class TemplateBenchmarkEnv(EnvAdapter):
|
||||
@@ -131,53 +127,12 @@ class TemplateBenchmarkEnv(EnvAdapter):
|
||||
)
|
||||
return results
|
||||
|
||||
# ── Reflect: turn rollout results into patch dicts ─────────────────
|
||||
|
||||
def reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
"""
|
||||
Turn rollouts into a list of raw patch dicts (or None to drop).
|
||||
|
||||
Each non-None dict MUST have:
|
||||
- "patch": {"edits": [...]} a Patch.to_dict() payload
|
||||
- "source_type": "failure" | "success"
|
||||
|
||||
Most benchmarks delegate to
|
||||
:func:`skillopt.gradient.reflect.run_minibatch_reflect` which
|
||||
will call the optimizer model with the
|
||||
``analyst_error_*`` / ``analyst_success_*`` prompts. To enable it,
|
||||
uncomment the import above and call:
|
||||
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=kwargs.get(
|
||||
"prediction_dir", os.path.join(out_dir, "predictions")
|
||||
),
|
||||
patches_dir=kwargs.get(
|
||||
"patches_dir", os.path.join(out_dir, "patches")
|
||||
),
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=kwargs.get("random_seed"),
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=kwargs.get("step_buffer_context", ""),
|
||||
update_mode=getattr(self, "_cfg", {}).get(
|
||||
"skill_update_mode", "patch"
|
||||
),
|
||||
)
|
||||
"""
|
||||
# Template default: produce no patches (no-op trainer step).
|
||||
return [None for _ in results]
|
||||
# ── Reflect (inherited) ─────────────────────────────────────────────
|
||||
#
|
||||
# ``reflect`` is inherited from ``EnvAdapter``: the default delegates to
|
||||
# ``skillopt.gradient.reflect.run_minibatch_reflect`` using your
|
||||
# ``analyst_error_*`` / ``analyst_success_*`` prompts. You do NOT need to
|
||||
# implement it — override only if your benchmark needs custom reflection.
|
||||
|
||||
# ── Stratification hint ────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from skillopt.envs.alfworld.rollout import (
|
||||
run_alfworld_batch,
|
||||
TASKS,
|
||||
)
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
from skillopt.utils import compute_score
|
||||
|
||||
|
||||
@@ -425,35 +424,5 @@ class ALFWorldAdapter(EnvAdapter):
|
||||
all_results.extend(chunk_results)
|
||||
return all_results
|
||||
|
||||
def reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
meta_skill_context = kwargs.get("meta_skill_context", "")
|
||||
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
)
|
||||
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
return list(TASKS)
|
||||
|
||||
+27
-7
@@ -231,7 +231,6 @@ class EnvAdapter(ABC):
|
||||
(float 0-1). May include env-specific fields.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
@@ -241,15 +240,36 @@ class EnvAdapter(ABC):
|
||||
) -> list[dict | None]:
|
||||
"""Analyze rollout results and produce patches.
|
||||
|
||||
Default implementation: delegate to the shared minibatch reflect
|
||||
stage. Every built-in benchmark uses this unchanged — override only
|
||||
if your environment needs custom reflection logic.
|
||||
|
||||
Each returned dict conforms to :class:`~skillopt.types.RawPatch`:
|
||||
``"patch"`` (with ``"edits"`` list) + ``"source_type"``
|
||||
(``"failure"`` or ``"success"``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[dict | None]
|
||||
Raw analyst outputs; ``None`` entries are filtered out.
|
||||
(``"failure"`` or ``"success"``); ``None`` entries are filtered out.
|
||||
"""
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=kwargs.get(
|
||||
"prediction_dir", os.path.join(out_dir, "predictions")
|
||||
),
|
||||
patches_dir=kwargs.get(
|
||||
"patches_dir", os.path.join(out_dir, "patches")
|
||||
),
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=kwargs.get("random_seed"),
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=kwargs.get("step_buffer_context", ""),
|
||||
meta_skill_context=kwargs.get("meta_skill_context", ""),
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def get_task_types(self) -> list[str]:
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs.docvqa.dataloader import DocVQADataLoader
|
||||
from skillopt.envs.docvqa.rollout import run_batch
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
|
||||
|
||||
class DocVQAAdapter(EnvAdapter):
|
||||
@@ -84,28 +81,6 @@ class DocVQAAdapter(EnvAdapter):
|
||||
task_timeout=self.exec_timeout,
|
||||
)
|
||||
|
||||
def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
seen: list[str] = []
|
||||
for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:
|
||||
|
||||
@@ -2,10 +2,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs.livemathematicianbench.dataloader import LiveMathematicianBenchDataLoader
|
||||
from skillopt.envs.livemathematicianbench.rollout import run_batch
|
||||
@@ -127,36 +125,5 @@ class LiveMathematicianBenchAdapter(EnvAdapter):
|
||||
task_timeout=self.exec_timeout,
|
||||
)
|
||||
|
||||
def reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
meta_skill_context = kwargs.get("meta_skill_context", "")
|
||||
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
return self.dataloader.get_task_types()
|
||||
|
||||
@@ -6,7 +6,6 @@ from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs.officeqa.dataloader import OfficeQADataLoader
|
||||
from skillopt.envs.officeqa.rollout import run_batch
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
|
||||
|
||||
class OfficeQAAdapter(EnvAdapter):
|
||||
@@ -104,28 +103,6 @@ class OfficeQAAdapter(EnvAdapter):
|
||||
diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
|
||||
)
|
||||
|
||||
def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
seen: list[str] = []
|
||||
for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:
|
||||
|
||||
@@ -2,13 +2,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs.searchqa.dataloader import SearchQADataLoader
|
||||
from skillopt.envs.searchqa.rollout import run_batch
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
from skillopt.model import get_target_backend
|
||||
|
||||
|
||||
@@ -94,36 +92,5 @@ class SearchQAAdapter(EnvAdapter):
|
||||
task_timeout=self.exec_timeout,
|
||||
)
|
||||
|
||||
def reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
meta_skill_context = kwargs.get("meta_skill_context", "")
|
||||
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
return ["qa"]
|
||||
|
||||
@@ -16,7 +16,6 @@ from skillopt.envs.spreadsheetbench.rollout import (
|
||||
run_spreadsheet_batch,
|
||||
run_spreadsheet_batch_codegen,
|
||||
)
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
from skillopt.model import get_target_backend, is_target_exec_backend
|
||||
|
||||
|
||||
@@ -156,37 +155,5 @@ class SpreadsheetBenchAdapter(EnvAdapter):
|
||||
|
||||
return results
|
||||
|
||||
def reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
"""Analyze rollout results and produce patches (minibatch mode)."""
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
meta_skill_context = kwargs.get("meta_skill_context", "")
|
||||
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
meta_skill_context=meta_skill_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
return list(TASK_TYPES)
|
||||
|
||||
@@ -163,6 +163,31 @@ def cmd_harvest(args) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_schedule(args) -> int:
|
||||
from skillopt_sleep.scheduler import schedule, list_scheduled
|
||||
cfg = _cfg_from_args(args)
|
||||
project = cfg.get("invoked_project") or os.getcwd()
|
||||
ok, msg = schedule(project, backend=cfg.get("backend", "mock"),
|
||||
hour=args.hour, minute=args.minute,
|
||||
extra=("--auto-adopt" if getattr(args, "auto_adopt", False) else ""))
|
||||
print("[sleep] " + msg)
|
||||
cur = list_scheduled()
|
||||
if cur:
|
||||
print("[sleep] currently scheduled:")
|
||||
for ln in cur:
|
||||
print(" " + ln[:140])
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def cmd_unschedule(args) -> int:
|
||||
from skillopt_sleep.scheduler import unschedule
|
||||
cfg = _cfg_from_args(args)
|
||||
project = cfg.get("invoked_project") or os.getcwd()
|
||||
ok, msg = unschedule(project, all_projects=getattr(args, "all", False))
|
||||
print("[sleep] " + msg)
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="skillopt_sleep", description="SkillOpt-Sleep nightly self-evolution")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
@@ -178,6 +203,13 @@ def main(argv=None) -> int:
|
||||
p_adopt.add_argument("--staging", default="", help="specific staging dir")
|
||||
p_harvest = sub.add_parser("harvest", help="debug: show mined tasks")
|
||||
_add_common(p_harvest)
|
||||
p_sched = sub.add_parser("schedule", help="install a nightly cron entry for this project")
|
||||
_add_common(p_sched)
|
||||
p_sched.add_argument("--hour", type=int, default=3)
|
||||
p_sched.add_argument("--minute", type=int, default=17)
|
||||
p_unsched = sub.add_parser("unschedule", help="remove the nightly cron entry")
|
||||
_add_common(p_unsched)
|
||||
p_unsched.add_argument("--all", action="store_true", help="remove all managed entries")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
if args.cmd == "run":
|
||||
@@ -190,6 +222,10 @@ def main(argv=None) -> int:
|
||||
return cmd_adopt(args)
|
||||
if args.cmd == "harvest":
|
||||
return cmd_harvest(args)
|
||||
if args.cmd == "schedule":
|
||||
return cmd_schedule(args)
|
||||
if args.cmd == "unschedule":
|
||||
return cmd_unschedule(args)
|
||||
parser.print_help()
|
||||
return 2
|
||||
|
||||
|
||||
+302
-16
@@ -41,7 +41,8 @@ class Backend:
|
||||
# Optional user preferences (free text) injected into reflect as a prior.
|
||||
preferences: str = ""
|
||||
|
||||
def attempt(self, task: TaskRecord, skill: str, memory: str) -> str:
|
||||
def attempt(self, task: TaskRecord, skill: str, memory: str,
|
||||
sample_id: int = 0) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def attempt_with_tools(
|
||||
@@ -151,7 +152,8 @@ class MockBackend(Backend):
|
||||
out.append(key)
|
||||
return out
|
||||
|
||||
def attempt(self, task: TaskRecord, skill: str, memory: str) -> str:
|
||||
def attempt(self, task: TaskRecord, skill: str, memory: str,
|
||||
sample_id: int = 0) -> str:
|
||||
ctx = (skill or "") + "\n" + (memory or "")
|
||||
rules = self._required_rules(task)
|
||||
# The "__harmful__" rule models a bad edit: even when present it makes
|
||||
@@ -191,6 +193,13 @@ class MockBackend(Backend):
|
||||
return resp, called
|
||||
|
||||
def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]:
|
||||
if task.reference_kind == "answer" and task.judge:
|
||||
try:
|
||||
from skillopt_sleep.experiments.real_eval import score_answer_judge
|
||||
except ImportError:
|
||||
score_answer_judge = None # research evaluators not bundled
|
||||
if score_answer_judge is not None:
|
||||
return score_answer_judge(task.judge, response)
|
||||
if task.reference_kind == "rule" and task.judge:
|
||||
from skillopt_sleep.judges import score_rule_judge
|
||||
return score_rule_judge(task.judge, response)
|
||||
@@ -253,6 +262,43 @@ def _extract_json(raw: str, kind: str):
|
||||
return None
|
||||
|
||||
|
||||
def _task_guardrail(pairs) -> str:
|
||||
"""Build an 'output contract' the optimizer must not violate.
|
||||
|
||||
``pairs`` is a list of (TaskRecord, ReplayResult). We surface the benchmark's
|
||||
own rollout system prompt (TaskRecord.system) plus a short, explicit list of
|
||||
invariants, so the optimizer cannot learn rules that the evaluator can never
|
||||
honor (the SpreadsheetBench failure mode: a learned "return ```vba```" or
|
||||
"ask the user for the range" rule scores 0 because the harness runs only
|
||||
```python``` openpyxl and cannot answer questions).
|
||||
|
||||
Returns "" when no task carries a system contract (e.g. mined daily cases),
|
||||
so non-benchmark runs are unchanged.
|
||||
"""
|
||||
sys_txt = ""
|
||||
for t, _ in pairs:
|
||||
s = getattr(t, "system", "") or ""
|
||||
if s.strip():
|
||||
sys_txt = s.strip()
|
||||
break
|
||||
if not sys_txt:
|
||||
return ""
|
||||
# the system prompt can be long; keep the rules portion concise for the optimizer
|
||||
contract = sys_txt
|
||||
if len(contract) > 900:
|
||||
contract = contract[:900] + " …"
|
||||
invariants = (
|
||||
"- Do NOT change the required output format or programming language.\n"
|
||||
"- Do NOT tell the agent to ask the user a question or request more info; "
|
||||
"it must always produce a best-effort answer from what is given.\n"
|
||||
"- Keep every rule consistent with the contract above."
|
||||
)
|
||||
return (
|
||||
"\n# Task output contract (rules MUST obey this — violating it scores 0)\n"
|
||||
f"{contract}\n{invariants}\n"
|
||||
)
|
||||
|
||||
|
||||
class CliBackend(Backend):
|
||||
"""Common logic for real CLI-driven backends (claude / codex).
|
||||
|
||||
@@ -283,24 +329,55 @@ class CliBackend(Backend):
|
||||
return out
|
||||
|
||||
# operations -----------------------------------------------------------
|
||||
def attempt(self, task: TaskRecord, skill: str, memory: str) -> str:
|
||||
def attempt(self, task: TaskRecord, skill: str, memory: str,
|
||||
sample_id: int = 0) -> str:
|
||||
# sample_id distinguishes repeated rollouts of the SAME (task, skill,
|
||||
# memory) in the cache key. Without it the attempt cache collapses all
|
||||
# K dream rollouts into one cached response (spread always 0), which
|
||||
# silently disables contrastive reflection. sample_id=0 keeps the old
|
||||
# key format so gate re-scoring still benefits from the cache.
|
||||
if task.system:
|
||||
# Benchmark carries its own (research-repo) rollout system prompt.
|
||||
# Use it verbatim with a neutral skill/memory section — this both
|
||||
# keeps scoring faithful and avoids the aggressive "OVERRIDE / HARD
|
||||
# CONSTRAINT" phrasing below, which Azure's content filter flags as a
|
||||
# jailbreak (HTTP 400) and silently zeroes the rollout.
|
||||
skill_section = f"## Skill\n{skill.strip()}\n\n" if skill.strip() else ""
|
||||
mem_section = f"## Memory\n{memory.strip()}\n\n" if memory.strip() else ""
|
||||
system = task.system.replace("{skill_section}", skill_section)
|
||||
if "{skill_section}" not in task.system and skill_section:
|
||||
system = skill_section + system
|
||||
body = task.intent + ("\n\n" + task.context_excerpt if task.context_excerpt else "")
|
||||
prompt = f"{system}{mem_section}\n{body}"
|
||||
salt = f"s{sample_id}:" if sample_id else ""
|
||||
key = "attempt:" + salt + skill_hash(prompt)
|
||||
return self._cached_call(key, prompt, max_tokens=512)
|
||||
# generic path (mined daily-case tasks): neutral, content-filter-safe
|
||||
# wording. Apply the skill/memory as guidance, not as adversarial
|
||||
# "OVERRIDE everything" directives.
|
||||
prompt = (
|
||||
"You are completing a recurring task for a user. Apply the skill and "
|
||||
"memory rules EXACTLY, including any output-format requirements. If the "
|
||||
"skill contains a 'Learned preferences' block, treat those rules as "
|
||||
"HARD CONSTRAINTS that OVERRIDE anything earlier in the skill they "
|
||||
"conflict with (e.g. an explicit length limit overrides 'be "
|
||||
"exhaustive'). Satisfy every such constraint even at the cost of "
|
||||
"brevity or detail.\n\n"
|
||||
"Complete the following task for the user. Follow the skill and memory "
|
||||
"guidance below, including any output-format and length requirements. "
|
||||
"When a 'Learned preferences' rule sets an explicit limit (e.g. a length "
|
||||
"cap), prefer that rule over more general advice it refines.\n\n"
|
||||
f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n"
|
||||
f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n"
|
||||
"Return ONLY the final answer text, nothing else."
|
||||
)
|
||||
# cache on (task, skill, memory) so identical hold-out re-scoring is free
|
||||
key = "attempt:" + skill_hash(prompt)
|
||||
salt = f"s{sample_id}:" if sample_id else ""
|
||||
key = "attempt:" + salt + skill_hash(prompt)
|
||||
return self._cached_call(key, prompt, max_tokens=512)
|
||||
|
||||
def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]:
|
||||
# real-benchmark correctness judge (searchqa/livemath/spreadsheet) — local
|
||||
if task.reference_kind == "answer" and task.judge:
|
||||
try:
|
||||
from skillopt_sleep.experiments.real_eval import score_answer_judge
|
||||
except ImportError:
|
||||
score_answer_judge = None # research evaluators not bundled
|
||||
if score_answer_judge is not None:
|
||||
return score_answer_judge(task.judge, response)
|
||||
# gbrain-style rule judge: scored locally, no API spend
|
||||
if task.reference_kind == "rule" and task.judge:
|
||||
from skillopt_sleep.judges import score_rule_judge
|
||||
@@ -389,6 +466,13 @@ class CliBackend(Backend):
|
||||
"\n# User preferences (honor these as priors when writing rules)\n"
|
||||
+ str(self.preferences).strip()
|
||||
)
|
||||
# Task GUARDRAIL: the optimizer must not invent rules that violate the
|
||||
# task's hard constraints (e.g. SpreadsheetBench answers MUST be a
|
||||
# ```python``` openpyxl block — a learned "return ```vba```" or "ask the
|
||||
# user for the range" rule scores 0 because the harness can't run VBA and
|
||||
# can't ask questions). We surface the benchmark's own rollout system
|
||||
# prompt (carried on TaskRecord.system) so proposed rules stay in-bounds.
|
||||
guard_text = _task_guardrail(failures)
|
||||
prompt = (
|
||||
"You are SkillOpt's optimizer. The agent keeps failing the recurring "
|
||||
f"tasks below. Propose at most {edit_budget} bounded edits to the "
|
||||
@@ -406,9 +490,15 @@ class CliBackend(Backend):
|
||||
"but outputs must be under a character limit), write an explicit, "
|
||||
"forceful OVERRIDE rule stating it supersedes the conflicting "
|
||||
"instruction, and put the hard requirement first.\n"
|
||||
"HARD CONSTRAINT: every rule you write MUST be consistent with the "
|
||||
"'Task output contract' below (if shown). NEVER propose a rule that "
|
||||
"changes the required output format/language, tells the agent to ask "
|
||||
"the user a question, or otherwise violates that contract — such a "
|
||||
"rule scores ZERO because the evaluator cannot honor it.\n"
|
||||
'Return ONLY a JSON array: '
|
||||
'[{"op":"add|replace|delete","content":"<rule>","anchor":"<text to replace/delete, optional>","rationale":"<why>"}].\n\n'
|
||||
f"# Current {target}\n{cur_doc}\n"
|
||||
f"{guard_text}"
|
||||
f"{criteria_text}\n"
|
||||
f"{pref_text}\n\n"
|
||||
f"# Recurring failures\n{fail_text}"
|
||||
@@ -717,8 +807,8 @@ class DualBackend(Backend):
|
||||
self.optimizer = optimizer
|
||||
self.name = f"target={target.name}/optimizer={optimizer.name}"
|
||||
|
||||
def attempt(self, task, skill, memory):
|
||||
return self.target.attempt(task, skill, memory)
|
||||
def attempt(self, task, skill, memory, sample_id: int = 0):
|
||||
return self.target.attempt(task, skill, memory, sample_id=sample_id)
|
||||
|
||||
def attempt_with_tools(self, task, skill, memory, tools):
|
||||
return self.target.attempt_with_tools(task, skill, memory, tools)
|
||||
@@ -741,18 +831,211 @@ class DualBackend(Backend):
|
||||
return self.target.tokens_used() + self.optimizer.tokens_used()
|
||||
|
||||
|
||||
# ── Azure OpenAI backend (gpt-5.x via managed identity) ───────────────────────
|
||||
|
||||
# Endpoint -> deployments, from the intern's avail_api.md. The backend picks the
|
||||
# first endpoint that hosts the requested deployment.
|
||||
_AZURE_ENDPOINTS = {
|
||||
"https://oaidr9.openai.azure.com/": {"gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano", "o3"},
|
||||
"https://t2vgoaigpt4o6.openai.azure.com/": {"gpt-5.5", "gpt-4o-mini", "o3", "o4-mini"},
|
||||
"https://oaidr21.openai.azure.com/": {"gpt-5.5", "o3", "o4-mini"},
|
||||
"https://searchagent5.cognitiveservices.azure.com/": {"gpt-5.4-mini", "gpt-4o-mini"},
|
||||
"https://t2vgoaigpt4o.openai.azure.com/": {"gpt-5.4", "gpt-5.4-nano", "gpt-5.2", "gpt-5.1", "o3", "o4-mini"},
|
||||
}
|
||||
_AZURE_MI_CLIENT_ID = "8cafa2b1-a2a7-4ad9-814a-ffe4aed7e800"
|
||||
|
||||
|
||||
class AzureOpenAIBackend(CliBackend):
|
||||
"""Drives Azure OpenAI gpt-5.x deployments via managed identity.
|
||||
|
||||
Mirrors the intern's blog_1 setup (avail_api.md): managed-identity auth, the
|
||||
same endpoints/deployments. Reuses CliBackend's attempt/judge/reflect prompts
|
||||
and JSON parsing; only _call() differs. openai + azure-identity are lazy
|
||||
imported so the mock/CLI paths stay dependency-free.
|
||||
"""
|
||||
|
||||
name = "azure"
|
||||
|
||||
def __init__(self, deployment: str = "", endpoint: str = "", timeout: int = 180,
|
||||
api_version: str = "2024-12-01-preview") -> None:
|
||||
super().__init__(model=deployment or "gpt-5.5", timeout=timeout)
|
||||
self.deployment = deployment or "gpt-5.5"
|
||||
self.endpoint = endpoint or self._endpoint_for(self.deployment)
|
||||
self.api_version = api_version
|
||||
self.name = f"azure:{self.deployment}"
|
||||
self._client = None
|
||||
|
||||
@staticmethod
|
||||
def _endpoint_for(deployment: str) -> str:
|
||||
for ep, deps in _AZURE_ENDPOINTS.items():
|
||||
if deployment in deps:
|
||||
return ep
|
||||
return "https://oaidr9.openai.azure.com/"
|
||||
|
||||
def _get_client(self):
|
||||
if self._client is None:
|
||||
from azure.identity import ManagedIdentityCredential, get_bearer_token_provider
|
||||
from openai import AzureOpenAI
|
||||
cred = ManagedIdentityCredential(client_id=_AZURE_MI_CLIENT_ID)
|
||||
tp = get_bearer_token_provider(cred, "https://cognitiveservices.azure.com/.default")
|
||||
self._client = AzureOpenAI(
|
||||
azure_endpoint=self.endpoint, azure_ad_token_provider=tp,
|
||||
api_version=self.api_version, max_retries=4,
|
||||
)
|
||||
return self._client
|
||||
|
||||
def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str:
|
||||
"""Call the deployment with bounded retries.
|
||||
|
||||
IMPORTANT: transient failures (429 rate-limit, timeouts, 5xx) must NOT be
|
||||
silently turned into an empty string — an empty response scores 0 and
|
||||
deflates every baseline/after measure. We retry with exponential backoff
|
||||
(mirroring the research repo's retries=5) and only return "" after the
|
||||
budget is exhausted. ``time``/``random`` are used for backoff; both are
|
||||
available here (this is library code, not a Workflow script sandbox).
|
||||
"""
|
||||
import random as _r
|
||||
import time as _t
|
||||
|
||||
client = self._get_client()
|
||||
last_exc = None
|
||||
for attempt in range(max(1, retries)):
|
||||
try:
|
||||
resp = client.chat.completions.create(
|
||||
model=self.deployment,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
max_completion_tokens=16384,
|
||||
)
|
||||
text = (resp.choices[0].message.content or "").strip()
|
||||
try:
|
||||
u = resp.usage
|
||||
self._tokens += (getattr(u, "prompt_tokens", 0) or 0) + (getattr(u, "completion_tokens", 0) or 0)
|
||||
except Exception:
|
||||
pass
|
||||
if text:
|
||||
return text
|
||||
# empty but no exception: model genuinely returned nothing — one
|
||||
# quick retry can help (reasoning models occasionally yield empty)
|
||||
last_exc = "empty-response"
|
||||
except Exception as e: # noqa: BLE001
|
||||
last_exc = e
|
||||
# backoff before next try (skip after the final attempt)
|
||||
if attempt < retries - 1:
|
||||
_t.sleep(min(8.0, (2 ** attempt) * 0.5) + _r.random() * 0.4)
|
||||
return ""
|
||||
|
||||
|
||||
class AzureResponsesBackend(AzureOpenAIBackend):
|
||||
"""gpt-5.x via the **Responses API** on the high-throughput gpt4v endpoints.
|
||||
|
||||
Differs from AzureOpenAIBackend in three ways, all required by the enhanced
|
||||
experiment:
|
||||
* Auth via ``AzureCliCredential`` (the logged-in user), not Managed Identity
|
||||
— the gpt4v-scus/swc accounts grant the data role to the CLI principal.
|
||||
* Calls ``client.responses.create`` (the /responses API) instead of
|
||||
chat.completions — these deployments are Responses-only.
|
||||
* Round-robins across multiple endpoints for parallel throughput; each
|
||||
worker thread binds a client for one endpoint (picked by thread index)
|
||||
so concurrent replay spreads load across all endpoints.
|
||||
|
||||
A single shared ``AzureCliCredential`` token provider is reused across all
|
||||
endpoint clients (the token is cached + auto-refreshed by the provider).
|
||||
"""
|
||||
|
||||
name = "azure-responses"
|
||||
|
||||
# the two parallel /responses endpoints (user-provided), both hosting gpt-5.5
|
||||
_RESP_ENDPOINTS = [
|
||||
"https://gpt4v-scus.openai.azure.com/",
|
||||
"https://gpt4v-swc.openai.azure.com/",
|
||||
]
|
||||
|
||||
def __init__(self, deployment: str = "", endpoints: Optional[List[str]] = None,
|
||||
timeout: int = 180, api_version: str = "2025-04-01-preview") -> None:
|
||||
super().__init__(deployment=deployment, endpoint=(endpoints or self._RESP_ENDPOINTS)[0],
|
||||
timeout=timeout, api_version=api_version)
|
||||
self.endpoints = list(endpoints or self._RESP_ENDPOINTS)
|
||||
self.name = f"azure-responses:{self.deployment}"
|
||||
self._token_provider = None
|
||||
self._clients: dict = {} # endpoint -> AzureOpenAI client
|
||||
import threading as _thr
|
||||
self._lock = _thr.Lock()
|
||||
self._rr = 0 # round-robin counter
|
||||
|
||||
def _get_provider(self):
|
||||
if self._token_provider is None:
|
||||
from azure.identity import AzureCliCredential, get_bearer_token_provider
|
||||
self._token_provider = get_bearer_token_provider(
|
||||
AzureCliCredential(), "https://cognitiveservices.azure.com/.default")
|
||||
return self._token_provider
|
||||
|
||||
def _client_for(self, endpoint: str):
|
||||
cl = self._clients.get(endpoint)
|
||||
if cl is None:
|
||||
from openai import AzureOpenAI
|
||||
cl = AzureOpenAI(
|
||||
azure_endpoint=endpoint, azure_ad_token_provider=self._get_provider(),
|
||||
api_version=self.api_version, max_retries=2,
|
||||
)
|
||||
self._clients[endpoint] = cl
|
||||
return cl
|
||||
|
||||
def _next_endpoint(self) -> str:
|
||||
# round-robin so concurrent calls spread across all endpoints
|
||||
with self._lock:
|
||||
ep = self.endpoints[self._rr % len(self.endpoints)]
|
||||
self._rr += 1
|
||||
return ep
|
||||
|
||||
def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str:
|
||||
import random as _r
|
||||
import time as _t
|
||||
last = None
|
||||
base_ep = self._next_endpoint() # this call's primary endpoint
|
||||
base_idx = self.endpoints.index(base_ep)
|
||||
for attempt in range(max(1, retries)):
|
||||
# on retry, fail over to the other endpoint(s)
|
||||
ep = self.endpoints[(base_idx + attempt) % len(self.endpoints)]
|
||||
try:
|
||||
client = self._client_for(ep)
|
||||
resp = client.responses.create(
|
||||
model=self.deployment, input=prompt,
|
||||
max_output_tokens=16384,
|
||||
)
|
||||
text = (getattr(resp, "output_text", "") or "").strip()
|
||||
try:
|
||||
u = resp.usage
|
||||
self._tokens += (getattr(u, "input_tokens", 0) or 0) + (getattr(u, "output_tokens", 0) or 0)
|
||||
except Exception:
|
||||
pass
|
||||
if text:
|
||||
return text
|
||||
last = "empty-response"
|
||||
except Exception as e: # noqa: BLE001
|
||||
last = e
|
||||
if attempt < retries - 1:
|
||||
_t.sleep(min(8.0, (2 ** attempt) * 0.5) + _r.random() * 0.4)
|
||||
return ""
|
||||
|
||||
|
||||
def get_backend(
|
||||
name: str,
|
||||
*,
|
||||
model: str = "",
|
||||
claude_path: str = "claude",
|
||||
codex_path: str = "",
|
||||
azure_endpoint: str = "",
|
||||
) -> Backend:
|
||||
n = (name or "mock").strip().lower()
|
||||
if n in {"claude", "anthropic", "claude_cli", "claude_code"}:
|
||||
return ClaudeCliBackend(model=model, claude_path=claude_path)
|
||||
if n in {"codex", "codex_cli", "openai_codex"}:
|
||||
return CodexCliBackend(model=model, codex_path=codex_path)
|
||||
if n in {"azure", "azure_openai", "aoai"}:
|
||||
return AzureOpenAIBackend(deployment=model, endpoint=azure_endpoint)
|
||||
if n in {"azure-responses", "azure_responses", "aoai-responses", "responses"}:
|
||||
eps = [e.strip() for e in azure_endpoint.split(",") if e.strip()] or None
|
||||
return AzureResponsesBackend(deployment=model, endpoints=eps)
|
||||
return MockBackend()
|
||||
|
||||
|
||||
@@ -765,6 +1048,7 @@ def build_backend(
|
||||
target_backend: str = "",
|
||||
target_model: str = "",
|
||||
codex_path: str = "",
|
||||
azure_endpoint: str = "",
|
||||
preferences: str = "",
|
||||
) -> Backend:
|
||||
"""Build a single or dual backend.
|
||||
@@ -776,11 +1060,13 @@ def build_backend(
|
||||
"""
|
||||
has_split = any([optimizer_backend, optimizer_model, target_backend, target_model])
|
||||
if not has_split:
|
||||
be = get_backend(backend, model=model, codex_path=codex_path)
|
||||
be = get_backend(backend, model=model, codex_path=codex_path, azure_endpoint=azure_endpoint)
|
||||
be.preferences = preferences
|
||||
return be
|
||||
tgt = get_backend(target_backend or backend, model=target_model or model, codex_path=codex_path)
|
||||
opt = get_backend(optimizer_backend or backend, model=optimizer_model or model, codex_path=codex_path)
|
||||
tgt = get_backend(target_backend or backend, model=target_model or model,
|
||||
codex_path=codex_path, azure_endpoint=azure_endpoint)
|
||||
opt = get_backend(optimizer_backend or backend, model=optimizer_model or model,
|
||||
codex_path=codex_path, azure_endpoint=azure_endpoint)
|
||||
opt.preferences = preferences # reflect runs on the optimizer
|
||||
dual = DualBackend(target=tgt, optimizer=opt)
|
||||
dual.preferences = preferences
|
||||
|
||||
@@ -89,8 +89,15 @@ def consolidate(
|
||||
gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"}
|
||||
|
||||
# ── baseline on the VAL slice (the gate reference) ────────────────────
|
||||
base_pairs = replay_batch(backend, val_tasks, skill, memory)
|
||||
base_hard, base_soft = aggregate_scores(base_pairs)
|
||||
# When the gate is OFF the user has opted out of holding out a validation set
|
||||
# (the daily-use design): we accept edits greedily and judge quality only on
|
||||
# the real test set, scored by the caller. So we SKIP all val scoring — it is
|
||||
# both wasted cost and contrary to the "no val set required" design.
|
||||
if gate_off:
|
||||
base_hard, base_soft = 0.0, 0.0
|
||||
else:
|
||||
base_pairs = replay_batch(backend, val_tasks, skill, memory)
|
||||
base_hard, base_soft = aggregate_scores(base_pairs)
|
||||
base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
|
||||
|
||||
# ── reflect over TRAIN-split failures/successes ───────────────────────
|
||||
@@ -109,14 +116,17 @@ def consolidate(
|
||||
new_doc, applied = apply_edits(doc, edits)
|
||||
if not applied:
|
||||
return doc
|
||||
# score the candidate on the VAL slice
|
||||
# gate OFF: accept greedily with NO val scoring (the daily-use path)
|
||||
if gate_off:
|
||||
all_applied.extend(applied)
|
||||
return new_doc
|
||||
# gate ON: score the candidate on the VAL slice, keep only if it improves
|
||||
trial_skill = new_doc if which == "skill" else cand_skill
|
||||
trial_memory = new_doc if which == "memory" else cand_memory
|
||||
pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory)
|
||||
h, s = aggregate_scores(pairs)
|
||||
cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight)
|
||||
# gate OFF: accept greedily (no regression check); gate ON: strict improve
|
||||
if gate_off or cand_score > base_score:
|
||||
if cand_score > base_score:
|
||||
base_score = max(base_score, cand_score)
|
||||
all_applied.extend(applied)
|
||||
return new_doc
|
||||
@@ -128,8 +138,28 @@ def consolidate(
|
||||
# multi-rollout contrastive reflection: run each train task K times
|
||||
# and distill a rule from the good-vs-bad contrast (the imagination signal).
|
||||
from skillopt_sleep.rollout import multi_rollout, contrastive_reflect
|
||||
sets = [multi_rollout(backend, t, cand_skill, cand_memory, k=rollouts_k)
|
||||
for t in train_tasks]
|
||||
# Parallelize across tasks (each multi_rollout also parallelizes its K
|
||||
# attempts). This dream phase is the dominant cost; serial execution
|
||||
# times out on real backends. Cap total in-flight at the worker env.
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
try:
|
||||
_w = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1"))
|
||||
except ValueError:
|
||||
_w = 1
|
||||
if _w > 1 and len(train_tasks) > 1:
|
||||
# split the worker budget between task-parallelism and per-task K
|
||||
task_workers = max(1, min(len(train_tasks), _w))
|
||||
per_task = max(1, _w // task_workers)
|
||||
with ThreadPoolExecutor(max_workers=task_workers) as ex:
|
||||
sets = list(ex.map(
|
||||
lambda t: multi_rollout(backend, t, cand_skill, cand_memory,
|
||||
k=rollouts_k, workers=per_task),
|
||||
train_tasks))
|
||||
else:
|
||||
sets = [multi_rollout(backend, t, cand_skill, cand_memory,
|
||||
k=rollouts_k, workers=1)
|
||||
for t in train_tasks]
|
||||
edits = contrastive_reflect(
|
||||
backend, sets, cand_skill, cand_memory,
|
||||
edit_budget=edit_budget, target="skill",
|
||||
@@ -158,40 +188,41 @@ def consolidate(
|
||||
)
|
||||
cand_memory = _gate_apply(cand_memory, edits_m, "memory")
|
||||
|
||||
# ── final decision, scored on the VAL slice ───────────────────────────
|
||||
final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory)
|
||||
final_hard, final_soft = aggregate_scores(final_pairs)
|
||||
final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight)
|
||||
base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
|
||||
|
||||
# ── final decision ────────────────────────────────────────────────────
|
||||
if gate_off:
|
||||
# greedy mode: keep whatever edits we applied; report quality movement
|
||||
# greedy mode: no val scoring at all. Keep whatever edits we applied; the
|
||||
# caller measures real quality on the test set. We report holdout_candidate
|
||||
# as 0.0 (val intentionally not computed in this variant).
|
||||
final_hard, final_soft = 0.0, 0.0
|
||||
final_score = 0.0
|
||||
accepted = bool(all_applied)
|
||||
if final_score > base_gate_score:
|
||||
action = "greedy_improved"
|
||||
elif final_score < base_gate_score:
|
||||
action = "greedy_regressed"
|
||||
else:
|
||||
action = "greedy_flat" if all_applied else "greedy_noop"
|
||||
elif _HAVE_REPO_GATE:
|
||||
gate = evaluate_gate(
|
||||
candidate_skill=cand_skill,
|
||||
cand_hard=final_hard,
|
||||
current_skill=skill,
|
||||
current_score=base_gate_score,
|
||||
best_skill=skill,
|
||||
best_score=base_gate_score,
|
||||
best_step=night - 1,
|
||||
global_step=night,
|
||||
cand_soft=final_soft,
|
||||
metric=gate_metric,
|
||||
mixed_weight=gate_mixed_weight,
|
||||
)
|
||||
action = gate.action
|
||||
accepted = bool(all_applied) and final_score > base_gate_score
|
||||
action = "greedy_applied" if all_applied else "greedy_noop"
|
||||
base_gate_score = 0.0
|
||||
else:
|
||||
action = "accept" if final_score > base_gate_score else "reject"
|
||||
accepted = bool(all_applied) and final_score > base_gate_score
|
||||
# scored on the VAL slice (the gate reference)
|
||||
final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory)
|
||||
final_hard, final_soft = aggregate_scores(final_pairs)
|
||||
final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight)
|
||||
base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
|
||||
if _HAVE_REPO_GATE:
|
||||
gate = evaluate_gate(
|
||||
candidate_skill=cand_skill,
|
||||
cand_hard=final_hard,
|
||||
current_skill=skill,
|
||||
current_score=base_gate_score,
|
||||
best_skill=skill,
|
||||
best_score=base_gate_score,
|
||||
best_step=night - 1,
|
||||
global_step=night,
|
||||
cand_soft=final_soft,
|
||||
metric=gate_metric,
|
||||
mixed_weight=gate_mixed_weight,
|
||||
)
|
||||
action = gate.action
|
||||
accepted = bool(all_applied) and final_score > base_gate_score
|
||||
else:
|
||||
action = "accept" if final_score > base_gate_score else "reject"
|
||||
accepted = bool(all_applied) and final_score > base_gate_score
|
||||
|
||||
return ConsolidationResult(
|
||||
accepted=accepted,
|
||||
|
||||
@@ -26,7 +26,11 @@ def _required_tools(task: TaskRecord) -> List[str]:
|
||||
return tools
|
||||
|
||||
|
||||
def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str) -> ReplayResult:
|
||||
def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str,
|
||||
sample_id: int = 0) -> ReplayResult:
|
||||
"""``sample_id`` distinguishes repeated dream rollouts of the same
|
||||
(task, skill, memory) in the attempt cache — without it all K rollouts
|
||||
collapse to one cached response and the contrastive signal is always 0."""
|
||||
import time
|
||||
tools = _required_tools(task)
|
||||
tools_called: List[str] = []
|
||||
@@ -35,7 +39,7 @@ def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str) -> R
|
||||
if tools:
|
||||
response, tools_called = backend.attempt_with_tools(task, skill, memory, tools)
|
||||
else:
|
||||
response = backend.attempt(task, skill, memory)
|
||||
response = backend.attempt(task, skill, memory, sample_id=sample_id)
|
||||
latency_ms = (time.time() - t0) * 1000.0
|
||||
tokens = max(0, backend.tokens_used() - tok_before)
|
||||
# if the backend doesn't track tokens (e.g. mock), approximate from text length
|
||||
@@ -63,13 +67,37 @@ def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str) -> R
|
||||
)
|
||||
|
||||
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
def replay_batch(
|
||||
backend: Backend,
|
||||
tasks: List[TaskRecord],
|
||||
skill: str,
|
||||
memory: str,
|
||||
*,
|
||||
workers: int = 0,
|
||||
) -> List[Tuple[TaskRecord, ReplayResult]]:
|
||||
return [(t, replay_one(backend, t, skill, memory)) for t in tasks]
|
||||
"""Replay tasks, optionally in parallel.
|
||||
|
||||
Real backends are network-bound, so a thread pool gives a large speedup on
|
||||
big test sets (like the research harness's --workers). ``workers`` defaults
|
||||
to env SKILLOPT_SLEEP_WORKERS or 1 (sequential). Mock stays sequential
|
||||
(deterministic) unless asked otherwise.
|
||||
"""
|
||||
if workers <= 0:
|
||||
workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1") or "1")
|
||||
if workers <= 1 or len(tasks) <= 1:
|
||||
return [(t, replay_one(backend, t, skill, memory)) for t in tasks]
|
||||
results: List = [None] * len(tasks)
|
||||
with ThreadPoolExecutor(max_workers=min(workers, len(tasks))) as ex:
|
||||
futs = {ex.submit(replay_one, backend, t, skill, memory): i
|
||||
for i, t in enumerate(tasks)}
|
||||
for fut in futs:
|
||||
i = futs[fut]
|
||||
results[i] = (tasks[i], fut.result())
|
||||
return results
|
||||
|
||||
|
||||
def aggregate_scores(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]:
|
||||
|
||||
@@ -58,12 +58,34 @@ def multi_rollout(
|
||||
memory: str,
|
||||
*,
|
||||
k: int = 3,
|
||||
workers: int = 0,
|
||||
) -> RolloutSet:
|
||||
"""Run ``task`` K times. replay_one is deterministic for mock; for real
|
||||
backends the model's own sampling yields variation across attempts."""
|
||||
backends the model's own sampling yields variation across attempts.
|
||||
|
||||
The K attempts are independent, so they run concurrently (this is the dream
|
||||
phase's dominant cost). ``workers`` defaults to the SKILLOPT_SLEEP_WORKERS
|
||||
env (capped at k); set to 1 to force serial (used by the mock tests).
|
||||
"""
|
||||
import os
|
||||
rs = RolloutSet(task=task)
|
||||
for _ in range(max(1, k)):
|
||||
rs.attempts.append(replay_one(backend, task, skill, memory))
|
||||
k = max(1, k)
|
||||
if workers <= 0:
|
||||
try:
|
||||
workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1"))
|
||||
except ValueError:
|
||||
workers = 1
|
||||
workers = max(1, min(workers, k))
|
||||
if workers == 1:
|
||||
for i in range(k):
|
||||
rs.attempts.append(replay_one(backend, task, skill, memory, sample_id=i))
|
||||
return rs
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
futs = [ex.submit(replay_one, backend, task, skill, memory, sample_id=i)
|
||||
for i in range(k)]
|
||||
for f in futs:
|
||||
rs.attempts.append(f.result())
|
||||
return rs
|
||||
|
||||
|
||||
@@ -97,6 +119,11 @@ def contrastive_reflect(
|
||||
f"- BAD attempt (score {rs.worst.hard:.1f}): {rs.worst.response[:200]}\n"
|
||||
f" (bad failed: {rs.worst.fail_reason[:100]})"
|
||||
)
|
||||
# the output contract the proposed rules must not violate (same guardrail the
|
||||
# single-shot reflect uses — prevents harness-violating rules like "return VBA"
|
||||
# or "ask the user for the range" on SpreadsheetBench).
|
||||
from skillopt_sleep.backend import _task_guardrail
|
||||
guard = _task_guardrail([(rs.task, rs.best) for rs in informative])
|
||||
prompt = (
|
||||
"You are SkillOpt's optimizer doing CONTRASTIVE reflection. For each task "
|
||||
"below the agent was run multiple times; some attempts succeeded and some "
|
||||
@@ -104,6 +131,10 @@ def contrastive_reflect(
|
||||
f"and propose at most {edit_budget} SHORT, GENERAL, reusable rules for the "
|
||||
f"{target} that would make the good behavior reliable every time. Quote "
|
||||
"concrete thresholds/formats verbatim; do not paraphrase vaguely. "
|
||||
"Every rule MUST obey the task output contract (if shown) — never propose "
|
||||
"a rule that changes the required output format/language or tells the agent "
|
||||
"to ask the user a question; such a rule scores ZERO.\n"
|
||||
f"{guard}"
|
||||
'Return ONLY a JSON array: '
|
||||
'[{"op":"add","content":"<rule>","rationale":"<what good did that bad didnt>"}].\n\n'
|
||||
+ "\n\n".join(blocks)
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""SkillOpt-Sleep — built-in nightly scheduler.
|
||||
|
||||
Installs/removes a crontab entry that runs the sleep cycle automatically, so the
|
||||
user doesn't have to wire cron themselves. Idempotent: a managed block delimited
|
||||
by marker comments is added/replaced/removed in the user's crontab.
|
||||
|
||||
Design choices:
|
||||
* Off-:00 minute (3:17 local by default) so many users don't all hit the API
|
||||
at the same instant.
|
||||
* The entry runs `python -m skillopt_sleep run` for a specific project and
|
||||
appends to <project>/.skillopt-sleep/cron.log.
|
||||
* `schedule` is additive per project (keyed by project path); `unschedule`
|
||||
removes the project's line (or the whole managed block with --all).
|
||||
|
||||
cron is the portable mechanism on Linux/macOS. On systems without `crontab`,
|
||||
`schedule` prints the line and instructions instead of failing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
_BEGIN = "# >>> skillopt-sleep (managed) >>>"
|
||||
_END = "# <<< skillopt-sleep (managed) <<<"
|
||||
|
||||
|
||||
def _have_crontab() -> bool:
|
||||
return shutil.which("crontab") is not None
|
||||
|
||||
|
||||
def _read_crontab() -> str:
|
||||
try:
|
||||
proc = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
|
||||
return proc.stdout if proc.returncode == 0 else ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _write_crontab(content: str) -> bool:
|
||||
try:
|
||||
proc = subprocess.run(["crontab", "-"], input=content, text=True,
|
||||
capture_output=True)
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _split_managed(crontab: str) -> Tuple[str, List[str]]:
|
||||
"""Return (text_outside_block, managed_lines_inside_block)."""
|
||||
lines = crontab.splitlines()
|
||||
outside: List[str] = []
|
||||
managed: List[str] = []
|
||||
in_block = False
|
||||
for ln in lines:
|
||||
if ln.strip() == _BEGIN:
|
||||
in_block = True
|
||||
continue
|
||||
if ln.strip() == _END:
|
||||
in_block = False
|
||||
continue
|
||||
(managed if in_block else outside).append(ln)
|
||||
return "\n".join(outside).rstrip(), managed
|
||||
|
||||
|
||||
def _runner_cmd(project: str, backend: str, extra: str, python: str) -> str:
|
||||
logdir = os.path.join(project, ".skillopt-sleep")
|
||||
log = os.path.join(logdir, "cron.log")
|
||||
# use absolute python + -m so cron's minimal env still works
|
||||
cmd = (f'{python} -m skillopt_sleep run --project "{project}" '
|
||||
f'--scope invoked --backend {backend} {extra}'.rstrip())
|
||||
return f'mkdir -p "{logdir}"; cd "{_repo_root()}" && {cmd} >> "{log}" 2>&1'
|
||||
|
||||
|
||||
def _repo_root() -> str:
|
||||
# the package lives at <repo>/skillopt_sleep/; repo root is its parent
|
||||
return os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
|
||||
def _project_marker(project: str) -> str:
|
||||
return f"# project={os.path.abspath(project)}"
|
||||
|
||||
|
||||
def schedule(project: str, *, backend: str = "mock", hour: int = 3, minute: int = 17,
|
||||
extra: str = "", python: Optional[str] = None) -> Tuple[bool, str]:
|
||||
"""Install (or replace) the nightly entry for ``project``.
|
||||
|
||||
Returns (installed, message). If crontab is unavailable, installed=False and
|
||||
the message contains the line to add manually.
|
||||
"""
|
||||
project = os.path.abspath(project)
|
||||
python = python or sys.executable or "python3"
|
||||
cron_line = f"{minute} {hour} * * * {_runner_cmd(project, backend, extra, python)} {_project_marker(project)}"
|
||||
|
||||
if not _have_crontab():
|
||||
return False, ("crontab not found on this system. Add this line to your "
|
||||
"scheduler manually:\n" + cron_line)
|
||||
|
||||
outside, managed = _split_managed(_read_crontab())
|
||||
# drop any existing line for this project, then add the new one
|
||||
marker = _project_marker(project)
|
||||
managed = [ln for ln in managed if marker not in ln and ln.strip()]
|
||||
managed.append(cron_line)
|
||||
|
||||
block = _BEGIN + "\n" + "\n".join(managed) + "\n" + _END
|
||||
new_crontab = (outside + "\n\n" + block + "\n").lstrip("\n")
|
||||
ok = _write_crontab(new_crontab)
|
||||
if ok:
|
||||
return True, (f"Scheduled nightly at {hour:02d}:{minute:02d} for {project} "
|
||||
f"(backend={backend}). Logs -> {project}/.skillopt-sleep/cron.log\n"
|
||||
f"Runs `skillopt_sleep run`; it only STAGES a proposal — adopt is still manual.")
|
||||
return False, "Failed to write crontab. Line to add manually:\n" + cron_line
|
||||
|
||||
|
||||
def unschedule(project: Optional[str] = None, *, all_projects: bool = False) -> Tuple[bool, str]:
|
||||
"""Remove the entry for ``project`` (or the whole managed block with all_projects)."""
|
||||
if not _have_crontab():
|
||||
return False, "crontab not found; nothing to remove."
|
||||
outside, managed = _split_managed(_read_crontab())
|
||||
if all_projects:
|
||||
managed = []
|
||||
elif project:
|
||||
marker = _project_marker(project)
|
||||
managed = [ln for ln in managed if marker not in ln and ln.strip()]
|
||||
if managed:
|
||||
block = _BEGIN + "\n" + "\n".join(managed) + "\n" + _END
|
||||
new_crontab = (outside + "\n\n" + block + "\n").lstrip("\n")
|
||||
else:
|
||||
new_crontab = outside.rstrip() + "\n"
|
||||
ok = _write_crontab(new_crontab)
|
||||
return ok, ("Removed." if ok else "Failed to update crontab.")
|
||||
|
||||
|
||||
def list_scheduled() -> List[str]:
|
||||
_outside, managed = _split_managed(_read_crontab())
|
||||
return [ln for ln in managed if ln.strip()]
|
||||
@@ -54,6 +54,12 @@ class TaskRecord:
|
||||
project: str
|
||||
intent: str # what the user wanted (the "question")
|
||||
context_excerpt: str = "" # minimal context needed to attempt it
|
||||
# Optional system framing for the rollout. When set (e.g. real benchmarks
|
||||
# carrying the research repo's exact rollout_system), the backend uses THIS
|
||||
# verbatim instead of its generic instruction wrapper — this keeps scoring
|
||||
# faithful to the source task and avoids re-deriving framing the benchmark
|
||||
# already bakes in.
|
||||
system: str = ""
|
||||
attempted_solution: str = "" # what the agent produced before
|
||||
outcome: str = "unknown" # success | fail | mixed | unknown
|
||||
reference_kind: str = "none" # exact | rubric | rule | none
|
||||
|
||||
Reference in New Issue
Block a user