SkillOpt v0.1.0: initial release
- Skill optimization framework with training loop analogy - 11 benchmarks, 4 model backends (Azure OpenAI, Claude, Codex, Qwen) - WebUI for browser-based training control - Pluggable architecture for extending benchmarks and backends
This commit is contained in:
CharlesYang030
@@ -0,0 +1,15 @@
|
||||
"""ReflACT Optimizer -- skill update operations.
|
||||
|
||||
Analogous to the optimizer in neural network training: applies the computed
|
||||
"gradient" (patches) to the current skill document to produce an updated
|
||||
candidate skill.
|
||||
|
||||
Modules
|
||||
-------
|
||||
- skill: edit application (optimizer.step() / parameter update)
|
||||
- clip: edit ranking and selection (gradient clipping)
|
||||
- meta_reflect: epoch-level macro refinement (momentum)
|
||||
- slow_update: longitudinal comparison and guidance (EMA / regularization)
|
||||
"""
|
||||
from skillopt.optimizer.skill import apply_edit, apply_patch # noqa: F401
|
||||
from skillopt.optimizer.clip import rank_and_select # noqa: F401
|
||||
@@ -0,0 +1,109 @@
|
||||
"""ReflACT gradient clipping — LLM-driven edit ranking and selection.
|
||||
|
||||
Analogous to gradient clipping in neural network training: ranks candidate
|
||||
edits by importance and selects the top-L to apply, controlling the
|
||||
effective step size. Previously core/select.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.optimizer.meta_skill import format_meta_skill_context
|
||||
from skillopt.optimizer.update_modes import (
|
||||
describe_item,
|
||||
get_payload_items,
|
||||
is_rewrite_mode,
|
||||
normalize_update_mode,
|
||||
payload_key,
|
||||
payload_label,
|
||||
)
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.utils import extract_json
|
||||
|
||||
|
||||
# ── Public API ────────────────────────────────────────────────────────────────
|
||||
|
||||
def rank_and_select(
|
||||
skill_content: str,
|
||||
patch: dict,
|
||||
max_edits: int,
|
||||
meta_skill_context: str = "",
|
||||
update_mode: str = "patch",
|
||||
) -> dict:
|
||||
"""Use a teacher LLM to rank edits by importance, then keep top-L.
|
||||
|
||||
If the edit pool is within budget, returns the patch unchanged.
|
||||
Otherwise, calls the teacher to rank and select the most impactful edits.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
skill_content : str
|
||||
Current skill document.
|
||||
patch : dict
|
||||
Merged :class:`~skillopt.types.Patch` dict with ``edits`` list.
|
||||
max_edits : int
|
||||
Maximum number of edits to keep (the "edit budget").
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
:class:`~skillopt.types.Patch` dict with selected edits and
|
||||
optional ``ranking_details``.
|
||||
"""
|
||||
update_mode = normalize_update_mode(update_mode)
|
||||
edits = get_payload_items(patch, update_mode)
|
||||
if len(edits) <= max_edits:
|
||||
return patch
|
||||
|
||||
# Build the edit pool description for the teacher
|
||||
edits_desc = []
|
||||
for i, edit in enumerate(edits):
|
||||
edits_desc.append(f"[{i}] {describe_item(edit, update_mode, max_chars=500)}")
|
||||
|
||||
user = (
|
||||
f"## Current Skill\n{skill_content}\n\n"
|
||||
f"## {payload_label(update_mode, title=True)} Pool ({len(edits)} {payload_label(update_mode)}, budget={max_edits})\n"
|
||||
+ "\n".join(edits_desc)
|
||||
+ f"\n\nSelect the {max_edits} most important {payload_label(update_mode)}. "
|
||||
f"Return their 0-based indices in priority order."
|
||||
)
|
||||
teacher_ctx = format_meta_skill_context(meta_skill_context)
|
||||
if teacher_ctx:
|
||||
user = f"{teacher_ctx}\n\n{user}"
|
||||
prompt_name = "ranking_rewrite" if is_rewrite_mode(update_mode) else "ranking"
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
system=load_prompt(prompt_name), user=user,
|
||||
max_completion_tokens=2048, retries=3, stage="ranking",
|
||||
)
|
||||
result = extract_json(response)
|
||||
if result and "selected_indices" in result:
|
||||
indices = result["selected_indices"]
|
||||
selected = []
|
||||
seen: set[int] = set()
|
||||
for idx in indices:
|
||||
if (
|
||||
isinstance(idx, int)
|
||||
and 0 <= idx < len(edits)
|
||||
and idx not in seen
|
||||
):
|
||||
selected.append(edits[idx])
|
||||
seen.add(idx)
|
||||
if len(selected) >= max_edits:
|
||||
break
|
||||
if selected:
|
||||
return {
|
||||
"reasoning": patch.get("reasoning", "")
|
||||
+ f" [teacher-ranked: selected {len(selected)}/{len(edits)} {payload_label(update_mode)}]",
|
||||
payload_key(update_mode): selected,
|
||||
"ranking_details": result,
|
||||
}
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
# Fallback: simple truncation
|
||||
return {
|
||||
"reasoning": patch.get("reasoning", "")
|
||||
+ f" [fallback truncated {len(edits)}->{max_edits} {payload_label(update_mode)}]",
|
||||
payload_key(update_mode): edits[:max_edits],
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Teacher-driven autonomous update-size decisions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.optimizer.meta_skill import format_meta_skill_context
|
||||
from skillopt.optimizer.update_modes import describe_item, get_payload_items, payload_label
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.utils import extract_json
|
||||
|
||||
|
||||
def _coerce_nonnegative_int(value: Any) -> int | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return max(0, value)
|
||||
if isinstance(value, float) and value.is_integer():
|
||||
return max(0, int(value))
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
match = re.search(r"-?\d+", text)
|
||||
if not match:
|
||||
return None
|
||||
return max(0, int(match.group(0)))
|
||||
|
||||
|
||||
def decide_autonomous_learning_rate(
|
||||
*,
|
||||
skill_content: str,
|
||||
merged_patch: dict,
|
||||
update_mode: str,
|
||||
rollout_hard: float,
|
||||
rollout_soft: float,
|
||||
rollout_n: int,
|
||||
step_buffer_context: str = "",
|
||||
meta_skill_context: str = "",
|
||||
) -> dict:
|
||||
"""Ask the teacher to choose the number of update items for this step.
|
||||
|
||||
The prompt intentionally avoids default budgets, candidate budget lists, or
|
||||
scheduler history. The only hard post-processing is validity: the returned
|
||||
integer is clamped to the available item count.
|
||||
"""
|
||||
items = get_payload_items(merged_patch, update_mode)
|
||||
available = len(items)
|
||||
item_lines = [
|
||||
f"[{idx}] {describe_item(item, update_mode, max_chars=700)}"
|
||||
for idx, item in enumerate(items)
|
||||
]
|
||||
user = (
|
||||
f"## Current Skill\n{skill_content}\n\n"
|
||||
f"## Current Step Evidence\n"
|
||||
f"rollout_n={rollout_n}\n"
|
||||
f"rollout_hard={rollout_hard:.6f}\n"
|
||||
f"rollout_soft={rollout_soft:.6f}\n"
|
||||
f"proposed_update_items={available}\n"
|
||||
f"update_item_type={payload_label(update_mode)}\n\n"
|
||||
f"## Proposed Update Items\n"
|
||||
+ "\n".join(item_lines)
|
||||
+ "\n\nDecide how many proposed update items should be applied now."
|
||||
)
|
||||
if step_buffer_context.strip():
|
||||
user += f"\n\n## Previous Steps in This Epoch\n{step_buffer_context}"
|
||||
teacher_ctx = format_meta_skill_context(meta_skill_context)
|
||||
if teacher_ctx:
|
||||
user = f"{teacher_ctx}\n\n{user}"
|
||||
|
||||
response = ""
|
||||
parsed: dict | None = None
|
||||
decision: int | None = None
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
system=load_prompt("lr_autonomous"),
|
||||
user=user,
|
||||
max_completion_tokens=2048,
|
||||
retries=3,
|
||||
stage="lr_autonomous",
|
||||
)
|
||||
parsed = extract_json(response)
|
||||
if parsed:
|
||||
decision = _coerce_nonnegative_int(parsed.get("learning_rate"))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
parsed = {"error": str(exc)}
|
||||
|
||||
fallback = False
|
||||
if decision is None:
|
||||
decision = 0
|
||||
fallback = True
|
||||
|
||||
chosen = min(decision, available)
|
||||
record = {
|
||||
"learning_rate": chosen,
|
||||
"raw_learning_rate": decision,
|
||||
"available_update_items": available,
|
||||
"clamped": chosen != decision,
|
||||
"fallback": fallback,
|
||||
"reasoning": (parsed or {}).get("reasoning", ""),
|
||||
"confidence": (parsed or {}).get("confidence", ""),
|
||||
"risk_notes": (parsed or {}).get("risk_notes", []),
|
||||
"raw_response": response,
|
||||
}
|
||||
if parsed and "error" in parsed:
|
||||
record["error"] = parsed["error"]
|
||||
return record
|
||||
@@ -0,0 +1,198 @@
|
||||
"""ReflACT Meta-Reflect — epoch-level skill refinement with momentum.
|
||||
|
||||
After each epoch, the meta-reflect stage reviews the epoch's step history
|
||||
(applied edits + gate scores) and performs high-level skill edits:
|
||||
merging redundant rules, removing ineffective ones, and distilling
|
||||
cross-step strategic patterns.
|
||||
|
||||
This is analogous to momentum in neural network optimization:
|
||||
- Fast update (per step): analyst edits fix local issues from current batch
|
||||
- Slow update (per epoch): meta-reflect refines the skill based on what
|
||||
worked and what didn't across the full epoch
|
||||
|
||||
The meta-reflect also maintains a ``meta_summary`` — a compact memory
|
||||
passed between epochs that captures directional insights (which editing
|
||||
directions are effective, which are not). This is the "momentum buffer".
|
||||
|
||||
Public API
|
||||
----------
|
||||
- :func:`build_epoch_history` — format an epoch's step records for meta-reflect
|
||||
- :func:`run_meta_reflect` — one teacher call to produce high-level edits + meta_summary
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import traceback
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.optimizer.update_modes import (
|
||||
describe_item,
|
||||
get_payload_items,
|
||||
normalize_update_mode,
|
||||
payload_label,
|
||||
truncate_payload,
|
||||
)
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.utils import extract_json
|
||||
|
||||
|
||||
# ── Epoch history formatting ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_epoch_history(
|
||||
epoch_step_records: list[dict],
|
||||
out_root: str,
|
||||
*,
|
||||
update_mode: str = "patch",
|
||||
) -> str:
|
||||
"""Format an epoch's step records into text for the meta-reflect teacher.
|
||||
|
||||
For each step, includes the exact edits applied (read from
|
||||
``ranked_edits.json``) and the gate evaluation result.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
epoch_step_records : list[dict]
|
||||
Step record dicts from ``history.json`` belonging to this epoch.
|
||||
out_root : str
|
||||
Training output root directory (to locate ``ranked_edits.json``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Formatted epoch history text.
|
||||
"""
|
||||
update_mode = normalize_update_mode(update_mode)
|
||||
parts: list[str] = []
|
||||
for rec in epoch_step_records:
|
||||
step = rec["step"]
|
||||
action = rec.get("action", "unknown")
|
||||
gate_score = rec.get("selection_hard", rec.get("current_score", "?"))
|
||||
best_score = rec.get("best_score", "?")
|
||||
|
||||
header = (
|
||||
f"### Step {step} — "
|
||||
f"gate: {gate_score}, {action.upper()}, "
|
||||
f"best_so_far: {best_score}"
|
||||
)
|
||||
|
||||
# Read the actual applied edits
|
||||
ranked_path = os.path.join(
|
||||
out_root, "steps", f"step_{step:04d}", "ranked_edits.json",
|
||||
)
|
||||
edits_text = ""
|
||||
if os.path.exists(ranked_path):
|
||||
try:
|
||||
with open(ranked_path) as f:
|
||||
ranked = json.load(f)
|
||||
edits = get_payload_items(ranked, update_mode)
|
||||
if edits:
|
||||
lines = [f"Selected {payload_label(update_mode)}:"]
|
||||
for i, edit in enumerate(edits, 1):
|
||||
lines.append(f" {i}. {describe_item(edit, update_mode, max_chars=220)}")
|
||||
edits_text = "\n".join(lines)
|
||||
else:
|
||||
edits_text = f"Selected {payload_label(update_mode)}: (none)"
|
||||
except Exception:
|
||||
edits_text = f"Selected {payload_label(update_mode)}: (could not read)"
|
||||
else:
|
||||
# Step may have been skipped
|
||||
if "skip" in action:
|
||||
edits_text = f"Selected {payload_label(update_mode)}: (skipped)"
|
||||
else:
|
||||
edits_text = f"Selected {payload_label(update_mode)}: (file not found)"
|
||||
|
||||
parts.append(f"{header}\n{edits_text}")
|
||||
|
||||
# Append trajectory failure digest if available
|
||||
digest_path = os.path.join(
|
||||
out_root, "steps", f"step_{step:04d}", "trajectory_digest.json",
|
||||
)
|
||||
if os.path.exists(digest_path):
|
||||
try:
|
||||
with open(digest_path) as f:
|
||||
digest = json.load(f)
|
||||
patterns = digest.get("failure_patterns", [])
|
||||
if patterns:
|
||||
n_fail = digest.get("n_fail", "?")
|
||||
n_total = digest.get("n_total", "?")
|
||||
lines = [f"Failure patterns ({n_fail}/{n_total} tasks failed):"]
|
||||
for p in patterns:
|
||||
lines.append(
|
||||
f' - "{p["pattern"]}" (×{p["count"]})'
|
||||
)
|
||||
parts[-1] += "\n" + "\n".join(lines)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
# ── Meta-reflect teacher call ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def run_meta_reflect(
|
||||
skill_content: str,
|
||||
epoch_history_text: str,
|
||||
prev_meta_summary: str,
|
||||
meta_edit_budget: int = 4,
|
||||
*,
|
||||
system_prompt: str | None = None,
|
||||
update_mode: str = "patch",
|
||||
) -> dict | None:
|
||||
"""Run one meta-reflect teacher call for an epoch.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
skill_content : str
|
||||
Current skill document (after the epoch's fast updates).
|
||||
epoch_history_text : str
|
||||
Formatted epoch history from :func:`build_epoch_history`.
|
||||
prev_meta_summary : str
|
||||
Meta summary from the previous epoch ("" if first epoch).
|
||||
meta_edit_budget : int
|
||||
Maximum number of high-level edits.
|
||||
system_prompt : str | None
|
||||
Custom system prompt. ``None`` = use generic default.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict | None
|
||||
Conforms to :class:`~skillopt.types.MetaReflectResult`:
|
||||
``"meta_summary"`` (str) and ``"patch"`` (:class:`~skillopt.types.Patch`
|
||||
dict), or ``None`` on failure.
|
||||
"""
|
||||
mode = normalize_update_mode(update_mode)
|
||||
actual_system = system_prompt if system_prompt is not None else load_prompt(
|
||||
"meta_reflect_rewrite" if mode == "rewrite_from_suggestions" else "meta_reflect"
|
||||
)
|
||||
|
||||
prev_section = prev_meta_summary.strip() if prev_meta_summary else "(First epoch — no previous summary)"
|
||||
|
||||
user = (
|
||||
f"## Previous Meta Summary\n{prev_section}\n\n"
|
||||
f"## Current Skill Document\n{skill_content}\n\n"
|
||||
f"## {payload_label(mode, title=True)} Budget\n"
|
||||
f"Produce at most {meta_edit_budget} high-level {payload_label(mode)}.\n\n"
|
||||
f"## This Epoch's Step History\n{epoch_history_text}"
|
||||
)
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
system=actual_system,
|
||||
user=user,
|
||||
max_completion_tokens=4096,
|
||||
retries=3,
|
||||
stage="meta_reflect",
|
||||
)
|
||||
result = extract_json(response)
|
||||
if result and "patch" in result:
|
||||
truncate_payload(result["patch"], meta_edit_budget, mode)
|
||||
if "meta_summary" not in result:
|
||||
result["meta_summary"] = ""
|
||||
return result
|
||||
except Exception: # noqa: BLE001
|
||||
traceback.print_exc()
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Teacher-side meta skill memory for cross-epoch optimization guidance.
|
||||
|
||||
This module maintains a compact teacher-facing memory distilled from
|
||||
adjacent-epoch skill comparisons. Unlike ``slow_update``, it does not
|
||||
modify the student skill document. Instead, it produces guidance meant to
|
||||
improve future teacher behavior when proposing, merging, and ranking edits.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import traceback
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.optimizer.slow_update import format_comparison_text
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.utils import extract_json
|
||||
|
||||
|
||||
def format_meta_skill_context(meta_skill_content: str) -> str:
|
||||
"""Render teacher memory into a prompt-ready context block."""
|
||||
content = (meta_skill_content or "").strip()
|
||||
if not content:
|
||||
return ""
|
||||
return (
|
||||
"## Teacher Meta Skill\n"
|
||||
"This is teacher-side memory distilled from prior epoch transitions in "
|
||||
"this environment. Use it to improve how you propose, merge, and rank "
|
||||
"skill edits. Prefer it when the current evidence is ambiguous, but do "
|
||||
"not force it if the current trajectories clearly contradict it.\n\n"
|
||||
f"{content}"
|
||||
)
|
||||
|
||||
|
||||
def run_meta_skill(
|
||||
prev_skill: str,
|
||||
curr_skill: str,
|
||||
comparison_pairs: list[dict],
|
||||
*,
|
||||
prev_meta_skill_content: str = "",
|
||||
system_prompt: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Produce updated teacher-side meta skill from adjacent epochs."""
|
||||
actual_system = system_prompt if system_prompt is not None else load_prompt("meta_skill")
|
||||
|
||||
prev_skill_display = prev_skill
|
||||
if len(prev_skill_display) > 6000:
|
||||
prev_skill_display = prev_skill_display[:6000] + "\n...[truncated]..."
|
||||
|
||||
curr_skill_display = curr_skill
|
||||
if len(curr_skill_display) > 6000:
|
||||
curr_skill_display = curr_skill_display[:6000] + "\n...[truncated]..."
|
||||
|
||||
prev_meta_section = (
|
||||
prev_meta_skill_content.strip()
|
||||
if prev_meta_skill_content and prev_meta_skill_content.strip()
|
||||
else "(No previous teacher meta skill — this is the first update.)"
|
||||
)
|
||||
|
||||
comparison_text = format_comparison_text(comparison_pairs)
|
||||
user = (
|
||||
f"## Previous Epoch Last-Step Skill\n{prev_skill_display}\n\n"
|
||||
f"## Current Epoch Last-Step Skill\n{curr_skill_display}\n\n"
|
||||
f"## Previous Teacher Meta Skill\n"
|
||||
f"The following teacher memory was available during the current epoch. "
|
||||
f"Reflect on whether it improved or harmed the quality of edits.\n\n"
|
||||
f"{prev_meta_section}\n\n"
|
||||
f"## Longitudinal Comparison (same tasks, two last-step skills)\n"
|
||||
f"{comparison_text}"
|
||||
)
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
system=actual_system,
|
||||
user=user,
|
||||
max_completion_tokens=3072,
|
||||
retries=3,
|
||||
stage="meta_skill",
|
||||
)
|
||||
result = extract_json(response)
|
||||
if result and result.get("meta_skill_content"):
|
||||
return {
|
||||
"reasoning": str(result.get("reasoning", "")).strip(),
|
||||
"meta_skill_content": str(result["meta_skill_content"]).strip(),
|
||||
}
|
||||
except Exception: # noqa: BLE001
|
||||
traceback.print_exc()
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Teacher-driven full skill rewrite from selected revise_suggestions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.optimizer.update_modes import get_payload_items
|
||||
from skillopt.utils import extract_json
|
||||
|
||||
|
||||
def rewrite_skill_from_suggestions(
|
||||
skill_content: str,
|
||||
patch: dict,
|
||||
*,
|
||||
system_prompt: str | None = None,
|
||||
step_buffer_context: str = "",
|
||||
env: str | None = None,
|
||||
reasoning_effort: str | None = "high",
|
||||
max_completion_tokens: int = 64000,
|
||||
) -> dict | None:
|
||||
suggestions = get_payload_items(patch, "rewrite_from_suggestions")
|
||||
if not suggestions:
|
||||
return None
|
||||
|
||||
user = (
|
||||
f"## Current Skill\n{skill_content}\n\n"
|
||||
f"## Selected Revise Suggestions ({len(suggestions)} total)\n"
|
||||
f"{json.dumps(suggestions, ensure_ascii=False, indent=2)}\n\n"
|
||||
)
|
||||
if step_buffer_context.strip():
|
||||
user += f"## Previous Steps in This Epoch\n{step_buffer_context}\n\n"
|
||||
user += (
|
||||
"Rewrite the full skill document so it integrates the selected suggestions. "
|
||||
"Return the complete new skill in `new_skill`."
|
||||
)
|
||||
|
||||
actual_system = system_prompt if system_prompt is not None else load_prompt(
|
||||
"rewrite_skill", env=env,
|
||||
)
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
system=actual_system,
|
||||
user=user,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
retries=3,
|
||||
stage="rewrite",
|
||||
reasoning_effort=reasoning_effort,
|
||||
)
|
||||
result = extract_json(response)
|
||||
if result and str(result.get("new_skill", "")).strip():
|
||||
result["new_skill"] = str(result["new_skill"]).rstrip() + "\n"
|
||||
if "change_summary" not in result or not isinstance(result["change_summary"], list):
|
||||
result["change_summary"] = []
|
||||
return result
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return None
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Learning-rate (edit budget) schedulers for ReflACT.
|
||||
|
||||
The "learning rate" in ReflACT is the maximum number of skill edits allowed
|
||||
per optimization step. A scheduler controls how this budget changes over
|
||||
the course of training.
|
||||
|
||||
Supported modes
|
||||
---------------
|
||||
- ``constant`` : Fixed budget throughout training.
|
||||
- ``linear`` : Linear decay from ``max_lr`` to ``min_lr``.
|
||||
- ``cosine`` : Cosine annealing from ``max_lr`` to ``min_lr``.
|
||||
- ``autonomous`` : No limit — the model decides how many edits to make.
|
||||
|
||||
Usage::
|
||||
|
||||
scheduler = build_scheduler(cfg)
|
||||
for step in range(1, total_steps + 1):
|
||||
lr = scheduler.step() # returns edit budget for this step
|
||||
# ... use lr as max_edits ...
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class LRScheduler(ABC):
|
||||
"""Base class for edit-budget schedulers."""
|
||||
|
||||
def __init__(self, max_lr: int, min_lr: int, total_steps: int) -> None:
|
||||
self.max_lr = max_lr
|
||||
self.min_lr = min_lr
|
||||
self.total_steps = total_steps
|
||||
self._current_step = 0
|
||||
|
||||
@abstractmethod
|
||||
def _compute_lr(self, step: int) -> int:
|
||||
"""Return the edit budget for the given 1-indexed step."""
|
||||
|
||||
def step(self) -> int:
|
||||
"""Advance one step and return the edit budget."""
|
||||
self._current_step += 1
|
||||
return self._compute_lr(self._current_step)
|
||||
|
||||
def get_lr(self, step: int) -> int:
|
||||
"""Return the edit budget for an arbitrary step (1-indexed)."""
|
||||
return self._compute_lr(step)
|
||||
|
||||
def state_dict(self) -> dict:
|
||||
return {"current_step": self._current_step}
|
||||
|
||||
def load_state_dict(self, state: dict) -> None:
|
||||
self._current_step = state.get("current_step", 0)
|
||||
|
||||
|
||||
class ConstantScheduler(LRScheduler):
|
||||
"""Fixed edit budget throughout training."""
|
||||
|
||||
def _compute_lr(self, step: int) -> int:
|
||||
return self.max_lr
|
||||
|
||||
|
||||
class LinearScheduler(LRScheduler):
|
||||
"""Linear decay from ``max_lr`` to ``min_lr`` over ``total_steps``."""
|
||||
|
||||
def _compute_lr(self, step: int) -> int:
|
||||
if self.total_steps <= 1:
|
||||
return self.max_lr
|
||||
t = min(step, self.total_steps) / self.total_steps
|
||||
lr = self.max_lr + (self.min_lr - self.max_lr) * t
|
||||
return max(self.min_lr, round(lr))
|
||||
|
||||
|
||||
class CosineScheduler(LRScheduler):
|
||||
"""Cosine annealing from ``max_lr`` to ``min_lr`` over ``total_steps``."""
|
||||
|
||||
def _compute_lr(self, step: int) -> int:
|
||||
if self.total_steps <= 1:
|
||||
return self.max_lr
|
||||
t = min(step, self.total_steps) / self.total_steps
|
||||
lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * (1 + math.cos(math.pi * t))
|
||||
return max(self.min_lr, round(lr))
|
||||
|
||||
|
||||
class AutonomousScheduler(LRScheduler):
|
||||
"""No edit limit — the model decides freely."""
|
||||
|
||||
NO_LIMIT = 999
|
||||
|
||||
def _compute_lr(self, step: int) -> int:
|
||||
return self.NO_LIMIT
|
||||
|
||||
|
||||
# ── Factory ──────────────────────────────────────────────────────────────
|
||||
|
||||
_REGISTRY: dict[str, type[LRScheduler]] = {
|
||||
"constant": ConstantScheduler,
|
||||
"linear": LinearScheduler,
|
||||
"cosine": CosineScheduler,
|
||||
"autonomous": AutonomousScheduler,
|
||||
}
|
||||
|
||||
|
||||
def build_scheduler(
|
||||
mode: str = "constant",
|
||||
max_lr: int = 8,
|
||||
min_lr: int = 2,
|
||||
total_steps: int = 8,
|
||||
) -> LRScheduler:
|
||||
"""Build a scheduler from config parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mode : str
|
||||
One of ``constant``, ``linear``, ``cosine``, ``autonomous``.
|
||||
max_lr : int
|
||||
Initial / maximum edit budget.
|
||||
min_lr : int
|
||||
Minimum edit budget (for decay modes).
|
||||
total_steps : int
|
||||
Total number of optimization steps in training.
|
||||
"""
|
||||
if mode not in _REGISTRY:
|
||||
raise ValueError(
|
||||
f"Unknown scheduler mode '{mode}'. Available: {list(_REGISTRY.keys())}"
|
||||
)
|
||||
return _REGISTRY[mode](max_lr=max_lr, min_lr=min_lr, total_steps=total_steps)
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Backward-compat stub — moved to skillopt.optimizer.clip."""
|
||||
from skillopt.optimizer.clip import rank_and_select # noqa: F401
|
||||
|
||||
__all__ = ["rank_and_select"]
|
||||
@@ -0,0 +1,154 @@
|
||||
"""ReflACT skill operations — edit application and patch processing.
|
||||
|
||||
The Update stage (⑤) of the ReflACT pipeline: apply a ranked set of
|
||||
edits to the current skill document, producing an updated candidate.
|
||||
Analogous to optimizer.step() in neural network training.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from skillopt.types import Edit as EditType, Patch as PatchType
|
||||
|
||||
SLOW_UPDATE_START = "<!-- SLOW_UPDATE_START -->"
|
||||
SLOW_UPDATE_END = "<!-- SLOW_UPDATE_END -->"
|
||||
|
||||
|
||||
def _is_in_slow_update_region(skill: str, target: str) -> bool:
|
||||
"""Check if *target* text falls within the protected slow update region."""
|
||||
start_idx = skill.find(SLOW_UPDATE_START)
|
||||
end_idx = skill.find(SLOW_UPDATE_END)
|
||||
if start_idx == -1 or end_idx == -1:
|
||||
return False
|
||||
target_idx = skill.find(target)
|
||||
if target_idx == -1:
|
||||
return False
|
||||
region_end = end_idx + len(SLOW_UPDATE_END)
|
||||
return start_idx <= target_idx < region_end
|
||||
|
||||
|
||||
def _edit_fields(edit: EditType | dict) -> tuple[str, str, str]:
|
||||
op = edit.op if hasattr(edit, "op") else edit.get("op", "")
|
||||
content = (edit.content if hasattr(edit, "content") else edit.get("content", "")).strip()
|
||||
target = edit.target if hasattr(edit, "target") else edit.get("target", "")
|
||||
return op, content, target
|
||||
|
||||
|
||||
def _apply_edit_with_report(skill: str, edit: EditType | dict) -> tuple[str, dict]:
|
||||
op, content, target = _edit_fields(edit)
|
||||
report = {
|
||||
"op": op,
|
||||
"target": target[:200],
|
||||
"content_preview": content[:200],
|
||||
"status": "unknown",
|
||||
}
|
||||
|
||||
if target and _is_in_slow_update_region(skill, target):
|
||||
report["status"] = "skipped_protected_slow_update_region"
|
||||
return skill, report
|
||||
|
||||
if op == "append":
|
||||
su_start = skill.find(SLOW_UPDATE_START)
|
||||
if su_start != -1:
|
||||
before = skill[:su_start].rstrip()
|
||||
after = skill[su_start:]
|
||||
report["status"] = "applied_append_before_slow_update"
|
||||
return before + "\n\n" + content + "\n\n" + after, report
|
||||
report["status"] = "applied_append"
|
||||
return skill.rstrip() + "\n\n" + content + "\n", report
|
||||
|
||||
if op == "insert_after":
|
||||
if not target or target not in skill:
|
||||
su_start = skill.find(SLOW_UPDATE_START)
|
||||
if su_start != -1:
|
||||
before = skill[:su_start].rstrip()
|
||||
after = skill[su_start:]
|
||||
report["status"] = "applied_insert_after_fallback_before_slow_update"
|
||||
return before + "\n\n" + content + "\n\n" + after, report
|
||||
report["status"] = "applied_insert_after_fallback_append"
|
||||
return skill.rstrip() + "\n\n" + content + "\n", report
|
||||
idx = skill.index(target) + len(target)
|
||||
newline = skill.find("\n", idx)
|
||||
insert_at = newline + 1 if newline != -1 else len(skill)
|
||||
report["status"] = "applied_insert_after"
|
||||
return skill[:insert_at] + "\n" + content + "\n" + skill[insert_at:], report
|
||||
|
||||
if op == "replace":
|
||||
if not target:
|
||||
report["status"] = "skipped_replace_missing_target"
|
||||
return skill, report
|
||||
if target not in skill:
|
||||
report["status"] = "skipped_replace_target_not_found"
|
||||
return skill, report
|
||||
report["status"] = "applied_replace"
|
||||
return skill.replace(target, content, 1), report
|
||||
|
||||
if op == "delete":
|
||||
if not target:
|
||||
report["status"] = "skipped_delete_missing_target"
|
||||
return skill, report
|
||||
if target not in skill:
|
||||
report["status"] = "skipped_delete_target_not_found"
|
||||
return skill, report
|
||||
report["status"] = "applied_delete"
|
||||
return skill.replace(target, "", 1), report
|
||||
|
||||
report["status"] = "skipped_unknown_op"
|
||||
return skill, report
|
||||
|
||||
|
||||
def apply_edit(skill: str, edit: EditType | dict) -> str:
|
||||
"""Apply a single edit operation to the skill document.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
skill : str
|
||||
Current skill document content.
|
||||
edit : Edit | dict
|
||||
An :class:`~skillopt.types.Edit` instance or a plain dict with
|
||||
keys ``op``, ``content``, ``target``.
|
||||
|
||||
Edits targeting the protected slow-update region are silently skipped.
|
||||
"""
|
||||
updated_skill, _ = _apply_edit_with_report(skill, edit)
|
||||
return updated_skill
|
||||
|
||||
|
||||
def apply_patch_with_report(
|
||||
skill: str,
|
||||
patch: PatchType | dict,
|
||||
) -> tuple[str, list[dict]]:
|
||||
"""Apply a patch and return a per-edit report for observability."""
|
||||
edits = patch.edits if hasattr(patch, "edits") else patch.get("edits", [])
|
||||
reports: list[dict] = []
|
||||
for idx, edit in enumerate(edits, 1):
|
||||
try:
|
||||
skill, report = _apply_edit_with_report(skill, edit)
|
||||
report["index"] = idx
|
||||
except Exception as exc: # noqa: BLE001
|
||||
report = {
|
||||
"index": idx,
|
||||
"op": "",
|
||||
"target": "",
|
||||
"content_preview": "",
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
reports.append(report)
|
||||
return skill, reports
|
||||
|
||||
|
||||
def apply_patch(skill: str, patch: PatchType | dict) -> str:
|
||||
"""Apply a patch (list of edits) to the skill document sequentially.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
skill : str
|
||||
Current skill document content.
|
||||
patch : Patch | dict
|
||||
A :class:`~skillopt.types.Patch` instance or a plain dict with
|
||||
key ``edits`` containing a list of edit operations.
|
||||
"""
|
||||
updated_skill, _ = apply_patch_with_report(skill, patch)
|
||||
return updated_skill
|
||||
@@ -0,0 +1,374 @@
|
||||
"""ReflACT Slow Update — epoch-level longitudinal skill refinement.
|
||||
|
||||
At the end of each epoch, the slow update compares rollout performance of the
|
||||
same sample set under the previous epoch's skill vs. the current epoch's skill
|
||||
(Markov: only adjacent epochs). A teacher analyzes regressions, improvements,
|
||||
and persistent failures, then writes a free-form guidance block into a
|
||||
**protected** section of the skill document. This section cannot be modified by
|
||||
step-level analyst edits — only the slow update process overwrites it.
|
||||
|
||||
Public API
|
||||
----------
|
||||
- :func:`inject_empty_slow_update_field` — add empty placeholder (epoch 1)
|
||||
- :func:`extract_slow_update_field` — read current content
|
||||
- :func:`replace_slow_update_field` — overwrite content
|
||||
- :func:`has_slow_update_field` — check if markers are present
|
||||
- :func:`build_comparison_text` — format side-by-side rollout results
|
||||
- :func:`run_slow_update` — teacher call to produce guidance
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import traceback
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.utils import extract_json
|
||||
|
||||
# ── Protected field markers ─────────────────────────────────────────────────
|
||||
|
||||
SLOW_UPDATE_START = "<!-- SLOW_UPDATE_START -->"
|
||||
SLOW_UPDATE_END = "<!-- SLOW_UPDATE_END -->"
|
||||
|
||||
# ── Field manipulation helpers ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def has_slow_update_field(skill: str) -> bool:
|
||||
return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill
|
||||
|
||||
|
||||
def inject_empty_slow_update_field(skill: str) -> str:
|
||||
if has_slow_update_field(skill):
|
||||
return skill
|
||||
block = (
|
||||
f"\n\n{SLOW_UPDATE_START}\n"
|
||||
f"{SLOW_UPDATE_END}\n"
|
||||
)
|
||||
return skill.rstrip() + block
|
||||
|
||||
|
||||
def extract_slow_update_field(skill: str) -> str:
|
||||
start = skill.find(SLOW_UPDATE_START)
|
||||
end = skill.find(SLOW_UPDATE_END)
|
||||
if start == -1 or end == -1:
|
||||
return ""
|
||||
inner_start = start + len(SLOW_UPDATE_START)
|
||||
return skill[inner_start:end].strip()
|
||||
|
||||
|
||||
def replace_slow_update_field(skill: str, new_content: str) -> str:
|
||||
start = skill.find(SLOW_UPDATE_START)
|
||||
end = skill.find(SLOW_UPDATE_END)
|
||||
if start == -1 or end == -1:
|
||||
skill = inject_empty_slow_update_field(skill)
|
||||
start = skill.find(SLOW_UPDATE_START)
|
||||
end = skill.find(SLOW_UPDATE_END)
|
||||
before = skill[:start + len(SLOW_UPDATE_START)]
|
||||
after = skill[end:]
|
||||
return before + "\n" + new_content.strip() + "\n" + after
|
||||
|
||||
|
||||
# ── Comparison text builder ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
_MAX_TRAJ_CHARS = 3000
|
||||
|
||||
|
||||
def _clip_text(value, limit: int) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return str(value)[:limit]
|
||||
|
||||
|
||||
def _read_trajectory(rollout_dir: str, task_id: str) -> str:
|
||||
"""Read and format a single trajectory from a rollout directory."""
|
||||
conv_path = os.path.join(rollout_dir, "predictions", task_id, "conversation.json")
|
||||
if not os.path.exists(conv_path):
|
||||
return "(trajectory not available)"
|
||||
try:
|
||||
with open(conv_path) as f:
|
||||
conversation = json.load(f)
|
||||
except Exception:
|
||||
return "(trajectory read error)"
|
||||
if not conversation:
|
||||
return "(empty trajectory)"
|
||||
|
||||
lines: list[str] = []
|
||||
for entry in conversation:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if entry.get("type") == "tool_call":
|
||||
cmd = _clip_text(entry.get("cmd"), 500)
|
||||
obs = _clip_text(entry.get("obs"), 800)
|
||||
lines.append(f"[action] {cmd}")
|
||||
lines.append(f"[obs] {obs}")
|
||||
elif "action" in entry and "env_feedback" in entry:
|
||||
step = entry.get("step", "?")
|
||||
reasoning = _clip_text(entry.get("reasoning"), 300)
|
||||
action = _clip_text(entry.get("action"), 200)
|
||||
feedback = _clip_text(entry.get("env_feedback"), 500)
|
||||
if reasoning:
|
||||
lines.append(f"[step {step} think] {reasoning}")
|
||||
lines.append(f"[step {step} action] {action}")
|
||||
lines.append(f"[step {step} obs] {feedback}")
|
||||
elif entry.get("role") == "system":
|
||||
msg = _clip_text(entry.get("content"), 1000)
|
||||
lines.append(f"[verification] {msg}")
|
||||
else:
|
||||
msg = _clip_text(entry.get("content"), 500)
|
||||
role = entry.get("role", "agent")
|
||||
lines.append(f"[{role}] {msg}")
|
||||
|
||||
text = "\n".join(lines)
|
||||
if len(text) > _MAX_TRAJ_CHARS:
|
||||
half = _MAX_TRAJ_CHARS // 2
|
||||
text = text[:half] + "\n...[truncated]...\n" + text[-half:]
|
||||
return text
|
||||
|
||||
|
||||
# ── Structured comparison pairs ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_comparison_pairs(
|
||||
results_prev: list[dict],
|
||||
results_curr: list[dict],
|
||||
items: list[dict],
|
||||
prev_rollout_dir: str = "",
|
||||
curr_rollout_dir: str = "",
|
||||
) -> list[dict]:
|
||||
"""Build a structured list of per-sample comparison entries.
|
||||
|
||||
Each entry bundles the original item, both rollout results, the change
|
||||
category, and both trajectories into one dict — the single source of
|
||||
truth for this sample's longitudinal comparison.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[dict]
|
||||
One dict per sample with keys:
|
||||
``id, task, category, prev, curr, prev_trajectory, curr_trajectory``
|
||||
"""
|
||||
prev_by_id = {str(r["id"]): r for r in results_prev}
|
||||
curr_by_id = {str(r["id"]): r for r in results_curr}
|
||||
|
||||
pairs: list[dict] = []
|
||||
for item in items:
|
||||
tid = str(item.get("id", ""))
|
||||
prev = prev_by_id.get(tid, {})
|
||||
curr = curr_by_id.get(tid, {})
|
||||
prev_ok = bool(prev.get("hard", 0))
|
||||
curr_ok = bool(curr.get("hard", 0))
|
||||
|
||||
if not prev_ok and curr_ok:
|
||||
category = "improved"
|
||||
elif prev_ok and not curr_ok:
|
||||
category = "regressed"
|
||||
elif not prev_ok and not curr_ok:
|
||||
category = "persistent_fail"
|
||||
else:
|
||||
category = "stable_success"
|
||||
|
||||
pairs.append({
|
||||
"id": tid,
|
||||
"task": item.get("question", item.get("task_description", item.get("instruction", tid))),
|
||||
"category": category,
|
||||
"prev": {
|
||||
"hard": int(prev_ok),
|
||||
"soft": float(prev.get("soft", 0.0)),
|
||||
"predicted_answer": prev.get("predicted_answer", prev.get("answer", "N/A")),
|
||||
"fail_reason": prev.get("fail_reason", ""),
|
||||
},
|
||||
"curr": {
|
||||
"hard": int(curr_ok),
|
||||
"soft": float(curr.get("soft", 0.0)),
|
||||
"predicted_answer": curr.get("predicted_answer", curr.get("answer", "N/A")),
|
||||
"fail_reason": curr.get("fail_reason", ""),
|
||||
},
|
||||
"prev_trajectory": (
|
||||
_read_trajectory(prev_rollout_dir, tid) if prev_rollout_dir else ""
|
||||
),
|
||||
"curr_trajectory": (
|
||||
_read_trajectory(curr_rollout_dir, tid) if curr_rollout_dir else ""
|
||||
),
|
||||
})
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def save_comparison_pairs(pairs: list[dict], out_path: str) -> None:
|
||||
"""Persist comparison pairs to JSON (without trajectory text to save space)."""
|
||||
slim = []
|
||||
for p in pairs:
|
||||
slim.append({
|
||||
"id": p["id"],
|
||||
"task": p["task"][:300],
|
||||
"category": p["category"],
|
||||
"prev": p["prev"],
|
||||
"curr": p["curr"],
|
||||
})
|
||||
with open(out_path, "w") as f:
|
||||
json.dump(slim, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def format_comparison_text(pairs: list[dict]) -> str:
|
||||
"""Format structured comparison pairs into teacher-readable text."""
|
||||
by_cat: dict[str, list[dict]] = {
|
||||
"regressed": [],
|
||||
"persistent_fail": [],
|
||||
"improved": [],
|
||||
"stable_success": [],
|
||||
}
|
||||
for p in pairs:
|
||||
by_cat.setdefault(p["category"], []).append(p)
|
||||
|
||||
total = len(pairs)
|
||||
parts = [
|
||||
f"## Longitudinal Comparison Summary\n"
|
||||
f"Total samples: {total}\n"
|
||||
f"- Improved (wrong→right): {len(by_cat['improved'])}\n"
|
||||
f"- Regressed (right→wrong): {len(by_cat['regressed'])}\n"
|
||||
f"- Persistent failures (wrong→wrong): {len(by_cat['persistent_fail'])}\n"
|
||||
f"- Stable successes (right→right): {len(by_cat['stable_success'])}\n"
|
||||
]
|
||||
|
||||
categories = [
|
||||
("regressed", "Regressions (right→wrong) — HIGHEST PRIORITY", True),
|
||||
("persistent_fail", "Persistent Failures (wrong→wrong)", True),
|
||||
("improved", "Improvements (wrong→right)", True),
|
||||
("stable_success", "Stable Successes (right→right)", False),
|
||||
]
|
||||
|
||||
for cat_key, label, show_traj in categories:
|
||||
entries = by_cat[cat_key]
|
||||
if not entries:
|
||||
parts.append(f"### {label}\n(none)\n")
|
||||
continue
|
||||
|
||||
lines = [f"### {label}"]
|
||||
for e in entries:
|
||||
prev = e["prev"]
|
||||
curr = e["curr"]
|
||||
lines.append(
|
||||
f"\n#### Task {e['id']}: {e['task'][:300]}\n"
|
||||
f"- Prev epoch: {'PASS' if prev['hard'] else 'FAIL'} "
|
||||
f"(soft={prev['soft']:.2f}) — answer: {str(prev['predicted_answer'])[:200]}\n"
|
||||
f"- Curr epoch: {'PASS' if curr['hard'] else 'FAIL'} "
|
||||
f"(soft={curr['soft']:.2f}) — answer: {str(curr['predicted_answer'])[:200]}"
|
||||
)
|
||||
if curr.get("fail_reason"):
|
||||
lines.append(f"- Curr fail reason: {curr['fail_reason'][:300]}")
|
||||
if prev.get("fail_reason") and not prev["hard"]:
|
||||
lines.append(f"- Prev fail reason: {prev['fail_reason'][:300]}")
|
||||
|
||||
if show_traj:
|
||||
if e.get("prev_trajectory"):
|
||||
lines.append(
|
||||
f"\n**Previous epoch trajectory:**\n```\n{e['prev_trajectory']}\n```"
|
||||
)
|
||||
if e.get("curr_trajectory"):
|
||||
lines.append(
|
||||
f"\n**Current epoch trajectory:**\n```\n{e['curr_trajectory']}\n```"
|
||||
)
|
||||
|
||||
parts.append("\n".join(lines))
|
||||
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
|
||||
# ── Teacher call ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def run_slow_update(
|
||||
skill_content: str,
|
||||
results_prev: list[dict],
|
||||
results_curr: list[dict],
|
||||
items: list[dict],
|
||||
*,
|
||||
prev_skill: str = "",
|
||||
prev_slow_update_content: str = "",
|
||||
prev_rollout_dir: str = "",
|
||||
curr_rollout_dir: str = "",
|
||||
comparison_pairs: list[dict] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Run the slow update teacher call for one epoch boundary.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
skill_content : str
|
||||
Current epoch's skill (after fast updates).
|
||||
results_prev : list[dict]
|
||||
Rollout results of the 20 samples under previous epoch's skill.
|
||||
results_curr : list[dict]
|
||||
Rollout results of the 20 samples under current epoch's skill.
|
||||
items : list[dict]
|
||||
The 20 sample items used for comparison.
|
||||
prev_skill : str
|
||||
Previous epoch's skill content.
|
||||
prev_slow_update_content : str
|
||||
The slow update guidance from the previous epoch (to reflect on).
|
||||
prev_rollout_dir : str
|
||||
Path to previous epoch rollout output (contains predictions/).
|
||||
curr_rollout_dir : str
|
||||
Path to current epoch rollout output (contains predictions/).
|
||||
system_prompt : str | None
|
||||
Custom system prompt override.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict | None
|
||||
Conforms to :class:`~skillopt.types.SlowUpdateResult`:
|
||||
``{"reasoning": str, "slow_update_content": str}`` or ``None``.
|
||||
"""
|
||||
actual_system = system_prompt if system_prompt is not None else load_prompt("slow_update")
|
||||
|
||||
pairs = comparison_pairs
|
||||
if pairs is None:
|
||||
pairs = build_comparison_pairs(
|
||||
results_prev, results_curr, items,
|
||||
prev_rollout_dir=prev_rollout_dir,
|
||||
curr_rollout_dir=curr_rollout_dir,
|
||||
)
|
||||
comparison_text = format_comparison_text(pairs)
|
||||
|
||||
prev_skill_display = prev_skill
|
||||
if len(prev_skill_display) > 6000:
|
||||
prev_skill_display = prev_skill_display[:6000] + "\n...[truncated]..."
|
||||
|
||||
prev_guidance_section = (
|
||||
prev_slow_update_content.strip()
|
||||
if prev_slow_update_content and prev_slow_update_content.strip()
|
||||
else "(No previous guidance — this is the first slow update.)"
|
||||
)
|
||||
|
||||
user = (
|
||||
f"## Previous Epoch's Skill\n{prev_skill_display}\n\n"
|
||||
f"## Current Epoch's Skill\n{skill_content}\n\n"
|
||||
f"## Previous Slow Update Guidance\n"
|
||||
f"The following guidance was active during the current epoch. "
|
||||
f"Reflect on its effectiveness before writing the new version.\n\n"
|
||||
f"{prev_guidance_section}\n\n"
|
||||
f"## Longitudinal Comparison (same 20 tasks, two skill versions)\n"
|
||||
f"{comparison_text}"
|
||||
)
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
system=actual_system,
|
||||
user=user,
|
||||
max_completion_tokens=4096,
|
||||
retries=3,
|
||||
stage="slow_update",
|
||||
)
|
||||
result = extract_json(response)
|
||||
if result and result.get("slow_update_content"):
|
||||
return {
|
||||
"reasoning": str(result.get("reasoning", "")).strip(),
|
||||
"slow_update_content": str(result["slow_update_content"]).strip(),
|
||||
}
|
||||
except Exception: # noqa: BLE001
|
||||
traceback.print_exc()
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Helpers for switching between patch edits and rewrite-from-suggestions."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
PATCH_MODE = "patch"
|
||||
REWRITE_MODE = "rewrite_from_suggestions"
|
||||
FULL_REWRITE_MINIBATCH_MODE = "full_rewrite_minibatch"
|
||||
|
||||
|
||||
def normalize_update_mode(mode: str | None) -> str:
|
||||
raw = str(mode or PATCH_MODE).strip().lower()
|
||||
aliases = {
|
||||
"patch": PATCH_MODE,
|
||||
"edits": PATCH_MODE,
|
||||
"rewrite": REWRITE_MODE,
|
||||
"rewrite_from_suggestions": REWRITE_MODE,
|
||||
"suggestions": REWRITE_MODE,
|
||||
"rewrite_suggestions": REWRITE_MODE,
|
||||
"full_rewrite": FULL_REWRITE_MINIBATCH_MODE,
|
||||
"full_rewrite_minibatch": FULL_REWRITE_MINIBATCH_MODE,
|
||||
"minibatch_full_rewrite": FULL_REWRITE_MINIBATCH_MODE,
|
||||
"skill_rewrite_minibatch": FULL_REWRITE_MINIBATCH_MODE,
|
||||
}
|
||||
return aliases.get(raw, PATCH_MODE)
|
||||
|
||||
|
||||
def is_rewrite_mode(mode: str | None) -> bool:
|
||||
return normalize_update_mode(mode) == REWRITE_MODE
|
||||
|
||||
|
||||
def is_full_rewrite_minibatch_mode(mode: str | None) -> bool:
|
||||
return normalize_update_mode(mode) == FULL_REWRITE_MINIBATCH_MODE
|
||||
|
||||
|
||||
def payload_key(mode: str | None) -> str:
|
||||
if is_full_rewrite_minibatch_mode(mode):
|
||||
return "skill_candidates"
|
||||
return "revise_suggestions" if is_rewrite_mode(mode) else "edits"
|
||||
|
||||
|
||||
def payload_label(mode: str | None, *, singular: bool = False, title: bool = False) -> str:
|
||||
if is_full_rewrite_minibatch_mode(mode):
|
||||
word = "skill candidate" if singular else "skill candidates"
|
||||
elif is_rewrite_mode(mode):
|
||||
word = "suggestion" if singular else "suggestions"
|
||||
else:
|
||||
word = "edit" if singular else "edits"
|
||||
return word.title() if title else word
|
||||
|
||||
|
||||
def get_payload_items(container: dict | None, mode: str | None) -> list[dict]:
|
||||
if not isinstance(container, dict):
|
||||
return []
|
||||
items = container.get(payload_key(mode), [])
|
||||
return items if isinstance(items, list) else []
|
||||
|
||||
|
||||
def set_payload_items(container: dict, items: list[dict], mode: str | None) -> dict:
|
||||
container[payload_key(mode)] = items
|
||||
return container
|
||||
|
||||
|
||||
def truncate_payload(container: dict, max_items: int, mode: str | None) -> dict:
|
||||
if max_items < 0:
|
||||
return container
|
||||
items = get_payload_items(container, mode)
|
||||
if len(items) > max_items:
|
||||
set_payload_items(container, items[:max_items], mode)
|
||||
return container
|
||||
|
||||
|
||||
def describe_item(item: dict, mode: str | None, *, max_chars: int = 240) -> str:
|
||||
if not isinstance(item, dict):
|
||||
return ""
|
||||
if is_full_rewrite_minibatch_mode(mode):
|
||||
parts = [
|
||||
f"title={item.get('title', '')!r}",
|
||||
f"change_summary={item.get('change_summary', [])!r}",
|
||||
]
|
||||
if item.get("source_type"):
|
||||
parts.append(f"source={item.get('source_type')}")
|
||||
if item.get("support_count") is not None:
|
||||
parts.append(f"support={item.get('support_count')}")
|
||||
new_skill = str(item.get("new_skill", "")).strip()
|
||||
if new_skill:
|
||||
parts.append(f"new_skill_preview={new_skill[:120]!r}")
|
||||
text = " ".join(parts)
|
||||
elif is_rewrite_mode(mode):
|
||||
parts = [
|
||||
f"type={item.get('type', '?')}",
|
||||
f"title={item.get('title', '')!r}",
|
||||
f"instruction={item.get('instruction', '')!r}",
|
||||
]
|
||||
if item.get("priority_hint"):
|
||||
parts.append(f"priority={item.get('priority_hint')}")
|
||||
if item.get("support_count") is not None:
|
||||
parts.append(f"support={item.get('support_count')}")
|
||||
text = " ".join(parts)
|
||||
else:
|
||||
op = item.get("op", "?")
|
||||
target = item.get("target", "")
|
||||
content = item.get("content", "")
|
||||
parts = [f"op={op}"]
|
||||
if target:
|
||||
parts.append(f"target={target!r}")
|
||||
if content:
|
||||
parts.append(f"content={content!r}")
|
||||
if item.get("support_count") is not None:
|
||||
parts.append(f"support={item.get('support_count')}")
|
||||
text = " ".join(parts)
|
||||
if len(text) <= max_chars:
|
||||
return text
|
||||
return text[: max_chars - 3].rstrip() + "..."
|
||||
|
||||
|
||||
def short_item_summary(item: dict, mode: str | None, *, max_chars: int = 200) -> dict[str, Any]:
|
||||
if is_full_rewrite_minibatch_mode(mode):
|
||||
return {
|
||||
"title": str(item.get("title", ""))[:max_chars],
|
||||
"change_summary": [
|
||||
str(x)[:max_chars] for x in item.get("change_summary", [])[:3]
|
||||
] if isinstance(item.get("change_summary"), list) else [],
|
||||
"source_type": item.get("source_type", ""),
|
||||
}
|
||||
if is_rewrite_mode(mode):
|
||||
return {
|
||||
"type": item.get("type", "?"),
|
||||
"title": str(item.get("title", ""))[:max_chars],
|
||||
"instruction": str(item.get("instruction", ""))[:max_chars],
|
||||
}
|
||||
return {
|
||||
"op": item.get("op", "?"),
|
||||
"content": str(item.get("content", ""))[:max_chars],
|
||||
"target": item.get("target", ""),
|
||||
}
|
||||
Reference in New Issue
Block a user