refactor: rename teacher/student to optimizer/target, remove best skills, fix slow update
- Rename teacher -> optimizer, student -> target across all code, configs, docs, prompts - CLI: --teacher_model -> --optimizer_model, --student_model -> --target_model - Remove best_skill files, keep only initial skills - Fix slow update gate (force write into skill) - Fix SLOW_UPDATE marker stripping - Remove deep_reflect and meta_reflect mechanisms - Update .env.example with export prefix and azure_cli docs - Add endpoint empty validation in azure_openai.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Cuzyoung
@@ -1,4 +1,4 @@
|
||||
"""ReflACT Optimizer -- skill update operations.
|
||||
"""SkillOpt Optimizer -- skill update operations.
|
||||
|
||||
Analogous to the optimizer in neural network training: applies the computed
|
||||
"gradient" (patches) to the current skill document to produce an updated
|
||||
@@ -8,8 +8,8 @@ Modules
|
||||
-------
|
||||
- skill: edit application (optimizer.step() / parameter update)
|
||||
- clip: edit ranking and selection (gradient clipping)
|
||||
- meta_reflect: epoch-level macro refinement (momentum)
|
||||
- slow_update: longitudinal comparison and guidance (EMA / regularization)
|
||||
- meta_skill: cross-epoch memory for optimizer context
|
||||
"""
|
||||
from skillopt.optimizer.skill import apply_edit, apply_patch # noqa: F401
|
||||
from skillopt.optimizer.clip import rank_and_select # noqa: F401
|
||||
|
||||
@@ -6,7 +6,7 @@ effective step size. Previously core/select.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.model import chat_optimizer
|
||||
from skillopt.optimizer.meta_skill import format_meta_skill_context
|
||||
from skillopt.optimizer.update_modes import (
|
||||
describe_item,
|
||||
@@ -29,10 +29,10 @@ def rank_and_select(
|
||||
meta_skill_context: str = "",
|
||||
update_mode: str = "patch",
|
||||
) -> dict:
|
||||
"""Use a teacher LLM to rank edits by importance, then keep top-L.
|
||||
"""Use a optimizer LLM to rank edits by importance, then keep top-L.
|
||||
|
||||
If the edit pool is within budget, returns the patch unchanged.
|
||||
Otherwise, calls the teacher to rank and select the most impactful edits.
|
||||
Otherwise, calls the optimizer to rank and select the most impactful edits.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -54,7 +54,7 @@ def rank_and_select(
|
||||
if len(edits) <= max_edits:
|
||||
return patch
|
||||
|
||||
# Build the edit pool description for the teacher
|
||||
# Build the edit pool description for the optimizer
|
||||
edits_desc = []
|
||||
for i, edit in enumerate(edits):
|
||||
edits_desc.append(f"[{i}] {describe_item(edit, update_mode, max_chars=500)}")
|
||||
@@ -66,13 +66,13 @@ def rank_and_select(
|
||||
+ f"\n\nSelect the {max_edits} most important {payload_label(update_mode)}. "
|
||||
f"Return their 0-based indices in priority order."
|
||||
)
|
||||
teacher_ctx = format_meta_skill_context(meta_skill_context)
|
||||
if teacher_ctx:
|
||||
user = f"{teacher_ctx}\n\n{user}"
|
||||
optimizer_ctx = format_meta_skill_context(meta_skill_context)
|
||||
if optimizer_ctx:
|
||||
user = f"{optimizer_ctx}\n\n{user}"
|
||||
prompt_name = "ranking_rewrite" if is_rewrite_mode(update_mode) else "ranking"
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
response, _ = chat_optimizer(
|
||||
system=load_prompt(prompt_name), user=user,
|
||||
max_completion_tokens=2048, retries=3, stage="ranking",
|
||||
)
|
||||
@@ -94,7 +94,7 @@ def rank_and_select(
|
||||
if selected:
|
||||
return {
|
||||
"reasoning": patch.get("reasoning", "")
|
||||
+ f" [teacher-ranked: selected {len(selected)}/{len(edits)} {payload_label(update_mode)}]",
|
||||
+ f" [optimizer-ranked: selected {len(selected)}/{len(edits)} {payload_label(update_mode)}]",
|
||||
payload_key(update_mode): selected,
|
||||
"ranking_details": result,
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
"""Teacher-driven autonomous update-size decisions."""
|
||||
"""Optimizer-driven autonomous update-size decisions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.model import chat_optimizer
|
||||
from skillopt.optimizer.meta_skill import format_meta_skill_context
|
||||
from skillopt.optimizer.update_modes import describe_item, get_payload_items, payload_label
|
||||
from skillopt.prompts import load_prompt
|
||||
@@ -39,7 +39,7 @@ def decide_autonomous_learning_rate(
|
||||
step_buffer_context: str = "",
|
||||
meta_skill_context: str = "",
|
||||
) -> dict:
|
||||
"""Ask the teacher to choose the number of update items for this step.
|
||||
"""Ask the optimizer to choose the number of update items for this step.
|
||||
|
||||
The prompt intentionally avoids default budgets, candidate budget lists, or
|
||||
scheduler history. The only hard post-processing is validity: the returned
|
||||
@@ -65,15 +65,15 @@ def decide_autonomous_learning_rate(
|
||||
)
|
||||
if step_buffer_context.strip():
|
||||
user += f"\n\n## Previous Steps in This Epoch\n{step_buffer_context}"
|
||||
teacher_ctx = format_meta_skill_context(meta_skill_context)
|
||||
if teacher_ctx:
|
||||
user = f"{teacher_ctx}\n\n{user}"
|
||||
optimizer_ctx = format_meta_skill_context(meta_skill_context)
|
||||
if optimizer_ctx:
|
||||
user = f"{optimizer_ctx}\n\n{user}"
|
||||
|
||||
response = ""
|
||||
parsed: dict | None = None
|
||||
decision: int | None = None
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
response, _ = chat_optimizer(
|
||||
system=load_prompt("lr_autonomous"),
|
||||
user=user,
|
||||
max_completion_tokens=2048,
|
||||
|
||||
@@ -1,28 +1,28 @@
|
||||
"""Teacher-side meta skill memory for cross-epoch optimization guidance.
|
||||
"""Optimizer-side meta skill memory for cross-epoch optimization guidance.
|
||||
|
||||
This module maintains a compact teacher-facing memory distilled from
|
||||
This module maintains a compact optimizer-facing memory distilled from
|
||||
adjacent-epoch skill comparisons. Unlike ``slow_update``, it does not
|
||||
modify the student skill document. Instead, it produces guidance meant to
|
||||
improve future teacher behavior when proposing, merging, and ranking edits.
|
||||
modify the target skill document. Instead, it produces guidance meant to
|
||||
improve future optimizer behavior when proposing, merging, and ranking edits.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import traceback
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.model import chat_optimizer
|
||||
from skillopt.optimizer.slow_update import format_comparison_text
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.utils import extract_json
|
||||
|
||||
|
||||
def format_meta_skill_context(meta_skill_content: str) -> str:
|
||||
"""Render teacher memory into a prompt-ready context block."""
|
||||
"""Render optimizer memory into a prompt-ready context block."""
|
||||
content = (meta_skill_content or "").strip()
|
||||
if not content:
|
||||
return ""
|
||||
return (
|
||||
"## Teacher Meta Skill\n"
|
||||
"This is teacher-side memory distilled from prior epoch transitions in "
|
||||
"## Optimizer Meta Skill\n"
|
||||
"This is optimizer-side memory distilled from prior epoch transitions in "
|
||||
"this environment. Use it to improve how you propose, merge, and rank "
|
||||
"skill edits. Prefer it when the current evidence is ambiguous, but do "
|
||||
"not force it if the current trajectories clearly contradict it.\n\n"
|
||||
@@ -38,7 +38,7 @@ def run_meta_skill(
|
||||
prev_meta_skill_content: str = "",
|
||||
system_prompt: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Produce updated teacher-side meta skill from adjacent epochs."""
|
||||
"""Produce updated optimizer-side meta skill from adjacent epochs."""
|
||||
actual_system = system_prompt if system_prompt is not None else load_prompt("meta_skill")
|
||||
|
||||
prev_skill_display = prev_skill
|
||||
@@ -52,15 +52,15 @@ def run_meta_skill(
|
||||
prev_meta_section = (
|
||||
prev_meta_skill_content.strip()
|
||||
if prev_meta_skill_content and prev_meta_skill_content.strip()
|
||||
else "(No previous teacher meta skill — this is the first update.)"
|
||||
else "(No previous optimizer meta skill — this is the first update.)"
|
||||
)
|
||||
|
||||
comparison_text = format_comparison_text(comparison_pairs)
|
||||
user = (
|
||||
f"## Previous Epoch Last-Step Skill\n{prev_skill_display}\n\n"
|
||||
f"## Current Epoch Last-Step Skill\n{curr_skill_display}\n\n"
|
||||
f"## Previous Teacher Meta Skill\n"
|
||||
f"The following teacher memory was available during the current epoch. "
|
||||
f"## Previous Optimizer Meta Skill\n"
|
||||
f"The following optimizer memory was available during the current epoch. "
|
||||
f"Reflect on whether it improved or harmed the quality of edits.\n\n"
|
||||
f"{prev_meta_section}\n\n"
|
||||
f"## Longitudinal Comparison (same tasks, two last-step skills)\n"
|
||||
@@ -68,7 +68,7 @@ def run_meta_skill(
|
||||
)
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
response, _ = chat_optimizer(
|
||||
system=actual_system,
|
||||
user=user,
|
||||
max_completion_tokens=3072,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
"""Teacher-driven full skill rewrite from selected revise_suggestions."""
|
||||
"""Optimizer-driven full skill rewrite from selected revise_suggestions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.model import chat_optimizer
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.optimizer.update_modes import get_payload_items
|
||||
from skillopt.utils import extract_json
|
||||
@@ -40,7 +40,7 @@ def rewrite_skill_from_suggestions(
|
||||
)
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
response, _ = chat_optimizer(
|
||||
system=actual_system,
|
||||
user=user,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
|
||||
@@ -28,9 +28,19 @@ def _is_in_slow_update_region(skill: str, target: str) -> bool:
|
||||
return start_idx <= target_idx < region_end
|
||||
|
||||
|
||||
def _strip_slow_update_markers(text: str) -> str:
|
||||
"""Remove any SLOW_UPDATE markers from edit content to prevent duplication."""
|
||||
return (
|
||||
text.replace(SLOW_UPDATE_START, "")
|
||||
.replace(SLOW_UPDATE_END, "")
|
||||
)
|
||||
|
||||
|
||||
def _edit_fields(edit: EditType | dict) -> tuple[str, str, str]:
|
||||
op = edit.op if hasattr(edit, "op") else edit.get("op", "")
|
||||
content = (edit.content if hasattr(edit, "content") else edit.get("content", "")).strip()
|
||||
content = _strip_slow_update_markers(
|
||||
(edit.content if hasattr(edit, "content") else edit.get("content", "")).strip()
|
||||
)
|
||||
target = edit.target if hasattr(edit, "target") else edit.get("target", "")
|
||||
return op, content, target
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
At the end of each epoch, the slow update compares rollout performance of the
|
||||
same sample set under the previous epoch's skill vs. the current epoch's skill
|
||||
(Markov: only adjacent epochs). A teacher analyzes regressions, improvements,
|
||||
(Markov: only adjacent epochs). A optimizer analyzes regressions, improvements,
|
||||
and persistent failures, then writes a free-form guidance block into a
|
||||
**protected** section of the skill document. This section cannot be modified by
|
||||
step-level analyst edits — only the slow update process overwrites it.
|
||||
@@ -14,7 +14,7 @@ Public API
|
||||
- :func:`replace_slow_update_field` — overwrite content
|
||||
- :func:`has_slow_update_field` — check if markers are present
|
||||
- :func:`build_comparison_text` — format side-by-side rollout results
|
||||
- :func:`run_slow_update` — teacher call to produce guidance
|
||||
- :func:`run_slow_update` — optimizer call to produce guidance
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -22,7 +22,7 @@ import json
|
||||
import os
|
||||
import traceback
|
||||
|
||||
from skillopt.model import chat_teacher
|
||||
from skillopt.model import chat_optimizer
|
||||
from skillopt.prompts import load_prompt
|
||||
from skillopt.utils import extract_json
|
||||
|
||||
@@ -57,16 +57,35 @@ def extract_slow_update_field(skill: str) -> str:
|
||||
return skill[inner_start:end].strip()
|
||||
|
||||
|
||||
def replace_slow_update_field(skill: str, new_content: str) -> str:
|
||||
start = skill.find(SLOW_UPDATE_START)
|
||||
end = skill.find(SLOW_UPDATE_END)
|
||||
if start == -1 or end == -1:
|
||||
skill = inject_empty_slow_update_field(skill)
|
||||
def _strip_all_slow_update_fields(skill: str) -> str:
|
||||
"""Remove every SLOW_UPDATE_START/END pair (and content between) from *skill*."""
|
||||
while True:
|
||||
start = skill.find(SLOW_UPDATE_START)
|
||||
end = skill.find(SLOW_UPDATE_END)
|
||||
before = skill[:start + len(SLOW_UPDATE_START)]
|
||||
after = skill[end:]
|
||||
return before + "\n" + new_content.strip() + "\n" + after
|
||||
if start == -1:
|
||||
break
|
||||
end = skill.find(SLOW_UPDATE_END, start)
|
||||
if end == -1:
|
||||
# Orphan start marker — remove it
|
||||
skill = skill[:start] + skill[start + len(SLOW_UPDATE_START):]
|
||||
break
|
||||
skill = skill[:start] + skill[end + len(SLOW_UPDATE_END):]
|
||||
# Clean up stray end markers
|
||||
skill = skill.replace(SLOW_UPDATE_END, "")
|
||||
# Collapse excess blank lines left behind
|
||||
while "\n\n\n" in skill:
|
||||
skill = skill.replace("\n\n\n", "\n\n")
|
||||
return skill.rstrip()
|
||||
|
||||
|
||||
def replace_slow_update_field(skill: str, new_content: str) -> str:
|
||||
# Remove all existing slow update regions first to guarantee exactly one.
|
||||
skill = _strip_all_slow_update_fields(skill)
|
||||
block = (
|
||||
f"\n\n{SLOW_UPDATE_START}\n"
|
||||
f"{new_content.strip()}\n"
|
||||
f"{SLOW_UPDATE_END}\n"
|
||||
)
|
||||
return skill + block
|
||||
|
||||
|
||||
# ── Comparison text builder ─────────────────────────────────────────────────
|
||||
@@ -212,7 +231,7 @@ def save_comparison_pairs(pairs: list[dict], out_path: str) -> None:
|
||||
|
||||
|
||||
def format_comparison_text(pairs: list[dict]) -> str:
|
||||
"""Format structured comparison pairs into teacher-readable text."""
|
||||
"""Format structured comparison pairs into optimizer-readable text."""
|
||||
by_cat: dict[str, list[dict]] = {
|
||||
"regressed": [],
|
||||
"persistent_fail": [],
|
||||
@@ -277,7 +296,7 @@ def format_comparison_text(pairs: list[dict]) -> str:
|
||||
|
||||
|
||||
|
||||
# ── Teacher call ────────────────────────────────────────────────────────────
|
||||
# ── Optimizer call ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def run_slow_update(
|
||||
@@ -293,7 +312,7 @@ def run_slow_update(
|
||||
comparison_pairs: list[dict] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Run the slow update teacher call for one epoch boundary.
|
||||
"""Run the slow update optimizer call for one epoch boundary.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -355,7 +374,7 @@ def run_slow_update(
|
||||
)
|
||||
|
||||
try:
|
||||
response, _ = chat_teacher(
|
||||
response, _ = chat_optimizer(
|
||||
system=actual_system,
|
||||
user=user,
|
||||
max_completion_tokens=4096,
|
||||
|
||||
Reference in New Issue
Block a user