feat(optimizer): skill-aware reflection (EmbodiSkill S_app), config-controlled and env-independent

Split failure reflections into SKILL_DEFECT (body edit) vs EXECUTION_LAPSE
(protected appendix note that re-emphasizes an existing rule, never edited
by step-level analysts). Toggle: optimizer.use_skill_aware_reflection
(default false; baseline byte-identical when off).

- optimizer/appendix.py: protected APPENDIX region (inject/extract/append
  with dedup), mirrors the slow_update protected-field pattern
- optimizer/skill_aware.py: analyst prompt augmentation, appendix_notes
  parsing, threshold-gated LLM consolidation, and a process-wide runtime
  switch (configure_skill_aware_reflection) set once by the trainer
- gradient/reflect.py: augment error/success analyst prompts at runtime;
  None-sentinel kwargs resolve from the global switch, so env adapters
  need no per-benchmark wiring (works for all envs, present and future)
- optimizer/skill.py: generalize the protected-region check to
  (slow_update, appendix); edits inside any protected region are skipped
- engine/trainer.py: inject appendix at init, flush per-step
  EXECUTION_LAPSE notes after the gate settles, optional consolidation
- tests: regression suite incl. toggle-off byte-identical guarantee and
  env-independent global-switch resolution (6/6 passing + live smoke)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

skillopt/optimizer/appendix.py

@@ -0,0 +1,156 @@
+"""Skill-Aware Reflection — protected appendix field (EmbodiSkill S_app).
+
+EmbodiSkill (paper 2605.10332v1) splits a skill into ``S = (S_body, S_app)``:
+the body holds the main prescriptive rules; the appendix only *emphasizes*
+existing valid rules that the executor failed to follow (EXECUTION_LAPSE), and
+**never introduces new rules**.
+
+This module owns the appendix region of the skill document. It mirrors the
+protected-field pattern of :mod:`skillopt.optimizer.slow_update`, with two
+differences:
+
+1. **Append semantics** (not replace): execution-lapse reminders accumulate
+   across steps within a run, so new notes are merged into the existing
+   appendix rather than overwriting it.
+2. **Lightweight dedup**: near-duplicate reminders are collapsed (inspired by
+   GMemory's ``_dedupe_preserve_order``) so the appendix stays compact.
+
+The appendix lives **inside** the skill markdown, between dedicated markers, so
+it is persisted by the normal ``_save_skill`` path and is resume-safe. Step-level
+analyst edits cannot modify it (enforced by the shared protected-region check in
+:mod:`skillopt.optimizer.skill`).
+
+Public API
+----------
+- :func:`has_appendix_field`        — check if markers are present
+- :func:`inject_empty_appendix_field` — add empty placeholder (skill init)
+- :func:`extract_appendix_notes`    — read current notes as a list
+- :func:`append_to_appendix_field`  — merge new notes (dedup) into the region
+"""
+from __future__ import annotations
+
+import re
+
+# ── Protected field markers ─────────────────────────────────────────────────
+
+APPENDIX_START = "<!-- APPENDIX_START -->"
+APPENDIX_END = "<!-- APPENDIX_END -->"
+
+# Heading shown inside the rendered appendix block (human-readable only).
+APPENDIX_HEADING = "## Execution Notes Appendix"
+
+# Each note is rendered as a markdown bullet so the target model reads it as
+# ordinary guidance.
+_NOTE_BULLET_PREFIX = "- "
+
+
+# ── Dedup helpers ───────────────────────────────────────────────────────────
+
+
+def _canonicalize(text: str) -> str:
+    """Normalize a note for duplicate detection (whitespace/punct/case-insensitive)."""
+    normalized = re.sub(r"\s+", " ", str(text or "").strip())
+    normalized = normalized.rstrip(" .;:,_-")
+    return normalized.casefold()
+
+
+def _dedupe_preserve_order(notes: list[str]) -> list[str]:
+    """Drop blanks and near-duplicates, preserving first-seen order."""
+    seen: set[str] = set()
+    deduped: list[str] = []
+    for note in notes:
+        text = re.sub(r"\s+", " ", str(note).strip())
+        if not text:
+            continue
+        key = _canonicalize(text)
+        if not key or key in seen:
+            continue
+        seen.add(key)
+        deduped.append(text)
+    return deduped
+
+
+# ── Field manipulation ──────────────────────────────────────────────────────
+
+
+def has_appendix_field(skill: str) -> bool:
+    return APPENDIX_START in skill and APPENDIX_END in skill
+
+
+def _render_block(notes: list[str]) -> str:
+    """Render the full marker-delimited appendix block for *notes*."""
+    lines = [APPENDIX_START, APPENDIX_HEADING]
+    for note in notes:
+        lines.append(f"{_NOTE_BULLET_PREFIX}{note}")
+    lines.append(APPENDIX_END)
+    return "\n".join(lines)
+
+
+def inject_empty_appendix_field(skill: str) -> str:
+    """Add an empty appendix placeholder at the end of *skill* (idempotent).
+
+    Mirrors ``inject_empty_slow_update_field``: called once at skill init so the
+    protected region exists before any note is written.
+    """
+    if has_appendix_field(skill):
+        return skill
+    block = f"\n\n{APPENDIX_START}\n{APPENDIX_HEADING}\n{APPENDIX_END}\n"
+    return skill.rstrip() + block
+
+
+def extract_appendix_notes(skill: str) -> list[str]:
+    """Return the current appendix notes as a list of strings (no markers/heading)."""
+    start = skill.find(APPENDIX_START)
+    end = skill.find(APPENDIX_END)
+    if start == -1 or end == -1:
+        return []
+    inner = skill[start + len(APPENDIX_START):end].strip()
+    notes: list[str] = []
+    for raw_line in inner.splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        if line == APPENDIX_HEADING or line.lstrip("#").strip() == APPENDIX_HEADING.lstrip("#").strip():
+            continue
+        if line.startswith(_NOTE_BULLET_PREFIX):
+            line = line[len(_NOTE_BULLET_PREFIX):].strip()
+        elif line.startswith("-") or line.startswith("*"):
+            line = line[1:].strip()
+        if line:
+            notes.append(line)
+    return notes
+
+
+def _strip_all_appendix_fields(skill: str) -> str:
+    """Remove every appendix marker pair (and content between) from *skill*."""
+    while True:
+        start = skill.find(APPENDIX_START)
+        if start == -1:
+            break
+        end = skill.find(APPENDIX_END, start)
+        if end == -1:
+            skill = skill[:start] + skill[start + len(APPENDIX_START):]
+            break
+        skill = skill[:end + len(APPENDIX_END)].rsplit(APPENDIX_START, 1)[0] + skill[end + len(APPENDIX_END):]
+    skill = skill.replace(APPENDIX_END, "")
+    while "\n\n\n" in skill:
+        skill = skill.replace("\n\n\n", "\n\n")
+    return skill.rstrip()
+
+
+def append_to_appendix_field(skill: str, new_notes: list[str]) -> str:
+    """Merge *new_notes* into the appendix region (dedup), returning updated skill.
+
+    - If no appendix region exists yet, one is created.
+    - Existing notes are preserved; new ones are appended after dedup against the
+      combined set, so order is stable and duplicates are dropped.
+    - Empty / whitespace-only notes are ignored. If the merged set is empty, an
+      empty placeholder region is still ensured.
+    """
+    incoming = _dedupe_preserve_order(list(new_notes or []))
+    existing = extract_appendix_notes(skill)
+    merged = _dedupe_preserve_order(existing + incoming)
+
+    base = _strip_all_appendix_fields(skill)
+    block = _render_block(merged)
+    return f"{base}\n\n{block}\n"

skillopt/optimizer/skill.py

@@ -14,25 +14,62 @@ if TYPE_CHECKING:
 SLOW_UPDATE_START = "<!-- SLOW_UPDATE_START -->"
 SLOW_UPDATE_END = "<!-- SLOW_UPDATE_END -->"
 
+# Skill-aware reflection (EmbodiSkill S_app) appendix region. Like the slow
+# update region, it is protected: step-level analyst edits must not modify it.
+APPENDIX_START = "<!-- APPENDIX_START -->"
+APPENDIX_END = "<!-- APPENDIX_END -->"
 
-def _is_in_slow_update_region(skill: str, target: str) -> bool:
-    """Check if *target* text falls within the protected slow update region."""
-    start_idx = skill.find(SLOW_UPDATE_START)
-    end_idx = skill.find(SLOW_UPDATE_END)
-    if start_idx == -1 or end_idx == -1:
+# All protected (start, end) marker pairs. Step-level edits cannot target text
+# inside any of these regions, and `append` / `insert_after`-fallback ops are
+# inserted before the earliest-occurring region so protected blocks stay at the
+# document tail. With only the slow-update region present, every helper reduces
+# to the original slow-update-only behavior (byte-identical skill output).
+_PROTECTED_REGIONS: tuple[tuple[str, str], ...] = (
+    (SLOW_UPDATE_START, SLOW_UPDATE_END),
+    (APPENDIX_START, APPENDIX_END),
+)
+
+
+def _earliest_protected_start(skill: str) -> int:
+    """Index of the earliest protected-region start marker, or -1 if none."""
+    positions = [
+        idx
+        for idx in (skill.find(start) for start, _ in _PROTECTED_REGIONS)
+        if idx != -1
+    ]
+    return min(positions) if positions else -1
+
+
+def _is_in_protected_region(skill: str, target: str) -> bool:
+    """Check if *target* text falls within any protected region."""
+    if not target:
         return False
     target_idx = skill.find(target)
     if target_idx == -1:
         return False
-    region_end = end_idx + len(SLOW_UPDATE_END)
-    return start_idx <= target_idx < region_end
+    for start_marker, end_marker in _PROTECTED_REGIONS:
+        start_idx = skill.find(start_marker)
+        end_idx = skill.find(end_marker)
+        if start_idx == -1 or end_idx == -1:
+            continue
+        region_end = end_idx + len(end_marker)
+        if start_idx <= target_idx < region_end:
+            return True
+    return False
+
+
+def _is_in_slow_update_region(skill: str, target: str) -> bool:
+    """Backward-compatible alias kept for any external callers/tests."""
+    return _is_in_protected_region(skill, target)
 
 
 def _strip_slow_update_markers(text: str) -> str:
-    """Remove any SLOW_UPDATE markers from edit content to prevent duplication."""
+    """Remove any protected-region markers from edit content to prevent duplication."""
     return (
         text.replace(SLOW_UPDATE_START, "")
             .replace(SLOW_UPDATE_END, "")
+            .replace(APPENDIX_START, "")
+            .replace(APPENDIX_END, "")
     )
 
 
@@ -54,27 +91,27 @@ def _apply_edit_with_report(skill: str, edit: EditType | dict) -> tuple[str, dic
         "status": "unknown",
     }
 
-    if target and _is_in_slow_update_region(skill, target):
-        report["status"] = "skipped_protected_slow_update_region"
+    if target and _is_in_protected_region(skill, target):
+        report["status"] = "skipped_protected_region"
         return skill, report
 
     if op == "append":
-        su_start = skill.find(SLOW_UPDATE_START)
-        if su_start != -1:
-            before = skill[:su_start].rstrip()
-            after = skill[su_start:]
-            report["status"] = "applied_append_before_slow_update"
+        prot_start = _earliest_protected_start(skill)
+        if prot_start != -1:
+            before = skill[:prot_start].rstrip()
+            after = skill[prot_start:]
+            report["status"] = "applied_append_before_protected_region"
             return before + "\n\n" + content + "\n\n" + after, report
         report["status"] = "applied_append"
         return skill.rstrip() + "\n\n" + content + "\n", report
 
     if op == "insert_after":
         if not target or target not in skill:
-            su_start = skill.find(SLOW_UPDATE_START)
-            if su_start != -1:
-                before = skill[:su_start].rstrip()
-                after = skill[su_start:]
-                report["status"] = "applied_insert_after_fallback_before_slow_update"
+            prot_start = _earliest_protected_start(skill)
+            if prot_start != -1:
+                before = skill[:prot_start].rstrip()
+                after = skill[prot_start:]
+                report["status"] = "applied_insert_after_fallback_before_protected_region"
                 return before + "\n\n" + content + "\n\n" + after, report
             report["status"] = "applied_insert_after_fallback_append"
             return skill.rstrip() + "\n\n" + content + "\n", report

skillopt/optimizer/skill_aware.py

@@ -0,0 +1,206 @@
+"""Skill-Aware Reflection — analyst prompt augmentation (EmbodiSkill).
+
+When ``use_skill_aware_reflection`` is enabled, the failure/success analysts are
+asked to additionally classify each reflection by EmbodiSkill type and to route
+**EXECUTION_LAPSE** reflections (the skill rule is correct, the executor just
+failed to follow it) into a separate ``appendix_notes`` list instead of the body
+patch. This module owns:
+
+1. the instruction text appended to the resolved analyst system prompt, and
+2. extraction of ``appendix_notes`` from the analyst JSON response.
+
+Design notes
+------------
+- The suffix is appended **at runtime, gated by the toggle**, so env-specific and
+  generic analyst prompts are augmented uniformly and — when the toggle is off —
+  remain byte-identical to baseline.
+- Discrimination follows the paper / GMemory: ``SKILL_DEFECT`` = the skill rule is
+  wrong / missing / underspecified (→ body edit); ``EXECUTION_LAPSE`` = the rule
+  is valid but the agent didn't follow it (→ appendix reminder, body untouched).
+  **When unsure, default to EXECUTION_LAPSE** (protect the body — never delete a
+  valid rule over a one-off execution slip).
+- Success reflections are labeled DISCOVERY / OPTIMIZATION for logging only; their
+  edit behavior is unchanged.
+"""
+from __future__ import annotations
+
+
+# ── Runtime switch (config-driven, env-independent) ─────────────────────────
+#
+# The trainer calls :func:`configure_skill_aware_reflection` once at startup
+# from the resolved config. ``run_minibatch_reflect`` then picks these values
+# up automatically, so env adapters never need to thread the toggle through —
+# the feature is controlled purely by ``optimizer.use_skill_aware_reflection``
+# regardless of benchmark. Mirrors the ``configure_azure_openai`` pattern in
+# :mod:`skillopt.model`. Explicit kwargs at a call site still take precedence
+# (backward compatible).
+
+_RUNTIME: dict = {"enabled": False, "appendix_source": "both"}
+
+
+def configure_skill_aware_reflection(
+    enabled: bool,
+    appendix_source: str = "both",
+) -> None:
+    """Set the process-wide skill-aware reflection switch from config."""
+    _RUNTIME["enabled"] = bool(enabled)
+    _RUNTIME["appendix_source"] = str(appendix_source or "both")
+
+
+def is_skill_aware_enabled() -> bool:
+    return bool(_RUNTIME["enabled"])
+
+
+def get_skill_aware_appendix_source() -> str:
+    return str(_RUNTIME["appendix_source"])
+
+
+# ── Prompt suffixes ─────────────────────────────────────────────────────────
+
+# Appended to the FAILURE analyst system prompt when the toggle is on.
+ERROR_SUFFIX = """
+
+## Skill-Aware Reflection (EmbodiSkill)
+
+Before proposing body edits, classify EACH failure pattern as one of:
+
+- **SKILL_DEFECT**: the current skill is wrong, missing, or underspecified for
+  this situation — i.e. an agent that *followed the skill* would still fail, or
+  the skill gives no relevant guidance. These become normal body `edits`.
+- **EXECUTION_LAPSE**: the skill ALREADY contains a relevant, correct rule that
+  would have avoided the failure, but the agent did not follow it (e.g. ignored a
+  rule, malformed output, copied the feedback text verbatim, emitted a non-action
+  token like "stop", or otherwise broke execution unrelated to skill content).
+
+Discrimination test: "Is there a rule in the current skill that, if followed,
+prevents this failure?" If yes → EXECUTION_LAPSE. If no (rule absent/wrong) →
+SKILL_DEFECT. **When genuinely unsure, choose EXECUTION_LAPSE** — do not edit or
+delete a valid rule over a one-off execution slip.
+
+Routing:
+- SKILL_DEFECT → put the fix in `patch.edits` (body), as usual.
+- EXECUTION_LAPSE → put a concise reminder in `appendix_notes` (a flat list of
+  strings). DO NOT add a body edit for it. Each note should re-emphasize the
+  existing valid rule the agent failed to follow; it must NOT introduce a new
+  rule. Keep notes short, concrete, and reusable.
+
+Add `appendix_notes` as a TOP-LEVEL key of your JSON output (a sibling of
+`patch`), e.g. `"appendix_notes": ["Follow the existing X rule before Y."]`.
+Use `[]` when there is no execution lapse. Body edits and appendix notes are
+independent: a batch may yield only edits, only notes, both, or neither.
+"""
+
+# Appended to the SUCCESS analyst system prompt when the toggle is on.
+SUCCESS_SUFFIX = """
+
+## Skill-Aware Reflection (EmbodiSkill)
+
+For each proposed edit, optionally label its `reflection_type` for logging:
+- **DISCOVERY**: a useful new rule not yet in the skill (typically an `append`).
+- **OPTIMIZATION**: a better way to perform an existing rule (typically a
+  `replace` of that rule).
+
+This labeling does not change edit behavior. You may also add a top-level
+`appendix_notes` list (flat strings) if a successful trajectory reveals an
+existing valid rule worth re-emphasizing; otherwise use `[]`.
+"""
+
+
+def augment_error_prompt(system_prompt: str) -> str:
+    """Append the failure-analyst skill-aware instruction."""
+    return system_prompt.rstrip() + "\n" + ERROR_SUFFIX
+
+
+def augment_success_prompt(system_prompt: str) -> str:
+    """Append the success-analyst skill-aware instruction."""
+    return system_prompt.rstrip() + "\n" + SUCCESS_SUFFIX
+
+
+# ── Response parsing ────────────────────────────────────────────────────────
+
+
+def extract_appendix_notes(result: dict | None) -> list[str]:
+    """Pull a clean list of appendix-note strings from an analyst JSON result.
+
+    Tolerant of shape: accepts a top-level ``appendix_notes`` list, a single
+    string, or items wrapped in dicts with a ``note``/``content`` field. Returns
+    ``[]`` for anything missing or malformed (so a non-compliant model degrades
+    gracefully to baseline body-only behavior).
+    """
+    if not isinstance(result, dict):
+        return []
+    raw = result.get("appendix_notes")
+    if raw is None:
+        return []
+    if isinstance(raw, str):
+        raw = [raw]
+    if not isinstance(raw, list):
+        return []
+    notes: list[str] = []
+    for item in raw:
+        if isinstance(item, str):
+            text = item.strip()
+        elif isinstance(item, dict):
+            text = str(item.get("note") or item.get("content") or "").strip()
+        else:
+            text = ""
+        if text:
+            notes.append(text)
+    return notes
+
+
+# ── Appendix consolidation (threshold-gated, paper Eq.11 UpdateSkillAppendix) ──
+
+_CONSOLIDATE_SYSTEM = (
+    "You compact the Execution Notes Appendix of an agent skill. Each note "
+    "re-emphasizes an existing skill rule the agent failed to follow. Your job "
+    "is a periodic compaction pass: remove duplicates and redundant overlap, "
+    "merge near-identical reminders into one, and simplify phrasing while keeping "
+    "each note concrete and operational. Do not invent new rules. Preserve the "
+    "distinct actionable content. Return valid JSON only."
+)
+
+
+def consolidate_appendix_notes(
+    notes: list[str],
+    *,
+    chat_fn,
+    max_completion_tokens: int = 4096,
+) -> list[str]:
+    """LLM-consolidate appendix notes: dedupe / merge / compact.
+
+    Mirrors GMemory ``_maybe_refactor_execution_notes`` and paper Eq.11. ``chat_fn``
+    is the optimizer chat callable ``(system, user, max_completion_tokens, retries,
+    stage) -> (text, meta)``. On ANY failure (parse, empty, exception) the original
+    notes are returned unchanged, so consolidation can never lose the appendix.
+    """
+    from skillopt.utils import extract_json  # local import to avoid cycles
+
+    clean = [str(n).strip() for n in (notes or []) if str(n).strip()]
+    if len(clean) < 2:
+        return clean
+
+    numbered = "\n".join(f"{i}. {n}" for i, n in enumerate(clean, 1))
+    user = (
+        f"## Current Execution Notes ({len(clean)} total)\n{numbered}\n\n"
+        "Compact these into a shorter list without losing distinct actionable "
+        "information. Merge duplicates and near-duplicates; keep each note short, "
+        "concrete, and reusable. Return valid JSON only with this schema:\n"
+        '{ "appendix_notes": ["compacted note 1", "compacted note 2"] }'
+    )
+    try:
+        response, _ = chat_fn(
+            system=_CONSOLIDATE_SYSTEM,
+            user=user,
+            max_completion_tokens=max_completion_tokens,
+            retries=2,
+            stage="appendix_consolidate",
+        )
+        result = extract_json(response)
+        compacted = extract_appendix_notes(result)
+        # Guard: only accept a non-empty result that actually shrinks the set.
+        if compacted and len(compacted) <= len(clean):
+            return compacted
+    except Exception:  # noqa: BLE001
+        pass
+    return clean