fix(sleep): isolate claude CLI calls; concrete+override-aware reflect; honor hard constraints
Critical correctness fix found by debugging the thorough-analyst failure: * `claude -p` was running with the AMBIENT Claude Code project context (the repo's CLAUDE.md, installed skills, tools). The optimizer/target calls were polluted — reflect once replied with a list of the user's installed skills instead of JSON edits. Now ClaudeCliBackend._call runs ISOLATED: a clean temp cwd, --disallowedTools '*', --exclude-dynamic-system-prompt-sections. This is essential for the backend to be trustworthy and reproducible. * reflect prompt: translate failing rule-judge criteria into plain English (max_chars=1200 -> "the ENTIRE response must be at most 1200 characters") and require CONCRETE, verbatim thresholds in proposed rules (not "respect limits"). * attempt prompt: treat the Learned-preferences block as HARD CONSTRAINTS that override earlier conflicting skill text. Earlier Claude results predate this fix and are being re-validated clean; the Codex backend was never affected (it runs in its own exec context). Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
@@ -253,7 +253,12 @@ class CliBackend(Backend):
|
|||||||
def attempt(self, task: TaskRecord, skill: str, memory: str) -> str:
|
def attempt(self, task: TaskRecord, skill: str, memory: str) -> str:
|
||||||
prompt = (
|
prompt = (
|
||||||
"You are completing a recurring task for a user. Apply the skill and "
|
"You are completing a recurring task for a user. Apply the skill and "
|
||||||
"memory rules EXACTLY, including any output-format requirements.\n\n"
|
"memory rules EXACTLY, including any output-format requirements. If the "
|
||||||
|
"skill contains a 'Learned preferences' block, treat those rules as "
|
||||||
|
"HARD CONSTRAINTS that OVERRIDE anything earlier in the skill they "
|
||||||
|
"conflict with (e.g. an explicit length limit overrides 'be "
|
||||||
|
"exhaustive'). Satisfy every such constraint even at the cost of "
|
||||||
|
"brevity or detail.\n\n"
|
||||||
f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n"
|
f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n"
|
||||||
f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n"
|
f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n"
|
||||||
"Return ONLY the final answer text, nothing else."
|
"Return ONLY the final answer text, nothing else."
|
||||||
@@ -319,11 +324,31 @@ class CliBackend(Backend):
|
|||||||
part = part.strip()
|
part = part.strip()
|
||||||
if part:
|
if part:
|
||||||
crit[part] += 1
|
crit[part] += 1
|
||||||
|
|
||||||
|
def _explain(c: str) -> str:
|
||||||
|
# translate an "op=arg" criterion into a plain-English requirement
|
||||||
|
if "=" in c:
|
||||||
|
op, _, arg = c.partition("=")
|
||||||
|
op = op.strip(); arg = arg.strip()
|
||||||
|
if op == "max_chars":
|
||||||
|
return f"the ENTIRE response must be at most {arg} characters long"
|
||||||
|
if op == "min_chars":
|
||||||
|
return f"the response must be at least {arg} characters long"
|
||||||
|
if op == "section_present":
|
||||||
|
return f"the response must contain a section/heading titled '{arg}'"
|
||||||
|
if op == "regex":
|
||||||
|
return f"the response must match the pattern /{arg}/ (e.g. include that label)"
|
||||||
|
if op == "contains":
|
||||||
|
return f"the response must contain the text '{arg}'"
|
||||||
|
if op == "tool_called":
|
||||||
|
return f"the agent must actually call the '{arg}' tool"
|
||||||
|
return c
|
||||||
|
|
||||||
criteria_text = ""
|
criteria_text = ""
|
||||||
if crit:
|
if crit:
|
||||||
criteria_text = (
|
criteria_text = (
|
||||||
"\n# Exact criteria the outputs are FAILING (fix these directly)\n"
|
"\n# Exact criteria the outputs are FAILING (fix these directly)\n"
|
||||||
+ "\n".join(f"- {c} (failed {n}x)" for c, n in crit.most_common())
|
+ "\n".join(f"- {_explain(c)} [{c}, failed {n}x]" for c, n in crit.most_common())
|
||||||
)
|
)
|
||||||
prompt = (
|
prompt = (
|
||||||
"You are SkillOpt's optimizer. The agent keeps failing the recurring "
|
"You are SkillOpt's optimizer. The agent keeps failing the recurring "
|
||||||
@@ -332,12 +357,16 @@ class CliBackend(Backend):
|
|||||||
"GENERAL, reusable rule or preference (never task-specific, never an "
|
"GENERAL, reusable rule or preference (never task-specific, never an "
|
||||||
"answer to a single task). If exact failing criteria are listed, your "
|
"answer to a single task). If exact failing criteria are listed, your "
|
||||||
"edits MUST make future outputs satisfy every one of them.\n"
|
"edits MUST make future outputs satisfy every one of them.\n"
|
||||||
|
"BE CONCRETE: quote the exact threshold, section name, or format from "
|
||||||
|
"the criteria verbatim in your rule (e.g. write 'keep the entire "
|
||||||
|
"response under 1200 characters', NOT 'respect length limits'). Vague "
|
||||||
|
"rules do not change behavior; specific numeric/structural rules do.\n"
|
||||||
"IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; "
|
"IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; "
|
||||||
"you CANNOT delete the existing instructions above. If the current "
|
"you CANNOT delete the existing instructions above. If the current "
|
||||||
f"{target} text conflicts with a criterion (e.g. it says 'be exhaustive' "
|
f"{target} text conflicts with a criterion (e.g. it says 'be exhaustive' "
|
||||||
"but outputs must be under a character limit), write an explicit, "
|
"but outputs must be under a character limit), write an explicit, "
|
||||||
"forceful OVERRIDE rule that says it supersedes the conflicting "
|
"forceful OVERRIDE rule stating it supersedes the conflicting "
|
||||||
"instruction. "
|
"instruction, and put the hard requirement first.\n"
|
||||||
'Return ONLY a JSON array: '
|
'Return ONLY a JSON array: '
|
||||||
'[{"op":"add|replace|delete","content":"<rule>","anchor":"<text to replace/delete, optional>","rationale":"<why>"}].\n\n'
|
'[{"op":"add|replace|delete","content":"<rule>","anchor":"<text to replace/delete, optional>","rationale":"<why>"}].\n\n'
|
||||||
f"# Current {target}\n{cur_doc}\n"
|
f"# Current {target}\n{cur_doc}\n"
|
||||||
@@ -381,14 +410,33 @@ class ClaudeCliBackend(CliBackend):
|
|||||||
self.claude_path = claude_path
|
self.claude_path = claude_path
|
||||||
|
|
||||||
def _call(self, prompt: str, *, max_tokens: int = 1024) -> str:
|
def _call(self, prompt: str, *, max_tokens: int = 1024) -> str:
|
||||||
cmd = [self.claude_path, "-p", "--output-format", "text"]
|
# Run ISOLATED: a clean temp cwd so the ambient project's CLAUDE.md /
|
||||||
|
# skills / tools do not leak into the optimizer/target call, no tools,
|
||||||
|
# and per-machine dynamic system-prompt sections excluded. Without this,
|
||||||
|
# `claude -p` answers with full Claude Code context and ignores our
|
||||||
|
# prompt (e.g. it lists the user's installed skills).
|
||||||
|
import tempfile
|
||||||
|
cmd = [
|
||||||
|
self.claude_path, "-p", "--output-format", "text",
|
||||||
|
"--disallowedTools", "*",
|
||||||
|
"--exclude-dynamic-system-prompt-sections",
|
||||||
|
]
|
||||||
if self.model:
|
if self.model:
|
||||||
cmd += ["--model", self.model]
|
cmd += ["--model", self.model]
|
||||||
cmd += ["--", prompt]
|
cmd += ["--", prompt]
|
||||||
|
clean_cwd = tempfile.mkdtemp(prefix="skillopt_sleep_claude_")
|
||||||
try:
|
try:
|
||||||
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=self.timeout)
|
proc = subprocess.run(
|
||||||
|
cmd, capture_output=True, text=True, timeout=self.timeout, cwd=clean_cwd,
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
return ""
|
return ""
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
import shutil
|
||||||
|
shutil.rmtree(clean_cwd, ignore_errors=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return (proc.stdout or "").strip()
|
return (proc.stdout or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user