SkillOpt v0.1.0: initial release
- Skill optimization framework with training loop analogy - 11 benchmarks, 4 model backends (Azure OpenAI, Claude, Codex, Qwen) - WebUI for browser-based training control - Pluggable architecture for extending benchmarks and backends
This commit is contained in:
CharlesYang030
@@ -0,0 +1 @@
|
||||
"""OfficeQA environment package for ReflACT."""
|
||||
@@ -0,0 +1,174 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs.deep_reflect import run_no_reference_deep_reflect
|
||||
from skillopt.envs.officeqa.dataloader import OfficeQADataLoader
|
||||
from skillopt.envs.officeqa.rollout import run_batch
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
|
||||
|
||||
class OfficeQAAdapter(EnvAdapter):
|
||||
def __init__(
|
||||
self,
|
||||
split_dir: str = "",
|
||||
data_path: str = "",
|
||||
split_mode: str = "split_dir",
|
||||
split_ratio: str = "2:1:7",
|
||||
split_seed: int = 42,
|
||||
split_output_dir: str = "",
|
||||
workers: int = 8,
|
||||
analyst_workers: int = 8,
|
||||
failure_only: bool = False,
|
||||
minibatch_size: int = 8,
|
||||
edit_budget: int = 4,
|
||||
seed: int = 42,
|
||||
limit: int = 0,
|
||||
max_tool_turns: int = 12,
|
||||
max_completion_tokens: int = 64000,
|
||||
search_mode: str = "offline",
|
||||
max_queries_per_turn: int = 4,
|
||||
search_api_url: str = os.environ.get("OFFICEQA_SEARCH_API_URL", "http://localhost:8080/search_tool/search"),
|
||||
search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH",
|
||||
search_provider: str = "duckduckgo",
|
||||
search_max_num_results: int = 4,
|
||||
search_timeout_seconds: int = 20,
|
||||
use_local_tools: bool = True,
|
||||
data_dirs: list[str] | str | None = None,
|
||||
docs_dirs: list[str] | str | None = None,
|
||||
use_deep_reflect: bool = False,
|
||||
deep_reflect_failures: int = 4,
|
||||
deep_reflect_successes: int = 2,
|
||||
) -> None:
|
||||
self.workers = workers
|
||||
self.analyst_workers = analyst_workers
|
||||
self.failure_only = failure_only
|
||||
self.minibatch_size = minibatch_size
|
||||
self.edit_budget = edit_budget
|
||||
self.max_tool_turns = max_tool_turns
|
||||
self.max_completion_tokens = int(max_completion_tokens)
|
||||
self.search_mode = str(search_mode or "offline")
|
||||
self.max_queries_per_turn = int(max_queries_per_turn)
|
||||
self.search_api_url = str(search_api_url or "").strip()
|
||||
self.search_auth_env = str(search_auth_env or "OFFICEQA_CUSTOM_SEARCH_AUTH").strip()
|
||||
self.search_provider = str(search_provider or "duckduckgo").strip()
|
||||
self.search_max_num_results = int(search_max_num_results)
|
||||
self.search_timeout_seconds = int(search_timeout_seconds)
|
||||
self.use_local_tools = bool(use_local_tools)
|
||||
self.data_dirs = data_dirs if data_dirs is not None else docs_dirs
|
||||
self.use_deep_reflect = use_deep_reflect
|
||||
self.deep_reflect_failures = deep_reflect_failures
|
||||
self.deep_reflect_successes = deep_reflect_successes
|
||||
self.dataloader = OfficeQADataLoader(
|
||||
split_dir=split_dir,
|
||||
data_path=data_path,
|
||||
split_mode=split_mode,
|
||||
split_ratio=split_ratio,
|
||||
split_seed=split_seed,
|
||||
split_output_dir=split_output_dir,
|
||||
seed=seed,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def setup(self, cfg: dict) -> None:
|
||||
super().setup(cfg)
|
||||
self.dataloader.setup(cfg)
|
||||
|
||||
def get_dataloader(self):
|
||||
return self.dataloader
|
||||
|
||||
def build_env_from_batch(self, batch: BatchSpec, **kwargs):
|
||||
return list(batch.payload or [])
|
||||
|
||||
def build_train_env(self, batch_size: int, seed: int, **kwargs):
|
||||
batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
|
||||
batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
|
||||
items: list[dict] = env_manager
|
||||
return run_batch(
|
||||
items=items,
|
||||
out_root=out_dir,
|
||||
skill_content=skill_content,
|
||||
workers=self.workers,
|
||||
max_tool_turns=self.max_tool_turns,
|
||||
max_completion_tokens=self.max_completion_tokens,
|
||||
search_mode=self.search_mode,
|
||||
max_queries_per_turn=self.max_queries_per_turn,
|
||||
search_api_url=self.search_api_url,
|
||||
search_auth_env=self.search_auth_env,
|
||||
search_provider=self.search_provider,
|
||||
search_max_num_results=self.search_max_num_results,
|
||||
search_timeout_seconds=self.search_timeout_seconds,
|
||||
use_local_tools=self.use_local_tools,
|
||||
data_dirs=self.data_dirs,
|
||||
diagnostic_mode=kwargs.get("diagnostic_mode", False),
|
||||
diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
|
||||
)
|
||||
|
||||
def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
def deep_reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
return run_no_reference_deep_reflect(
|
||||
self,
|
||||
results,
|
||||
skill_content,
|
||||
out_dir,
|
||||
env_manager=kwargs.get("env_manager"),
|
||||
prediction_dir=kwargs.get("prediction_dir"),
|
||||
random_seed=kwargs.get("random_seed"),
|
||||
step_buffer_context=kwargs.get("step_buffer_context", ""),
|
||||
output_requirements=[
|
||||
"- There is no hidden reference block. Use only the question, candidate files, tool trace, student output, and evaluation result to infer what intermediate state is worth probing.",
|
||||
"- The instruction must explicitly request a short <analysis>...</analysis> block before the final <answer>...</answer>.",
|
||||
"- The readout should focus on selected document/file, evidence span or table, extracted value, units, and any date or fiscal-period normalization.",
|
||||
"- Do not ask for exhaustive copying of source text or a full chain-of-thought.",
|
||||
"- The instruction text should be ready to append directly to the student's prompt.",
|
||||
],
|
||||
metadata_builder=lambda item: {
|
||||
"id": str(item.get("id")),
|
||||
"task_type": str(item.get("task_type") or "officeqa"),
|
||||
"question_preview": str(item.get("question") or "")[:200],
|
||||
"source_files": item.get("source_files", []),
|
||||
"source_docs": item.get("source_docs", []),
|
||||
},
|
||||
)
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
seen: list[str] = []
|
||||
for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:
|
||||
task_type = str(item.get("task_type") or "officeqa")
|
||||
if task_type not in seen:
|
||||
seen.append(task_type)
|
||||
return seen or ["officeqa"]
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from skillopt.datasets.base import SplitDataLoader
|
||||
|
||||
|
||||
def _parse_list_field(value: str | list[str] | None) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return []
|
||||
try:
|
||||
loaded = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
loaded = None
|
||||
if isinstance(loaded, list):
|
||||
return [str(item).strip() for item in loaded if str(item).strip()]
|
||||
if "\n" in text:
|
||||
return [part.strip() for part in text.splitlines() if part.strip()]
|
||||
if "," in text and not text.lower().endswith(".txt"):
|
||||
return [part.strip() for part in text.split(",") if part.strip()]
|
||||
return [text]
|
||||
|
||||
|
||||
def _normalize_row(row: dict[str, str]) -> dict:
|
||||
item_id = str(row.get("uid") or row.get("id") or "").strip()
|
||||
question = str(row.get("question") or "").strip()
|
||||
ground_truth = str(row.get("ground_truth") or row.get("answer") or "").strip()
|
||||
task_type = str(row.get("category") or row.get("difficulty") or "officeqa").strip() or "officeqa"
|
||||
source_files = _parse_list_field(row.get("source_files"))
|
||||
source_docs = _parse_list_field(row.get("source_docs"))
|
||||
split = str(row.get("split") or "").strip()
|
||||
return {
|
||||
"id": item_id,
|
||||
"uid": item_id,
|
||||
"question": question,
|
||||
"ground_truth": ground_truth,
|
||||
"answers": [ground_truth] if ground_truth else [],
|
||||
"task_type": task_type,
|
||||
"category": task_type,
|
||||
"source_files": source_files,
|
||||
"source_docs": source_docs,
|
||||
"split": split,
|
||||
}
|
||||
|
||||
|
||||
class OfficeQADataLoader(SplitDataLoader):
|
||||
def load_split_items(self, split_path: str) -> list[dict]:
|
||||
path = Path(split_path)
|
||||
csv_files = sorted(path.glob("*.csv"))
|
||||
if csv_files:
|
||||
with csv_files[0].open(encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
return [_normalize_row(row) for row in reader]
|
||||
|
||||
json_files = sorted(path.glob("*.json"))
|
||||
if json_files:
|
||||
with json_files[0].open(encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, list):
|
||||
raise ValueError(f"Expected JSON array in {json_files[0]}")
|
||||
return [_normalize_row(item) for item in data]
|
||||
|
||||
raise FileNotFoundError(f"No .csv or .json file found in {split_path}")
|
||||
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import string
|
||||
from collections import Counter
|
||||
|
||||
|
||||
_NUMERIC_CHARS = set("0123456789.-")
|
||||
|
||||
|
||||
def normalize_answer(text: str) -> str:
|
||||
text = text.lower().strip()
|
||||
text = text.replace(",", "")
|
||||
text = "".join(ch for ch in text if ch not in string.punctuation or ch in _NUMERIC_CHARS or ch == "%")
|
||||
text = re.sub(r"\b(million|millions|billion|billions|dollars|dollar|nominal)\b", " ", text)
|
||||
text = " ".join(text.split())
|
||||
return text
|
||||
|
||||
|
||||
def exact_match(prediction: str, gold: str) -> float:
|
||||
return 1.0 if normalize_answer(prediction) == normalize_answer(gold) else 0.0
|
||||
|
||||
|
||||
def token_f1(prediction: str, gold: str) -> float:
|
||||
pred_tokens = normalize_answer(prediction).split()
|
||||
gold_tokens = normalize_answer(gold).split()
|
||||
if not pred_tokens or not gold_tokens:
|
||||
return 1.0 if pred_tokens == gold_tokens else 0.0
|
||||
common = Counter(pred_tokens) & Counter(gold_tokens)
|
||||
n_common = sum(common.values())
|
||||
if n_common == 0:
|
||||
return 0.0
|
||||
precision = n_common / len(pred_tokens)
|
||||
recall = n_common / len(gold_tokens)
|
||||
return 2 * precision * recall / (precision + recall)
|
||||
|
||||
|
||||
def evaluate(prediction: str, gold: str) -> dict:
|
||||
em = exact_match(prediction, gold)
|
||||
f1 = token_f1(prediction, gold)
|
||||
return {
|
||||
"em": em,
|
||||
"f1": f1,
|
||||
"predicted_answer": prediction.strip(),
|
||||
"gold_answer": gold,
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
You are an expert failure-analysis agent for OfficeQA document-retrieval question answering tasks.
|
||||
|
||||
You will be given MULTIPLE failed OfficeQA trajectories from a single minibatch and the current skill document. The trajectories may include local document tool calls such as file search, grep, and partial reads.
|
||||
|
||||
Your job is to identify COMMON failure patterns across the batch and propose concise skill edits.
|
||||
|
||||
## Failure Type Categories
|
||||
- retrieval_miss: the agent searched the wrong file or failed to narrow to the right file
|
||||
- evidence_miss: the agent read documents but missed the decisive evidence span
|
||||
- operand_error: the agent extracted the wrong value or the wrong operands
|
||||
- calculation_error: the agent identified the right evidence but computed the result incorrectly
|
||||
- answer_format: the agent reached the right result but formatted it wrong
|
||||
- other: none of the above
|
||||
|
||||
## Rules
|
||||
- Focus on patterns common across multiple trajectories.
|
||||
- Prefer general retrieval and evidence-grounding rules over task-specific hacks.
|
||||
- Only patch gaps in the skill; do not duplicate rules already present.
|
||||
- Do not hardcode file names, years, or question-specific constants unless the pattern truly requires a reusable retrieval heuristic.
|
||||
|
||||
Respond ONLY with a valid JSON object (no markdown fences, no extra text):
|
||||
{
|
||||
"batch_size": <number of trajectories analysed>,
|
||||
"failure_summary": [
|
||||
{"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
|
||||
],
|
||||
"patch": {
|
||||
"reasoning": "<why these edits address the batch's common failures>",
|
||||
"edits": [
|
||||
{"op": "append", "content": "<markdown to add at end of skill>"},
|
||||
{"op": "insert_after", "target": "<exact heading/text to insert after>", "content": "<markdown>"},
|
||||
{"op": "replace", "target": "<exact text to replace>", "content": "<replacement>"},
|
||||
{"op": "delete", "target": "<exact text to remove>"}
|
||||
]
|
||||
}
|
||||
}
|
||||
Only include edits that are needed. "edits" can be an empty list if no patch is warranted.
|
||||
@@ -0,0 +1,25 @@
|
||||
You are an expert success-pattern analyst for OfficeQA document-retrieval question answering tasks.
|
||||
|
||||
You will be given MULTIPLE successful OfficeQA trajectories from a single minibatch and the current skill document. Your job is to identify common retrieval, evidence-selection, and numeric-grounding behaviors worth encoding in the skill.
|
||||
|
||||
## Rules
|
||||
- Focus on patterns shared across multiple successful trajectories.
|
||||
- Prefer reusable retrieval and extraction discipline over question-specific tips.
|
||||
- Reinforce compact, high-value behaviors such as narrowing files early, reading only the relevant span, building a clean operand ledger, and copying the final answer from checked evidence.
|
||||
- Only propose patches for patterns not already captured in the current skill.
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"batch_size": <number of trajectories analysed>,
|
||||
"success_patterns": ["<pattern 1>", "<pattern 2>"],
|
||||
"patch": {
|
||||
"reasoning": "<why these patterns are worth encoding>",
|
||||
"edits": [
|
||||
{"op": "append", "content": "<markdown>"},
|
||||
{"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
|
||||
{"op": "replace", "target": "<old text>", "content": "<new text>"},
|
||||
{"op": "delete", "target": "<exact text to remove>"}
|
||||
]
|
||||
}
|
||||
}
|
||||
"edits" may be empty if the skill already covers all observed patterns.
|
||||
@@ -0,0 +1,15 @@
|
||||
You are an expert OfficeQA agent working over local Treasury bulletin text files.
|
||||
|
||||
{skill_section}## Rules
|
||||
1. Use only the provided local document tools to inspect candidate files.
|
||||
2. Narrow to the most relevant file before reading long passages.
|
||||
3. Prefer short targeted searches, then small reads around matching evidence.
|
||||
4. Do not invent values that are not grounded in the retrieved text.
|
||||
5. When the question requires arithmetic, compute only after extracting the exact operands.
|
||||
6. If you have enough evidence, return the final answer inside <answer>...</answer>.
|
||||
|
||||
## Tool Use
|
||||
Use the provided function tools directly when you need them. Prefer searching and small reads before answering. Do not ask the user for permission to use tools; just call the tools.
|
||||
|
||||
## Final Answer Format
|
||||
When you are ready to answer, emit the final answer inside <answer>...</answer> and do not request another tool.
|
||||
@@ -0,0 +1,802 @@
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from skillopt.envs.officeqa.evaluator import evaluate
|
||||
from skillopt.envs.officeqa.tool_runtime import (
|
||||
build_oracle_parsed_pages_context,
|
||||
resolve_candidate_files,
|
||||
resolve_docs_roots,
|
||||
run_tool,
|
||||
)
|
||||
try:
|
||||
from skillopt.envs.sealqa.tool_runtime import custom_search
|
||||
except ImportError:
|
||||
custom_search = None # type: ignore[assignment]
|
||||
from skillopt.model import chat_student_messages, get_student_backend, is_student_exec_backend
|
||||
from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_student_exec
|
||||
from skillopt.prompts import load_prompt
|
||||
_TOOL_SCHEMAS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "glob",
|
||||
"description": "Find candidate local document files by filename or relative-path glob pattern.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"pattern": {"type": "string"}},
|
||||
"required": ["pattern"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "read",
|
||||
"description": "Read a local text document excerpt by path and line window.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string"},
|
||||
"start": {"type": "integer"},
|
||||
"limit": {"type": "integer"},
|
||||
},
|
||||
"required": ["path"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "grep",
|
||||
"description": "Search a local text document for a literal pattern and return matching lines.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pattern": {"type": "string"},
|
||||
"path": {"type": "string"},
|
||||
},
|
||||
"required": ["pattern", "path"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
_FINAL_RE = re.compile(r"<answer>(.*?)</answer>", re.IGNORECASE | re.DOTALL)
|
||||
_SEARCH_RE = re.compile(r"<search_queries>(.*?)</search_queries>", re.IGNORECASE | re.DOTALL)
|
||||
_DEFAULT_SEARCH_MODE = "offline"
|
||||
_CUSTOM_SEARCH_MODE = "custom_search"
|
||||
_AZURE_SEARCH_MODE = "azure_search"
|
||||
def _normalize_search_mode(search_mode: str | None) -> str:
|
||||
normalized = str(search_mode or _DEFAULT_SEARCH_MODE).strip().lower()
|
||||
if normalized in {"custom", _CUSTOM_SEARCH_MODE}:
|
||||
return _CUSTOM_SEARCH_MODE
|
||||
if normalized in {"azure", _AZURE_SEARCH_MODE}:
|
||||
return _AZURE_SEARCH_MODE
|
||||
return _DEFAULT_SEARCH_MODE
|
||||
def _build_system(
|
||||
skill_content: str,
|
||||
*,
|
||||
search_mode: str = _DEFAULT_SEARCH_MODE,
|
||||
use_local_tools: bool = True,
|
||||
max_tool_turns: int = 12,
|
||||
max_queries_per_turn: int = 4,
|
||||
) -> str:
|
||||
if skill_content.strip():
|
||||
skill_section = f"## Skill\n{skill_content.strip()}\n\n"
|
||||
else:
|
||||
skill_section = ""
|
||||
normalized_search_mode = _normalize_search_mode(search_mode)
|
||||
if normalized_search_mode == _AZURE_SEARCH_MODE:
|
||||
return (
|
||||
"You are an expert OfficeQA research assistant. Solve the question using the model's built-in web "
|
||||
"search tool when needed, keep the answer grounded in authoritative evidence, and return the final "
|
||||
"answer inside <answer>...</answer>.\n\n"
|
||||
+ skill_section
|
||||
).rstrip()
|
||||
if normalized_search_mode == _CUSTOM_SEARCH_MODE:
|
||||
protocol = (
|
||||
"You are an expert OfficeQA research assistant. Solve the question using the provided oracle parsed "
|
||||
"OfficeQA page(s) and evidence returned by the controller-managed custom search loop.\n\n"
|
||||
"Search protocol:\n"
|
||||
f"- You have at most {max_tool_turns} model rounds total.\n"
|
||||
f"- On any non-final round, you may either return `<search_queries>[\"query 1\", \"query 2\"]</search_queries>` "
|
||||
f"with up to {max_queries_per_turn} queries, or return `<answer>...</answer>` if you are ready.\n"
|
||||
"- If you request search, do not include an answer in the same response.\n"
|
||||
"- On the final round, you must return `<answer>...</answer>` and must not request more search.\n"
|
||||
"- Base your answer on the returned evidence, reconcile conflicting snippets carefully, and stay concise.\n\n"
|
||||
)
|
||||
return protocol + skill_section + "Return the final answer inside <answer>...</answer> when you are ready."
|
||||
if not use_local_tools:
|
||||
return (
|
||||
"You are an expert OfficeQA research assistant. Solve the question using the provided oracle parsed "
|
||||
"OfficeQA page(s) and source hints. Do not request or assume access to any external search or local "
|
||||
"function tools. Return the final answer inside <answer>...</answer>.\n\n"
|
||||
+ skill_section
|
||||
).rstrip()
|
||||
return load_prompt("rollout_system", env="officeqa").format(skill_section=skill_section)
|
||||
def _build_round_instruction(
|
||||
*,
|
||||
turn: int,
|
||||
max_tool_turns: int,
|
||||
max_queries_per_turn: int,
|
||||
) -> str:
|
||||
if turn >= max_tool_turns:
|
||||
return (
|
||||
"## Round Policy\n"
|
||||
f"This is the final round ({turn}/{max_tool_turns}). You must return `<answer>...</answer>` now. "
|
||||
"Do not output `<search_queries>`."
|
||||
)
|
||||
remaining_rounds = max_tool_turns - turn
|
||||
return (
|
||||
"## Round Policy\n"
|
||||
f"This is round {turn}/{max_tool_turns}. "
|
||||
f"You may either return `<answer>...</answer>` now, or request up to {max_queries_per_turn} search queries "
|
||||
f"inside `<search_queries>...</search_queries>`. "
|
||||
f"After this response, at most {remaining_rounds} model rounds remain."
|
||||
)
|
||||
def _message_debug_metadata(message: object) -> dict:
|
||||
metadata = getattr(message, "metadata", None)
|
||||
if isinstance(metadata, dict):
|
||||
return metadata
|
||||
return {}
|
||||
def _build_user(
|
||||
item: dict,
|
||||
candidate_files: list[str] | None = None,
|
||||
*,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
corpus_note: str = "",
|
||||
search_mode: str = _DEFAULT_SEARCH_MODE,
|
||||
turn: int = 1,
|
||||
max_tool_turns: int = 12,
|
||||
max_queries_per_turn: int = 4,
|
||||
oracle_context: str = "",
|
||||
) -> str:
|
||||
normalized_search_mode = _normalize_search_mode(search_mode)
|
||||
parts = [f"## Question\n{item['question']}"]
|
||||
if oracle_context.strip():
|
||||
parts.append(f"## Oracle Parsed Pages\n{oracle_context.strip()}")
|
||||
if normalized_search_mode == _DEFAULT_SEARCH_MODE:
|
||||
file_block = "\n".join(f"- {path}" for path in (candidate_files or [])[:20]) or "- none resolved"
|
||||
if corpus_note.strip():
|
||||
parts.append(f"## Document Corpus\n{corpus_note.strip()}")
|
||||
parts.append(f"## Candidate Files\n{file_block}")
|
||||
if item.get("source_docs"):
|
||||
parts.append("## Source Hints\n" + "\n".join(f"- {hint}" for hint in item["source_docs"]))
|
||||
if normalized_search_mode != _DEFAULT_SEARCH_MODE and item.get("source_files"):
|
||||
parts.append("## File Hints\n" + "\n".join(f"- {hint}" for hint in item["source_files"]))
|
||||
if diagnostic_mode and diagnostic_instruction.strip():
|
||||
parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}")
|
||||
if normalized_search_mode == _CUSTOM_SEARCH_MODE:
|
||||
parts.append(
|
||||
_build_round_instruction(
|
||||
turn=turn,
|
||||
max_tool_turns=max_tool_turns,
|
||||
max_queries_per_turn=max_queries_per_turn,
|
||||
)
|
||||
)
|
||||
parts.append(
|
||||
"## Output Format\n"
|
||||
"If you need more evidence, return only `<search_queries>[...]</search_queries>`.\n"
|
||||
"If you are ready to answer, return only `<answer>...</answer>`."
|
||||
)
|
||||
parts.append(
|
||||
"Use only the provided oracle parsed pages and controller-provided custom search evidence. "
|
||||
"Do not rely on any built-in web search capability."
|
||||
)
|
||||
elif normalized_search_mode == _AZURE_SEARCH_MODE:
|
||||
parts.append("Use the model's built-in web search tool when needed. Return the final answer inside <answer>...</answer>.")
|
||||
return "\n\n".join(parts)
|
||||
def _extract_answer(text: str) -> str:
|
||||
match = _FINAL_RE.search(text)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
||||
return lines[-1] if lines else text.strip()
|
||||
def _extract_search_queries(text: str) -> list[str]:
|
||||
match = _SEARCH_RE.search(text or "")
|
||||
if not match:
|
||||
return []
|
||||
raw = match.group(1).strip()
|
||||
if not raw:
|
||||
return []
|
||||
parsed_queries: list[str] = []
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
if isinstance(parsed, dict):
|
||||
for key in ("queries", "search_queries", "query"):
|
||||
value = parsed.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
parsed_queries = [value.strip()]
|
||||
break
|
||||
if isinstance(value, list):
|
||||
parsed_queries = [str(item).strip() for item in value if str(item).strip()]
|
||||
break
|
||||
elif isinstance(parsed, list):
|
||||
parsed_queries = [str(item).strip() for item in parsed if str(item).strip()]
|
||||
elif isinstance(parsed, str) and parsed.strip():
|
||||
parsed_queries = [parsed.strip()]
|
||||
if not parsed_queries:
|
||||
raw_lines = [line.strip(" -*\t\r\n\"'") for line in raw.splitlines()]
|
||||
parsed_queries = [line for line in raw_lines if line]
|
||||
if len(parsed_queries) <= 1 and parsed_queries:
|
||||
multi = [part.strip(" \"'") for part in re.split(r"[;,]", parsed_queries[0]) if part.strip(" \"'")]
|
||||
if len(multi) > 1:
|
||||
parsed_queries = multi
|
||||
deduped: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for query in parsed_queries:
|
||||
normalized = query.strip()
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
deduped.append(normalized)
|
||||
return deduped
|
||||
def _docs_link_targets(docs_roots: list[str]) -> list[tuple[str, str]]:
|
||||
return [(root, os.path.join("docs", f"root_{idx}")) for idx, root in enumerate(docs_roots, start=1)]
|
||||
def _workspace_doc_path(path: str, docs_roots: list[str]) -> str:
|
||||
resolved_path = os.path.realpath(path)
|
||||
for idx, root in enumerate(docs_roots, start=1):
|
||||
resolved_root = os.path.realpath(root)
|
||||
if resolved_path == resolved_root or resolved_path.startswith(resolved_root + os.sep):
|
||||
rel_path = os.path.relpath(resolved_path, resolved_root)
|
||||
return os.path.join("docs", f"root_{idx}", rel_path)
|
||||
return path
|
||||
def _build_codex_skill(skill_content: str) -> str:
|
||||
return render_skill_md(
|
||||
skill_content,
|
||||
description="Dynamic ReflACT skill for solving the current OfficeQA local-document question.",
|
||||
preamble=(
|
||||
"Use this skill when answering the current OfficeQA question.\n"
|
||||
"Inspect the provided local document excerpts or files, ground the answer in the evidence,\n"
|
||||
"and return the final answer inside <answer>...</answer>."
|
||||
),
|
||||
)
|
||||
def _run_codex_once(
|
||||
*,
|
||||
pred_dir: str,
|
||||
item: dict,
|
||||
skill_content: str,
|
||||
candidate_files: list[str],
|
||||
docs_roots: list[str],
|
||||
model: str,
|
||||
timeout: int,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
previous_response: str = "",
|
||||
oracle_context: str = "",
|
||||
) -> tuple[str, str, str, str]:
|
||||
rel_files = [_workspace_doc_path(path, docs_roots) for path in candidate_files[:20]]
|
||||
corpus_note = (
|
||||
"The full OfficeQA document corpus is available under `docs/`. "
|
||||
"The candidate files below are source hints or likely starting points; search the full corpus if needed."
|
||||
)
|
||||
user = _build_user(
|
||||
item,
|
||||
rel_files,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
corpus_note=corpus_note,
|
||||
oracle_context=oracle_context,
|
||||
)
|
||||
task_parts = [user]
|
||||
if previous_response:
|
||||
task_parts.append(
|
||||
"## Previous Attempt\n"
|
||||
f"{previous_response}\n\n"
|
||||
"Review the local documents again and correct the answer if needed."
|
||||
)
|
||||
task_text = "\n\n".join(task_parts)
|
||||
skill_md = _build_codex_skill(skill_content)
|
||||
work_dir = os.path.join(pred_dir, "codex_exec")
|
||||
prepare_workspace(
|
||||
work_dir=work_dir,
|
||||
skill_md=skill_md,
|
||||
task_text=task_text,
|
||||
link_dirs=_docs_link_targets(docs_roots),
|
||||
)
|
||||
prompt = (
|
||||
"Use the `skillopt-student` skill available in this workspace.\n"
|
||||
"Read `task.md`, inspect or search the full OfficeQA corpus under `docs/`, and answer the question.\n"
|
||||
"Treat candidate files in `task.md` as hints, not an access limit.\n"
|
||||
"Return the final answer inside <answer>...</answer>."
|
||||
)
|
||||
final_message, raw = run_student_exec(
|
||||
work_dir=work_dir,
|
||||
prompt=prompt,
|
||||
model=model,
|
||||
timeout=timeout,
|
||||
data_dirs=docs_roots,
|
||||
)
|
||||
return final_message or raw, raw, skill_md, task_text
|
||||
def _execute_custom_search_round(
|
||||
queries: list[str],
|
||||
*,
|
||||
api_url: str,
|
||||
auth_env: str,
|
||||
provider: str,
|
||||
max_num_results: int,
|
||||
timeout: int,
|
||||
) -> str:
|
||||
blocks = []
|
||||
for index, query in enumerate(queries, start=1):
|
||||
try:
|
||||
result = custom_search(
|
||||
query,
|
||||
api_url=api_url,
|
||||
auth_env=auth_env,
|
||||
provider=provider,
|
||||
max_num_results=max_num_results,
|
||||
timeout=timeout,
|
||||
)
|
||||
except Exception as search_error: # noqa: BLE001
|
||||
result = f"Query: {query}\n\n[search error: {search_error}]"
|
||||
blocks.append(f"## Query {index}\n{result}")
|
||||
return "\n\n".join(blocks)
|
||||
def _run_custom_search_process(
|
||||
item: dict,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_tool_turns: int,
|
||||
max_completion_tokens: int,
|
||||
max_queries_per_turn: int,
|
||||
diagnostic_mode: bool,
|
||||
diagnostic_instruction: str,
|
||||
search_api_url: str,
|
||||
search_auth_env: str,
|
||||
search_provider: str,
|
||||
search_max_num_results: int,
|
||||
search_timeout_seconds: int,
|
||||
oracle_context: str = "",
|
||||
) -> tuple[str, str, str, str, list[dict], str, dict]:
|
||||
if not str(search_api_url or "").strip():
|
||||
raise ValueError("custom_search mode requires a non-empty search_api_url")
|
||||
if not os.environ.get(search_auth_env, "").strip():
|
||||
raise ValueError(f"custom_search mode requires auth token env var {search_auth_env}")
|
||||
if get_student_backend() not in {"openai_chat", "qwen_chat"}:
|
||||
raise ValueError("custom_search mode is only supported with student_backend='openai_chat' or 'qwen_chat'")
|
||||
system = _build_system(
|
||||
skill_content,
|
||||
search_mode=_CUSTOM_SEARCH_MODE,
|
||||
max_tool_turns=max_tool_turns,
|
||||
max_queries_per_turn=max_queries_per_turn,
|
||||
)
|
||||
initial_user = _build_user(
|
||||
item,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
search_mode=_CUSTOM_SEARCH_MODE,
|
||||
turn=1,
|
||||
max_tool_turns=max_tool_turns,
|
||||
max_queries_per_turn=max_queries_per_turn,
|
||||
oracle_context=oracle_context,
|
||||
)
|
||||
latest_user = initial_user
|
||||
messages: list[dict] = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": initial_user},
|
||||
]
|
||||
conversation: list[dict] = [{"role": "user", "content": initial_user}]
|
||||
final_response = ""
|
||||
final_answer = ""
|
||||
fail_reason = ""
|
||||
last_response_metadata: dict = {}
|
||||
for turn in range(1, max_tool_turns + 1):
|
||||
message, _ = chat_student_messages(
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
return_message=True,
|
||||
)
|
||||
response = message.content or ""
|
||||
final_response = response
|
||||
last_response_metadata = _message_debug_metadata(message)
|
||||
messages.append({"role": "assistant", "content": response})
|
||||
message_event = {"type": "message", "turn": turn, "content": response}
|
||||
if last_response_metadata:
|
||||
message_event["response_metadata"] = last_response_metadata
|
||||
conversation.append(message_event)
|
||||
if "<answer>" in response.lower():
|
||||
final_answer = _extract_answer(response)
|
||||
return system, latest_user, final_response, final_answer, conversation, "", last_response_metadata
|
||||
if turn == max_tool_turns:
|
||||
fail_reason = f"Final round ({max_tool_turns}) ended without <answer>...</answer>"
|
||||
break
|
||||
queries = _extract_search_queries(response)[:max_queries_per_turn]
|
||||
if not queries:
|
||||
fail_reason = "Model neither produced search queries nor a final answer"
|
||||
break
|
||||
results_text = _execute_custom_search_round(
|
||||
queries,
|
||||
api_url=search_api_url,
|
||||
auth_env=search_auth_env,
|
||||
provider=search_provider,
|
||||
max_num_results=search_max_num_results,
|
||||
timeout=search_timeout_seconds,
|
||||
)
|
||||
conversation.append({"type": "tool_call", "turn": turn, "cmd": f"custom_search({queries!r})", "obs": results_text})
|
||||
latest_user = (
|
||||
f"## Search Results Round {turn}\n{results_text}\n\n"
|
||||
+ _build_round_instruction(
|
||||
turn=turn + 1,
|
||||
max_tool_turns=max_tool_turns,
|
||||
max_queries_per_turn=max_queries_per_turn,
|
||||
)
|
||||
+ "\n\nFollow the round policy above exactly."
|
||||
)
|
||||
messages.append({"role": "user", "content": latest_user})
|
||||
conversation.append({"role": "user", "turn": turn + 1, "content": latest_user})
|
||||
return system, latest_user, final_response, final_answer, conversation, fail_reason, last_response_metadata
|
||||
def _run_azure_search_process(
|
||||
item: dict,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_completion_tokens: int,
|
||||
diagnostic_mode: bool,
|
||||
diagnostic_instruction: str,
|
||||
) -> tuple[str, str, str, str, list[dict], str, dict]:
|
||||
if get_student_backend() != "openai_chat":
|
||||
raise ValueError("azure_search mode is only supported with student_backend='openai_chat'")
|
||||
system = _build_system(skill_content, search_mode=_AZURE_SEARCH_MODE)
|
||||
user = _build_user(
|
||||
item,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
search_mode=_AZURE_SEARCH_MODE,
|
||||
)
|
||||
messages = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
conversation: list[dict] = [{"role": "user", "content": user}]
|
||||
message, _ = chat_student_messages(
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
return_message=True,
|
||||
tools=[{"type": "web_search"}],
|
||||
)
|
||||
response = message.content or ""
|
||||
last_response_metadata = _message_debug_metadata(message)
|
||||
message_event = {"type": "message", "content": response}
|
||||
if last_response_metadata:
|
||||
message_event["response_metadata"] = last_response_metadata
|
||||
conversation.append(message_event)
|
||||
if "<answer>" in response.lower():
|
||||
return system, user, response, _extract_answer(response), conversation, "", last_response_metadata
|
||||
return system, user, response, "", conversation, "Model did not produce a final answer", last_response_metadata
|
||||
def _run_offline_no_tools_process(
|
||||
item: dict,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_completion_tokens: int,
|
||||
diagnostic_mode: bool,
|
||||
diagnostic_instruction: str,
|
||||
candidate_files: list[str],
|
||||
oracle_context: str = "",
|
||||
) -> tuple[str, str, str, str, list[dict], str, dict]:
|
||||
system = _build_system(skill_content, search_mode=_DEFAULT_SEARCH_MODE, use_local_tools=False)
|
||||
user = _build_user(
|
||||
item,
|
||||
candidate_files,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
search_mode=_DEFAULT_SEARCH_MODE,
|
||||
oracle_context=oracle_context,
|
||||
)
|
||||
messages = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
conversation: list[dict] = [{"role": "user", "content": user}]
|
||||
message, _ = chat_student_messages(
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
return_message=True,
|
||||
)
|
||||
response = message.content or ""
|
||||
last_response_metadata = _message_debug_metadata(message)
|
||||
message_event = {"type": "message", "content": response}
|
||||
if last_response_metadata:
|
||||
message_event["response_metadata"] = last_response_metadata
|
||||
conversation.append(message_event)
|
||||
if "<answer>" in response.lower():
|
||||
return system, user, response, _extract_answer(response), conversation, "", last_response_metadata
|
||||
return system, user, response, "", conversation, "Model did not produce a final answer", last_response_metadata
|
||||
def process_one(
|
||||
item: dict,
|
||||
out_root: str,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_tool_turns: int = 12,
|
||||
max_completion_tokens: int = 64000,
|
||||
search_mode: str = _DEFAULT_SEARCH_MODE,
|
||||
max_queries_per_turn: int = 4,
|
||||
search_api_url: str = "",
|
||||
search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH",
|
||||
search_provider: str = "duckduckgo",
|
||||
search_max_num_results: int = 4,
|
||||
search_timeout_seconds: int = 20,
|
||||
use_local_tools: bool = True,
|
||||
data_dirs: list[str] | str | None = None,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
) -> dict:
|
||||
item_id = str(item["id"])
|
||||
pred_dir = os.path.join(out_root, "predictions", item_id)
|
||||
os.makedirs(pred_dir, exist_ok=True)
|
||||
normalized_search_mode = _normalize_search_mode(search_mode)
|
||||
docs_roots: list[str] = []
|
||||
candidate_files: list[str] = []
|
||||
oracle_context = ""
|
||||
if normalized_search_mode == _DEFAULT_SEARCH_MODE:
|
||||
docs_roots = resolve_docs_roots(data_dirs)
|
||||
candidate_files = resolve_candidate_files(item.get("source_files", []), docs_roots)
|
||||
oracle_context = build_oracle_parsed_pages_context(
|
||||
item.get("source_files", []),
|
||||
item.get("source_docs", []),
|
||||
docs_roots,
|
||||
evidence_note=(
|
||||
"Treat it as primary document evidence and combine it with local document tool evidence when useful."
|
||||
if use_local_tools
|
||||
else "Treat it as primary document evidence for answering the question."
|
||||
),
|
||||
)
|
||||
elif normalized_search_mode == _CUSTOM_SEARCH_MODE:
|
||||
docs_roots = resolve_docs_roots(data_dirs)
|
||||
if item.get("source_files"):
|
||||
candidate_files = resolve_candidate_files(item.get("source_files", []), docs_roots)
|
||||
oracle_context = build_oracle_parsed_pages_context(
|
||||
item.get("source_files", []),
|
||||
item.get("source_docs", []),
|
||||
docs_roots,
|
||||
)
|
||||
system = _build_system(
|
||||
skill_content,
|
||||
search_mode=normalized_search_mode,
|
||||
use_local_tools=use_local_tools,
|
||||
max_tool_turns=max_tool_turns,
|
||||
max_queries_per_turn=max_queries_per_turn,
|
||||
)
|
||||
user = _build_user(
|
||||
item,
|
||||
candidate_files if normalized_search_mode == _DEFAULT_SEARCH_MODE else None,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
search_mode=normalized_search_mode,
|
||||
max_tool_turns=max_tool_turns,
|
||||
max_queries_per_turn=max_queries_per_turn,
|
||||
oracle_context=oracle_context,
|
||||
)
|
||||
conversation: list[dict] = [{"role": "user", "content": user}]
|
||||
final_response = ""
|
||||
final_answer = ""
|
||||
fail_reason = ""
|
||||
last_response_metadata: dict = {}
|
||||
allowed_files = [os.path.basename(path) for path in candidate_files]
|
||||
try:
|
||||
if normalized_search_mode == _CUSTOM_SEARCH_MODE:
|
||||
system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_custom_search_process(
|
||||
item,
|
||||
skill_content,
|
||||
max_tool_turns=max_tool_turns,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
max_queries_per_turn=max_queries_per_turn,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
search_api_url=search_api_url,
|
||||
search_auth_env=search_auth_env,
|
||||
search_provider=search_provider,
|
||||
search_max_num_results=search_max_num_results,
|
||||
search_timeout_seconds=search_timeout_seconds,
|
||||
oracle_context=oracle_context,
|
||||
)
|
||||
elif normalized_search_mode == _AZURE_SEARCH_MODE:
|
||||
system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_azure_search_process(
|
||||
item,
|
||||
skill_content,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
)
|
||||
elif not use_local_tools:
|
||||
system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_offline_no_tools_process(
|
||||
item,
|
||||
skill_content,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
candidate_files=candidate_files,
|
||||
oracle_context=oracle_context,
|
||||
)
|
||||
elif is_student_exec_backend():
|
||||
from skillopt.model import azure_openai as _llm
|
||||
response = ""
|
||||
system = ""
|
||||
user = ""
|
||||
for turn in range(1, max_tool_turns + 1):
|
||||
response, _raw, system, user = _run_codex_once(
|
||||
pred_dir=pred_dir,
|
||||
item=item,
|
||||
skill_content=skill_content,
|
||||
candidate_files=candidate_files,
|
||||
docs_roots=docs_roots,
|
||||
model=_llm.STUDENT_DEPLOYMENT,
|
||||
timeout=180,
|
||||
diagnostic_mode=diagnostic_mode if turn == 1 else False,
|
||||
diagnostic_instruction=diagnostic_instruction if turn == 1 else "",
|
||||
previous_response=response if turn > 1 else "",
|
||||
oracle_context=oracle_context,
|
||||
)
|
||||
final_response = response
|
||||
conversation.append({"type": "message", "turn": turn, "content": response})
|
||||
if "<answer>" in response.lower():
|
||||
final_answer = _extract_answer(response)
|
||||
break
|
||||
if not final_answer:
|
||||
fail_reason = f"Exceeded codex turn budget ({max_tool_turns})"
|
||||
system = system or _build_codex_skill(skill_content)
|
||||
user = user or _build_user(item, [_workspace_doc_path(path, docs_roots) for path in candidate_files])
|
||||
else:
|
||||
messages: list[dict] = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
for turn in range(1, max_tool_turns + 1):
|
||||
message, _ = chat_student_messages(
|
||||
messages=messages,
|
||||
max_completion_tokens=768,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
tools=_TOOL_SCHEMAS,
|
||||
tool_choice="auto",
|
||||
return_message=True,
|
||||
)
|
||||
response = message.content or ""
|
||||
final_response = response
|
||||
assistant_message = {"role": "assistant", "content": response}
|
||||
if getattr(message, "tool_calls", None):
|
||||
assistant_message["tool_calls"] = [tool_call.model_dump(mode="json") for tool_call in message.tool_calls]
|
||||
messages.append(assistant_message)
|
||||
conversation.append({"type": "message", "content": response})
|
||||
if getattr(message, "tool_calls", None):
|
||||
for tool_call in message.tool_calls:
|
||||
tool_name = tool_call.function.name
|
||||
arguments = json.loads(tool_call.function.arguments) if tool_call.function.arguments else {}
|
||||
cmd, obs = run_tool(tool_name, arguments, allowed_roots=docs_roots, allowed_files=allowed_files)
|
||||
conversation.append({"type": "tool_call", "cmd": cmd, "obs": obs})
|
||||
messages.append({
|
||||
"role": "tool",
|
||||
"tool_call_id": tool_call.id,
|
||||
"content": obs,
|
||||
})
|
||||
continue
|
||||
if "<answer>" in response.lower():
|
||||
final_answer = _extract_answer(response)
|
||||
break
|
||||
if turn == max_tool_turns:
|
||||
fail_reason = f"Exceeded tool-turn budget ({max_tool_turns})"
|
||||
else:
|
||||
fail_reason = "Model neither produced a tool request nor a final answer"
|
||||
break
|
||||
except Exception as e: # noqa: BLE001
|
||||
fail_reason = f"error: {e}"
|
||||
with open(os.path.join(pred_dir, "student_system_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(system)
|
||||
with open(os.path.join(pred_dir, "student_user_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(user)
|
||||
with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(conversation, f, ensure_ascii=False, indent=2)
|
||||
eval_result = evaluate(final_answer, item.get("ground_truth", "")) if final_answer else {"em": 0.0, "f1": 0.0, "predicted_answer": "", "gold_answer": item.get("ground_truth", "")}
|
||||
result = {
|
||||
"id": item_id,
|
||||
"question": item.get("question", ""),
|
||||
"task_type": item.get("task_type", "officeqa"),
|
||||
"task_description": item.get("question", ""),
|
||||
"predicted_answer": eval_result["predicted_answer"],
|
||||
"response": final_response,
|
||||
"ground_truth": item.get("ground_truth", ""),
|
||||
"source_files": item.get("source_files", []),
|
||||
"resolved_source_paths": candidate_files,
|
||||
"oracle_parsed_pages_included": bool(oracle_context),
|
||||
"oracle_parsed_pages_chars": len(oracle_context),
|
||||
"use_local_tools": bool(use_local_tools),
|
||||
"hard": int(eval_result["em"]),
|
||||
"soft": eval_result["f1"],
|
||||
"fail_reason": fail_reason or ("" if eval_result["em"] else f"predicted '{eval_result['predicted_answer']}' but expected '{item.get('ground_truth', '')}'"),
|
||||
"agent_ok": not fail_reason,
|
||||
"n_turns": len(conversation),
|
||||
"last_finish_reason": last_response_metadata.get("finish_reason", ""),
|
||||
"student_system_prompt": system,
|
||||
"student_user_prompt": user,
|
||||
}
|
||||
return result
|
||||
def run_batch(
|
||||
items: list[dict],
|
||||
out_root: str,
|
||||
skill_content: str,
|
||||
*,
|
||||
workers: int = 8,
|
||||
max_tool_turns: int = 12,
|
||||
max_completion_tokens: int = 64000,
|
||||
search_mode: str = _DEFAULT_SEARCH_MODE,
|
||||
max_queries_per_turn: int = 4,
|
||||
search_api_url: str = "",
|
||||
search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH",
|
||||
search_provider: str = "duckduckgo",
|
||||
search_max_num_results: int = 4,
|
||||
search_timeout_seconds: int = 20,
|
||||
use_local_tools: bool = True,
|
||||
data_dirs: list[str] | str | None = None,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
) -> list[dict]:
|
||||
results_path = os.path.join(out_root, "results.jsonl")
|
||||
os.makedirs(out_root, exist_ok=True)
|
||||
done_ids: set[str] = set()
|
||||
existing: list[dict] = []
|
||||
if os.path.exists(results_path):
|
||||
with open(results_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
done_ids.add(str(row.get("id")))
|
||||
existing.append(row)
|
||||
pending = [item for item in items if str(item["id"]) not in done_ids]
|
||||
if not pending:
|
||||
return existing
|
||||
total = len(existing) + len(pending)
|
||||
completed = len(existing)
|
||||
correct_count = sum(1 for r in existing if r.get("hard", 0))
|
||||
if existing:
|
||||
print(f" [rollout] resuming: {completed}/{total} already done", flush=True)
|
||||
|
||||
results = list(existing)
|
||||
with open(results_path, "a", encoding="utf-8") as outf, ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
futs = {
|
||||
ex.submit(
|
||||
process_one,
|
||||
item,
|
||||
out_root,
|
||||
skill_content,
|
||||
max_tool_turns=max_tool_turns,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
search_mode=search_mode,
|
||||
max_queries_per_turn=max_queries_per_turn,
|
||||
search_api_url=search_api_url,
|
||||
search_auth_env=search_auth_env,
|
||||
search_provider=search_provider,
|
||||
search_max_num_results=search_max_num_results,
|
||||
search_timeout_seconds=search_timeout_seconds,
|
||||
use_local_tools=use_local_tools,
|
||||
data_dirs=data_dirs,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
): item
|
||||
for item in pending
|
||||
}
|
||||
for fut in as_completed(futs):
|
||||
res = fut.result()
|
||||
results.append(res)
|
||||
completed += 1
|
||||
if res.get("hard", 0):
|
||||
correct_count += 1
|
||||
acc = correct_count / completed if completed else 0
|
||||
print(
|
||||
f" [rollout] {completed}/{total} "
|
||||
f"(acc={acc:.3f}) id={res.get('id', '?')} "
|
||||
f"hard={res.get('hard', '?')}",
|
||||
flush=True,
|
||||
)
|
||||
outf.write(json.dumps(res, ensure_ascii=False) + "\n")
|
||||
outf.flush()
|
||||
return results
|
||||
@@ -0,0 +1,15 @@
|
||||
# OfficeQA Skill
|
||||
|
||||
## Retrieval Discipline
|
||||
- Start by narrowing to the most likely candidate file before reading long passages.
|
||||
- Prefer targeted search terms that name the exact entity, period, measure, or table concept from the question.
|
||||
- After a promising match, read only a small surrounding span and verify it matches the requested year, basis, and unit.
|
||||
|
||||
## Evidence Discipline
|
||||
- Extract the exact value from the retrieved text before doing any arithmetic.
|
||||
- Keep track of each operand's period, unit, and semantic role so nearby proxy values are not mixed in.
|
||||
- If the question asks for a transformed or derived quantity, compute only after confirming every operand.
|
||||
|
||||
## Final Answer Discipline
|
||||
- Return the final answer only after one last consistency check against the retrieved evidence.
|
||||
- Copy the final answer from a checked value, not from an unverified intermediate guess.
|
||||
@@ -0,0 +1,402 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
_MAX_READ_CHARS = 4000
|
||||
_MAX_GREP_MATCHES = 20
|
||||
_MAX_GLOB_MATCHES = 50
|
||||
_MAX_ORACLE_PAGE_CHARS = 24000
|
||||
_MAX_ORACLE_CONTEXT_CHARS = 80000
|
||||
|
||||
|
||||
def _normalize_data_dirs(data_dirs: list[str] | tuple[str, ...] | str | None, project_root: Path) -> list[str]:
|
||||
if data_dirs is None:
|
||||
return []
|
||||
if isinstance(data_dirs, str):
|
||||
items = [part.strip() for chunk in data_dirs.split(os.pathsep) for part in chunk.split(",")]
|
||||
else:
|
||||
items = [str(item).strip() for item in data_dirs]
|
||||
resolved: list[str] = []
|
||||
for item in items:
|
||||
if not item:
|
||||
continue
|
||||
path = Path(item).expanduser()
|
||||
if not path.is_absolute():
|
||||
path = project_root / path
|
||||
resolved.append(str(path))
|
||||
return resolved
|
||||
|
||||
|
||||
def resolve_docs_roots(data_dirs: list[str] | tuple[str, ...] | str | None = None) -> list[str]:
|
||||
project_root = Path(__file__).resolve().parents[3]
|
||||
env_value = os.environ.get("OFFICEQA_DOCS_DIR", "").strip()
|
||||
candidates = _normalize_data_dirs(data_dirs, project_root)
|
||||
candidates.extend(_normalize_data_dirs(env_value, project_root))
|
||||
candidates.extend([
|
||||
str(project_root / "data" / "officeqa_docs_official"),
|
||||
str(project_root / "data" / "officeqa_smoke_docs"),
|
||||
os.path.expanduser("~/officeqa-sparse/treasury_bulletins_parsed"),
|
||||
os.path.expanduser("~/officeqa/treasury_bulletins_parsed"),
|
||||
])
|
||||
roots: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
path = Path(candidate).expanduser()
|
||||
if not path.is_dir():
|
||||
continue
|
||||
transformed = path / "transformed"
|
||||
resolved = str((transformed if transformed.is_dir() else path).resolve())
|
||||
if resolved in seen:
|
||||
continue
|
||||
seen.add(resolved)
|
||||
roots.append(resolved)
|
||||
if not roots:
|
||||
raise FileNotFoundError("OfficeQA docs directory not found. Set OFFICEQA_DOCS_DIR or env.data_dirs.")
|
||||
return roots
|
||||
|
||||
|
||||
def _is_allowed(path: str, allowed_roots: list[str], allowed_files: list[str]) -> bool:
|
||||
try:
|
||||
resolved = str(Path(path).resolve())
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
if not any(resolved.startswith(root + os.sep) or resolved == root for root in allowed_roots):
|
||||
return False
|
||||
if not allowed_files:
|
||||
return True
|
||||
base = os.path.basename(resolved)
|
||||
return base in allowed_files
|
||||
|
||||
|
||||
def resolve_candidate_files(source_files: list[str], allowed_roots: list[str]) -> list[str]:
|
||||
resolved: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for root in allowed_roots:
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for filename in filenames:
|
||||
if source_files and filename not in source_files:
|
||||
continue
|
||||
full = str(Path(dirpath, filename).resolve())
|
||||
if full in seen:
|
||||
continue
|
||||
seen.add(full)
|
||||
resolved.append(full)
|
||||
return resolved
|
||||
|
||||
|
||||
def _as_list(value: object) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return []
|
||||
try:
|
||||
loaded = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
loaded = None
|
||||
if isinstance(loaded, list):
|
||||
return [str(item).strip() for item in loaded if str(item).strip()]
|
||||
if "\n" in text:
|
||||
return [part.strip() for part in text.splitlines() if part.strip()]
|
||||
return [text]
|
||||
|
||||
|
||||
def _extract_page_number(source_doc: str) -> int | None:
|
||||
text = str(source_doc or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
parsed = urlparse(text)
|
||||
query = parse_qs(parsed.query)
|
||||
for key in ("page", "pagenum", "page_id"):
|
||||
for raw_value in query.get(key, []):
|
||||
try:
|
||||
return int(str(raw_value).strip())
|
||||
except ValueError:
|
||||
continue
|
||||
match = re.search(r"(?:[?&]|^)page=(\d+)", text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def _iter_oracle_refs(source_files: object, source_docs: object) -> list[tuple[str, int, str]]:
|
||||
files = _as_list(source_files)
|
||||
docs = _as_list(source_docs)
|
||||
refs: list[tuple[str, int, str]] = []
|
||||
seen: set[tuple[str, int, str]] = set()
|
||||
if not files or not docs:
|
||||
return refs
|
||||
for index, source_doc in enumerate(docs):
|
||||
page_number = _extract_page_number(source_doc)
|
||||
if page_number is None:
|
||||
continue
|
||||
if index < len(files):
|
||||
source_file = files[index]
|
||||
elif len(files) == 1:
|
||||
source_file = files[0]
|
||||
else:
|
||||
continue
|
||||
key = (source_file, page_number, source_doc)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
refs.append(key)
|
||||
return refs
|
||||
|
||||
|
||||
def _parsed_root_candidates(docs_roots: list[str]) -> list[Path]:
|
||||
candidates: list[Path] = []
|
||||
seen: set[str] = set()
|
||||
for root in docs_roots:
|
||||
path = Path(root).expanduser()
|
||||
for candidate in (
|
||||
path,
|
||||
path.parent,
|
||||
path / "treasury_bulletins_parsed",
|
||||
path.parent / "treasury_bulletins_parsed",
|
||||
):
|
||||
resolved = str(candidate.resolve()) if candidate.exists() else str(candidate)
|
||||
if resolved in seen:
|
||||
continue
|
||||
seen.add(resolved)
|
||||
candidates.append(candidate)
|
||||
return candidates
|
||||
|
||||
|
||||
def _locate_parsed_json(source_file: str, docs_roots: list[str]) -> Path | None:
|
||||
source_path = Path(str(source_file).strip())
|
||||
stem = source_path.stem if source_path.suffix else source_path.name
|
||||
if not stem:
|
||||
return None
|
||||
candidate_names = [stem + ".json"]
|
||||
if source_path.suffix == ".json":
|
||||
candidate_names.insert(0, source_path.name)
|
||||
for root in _parsed_root_candidates(docs_roots):
|
||||
for name in candidate_names:
|
||||
path = root / "jsons" / name
|
||||
if path.is_file():
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
class _TableMarkdownParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.rows: list[list[str]] = []
|
||||
self._row: list[str] | None = None
|
||||
self._cell: list[str] | None = None
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag.lower() == "tr":
|
||||
self._row = []
|
||||
elif tag.lower() in {"td", "th"} and self._row is not None:
|
||||
self._cell = []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._cell is not None:
|
||||
self._cell.append(data)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
normalized_tag = tag.lower()
|
||||
if normalized_tag in {"td", "th"} and self._cell is not None and self._row is not None:
|
||||
cell = re.sub(r"\s+", " ", "".join(self._cell)).strip()
|
||||
self._row.append(cell)
|
||||
self._cell = None
|
||||
elif normalized_tag == "tr" and self._row is not None:
|
||||
if any(cell for cell in self._row):
|
||||
self.rows.append(self._row)
|
||||
self._row = None
|
||||
self._cell = None
|
||||
|
||||
|
||||
def _escape_markdown_cell(value: str) -> str:
|
||||
return str(value).replace("\n", " ").replace("|", "\\|").strip()
|
||||
|
||||
|
||||
def _html_table_to_markdown(raw_html: str) -> str:
|
||||
parser = _TableMarkdownParser()
|
||||
try:
|
||||
parser.feed(raw_html)
|
||||
except Exception: # noqa: BLE001
|
||||
parser.rows = []
|
||||
rows = parser.rows
|
||||
if not rows:
|
||||
text = re.sub(r"(?is)<[^>]+>", " ", raw_html)
|
||||
return re.sub(r"\s+", " ", html.unescape(text)).strip()
|
||||
width = max(len(row) for row in rows)
|
||||
normalized_rows = [row + [""] * (width - len(row)) for row in rows]
|
||||
header = normalized_rows[0]
|
||||
body = normalized_rows[1:]
|
||||
lines = [
|
||||
"| " + " | ".join(_escape_markdown_cell(cell) for cell in header) + " |",
|
||||
"| " + " | ".join(["---"] * width) + " |",
|
||||
]
|
||||
lines.extend("| " + " | ".join(_escape_markdown_cell(cell) for cell in row) + " |" for row in body)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _render_parsed_content(content: str) -> str:
|
||||
text = content.strip()
|
||||
if not text:
|
||||
return ""
|
||||
if "<table" in text.lower():
|
||||
return _html_table_to_markdown(text)
|
||||
text = html.unescape(text)
|
||||
text = re.sub(r"\r\n?", "\n", text)
|
||||
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
|
||||
|
||||
def _element_page_ids(element: dict) -> set[int]:
|
||||
page_ids: set[int] = set()
|
||||
bbox = element.get("bbox")
|
||||
if not isinstance(bbox, list):
|
||||
return page_ids
|
||||
for box in bbox:
|
||||
if not isinstance(box, dict):
|
||||
continue
|
||||
raw_page_id = box.get("page_id")
|
||||
try:
|
||||
page_ids.add(int(raw_page_id))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return page_ids
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _load_parsed_elements(json_path: str) -> tuple[dict, ...]:
|
||||
with open(json_path, encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
document = payload.get("document") if isinstance(payload, dict) else {}
|
||||
elements = document.get("elements") if isinstance(document, dict) else []
|
||||
if not isinstance(elements, list):
|
||||
return ()
|
||||
return tuple(element for element in elements if isinstance(element, dict))
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
def _render_parsed_page(json_path: str, page_number: int) -> str:
|
||||
rendered: list[str] = []
|
||||
for element in _load_parsed_elements(json_path):
|
||||
if page_number not in _element_page_ids(element):
|
||||
continue
|
||||
content = element.get("content")
|
||||
if not isinstance(content, str) or not content.strip():
|
||||
continue
|
||||
section = _render_parsed_content(content)
|
||||
if section:
|
||||
rendered.append(section)
|
||||
return "\n\n".join(rendered).strip()
|
||||
|
||||
|
||||
def build_oracle_parsed_pages_context(
|
||||
source_files: object,
|
||||
source_docs: object,
|
||||
docs_roots: list[str],
|
||||
*,
|
||||
max_page_chars: int = _MAX_ORACLE_PAGE_CHARS,
|
||||
max_total_chars: int = _MAX_ORACLE_CONTEXT_CHARS,
|
||||
evidence_note: str = "Treat it as primary document evidence and combine it with custom web search results when useful.",
|
||||
) -> str:
|
||||
"""Render oracle parsed OfficeQA pages referenced by source_docs/source_files."""
|
||||
refs = _iter_oracle_refs(source_files, source_docs)
|
||||
if not refs:
|
||||
return ""
|
||||
|
||||
blocks: list[str] = []
|
||||
total_chars = 0
|
||||
seen_pages: set[tuple[str, int]] = set()
|
||||
for source_file, page_number, source_doc in refs:
|
||||
json_path = _locate_parsed_json(source_file, docs_roots)
|
||||
if json_path is None:
|
||||
continue
|
||||
page_key = (str(json_path), page_number)
|
||||
if page_key in seen_pages:
|
||||
continue
|
||||
seen_pages.add(page_key)
|
||||
page_text = _render_parsed_page(str(json_path), page_number)
|
||||
if not page_text:
|
||||
continue
|
||||
if len(page_text) > max_page_chars:
|
||||
omitted = len(page_text) - max_page_chars
|
||||
page_text = page_text[:max_page_chars].rstrip() + f"\n\n[... {omitted} characters omitted from this parsed page ...]"
|
||||
block = (
|
||||
f"### {source_file} page {page_number}\n"
|
||||
f"Source URL: {source_doc}\n\n"
|
||||
f"{page_text}"
|
||||
)
|
||||
if total_chars + len(block) > max_total_chars:
|
||||
remaining = max_total_chars - total_chars
|
||||
if remaining <= 0:
|
||||
break
|
||||
block = block[:remaining].rstrip() + "\n\n[... oracle parsed page context truncated ...]"
|
||||
blocks.append(block)
|
||||
break
|
||||
blocks.append(block)
|
||||
total_chars += len(block)
|
||||
if not blocks:
|
||||
return ""
|
||||
return (
|
||||
"The following content is pre-parsed from the oracle OfficeQA source page(s). "
|
||||
f"{evidence_note.strip()}\n\n"
|
||||
+ "\n\n".join(blocks)
|
||||
)
|
||||
|
||||
|
||||
def run_tool(name: str, arguments: dict, *, allowed_roots: list[str], allowed_files: list[str]) -> tuple[str, str]:
|
||||
if name == "glob":
|
||||
pattern = str(arguments.get("pattern") or "*")
|
||||
matches: list[str] = []
|
||||
for root in allowed_roots:
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for filename in filenames:
|
||||
if allowed_files and filename not in allowed_files:
|
||||
continue
|
||||
rel = os.path.relpath(os.path.join(dirpath, filename), root)
|
||||
if fnmatch.fnmatch(rel, pattern) or fnmatch.fnmatch(filename, pattern):
|
||||
matches.append(os.path.join(dirpath, filename))
|
||||
if len(matches) >= _MAX_GLOB_MATCHES:
|
||||
break
|
||||
if len(matches) >= _MAX_GLOB_MATCHES:
|
||||
break
|
||||
return f"glob(pattern={pattern!r})", "\n".join(matches) if matches else "[no matches]"
|
||||
|
||||
if name == "read":
|
||||
path = str(arguments.get("path") or "")
|
||||
if not path:
|
||||
return "read(path='')", "[read error: missing path]"
|
||||
if not _is_allowed(path, allowed_roots, allowed_files):
|
||||
return f"read(path={path!r})", "[read error: path not allowed]"
|
||||
start = max(int(arguments.get("start") or 1), 1)
|
||||
limit = max(int(arguments.get("limit") or 80), 1)
|
||||
with open(path, encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
excerpt = "".join(lines[start - 1:start - 1 + limit])
|
||||
return f"read(path={path!r}, start={start}, limit={limit})", excerpt[:_MAX_READ_CHARS] or "[empty file]"
|
||||
|
||||
if name == "grep":
|
||||
pattern = str(arguments.get("pattern") or "").lower()
|
||||
path = str(arguments.get("path") or "")
|
||||
if not pattern or not path:
|
||||
return f"grep(pattern={pattern!r}, path={path!r})", "[grep error: missing pattern or path]"
|
||||
if not _is_allowed(path, allowed_roots, allowed_files):
|
||||
return f"grep(pattern={pattern!r}, path={path!r})", "[grep error: path not allowed]"
|
||||
matches: list[str] = []
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for idx, line in enumerate(f, start=1):
|
||||
if pattern in line.lower():
|
||||
matches.append(f"{idx}: {line.rstrip()}")
|
||||
if len(matches) >= _MAX_GREP_MATCHES:
|
||||
break
|
||||
return f"grep(pattern={pattern!r}, path={path!r})", "\n".join(matches) if matches else "[no matches]"
|
||||
|
||||
return name, f"[tool error: unknown tool {name}]"
|
||||
Reference in New Issue
Block a user