SkillOpt v0.1.0: initial release
- Skill optimization framework with training loop analogy - 11 benchmarks, 4 model backends (Azure OpenAI, Claude, Codex, Qwen) - WebUI for browser-based training control - Pluggable architecture for extending benchmarks and backends
This commit is contained in:
CharlesYang030
@@ -0,0 +1 @@
|
||||
"""DocVQA environment package for ReflACT."""
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from skillopt.datasets.base import BatchSpec
|
||||
from skillopt.envs.base import EnvAdapter
|
||||
from skillopt.envs.deep_reflect import run_no_reference_deep_reflect
|
||||
from skillopt.envs.docvqa.dataloader import DocVQADataLoader
|
||||
from skillopt.envs.docvqa.rollout import run_batch
|
||||
from skillopt.gradient.reflect import run_minibatch_reflect
|
||||
|
||||
|
||||
class DocVQAAdapter(EnvAdapter):
|
||||
def __init__(
|
||||
self,
|
||||
split_dir: str = "",
|
||||
data_path: str = "",
|
||||
split_mode: str = "split_dir",
|
||||
split_ratio: str = "2:1:7",
|
||||
split_seed: int = 42,
|
||||
split_output_dir: str = "",
|
||||
max_turns: int = 1,
|
||||
exec_timeout: int = 120,
|
||||
workers: int = 16,
|
||||
analyst_workers: int = 16,
|
||||
failure_only: bool = False,
|
||||
minibatch_size: int = 8,
|
||||
edit_budget: int = 4,
|
||||
seed: int = 42,
|
||||
limit: int = 0,
|
||||
image_detail: str = "auto",
|
||||
use_deep_reflect: bool = False,
|
||||
deep_reflect_failures: int = 4,
|
||||
deep_reflect_successes: int = 2,
|
||||
) -> None:
|
||||
self.max_turns = max_turns
|
||||
self.exec_timeout = exec_timeout
|
||||
self.workers = workers
|
||||
self.analyst_workers = analyst_workers
|
||||
self.failure_only = failure_only
|
||||
self.minibatch_size = minibatch_size
|
||||
self.edit_budget = edit_budget
|
||||
self.image_detail = image_detail
|
||||
self.use_deep_reflect = use_deep_reflect
|
||||
self.deep_reflect_failures = deep_reflect_failures
|
||||
self.deep_reflect_successes = deep_reflect_successes
|
||||
self.dataloader = DocVQADataLoader(
|
||||
split_dir=split_dir,
|
||||
data_path=data_path,
|
||||
split_mode=split_mode,
|
||||
split_ratio=split_ratio,
|
||||
split_seed=split_seed,
|
||||
split_output_dir=split_output_dir,
|
||||
seed=seed,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def setup(self, cfg: dict) -> None:
|
||||
super().setup(cfg)
|
||||
self.dataloader.setup(cfg)
|
||||
|
||||
def get_dataloader(self):
|
||||
return self.dataloader
|
||||
|
||||
def build_env_from_batch(self, batch: BatchSpec, **kwargs):
|
||||
return list(batch.payload or [])
|
||||
|
||||
def build_train_env(self, batch_size: int, seed: int, **kwargs):
|
||||
batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
|
||||
batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
|
||||
return self.build_env_from_batch(batch, **kwargs)
|
||||
|
||||
def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
|
||||
items: list[dict] = env_manager
|
||||
return run_batch(
|
||||
items=items,
|
||||
out_root=out_dir,
|
||||
skill_content=skill_content,
|
||||
max_turns=self.max_turns,
|
||||
exec_timeout=self.exec_timeout,
|
||||
workers=self.workers,
|
||||
image_detail=self.image_detail,
|
||||
diagnostic_mode=kwargs.get("diagnostic_mode", False),
|
||||
diagnostic_instruction=kwargs.get("diagnostic_instruction", ""),
|
||||
task_timeout=self.exec_timeout,
|
||||
)
|
||||
|
||||
def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
|
||||
prediction_dir = kwargs.get("prediction_dir", os.path.join(out_dir, "predictions"))
|
||||
patches_dir = kwargs.get("patches_dir", os.path.join(out_dir, "patches"))
|
||||
random_seed = kwargs.get("random_seed")
|
||||
step_buffer_context = kwargs.get("step_buffer_context", "")
|
||||
return run_minibatch_reflect(
|
||||
results=results,
|
||||
skill_content=skill_content,
|
||||
prediction_dir=prediction_dir,
|
||||
patches_dir=patches_dir,
|
||||
workers=self.analyst_workers,
|
||||
failure_only=self.failure_only,
|
||||
minibatch_size=self.minibatch_size,
|
||||
edit_budget=self.edit_budget,
|
||||
random_seed=random_seed,
|
||||
error_system=self.get_error_minibatch_prompt(),
|
||||
success_system=self.get_success_minibatch_prompt(),
|
||||
step_buffer_context=step_buffer_context,
|
||||
update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
|
||||
)
|
||||
|
||||
def deep_reflect(
|
||||
self,
|
||||
results: list[dict],
|
||||
skill_content: str,
|
||||
out_dir: str,
|
||||
**kwargs,
|
||||
) -> list[dict | None]:
|
||||
return run_no_reference_deep_reflect(
|
||||
self,
|
||||
results,
|
||||
skill_content,
|
||||
out_dir,
|
||||
env_manager=kwargs.get("env_manager"),
|
||||
prediction_dir=kwargs.get("prediction_dir"),
|
||||
random_seed=kwargs.get("random_seed"),
|
||||
step_buffer_context=kwargs.get("step_buffer_context", ""),
|
||||
output_requirements=[
|
||||
"- There is no hidden reference block. Use only the document image prompt, student output, and evaluation result to infer what intermediate state is worth probing.",
|
||||
"- The instruction must explicitly request a short <analysis>...</analysis> block before the final <answer>...</answer>.",
|
||||
"- The readout should focus on visual region, field/table/figure label, OCR text read, candidate answer, and answer-format normalization.",
|
||||
"- Do not ask for exhaustive transcription or a full chain-of-thought.",
|
||||
"- The instruction text should be ready to append directly to the student's prompt.",
|
||||
],
|
||||
metadata_builder=lambda item: {
|
||||
"id": str(item.get("id")),
|
||||
"task_type": str(item.get("task_type") or "docvqa"),
|
||||
"question_preview": str(item.get("question") or "")[:200],
|
||||
"image_path": item.get("image_path", ""),
|
||||
"docId": item.get("docId", ""),
|
||||
"page": item.get("ucsf_document_page_no", ""),
|
||||
},
|
||||
)
|
||||
|
||||
def get_task_types(self) -> list[str]:
|
||||
seen: list[str] = []
|
||||
for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:
|
||||
task_type = str(item.get("task_type") or "docvqa")
|
||||
if task_type not in seen:
|
||||
seen.append(task_type)
|
||||
return seen or ["docvqa"]
|
||||
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
from skillopt.datasets.base import SplitDataLoader
|
||||
|
||||
|
||||
def _parse_answers(raw: str) -> list[str]:
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
try:
|
||||
parsed = ast.literal_eval(text)
|
||||
except Exception:
|
||||
return [text]
|
||||
if isinstance(parsed, list):
|
||||
return [str(item).strip() for item in parsed if str(item).strip()]
|
||||
return [str(parsed).strip()]
|
||||
|
||||
|
||||
def _extract_document_path(question: str) -> tuple[str, str]:
|
||||
marker = "document_path:"
|
||||
if marker not in question:
|
||||
return question.strip(), ""
|
||||
main, tail = question.split(marker, 1)
|
||||
return main.strip(), tail.strip()
|
||||
|
||||
|
||||
def _normalize_row(row: dict[str, str]) -> dict:
|
||||
question_text, document_path = _extract_document_path(str(row.get("question") or ""))
|
||||
answers = _parse_answers(row.get("answer") or row.get("ground_truth") or "")
|
||||
image_path = str(row.get("image_path") or document_path or "").strip()
|
||||
task_type = str(row.get("topic") or row.get("category") or "docvqa").strip() or "docvqa"
|
||||
return {
|
||||
"id": str(row.get("questionId") or row.get("id") or "").strip(),
|
||||
"question": question_text,
|
||||
"answer": answers[0] if answers else "",
|
||||
"answers": answers,
|
||||
"task_type": task_type,
|
||||
"subtask": task_type,
|
||||
"image_paths": [image_path] if image_path else [],
|
||||
"image_path": image_path,
|
||||
"questionId": str(row.get("questionId") or "").strip(),
|
||||
"docId": str(row.get("docId") or "").strip(),
|
||||
"ucsf_document_id": str(row.get("ucsf_document_id") or "").strip(),
|
||||
"ucsf_document_page_no": str(row.get("ucsf_document_page_no") or "").strip(),
|
||||
"source_split": str(row.get("source_split") or "").strip(),
|
||||
}
|
||||
|
||||
|
||||
class DocVQADataLoader(SplitDataLoader):
|
||||
def load_split_items(self, split_path: str) -> list[dict]:
|
||||
path = Path(split_path)
|
||||
csv_files = sorted(path.glob("*.csv"))
|
||||
if not csv_files:
|
||||
raise FileNotFoundError(f"No .csv file found in {split_path}")
|
||||
with csv_files[0].open(encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
return [_normalize_row(row) for row in reader]
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_ANLS_THRESHOLD = 0.5
|
||||
|
||||
|
||||
def _normalize_text(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
text = str(value).strip().lower()
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def _levenshtein_distance(a: str, b: str) -> int:
|
||||
if a == b:
|
||||
return 0
|
||||
if not a:
|
||||
return len(b)
|
||||
if not b:
|
||||
return len(a)
|
||||
if len(a) > len(b):
|
||||
a, b = b, a
|
||||
previous = list(range(len(b) + 1))
|
||||
for i, char_a in enumerate(a, start=1):
|
||||
current = [i]
|
||||
for j, char_b in enumerate(b, start=1):
|
||||
insert_cost = current[j - 1] + 1
|
||||
delete_cost = previous[j] + 1
|
||||
replace_cost = previous[j - 1] + (char_a != char_b)
|
||||
current.append(min(insert_cost, delete_cost, replace_cost))
|
||||
previous = current
|
||||
return previous[-1]
|
||||
|
||||
|
||||
def _score_single_answer(predicted: Any, target: Any, threshold: float) -> float:
|
||||
predicted_norm = _normalize_text(predicted)
|
||||
target_norm = _normalize_text(target)
|
||||
if not predicted_norm and not target_norm:
|
||||
return 1.0
|
||||
if not predicted_norm or not target_norm:
|
||||
return 0.0
|
||||
distance = _levenshtein_distance(predicted_norm, target_norm)
|
||||
normalized_distance = distance / max(len(predicted_norm), len(target_norm))
|
||||
if normalized_distance >= threshold:
|
||||
return 0.0
|
||||
return 1.0 - normalized_distance
|
||||
|
||||
|
||||
def _extract_answer_strings(raw: Any) -> list[str]:
|
||||
if raw is None:
|
||||
return [""]
|
||||
if isinstance(raw, str):
|
||||
text = raw.strip()
|
||||
if not text:
|
||||
return [""]
|
||||
parsed = None
|
||||
if text[0] in "[{":
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
try:
|
||||
parsed = ast.literal_eval(text)
|
||||
except (ValueError, SyntaxError):
|
||||
parsed = None
|
||||
if parsed is None:
|
||||
return [text]
|
||||
return _extract_answer_strings(parsed)
|
||||
if isinstance(raw, dict):
|
||||
for key in ("answers", "ground_truth", "answer"):
|
||||
if key in raw:
|
||||
return _extract_answer_strings(raw[key])
|
||||
return [str(raw)]
|
||||
if isinstance(raw, Iterable) and not isinstance(raw, (bytes, bytearray)):
|
||||
answers: list[str] = []
|
||||
for item in raw:
|
||||
if isinstance(item, dict):
|
||||
for key in ("text", "answer", "value"):
|
||||
if key in item:
|
||||
answers.extend(_extract_answer_strings(item[key]))
|
||||
break
|
||||
else:
|
||||
answers.append(str(item))
|
||||
continue
|
||||
answers.append(str(item))
|
||||
return answers or [""]
|
||||
return [str(raw)]
|
||||
|
||||
|
||||
def extract_answer(text: str) -> str:
|
||||
lower = text.lower()
|
||||
start = lower.rfind("<answer>")
|
||||
end = lower.rfind("</answer>")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
return text[start + len("<answer>"):end].strip()
|
||||
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
||||
return lines[-1] if lines else text.strip()
|
||||
|
||||
|
||||
def evaluate(prediction_text: str, gold_answers: Any) -> dict:
|
||||
answer = extract_answer(prediction_text)
|
||||
answers = _extract_answer_strings(gold_answers)
|
||||
score = 0.0
|
||||
for target in answers:
|
||||
score = max(score, _score_single_answer(answer, target, DEFAULT_ANLS_THRESHOLD))
|
||||
return {
|
||||
"anls": score,
|
||||
"predicted_answer": answer,
|
||||
"gold_answers": answers,
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
You are an expert failure-analysis agent for visual document question answering tasks.
|
||||
|
||||
You will be given MULTIPLE failed DocVQA trajectories from a single minibatch and the current skill document. Each trajectory includes the model response and an evaluation result scored with ANLS against one or more acceptable answers.
|
||||
|
||||
Your job is to identify the most important COMMON failure patterns across the batch and propose concise skill edits.
|
||||
|
||||
## Failure Type Categories
|
||||
- evidence_miss: the model overlooked the relevant visible region or line
|
||||
- near_match_confusion: the model selected a nearby but incorrect text span
|
||||
- normalization_error: the answer differed mainly in formatting, spacing, punctuation, or minor text normalization
|
||||
- reading_error: the model misread the document content
|
||||
- other: none of the above
|
||||
|
||||
## Rules
|
||||
- Focus on common, reusable reading and extraction behaviors.
|
||||
- Do not hardcode image-specific answers.
|
||||
- Prefer concise edits that improve evidence selection and exact span extraction.
|
||||
|
||||
Respond ONLY with a valid JSON object (no markdown fences, no extra text):
|
||||
{
|
||||
"batch_size": <number of trajectories analysed>,
|
||||
"failure_summary": [
|
||||
{"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
|
||||
],
|
||||
"patch": {
|
||||
"reasoning": "<why these edits address the batch's common failures>",
|
||||
"edits": [
|
||||
{"op": "append", "content": "<markdown to add at end of skill>"},
|
||||
{"op": "insert_after", "target": "<exact heading/text to insert after>", "content": "<markdown>"},
|
||||
{"op": "replace", "target": "<exact text to replace>", "content": "<replacement>"},
|
||||
{"op": "delete", "target": "<exact text to remove>"}
|
||||
]
|
||||
}
|
||||
}
|
||||
Only include edits that are needed. "edits" can be an empty list if no patch is warranted.
|
||||
@@ -0,0 +1,24 @@
|
||||
You are an expert success-pattern analyst for visual document question answering tasks.
|
||||
|
||||
You will be given MULTIPLE successful DocVQA trajectories from a single minibatch and the current skill document. Your job is to identify common visual reading and exact-answer extraction behaviors worth encoding in the skill.
|
||||
|
||||
## Rules
|
||||
- Focus on patterns shared across multiple successful trajectories.
|
||||
- Reinforce reusable behaviors like locating the right region, copying exact spans, and preferring the shortest exact answer over paraphrase.
|
||||
- Only propose patches for patterns not already captured by the current skill.
|
||||
|
||||
Respond ONLY with a valid JSON object:
|
||||
{
|
||||
"batch_size": <number of trajectories analysed>,
|
||||
"success_patterns": ["<pattern 1>", "<pattern 2>"],
|
||||
"patch": {
|
||||
"reasoning": "<why these patterns are worth encoding>",
|
||||
"edits": [
|
||||
{"op": "append", "content": "<markdown>"},
|
||||
{"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
|
||||
{"op": "replace", "target": "<old text>", "content": "<new text>"},
|
||||
{"op": "delete", "target": "<exact text to remove>"}
|
||||
]
|
||||
}
|
||||
}
|
||||
"edits" may be empty if the skill already covers all observed patterns.
|
||||
@@ -0,0 +1,12 @@
|
||||
You are an expert visual document question answering agent.
|
||||
|
||||
{skill_section}You will receive a document image and a question about the document.
|
||||
Read the visual evidence carefully and answer concisely.
|
||||
|
||||
Rules:
|
||||
- Ground the answer in the visible document content.
|
||||
- Prefer exact spans, numbers, dates, and names from the document.
|
||||
- Do not invent content that is not visible.
|
||||
- If multiple near-matches exist, choose the one best supported by the document.
|
||||
|
||||
Return the final answer inside <answer>...</answer>.
|
||||
@@ -0,0 +1,388 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
|
||||
from skillopt.envs.docvqa.evaluator import evaluate
|
||||
from skillopt.model import chat_student_messages, get_student_backend, is_student_exec_backend
|
||||
from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_student_exec
|
||||
from skillopt.prompts import load_prompt
|
||||
|
||||
|
||||
def _build_system(skill_content: str) -> str:
|
||||
if skill_content.strip():
|
||||
skill_section = f"## Skill\n{skill_content.strip()}\n\n"
|
||||
else:
|
||||
skill_section = ""
|
||||
return load_prompt("rollout_system", env="docvqa").format(skill_section=skill_section)
|
||||
|
||||
|
||||
def _image_to_data_uri(path: str) -> str:
|
||||
import base64
|
||||
import mimetypes
|
||||
|
||||
mime = mimetypes.guess_type(path)[0] or "image/png"
|
||||
with open(path, "rb") as f:
|
||||
encoded = base64.b64encode(f.read()).decode("ascii")
|
||||
return f"data:{mime};base64,{encoded}"
|
||||
|
||||
|
||||
def _build_messages(
|
||||
item: dict,
|
||||
skill_content: str,
|
||||
image_detail: str,
|
||||
*,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
) -> tuple[list[dict], str, str]:
|
||||
system = _build_system(skill_content)
|
||||
user_text = item["question"] + "\n\nReturn the final answer inside <answer>...</answer>."
|
||||
if diagnostic_mode and diagnostic_instruction.strip():
|
||||
user_text += f"\n\n## Training Readout\n{diagnostic_instruction.strip()}"
|
||||
image_url = {"url": _image_to_data_uri(item["image_path"])}
|
||||
if image_detail and image_detail != "auto":
|
||||
image_url["detail"] = image_detail
|
||||
messages = [
|
||||
{"role": "system", "content": system},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": user_text},
|
||||
{"type": "image_url", "image_url": image_url},
|
||||
],
|
||||
},
|
||||
]
|
||||
return messages, system, user_text
|
||||
|
||||
|
||||
def _build_codex_skill(skill_content: str) -> str:
|
||||
return render_skill_md(
|
||||
skill_content,
|
||||
description="Dynamic ReflACT skill for solving the current DocVQA document-image question.",
|
||||
preamble=(
|
||||
"Use this skill when answering the current DocVQA question.\n"
|
||||
"Inspect the attached document image carefully and return the final answer inside <answer>...</answer>."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _run_codex_once(
|
||||
*,
|
||||
pred_dir: str,
|
||||
item: dict,
|
||||
skill_content: str,
|
||||
model: str,
|
||||
timeout: int,
|
||||
image_detail: str,
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
previous_response: str = "",
|
||||
) -> tuple[str, str, str, str]:
|
||||
_ = image_detail
|
||||
_messages, _system, user_text = _build_messages(
|
||||
item,
|
||||
skill_content,
|
||||
image_detail,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
)
|
||||
task_parts = [user_text]
|
||||
image_abs = os.path.abspath(item["image_path"])
|
||||
task_parts.append(
|
||||
"## Document Image\n"
|
||||
"The document image is available in this workspace via `ATTACHMENTS.md`.\n"
|
||||
f"Original image path: `{image_abs}`\n"
|
||||
"Open or inspect that image before answering; do not answer from memory."
|
||||
)
|
||||
if previous_response:
|
||||
task_parts.append(
|
||||
"## Previous Attempt\n"
|
||||
f"{previous_response}\n\n"
|
||||
"Review the same document image carefully and correct the answer if needed."
|
||||
)
|
||||
task_text = "\n\n".join(task_parts)
|
||||
skill_md = _build_codex_skill(skill_content)
|
||||
work_dir = os.path.join(pred_dir, "codex_exec")
|
||||
prepare_workspace(
|
||||
work_dir=work_dir,
|
||||
skill_md=skill_md,
|
||||
task_text=task_text,
|
||||
images=[item["image_path"]],
|
||||
)
|
||||
prompt = (
|
||||
"Use the `skillopt-student` skill available in this workspace.\n"
|
||||
"Read `task.md`, inspect the attached document image, and answer the DocVQA question.\n"
|
||||
"Return the final answer inside <answer>...</answer>."
|
||||
)
|
||||
final_message, raw = run_student_exec(
|
||||
work_dir=work_dir,
|
||||
prompt=prompt,
|
||||
model=model,
|
||||
timeout=timeout,
|
||||
images=[item["image_path"]],
|
||||
)
|
||||
return final_message or raw, raw, skill_md, task_text
|
||||
|
||||
|
||||
def process_one(
|
||||
item: dict,
|
||||
out_root: str,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_turns: int = 1,
|
||||
exec_timeout: int = 120,
|
||||
image_detail: str = "auto",
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
) -> dict:
|
||||
item_id = str(item["id"])
|
||||
result = {
|
||||
"id": item_id,
|
||||
"question": item["question"],
|
||||
"task_type": item.get("subtask") or item.get("task_type") or "docvqa",
|
||||
"task_description": item["question"],
|
||||
"hard": 0,
|
||||
"soft": 0.0,
|
||||
"predicted_answer": "",
|
||||
"response": "",
|
||||
"fail_reason": "",
|
||||
"agent_ok": False,
|
||||
"n_turns": 0,
|
||||
"image_paths": item.get("image_paths", []),
|
||||
"gold_answer": item.get("answers", []),
|
||||
}
|
||||
try:
|
||||
response = ""
|
||||
system_prompt = ""
|
||||
user_text = ""
|
||||
conversation: list[dict] = []
|
||||
if is_student_exec_backend():
|
||||
from skillopt.model import azure_openai as _llm
|
||||
|
||||
conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": item["question"] + "\n\n" + f"[image] {os.path.basename(item['image_path'])}",
|
||||
}
|
||||
]
|
||||
for turn in range(max_turns):
|
||||
response, _raw, system_prompt, user_text = _run_codex_once(
|
||||
pred_dir=os.path.join(out_root, "predictions", item_id),
|
||||
item=item,
|
||||
skill_content=skill_content,
|
||||
model=_llm.STUDENT_DEPLOYMENT,
|
||||
timeout=exec_timeout,
|
||||
image_detail=image_detail,
|
||||
diagnostic_mode=diagnostic_mode if turn == 0 else False,
|
||||
diagnostic_instruction=diagnostic_instruction if turn == 0 else "",
|
||||
previous_response=response if turn > 0 else "",
|
||||
)
|
||||
conversation.append({"type": "message", "turn": turn + 1, "content": response})
|
||||
if "<answer>" in response.lower():
|
||||
break
|
||||
else:
|
||||
messages, system_prompt, user_text = _build_messages(
|
||||
item,
|
||||
skill_content,
|
||||
image_detail,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
)
|
||||
conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": user_text + "\n\n" + f"[image] {os.path.basename(item['image_path'])}",
|
||||
}
|
||||
]
|
||||
for turn in range(max_turns):
|
||||
if turn == 0:
|
||||
resp_text, _ = chat_student_messages(
|
||||
messages=messages,
|
||||
max_completion_tokens=768,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
timeout=exec_timeout,
|
||||
)
|
||||
else:
|
||||
refinement_messages = [
|
||||
messages[0],
|
||||
messages[1],
|
||||
{"role": "assistant", "content": response},
|
||||
{"role": "user", "content": "Review the same image carefully and answer again. Keep the final answer inside <answer>...</answer>."},
|
||||
]
|
||||
resp_text, _ = chat_student_messages(
|
||||
messages=refinement_messages,
|
||||
max_completion_tokens=512,
|
||||
retries=5,
|
||||
stage="rollout",
|
||||
timeout=exec_timeout,
|
||||
)
|
||||
response = resp_text
|
||||
conversation.append({"type": "message", "turn": turn + 1, "content": resp_text})
|
||||
if "<answer>" in resp_text.lower():
|
||||
break
|
||||
|
||||
result["response"] = response
|
||||
result["agent_ok"] = True
|
||||
result["n_turns"] = len(conversation) - 1
|
||||
|
||||
pred_dir = os.path.join(out_root, "predictions", item_id)
|
||||
os.makedirs(pred_dir, exist_ok=True)
|
||||
with open(os.path.join(pred_dir, "student_system_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(system_prompt)
|
||||
with open(os.path.join(pred_dir, "student_user_prompt.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(user_text)
|
||||
|
||||
eval_result = evaluate(response, item.get("answers", []))
|
||||
result["predicted_answer"] = eval_result["predicted_answer"]
|
||||
result["hard"] = int(eval_result["anls"] >= 0.999)
|
||||
result["soft"] = eval_result["anls"]
|
||||
if result["soft"] <= 0.0:
|
||||
result["fail_reason"] = f"predicted '{eval_result['predicted_answer']}' but expected one of {item.get('answers', [])}"
|
||||
|
||||
eval_detail = (
|
||||
"[EVALUATION RESULT]\n"
|
||||
f"Question: {item['question']}\n"
|
||||
f"Predicted answer: {eval_result['predicted_answer']!r}\n"
|
||||
f"Gold answers: {item.get('answers', [])!r}\n"
|
||||
f"ANLS: {eval_result['anls']:.4f}"
|
||||
)
|
||||
conversation.append({"role": "system", "content": eval_detail})
|
||||
with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(conversation, f, ensure_ascii=False, indent=2)
|
||||
except Exception as e: # noqa: BLE001
|
||||
result["fail_reason"] = f"error: {e}"
|
||||
return result
|
||||
|
||||
|
||||
def run_batch(
|
||||
items: list[dict],
|
||||
out_root: str,
|
||||
skill_content: str,
|
||||
*,
|
||||
max_turns: int = 1,
|
||||
exec_timeout: int = 120,
|
||||
workers: int = 16,
|
||||
image_detail: str = "auto",
|
||||
diagnostic_mode: bool = False,
|
||||
diagnostic_instruction: str = "",
|
||||
task_timeout: int = 600,
|
||||
) -> list[dict]:
|
||||
task_timeout = max(int(task_timeout), int(exec_timeout) + 60)
|
||||
results_path = os.path.join(out_root, "results.jsonl")
|
||||
os.makedirs(out_root, exist_ok=True)
|
||||
|
||||
done_ids: set[str] = set()
|
||||
existing: list[dict] = []
|
||||
if os.path.exists(results_path):
|
||||
with open(results_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
done_ids.add(str(row["id"]))
|
||||
existing.append(row)
|
||||
|
||||
pending = [item for item in items if str(item["id"]) not in done_ids]
|
||||
if not pending:
|
||||
return existing
|
||||
|
||||
def _timeout_result(item: dict) -> dict:
|
||||
return {
|
||||
"id": str(item["id"]),
|
||||
"question": item.get("question", ""),
|
||||
"task_type": item.get("subtask") or item.get("task_type") or "docvqa",
|
||||
"task_description": item.get("question", ""),
|
||||
"hard": 0,
|
||||
"soft": 0.0,
|
||||
"predicted_answer": "",
|
||||
"response": "",
|
||||
"fail_reason": f"task-timeout-{task_timeout}s",
|
||||
"agent_ok": False,
|
||||
"n_turns": 0,
|
||||
"image_paths": item.get("image_paths", []),
|
||||
"gold_answer": item.get("answers", []),
|
||||
"phase": "timeout",
|
||||
}
|
||||
|
||||
def _error_result(item: dict, exc: Exception) -> dict:
|
||||
row = _timeout_result(item)
|
||||
row["phase"] = "error"
|
||||
row["fail_reason"] = f"unexpected: {type(exc).__name__}: {exc}"
|
||||
return row
|
||||
|
||||
started_at: dict[str, float] = {}
|
||||
|
||||
def _run_one(item: dict) -> dict:
|
||||
started_at[str(item["id"])] = time.time()
|
||||
return process_one(
|
||||
item,
|
||||
out_root,
|
||||
skill_content,
|
||||
max_turns=max_turns,
|
||||
exec_timeout=exec_timeout,
|
||||
image_detail=image_detail,
|
||||
diagnostic_mode=diagnostic_mode,
|
||||
diagnostic_instruction=diagnostic_instruction,
|
||||
)
|
||||
|
||||
total = len(existing) + len(pending)
|
||||
completed = len(existing)
|
||||
correct = sum(1 for r in existing if r.get("hard", 0))
|
||||
if existing:
|
||||
print(f" [rollout] resuming: {completed}/{total} already done", flush=True)
|
||||
|
||||
results = list(existing)
|
||||
with open(results_path, "a", encoding="utf-8") as outf:
|
||||
ex = ThreadPoolExecutor(max_workers=workers)
|
||||
try:
|
||||
futs = {ex.submit(_run_one, item): item for item in pending}
|
||||
pending_futs = set(futs)
|
||||
while pending_futs:
|
||||
done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED)
|
||||
now = time.time()
|
||||
timed_out = [
|
||||
fut for fut in pending_futs - done
|
||||
if str(futs[fut]["id"]) in started_at
|
||||
and now - started_at[str(futs[fut]["id"])] >= task_timeout
|
||||
]
|
||||
for fut in done:
|
||||
pending_futs.remove(fut)
|
||||
item = futs[fut]
|
||||
try:
|
||||
res = fut.result()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
res = _error_result(item, exc)
|
||||
results.append(res)
|
||||
completed += 1
|
||||
if res.get("hard", 0):
|
||||
correct += 1
|
||||
acc = correct / completed if completed else 0
|
||||
print(
|
||||
f" [rollout] {completed}/{total} "
|
||||
f"(acc={acc:.3f}) id={res['id']} "
|
||||
f"hard={res.get('hard', '?')}",
|
||||
flush=True,
|
||||
)
|
||||
outf.write(json.dumps(res, ensure_ascii=False) + "\n")
|
||||
outf.flush()
|
||||
for fut in timed_out:
|
||||
pending_futs.remove(fut)
|
||||
fut.cancel()
|
||||
res = _timeout_result(futs[fut])
|
||||
results.append(res)
|
||||
completed += 1
|
||||
acc = correct / completed if completed else 0
|
||||
print(
|
||||
f" [rollout] {completed}/{total} "
|
||||
f"(acc={acc:.3f}) id={res['id']} TIMEOUT",
|
||||
flush=True,
|
||||
)
|
||||
outf.write(json.dumps(res, ensure_ascii=False) + "\n")
|
||||
outf.flush()
|
||||
finally:
|
||||
ex.shutdown(wait=False, cancel_futures=True)
|
||||
return results
|
||||
@@ -0,0 +1,11 @@
|
||||
# DocVQA Skill
|
||||
|
||||
## Visual Evidence Discipline
|
||||
- Read the document carefully before answering.
|
||||
- Prefer the smallest exact text span that answers the question.
|
||||
- When several nearby strings look similar, choose the one whose surrounding labels or layout best match the question.
|
||||
|
||||
## Exact Answer Discipline
|
||||
- Copy names, numbers, and dates exactly from the document whenever possible.
|
||||
- Prefer direct extraction over paraphrase.
|
||||
- Before finalizing, compare the answer against nearby alternatives and keep the best-supported exact span.
|
||||
Reference in New Issue
Block a user