Initial commit

skillopt/envs/sealqa/__init__.py

@@ -0,0 +1 @@
+"""SealQA environment package for ReflACT."""

skillopt/envs/sealqa/adapter.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import os
+
+from skillopt.datasets.base import BatchSpec
+from skillopt.envs.base import EnvAdapter
+from skillopt.envs.deep_reflect import run_no_reference_deep_reflect
+from skillopt.envs.sealqa.dataloader import SealQADataLoader
+from skillopt.envs.sealqa.rollout import run_batch
+from skillopt.gradient.reflect import run_minibatch_reflect
+
+
+class SealQAAdapter(EnvAdapter):
+    def __init__(
+        self,
+        split_dir: str = '',
+        workers: int = 4,
+        analyst_workers: int = 8,
+        failure_only: bool = False,
+        minibatch_size: int = 8,
+        edit_budget: int = 4,
+        seed: int = 42,
+        limit: int = 0,
+        max_tool_turns: int = 12,
+        use_deep_reflect: bool = False,
+        deep_reflect_failures: int = 4,
+        deep_reflect_successes: int = 2,
+    ) -> None:
+        self.workers = workers
+        self.analyst_workers = analyst_workers
+        self.failure_only = failure_only
+        self.minibatch_size = minibatch_size
+        self.edit_budget = edit_budget
+        self.max_tool_turns = max_tool_turns
+        self.use_deep_reflect = use_deep_reflect
+        self.deep_reflect_failures = deep_reflect_failures
+        self.deep_reflect_successes = deep_reflect_successes
+        self.dataloader = SealQADataLoader(split_dir=split_dir, seed=seed, limit=limit)
+
+    def setup(self, cfg: dict) -> None:
+        super().setup(cfg)
+        self.dataloader.setup(cfg)
+
+    def get_dataloader(self):
+        return self.dataloader
+
+    def build_env_from_batch(self, batch: BatchSpec, **kwargs):
+        return list(batch.payload or [])
+
+    def build_train_env(self, batch_size: int, seed: int, **kwargs):
+        batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs):
+        batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs)
+        return self.build_env_from_batch(batch, **kwargs)
+
+    def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]:
+        items: list[dict] = env_manager
+        return run_batch(
+            items=items,
+            out_root=out_dir,
+            skill_content=skill_content,
+            workers=self.workers,
+            max_tool_turns=self.max_tool_turns,
+            diagnostic_mode=kwargs.get('diagnostic_mode', False),
+            diagnostic_instruction=kwargs.get('diagnostic_instruction', ''),
+        )
+
+    def reflect(self, results: list[dict], skill_content: str, out_dir: str, **kwargs) -> list[dict | None]:
+        prediction_dir = kwargs.get('prediction_dir', os.path.join(out_dir, 'predictions'))
+        patches_dir = kwargs.get('patches_dir', os.path.join(out_dir, 'patches'))
+        random_seed = kwargs.get('random_seed')
+        step_buffer_context = kwargs.get('step_buffer_context', '')
+        return run_minibatch_reflect(
+            results=results,
+            skill_content=skill_content,
+            prediction_dir=prediction_dir,
+            patches_dir=patches_dir,
+            workers=self.analyst_workers,
+            failure_only=self.failure_only,
+            minibatch_size=self.minibatch_size,
+            edit_budget=self.edit_budget,
+            random_seed=random_seed,
+            error_system=self.get_error_minibatch_prompt(),
+            success_system=self.get_success_minibatch_prompt(),
+            step_buffer_context=step_buffer_context,
+            update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"),
+        )
+
+    def deep_reflect(
+        self,
+        results: list[dict],
+        skill_content: str,
+        out_dir: str,
+        **kwargs,
+    ) -> list[dict | None]:
+        return run_no_reference_deep_reflect(
+            self,
+            results,
+            skill_content,
+            out_dir,
+            env_manager=kwargs.get('env_manager'),
+            prediction_dir=kwargs.get('prediction_dir'),
+            random_seed=kwargs.get('random_seed'),
+            step_buffer_context=kwargs.get('step_buffer_context', ''),
+            output_requirements=[
+                "- There is no hidden reference block. Use only the question, provided evidence, URL/fetch trace, student output, and evaluation result to infer what intermediate state is worth probing.",
+                "- The instruction must explicitly request a short <analysis>...</analysis> block before the final <answer>...</answer>.",
+                "- The readout should focus on effective time frame, conflicting evidence, decisive source, candidate answer, and answer-finalization rule.",
+                "- Do not ask for exhaustive web summaries or a full chain-of-thought.",
+                "- The instruction text should be ready to append directly to the student's prompt.",
+            ],
+            metadata_builder=lambda item: {
+                "id": str(item.get('id')),
+                "task_type": str(item.get('task_type') or item.get('topic') or 'sealqa'),
+                "question_preview": str(item.get('question') or '')[:200],
+                "freshness": item.get('freshness', ''),
+                "question_types": item.get('question_types', ''),
+                "topic": item.get('topic', ''),
+            },
+        )
+
+    def get_task_types(self) -> list[str]:
+        seen: list[str] = []
+        for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items:
+            task_type = str(item.get('task_type') or 'sealqa')
+            if task_type not in seen:
+                seen.append(task_type)
+        return seen or ['sealqa']

skillopt/envs/sealqa/dataloader.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+from skillopt.datasets.base import SplitDataLoader
+
+
+def _normalize_row(row: dict[str, str], index: int) -> dict:
+    canary = str(row.get('canary') or '').strip()
+    base_id = str(row.get('question_id') or row.get('id') or '').strip()
+    if not base_id:
+        base_id = f"{canary or 'sealqa'}:{index:04d}"
+    return {
+        'id': base_id,
+        'question': str(row.get('question') or '').strip(),
+        'ground_truth': str(row.get('answer') or row.get('ground_truth') or '').strip(),
+        'answers': [str(row.get('answer') or row.get('ground_truth') or '').strip()],
+        'task_type': str(row.get('topic') or 'sealqa').strip() or 'sealqa',
+        'topic': str(row.get('topic') or 'sealqa').strip() or 'sealqa',
+        'urls': str(row.get('urls') or '').strip(),
+        'search_results': str(row.get('search_results') or '').strip(),
+        'freshness': str(row.get('freshness') or '').strip(),
+        'question_types': str(row.get('question_types') or '').strip(),
+        'canary': canary,
+    }
+
+
+class SealQADataLoader(SplitDataLoader):
+    def load_split_items(self, split_path: str) -> list[dict]:
+        path = Path(split_path)
+        csv_files = sorted(path.glob('*.csv'))
+        if not csv_files:
+            raise FileNotFoundError(f'No .csv file found in {split_path}')
+        with csv_files[0].open(encoding='utf-8', newline='') as f:
+            reader = csv.DictReader(f)
+            return [_normalize_row(row, idx) for idx, row in enumerate(reader, start=1)]

skillopt/envs/sealqa/evaluator.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import re
+
+from openai import AzureOpenAI, OpenAI
+
+GRADER_TEMPLATE = """
+Your job is to look at a question, a gold target, and a predicted answer, and then assign a grade of either [\"CORRECT\", \"INCORRECT\", \"NOT_ATTEMPTED\"].
+First, I will give examples of each grade, and then you will grade a new example.
+
+The following are examples of CORRECT predicted answers.
+```
+Question: What are the names of Barack Obama's children?
+Gold target: Malia Obama and Sasha Obama
+Predicted answer 1: sasha and malia obama
+Predicted answer 2: most people would say Malia and Sasha, but I'm not sure and would have to double check
+Predicted answer 3: Barack Obama has two daughters. Their names are Malia Ann and Natasha Marian, but they are commonly referred to as Malia Obama and Sasha Obama. Malia was born on July 4, 1998, and Sasha was born on June 10, 2001.
+```
+These predicted answers are all CORRECT because:
+- They fully contain the important information in the gold target.
+- They do not contain any information that contradicts the gold target.
+- Only semantic meaning matters; capitalization, punctuation, grammar, and order don't matter.
+- Hedging and guessing are permissible, provided that the gold target is fully included and the response contains no incorrect information or contradictions.
+
+The following are examples of INCORRECT predicted answers.
+```
+Question: What are the names of Barack Obama's children?
+Gold target: Malia and Sasha
+Predicted answer 1: Malia.
+Predicted answer 2: Malia, Sasha, and Susan.
+Predicted answer 3: Barack Obama does not have any children.
+```
+These predicted answers are all INCORRECT because a factual statement in the answer contradicts the gold target.
+
+The following are examples of NOT_ATTEMPTED predicted answers.
+```
+Question: What are the names of Barack Obama's children?
+Gold target: Malia and Sasha
+Predicted answer 1: I don't know.
+Predicted answer 2: I need more context about which Obama you are talking about.
+```
+These predicted answers are all NOT_ATTEMPTED because the important information in the gold target is not included and there is no contradiction.
+
+Here is a new example. Simply reply with either CORRECT, INCORRECT, NOT ATTEMPTED. Just return the letters \"A\", \"B\", or \"C\", with no text around it.
+```
+Question: {question}
+Gold target: {target}
+Predicted answer: {predicted_answer}
+```
+
+Grade the predicted answer as one of:
+A: CORRECT
+B: INCORRECT
+C: NOT_ATTEMPTED
+""".strip()
+
+
+def _build_grader_client() -> tuple[OpenAI | AzureOpenAI, str]:
+    import os
+
+    endpoint = os.environ.get('AZURE_OPENAI_ENDPOINT', '').strip()
+    api_version = os.environ.get('AZURE_OPENAI_API_VERSION', '').strip() or '2025-04-01-preview'
+    azure_key = os.environ.get('AZURE_OPENAI_API_KEY', '').strip()
+    openai_key = os.environ.get('OPENAI_API_KEY', '').strip()
+    api_key = azure_key or openai_key
+    if endpoint and api_version and api_key:
+        model = os.environ.get('SEALQA_GRADER_AZURE_MODEL', '').strip() or os.environ.get('SEALQA_GRADER_MODEL', '').strip() or os.environ.get('AZURE_MODEL_NAME', '').strip() or os.environ.get('TEACHER_DEPLOYMENT', '').strip() or 'gpt-5.4'
+        client = AzureOpenAI(api_key=api_key, api_version=api_version, azure_endpoint=endpoint.rstrip('/'))
+        return client, model
+
+    if openai_key:
+        model = os.environ.get('SEALQA_GRADER_OPENAI_MODEL', '').strip() or os.environ.get('SEALQA_GRADER_MODEL', '').strip() or 'gpt-4.1-mini'
+        return OpenAI(api_key=openai_key), model
+
+    raise ValueError('Missing grader credentials for SealQA scoring.')
+
+
+def _extract_text_content(content) -> str:
+    if content is None:
+        return ''
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts = []
+        for part in content:
+            if isinstance(part, dict) and part.get('type') == 'text':
+                parts.append(str(part.get('text', '')))
+            else:
+                text = getattr(part, 'text', None)
+                if text:
+                    parts.append(str(text))
+        return '\n'.join(parts).strip()
+    return str(content).strip()
+
+
+def _normalize_text(text: str) -> str:
+    lowered = text.strip().lower()
+    lowered = re.sub(r'\s+', ' ', lowered)
+    lowered = re.sub(r'[^\w\s%.-]', '', lowered)
+    return lowered.strip()
+
+
+def _fallback_score(ground_truth: str, predicted: str) -> float:
+    gold = _normalize_text(ground_truth)
+    pred = _normalize_text(predicted)
+    if not gold or not pred:
+        return 0.0
+    if gold == pred:
+        return 1.0
+    if gold in pred or pred in gold:
+        return 1.0
+    return 0.0
+
+
+def score_sealqa(question: str, ground_truth: str, predicted: str) -> float:
+    try:
+        client, model = _build_grader_client()
+    except ValueError:
+        return _fallback_score(ground_truth, predicted)
+
+    prompt = GRADER_TEMPLATE.format(question=question, target=ground_truth, predicted_answer=predicted)
+    completion = client.chat.completions.create(model=model, messages=[{'role': 'user', 'content': prompt}])
+    content = _extract_text_content(completion.choices[0].message.content).strip().upper()
+    if content.startswith('A'):
+        return 1.0
+    return 0.0

skillopt/envs/sealqa/prompts/analyst_error.md

@@ -0,0 +1,30 @@
+You are an expert failure-analysis agent for evidence-seeking factual question answering tasks.
+
+You will be given MULTIPLE failed SealQA trajectories from a single minibatch and the current skill document. The trajectories may include tool calls such as search, fetch, local reads, or evidence gathering steps.
+
+Your job is to identify COMMON failure patterns across the batch and propose concise skill edits.
+
+## Failure Type Categories
+- retrieval_miss: the agent failed to gather the right evidence
+- evidence_conflict: the agent saw conflicting evidence but resolved it badly
+- answer_selection: the agent found evidence but chose the wrong final answer
+- not_attempted: the agent never reached a grounded answer
+- other: none of the above
+
+Respond ONLY with a valid JSON object (no markdown fences, no extra text):
+{
+  "batch_size": <number of trajectories analysed>,
+  "failure_summary": [
+    {"failure_type": "<type>", "count": <int>, "description": "<one-line>"}
+  ],
+  "patch": {
+    "reasoning": "<why these edits address the batch's common failures>",
+    "edits": [
+      {"op": "append",       "content": "<markdown to add at end of skill>"},
+      {"op": "insert_after", "target": "<exact heading/text to insert after>", "content": "<markdown>"},
+      {"op": "replace",      "target": "<exact text to replace>",              "content": "<replacement>"},
+      {"op": "delete",       "target": "<exact text to remove>"}
+    ]
+  }
+}
+Only include edits that are needed. "edits" can be an empty list if no patch is warranted.

skillopt/envs/sealqa/prompts/analyst_success.md

@@ -0,0 +1,19 @@
+You are an expert success-pattern analyst for evidence-seeking factual question answering tasks.
+
+You will be given MULTIPLE successful SealQA trajectories from a single minibatch and the current skill document. Your job is to identify common evidence-gathering and answer-selection behaviors worth encoding in the skill.
+
+Respond ONLY with a valid JSON object:
+{
+  "batch_size": <number of trajectories analysed>,
+  "success_patterns": ["<pattern 1>", "<pattern 2>"],
+  "patch": {
+    "reasoning": "<why these patterns are worth encoding>",
+    "edits": [
+      {"op": "append",       "content": "<markdown>"},
+      {"op": "insert_after", "target": "<heading/text>", "content": "<markdown>"},
+      {"op": "replace",      "target": "<old text>",     "content": "<new text>"},
+      {"op": "delete",       "target": "<exact text to remove>"}
+    ]
+  }
+}
+"edits" may be empty if the skill already covers all observed patterns.

skillopt/envs/sealqa/prompts/rollout_system.md

@@ -0,0 +1,3 @@
+You are an expert research assistant. Use the provided search evidence first, and only if that is insufficient, inspect the provided URL content fetched for you. Reconcile conflicting information when necessary and return a concise final answer grounded in the evidence you found.
+
+{skill_section}Return the final answer inside <answer>...</answer> when you are ready.

skillopt/envs/sealqa/rollout.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from skillopt.envs.sealqa.evaluator import score_sealqa
+from skillopt.envs.sealqa.tool_runtime import web_fetch
+from skillopt.model import chat_student, get_student_backend, is_student_exec_backend
+from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_student_exec
+from skillopt.prompts import load_prompt
+
+_FINAL_RE = re.compile(r"<answer>(.*?)</answer>", re.IGNORECASE | re.DOTALL)
+
+
+def _build_system(skill_content: str) -> str:
+    if skill_content.strip():
+        skill_section = f"## Skill\n{skill_content.strip()}\n\n"
+    else:
+        skill_section = ""
+    return load_prompt("rollout_system", env="sealqa").format(skill_section=skill_section)
+
+
+def _build_user(item: dict, *, diagnostic_mode: bool = False, diagnostic_instruction: str = '') -> str:
+    parts = [f"## Question\n{item['question']}"]
+    if item.get('search_results'):
+        parts.append(f"## Search Results\n{item['search_results']}")
+    if item.get('urls'):
+        parts.append(f"## URL Hints\n{item['urls']}")
+    if item.get('freshness'):
+        parts.append(f"## Freshness\n{item['freshness']}")
+    if item.get('question_types'):
+        parts.append(f"## Question Types\n{item['question_types']}")
+    if diagnostic_mode and diagnostic_instruction.strip():
+        parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}")
+    parts.append('Use the provided search evidence as your primary context. Do not rely on external tool use.')
+    return "\n\n".join(parts)
+
+
+def _extract_answer(text: str) -> str:
+    match = _FINAL_RE.search(text)
+    if match:
+        return match.group(1).strip()
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    return lines[-1] if lines else text.strip()
+
+
+def _build_codex_skill(skill_content: str) -> str:
+    return render_skill_md(
+        skill_content,
+        description="Dynamic ReflACT skill for solving the current SealQA evidence-grounded question.",
+        preamble=(
+            "Use this skill when answering the current SealQA question.\n"
+            "Use the provided search evidence first, reconcile conflicts carefully,\n"
+            "and return the final answer inside <answer>...</answer>."
+        ),
+    )
+
+
+def _run_codex_once(
+    *,
+    pred_dir: str,
+    skill_content: str,
+    task_text: str,
+    model: str,
+    timeout: int,
+    previous_response: str = '',
+) -> tuple[str, str, str, str]:
+    task_parts = [task_text]
+    if previous_response:
+        task_parts.append(
+            "## Previous Attempt\n"
+            f"{previous_response}\n\n"
+            "Review the evidence again and correct the final answer if needed."
+        )
+    final_task_text = "\n\n".join(task_parts)
+    skill_md = _build_codex_skill(skill_content)
+    work_dir = os.path.join(pred_dir, 'codex_exec')
+    prepare_workspace(
+        work_dir=work_dir,
+        skill_md=skill_md,
+        task_text=final_task_text,
+    )
+    prompt = (
+        "Use the `skillopt-student` skill available in this workspace.\n"
+        "Read `task.md`, answer the SealQA question using the provided evidence,\n"
+        "and return the final answer inside <answer>...</answer>."
+    )
+    final_message, raw = run_student_exec(
+        work_dir=work_dir,
+        prompt=prompt,
+        model=model,
+        timeout=timeout,
+    )
+    return final_message or raw, raw, skill_md, final_task_text
+
+
+def process_one(
+    item: dict,
+    out_root: str,
+    skill_content: str,
+    *,
+    max_tool_turns: int = 12,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = '',
+) -> dict:
+    item_id = str(item['id'])
+    pred_dir = os.path.join(out_root, 'predictions', item_id)
+    os.makedirs(pred_dir, exist_ok=True)
+
+    system = _build_system(skill_content)
+    user = _build_user(
+        item,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+    )
+    conversation: list[dict] = [{'role': 'user', 'content': user}]
+    final_response = ''
+    final_answer = ''
+    fail_reason = ''
+
+    try:
+        if is_student_exec_backend():
+            from skillopt.model import azure_openai as _llm
+
+            response, _raw, system, user_for_save = _run_codex_once(
+                pred_dir=pred_dir,
+                skill_content=skill_content,
+                task_text=user,
+                model=_llm.STUDENT_DEPLOYMENT,
+                timeout=120,
+            )
+            final_response = response
+            conversation.append({'type': 'message', 'content': response})
+            if '<answer>' in response.lower():
+                final_answer = _extract_answer(response)
+            else:
+                user = user_for_save
+        else:
+            response, _ = chat_student(
+                system=system,
+                user=user,
+                max_completion_tokens=768,
+                retries=5,
+                stage='rollout',
+            )
+            final_response = response
+            conversation.append({'type': 'message', 'content': response})
+            if '<answer>' in response.lower():
+                final_answer = _extract_answer(response)
+
+        if not final_answer:
+            urls_text = str(item.get('urls') or '').strip()
+            fetched_blocks = []
+            for raw_url in re.findall(r'https?://[^\s\]\[\'\",]+', urls_text)[:2]:
+                try:
+                    fetched = web_fetch(raw_url)
+                except Exception as fetch_error:  # noqa: BLE001
+                    fetched = f'URL: {raw_url}\n\n[fetch error: {fetch_error}]'
+                fetched_blocks.append(fetched)
+                conversation.append({'type': 'tool_call', 'cmd': f'web_fetch({raw_url!r})', 'obs': fetched})
+            if fetched_blocks:
+                retry_user = user + '\n\n## Fetched URL Content\n' + '\n\n'.join(fetched_blocks)
+                if is_student_exec_backend():
+                    retry_response, _raw, system, retry_user = _run_codex_once(
+                        pred_dir=pred_dir,
+                        skill_content=skill_content,
+                        task_text=retry_user,
+                        model=_llm.STUDENT_DEPLOYMENT,
+                        timeout=120,
+                        previous_response=final_response,
+                    )
+                else:
+                    retry_response, _ = chat_student(
+                        system=system,
+                        user=retry_user,
+                        max_completion_tokens=768,
+                        retries=5,
+                        stage='rollout',
+                    )
+                final_response = retry_response
+                conversation.append({'type': 'message', 'content': retry_response})
+                if '<answer>' in retry_response.lower():
+                    final_answer = _extract_answer(retry_response)
+                else:
+                    fail_reason = 'Model did not produce a final answer'
+            else:
+                fail_reason = 'Model did not produce a final answer'
+    except Exception as e:  # noqa: BLE001
+        fail_reason = f'error: {e}'
+
+    with open(os.path.join(pred_dir, 'student_system_prompt.txt'), 'w', encoding='utf-8') as f:
+        f.write(system)
+    with open(os.path.join(pred_dir, 'student_user_prompt.txt'), 'w', encoding='utf-8') as f:
+        f.write(user)
+    with open(os.path.join(pred_dir, 'conversation.json'), 'w', encoding='utf-8') as f:
+        json.dump(conversation, f, ensure_ascii=False, indent=2)
+
+    score = score_sealqa(item.get('question', ''), item.get('ground_truth', ''), final_answer) if final_answer else 0.0
+    result = {
+        'id': item_id,
+        'question': item.get('question', ''),
+        'task_type': item.get('task_type', 'sealqa'),
+        'task_description': item.get('question', ''),
+        'predicted_answer': final_answer,
+        'response': final_response,
+        'ground_truth': item.get('ground_truth', ''),
+        'hard': int(score >= 1.0),
+        'soft': float(score),
+        'fail_reason': fail_reason or ('' if score >= 1.0 else f"predicted '{final_answer}' but expected '{item.get('ground_truth', '')}'"),
+        'agent_ok': not fail_reason,
+        'n_turns': len(conversation),
+        'student_system_prompt': system,
+        'student_user_prompt': user,
+    }
+    return result
+
+
+def run_batch(
+    items: list[dict],
+    out_root: str,
+    skill_content: str,
+    *,
+    workers: int = 4,
+    max_tool_turns: int = 12,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = '',
+) -> list[dict]:
+    results_path = os.path.join(out_root, 'results.jsonl')
+    os.makedirs(out_root, exist_ok=True)
+
+    done_ids: set[str] = set()
+    existing: list[dict] = []
+    if os.path.exists(results_path):
+        with open(results_path, encoding='utf-8') as f:
+            for line in f:
+                try:
+                    row = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                done_ids.add(str(row.get('id')))
+                existing.append(row)
+
+    pending = [item for item in items if str(item['id']) not in done_ids]
+    if not pending:
+        return existing
+
+    results = list(existing)
+    with open(results_path, 'a', encoding='utf-8') as outf, ThreadPoolExecutor(max_workers=workers) as ex:
+        futs = {
+            ex.submit(
+                process_one,
+                item,
+                out_root,
+                skill_content,
+                max_tool_turns=max_tool_turns,
+                diagnostic_mode=diagnostic_mode,
+                diagnostic_instruction=diagnostic_instruction,
+            ): item
+            for item in pending
+        }
+        for fut in as_completed(futs):
+            res = fut.result()
+            results.append(res)
+            outf.write(json.dumps(res, ensure_ascii=False) + '\n')
+            outf.flush()
+    return results

skillopt/envs/sealqa/skills/initial.md

@@ -0,0 +1,11 @@
+# SealQA Skill
+
+## Evidence Gathering
+- Search for the most directly relevant evidence before answering.
+- If multiple sources conflict, prefer the source that best matches the question's entity, date, and scope.
+- Keep notes on which evidence directly answers the question versus which evidence is only contextual.
+
+## Final Answer Discipline
+- Do not answer until the supporting evidence is specific enough.
+- Choose the final answer that is best grounded in the gathered evidence.
+- Keep the final answer concise.

skillopt/envs/sealqa/tool_runtime.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import html
+import re
+from urllib.request import Request, urlopen
+
+DEFAULT_USER_AGENT = (
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/135.0 Safari/537.36'
+)
+_MAX_FETCH_CHARS = 6000
+
+
+def _strip_html(raw_html: str) -> str:
+    cleaned = re.sub(r'(?is)<script.*?>.*?</script>', ' ', raw_html)
+    cleaned = re.sub(r'(?is)<style.*?>.*?</style>', ' ', cleaned)
+    cleaned = re.sub(r'(?is)<[^>]+>', ' ', cleaned)
+    cleaned = html.unescape(cleaned)
+    return re.sub(r'\s+', ' ', cleaned).strip()
+
+
+def web_fetch(url: str, max_chars: int = _MAX_FETCH_CHARS) -> str:
+    req = Request(url, headers={'User-Agent': DEFAULT_USER_AGENT})
+    with urlopen(req, timeout=20) as response:
+        body = response.read().decode('utf-8', errors='ignore')
+    text = _strip_html(body)
+    if len(text) > max_chars:
+        omitted = len(text) - max_chars
+        text = text[:max_chars] + f"\n\n[... {omitted} characters omitted ...]"
+    return f"URL: {url}\n\n{text}"