Initial commit
This commit is contained in:
Cuzyoung
@@ -0,0 +1,93 @@
|
||||
# ReflACT default configuration — base for all environments.
|
||||
# Environment configs should inherit via: _base_: default.yaml
|
||||
|
||||
model:
|
||||
backend: azure_openai
|
||||
teacher: gpt-5.5
|
||||
student: gpt-5.5
|
||||
teacher_backend: openai_chat
|
||||
student_backend: openai_chat
|
||||
reasoning_effort: medium
|
||||
rewrite_reasoning_effort: ""
|
||||
rewrite_max_completion_tokens: 64000
|
||||
codex_exec_path: codex
|
||||
codex_exec_sandbox: workspace-write
|
||||
codex_exec_profile: ""
|
||||
codex_exec_full_auto: false
|
||||
codex_exec_reasoning_effort: none
|
||||
codex_exec_use_sdk: auto
|
||||
codex_exec_network_access: false
|
||||
codex_exec_web_search: false
|
||||
codex_exec_approval_policy: never
|
||||
claude_code_exec_path: claude
|
||||
claude_code_exec_profile: ""
|
||||
claude_code_exec_use_sdk: auto
|
||||
claude_code_exec_effort: medium
|
||||
claude_code_exec_max_thinking_tokens: 16384
|
||||
codex_trace_to_teacher: true
|
||||
azure_openai_endpoint: "https://t2vgoaigpt4o3.openai.azure.com/"
|
||||
azure_openai_api_version: "2024-12-01-preview"
|
||||
azure_openai_api_key: "" # Fill locally if you do not export AZURE_OPENAI_API_KEY
|
||||
azure_openai_auth_mode: azure_cli
|
||||
azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
|
||||
azure_openai_managed_identity_client_id: ""
|
||||
teacher_azure_openai_endpoint: "https://t2vgoaigpt4o3.openai.azure.com/"
|
||||
teacher_azure_openai_api_version: "2024-12-01-preview"
|
||||
teacher_azure_openai_api_key: ""
|
||||
teacher_azure_openai_auth_mode: azure_cli
|
||||
teacher_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
|
||||
teacher_azure_openai_managed_identity_client_id: ""
|
||||
student_azure_openai_endpoint: "https://t2vgoaigpt4o3.openai.azure.com/"
|
||||
student_azure_openai_api_version: "2024-12-01-preview"
|
||||
student_azure_openai_api_key: ""
|
||||
student_azure_openai_auth_mode: azure_cli
|
||||
student_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
|
||||
student_azure_openai_managed_identity_client_id: ""
|
||||
|
||||
train:
|
||||
num_epochs: 4
|
||||
train_size: 0 # 0 = derive from dataset split when available
|
||||
batch_size: 40
|
||||
accumulation: 1
|
||||
seed: 42
|
||||
|
||||
gradient:
|
||||
minibatch_size: 8
|
||||
merge_batch_size: 8
|
||||
analyst_workers: 16
|
||||
max_analyst_rounds: 3
|
||||
failure_only: false
|
||||
use_deep_reflect: false
|
||||
deep_reflect_failures: 4
|
||||
deep_reflect_successes: 2
|
||||
|
||||
optimizer:
|
||||
learning_rate: 4 # max edits per step (edit_budget)
|
||||
min_learning_rate: 2 # min edits for decay schedulers
|
||||
lr_scheduler: cosine # constant / linear / cosine / autonomous
|
||||
lr_control_mode: fixed # fixed / autonomous / none
|
||||
skill_update_mode: patch # patch / rewrite_from_suggestions / full_rewrite_minibatch
|
||||
use_meta_reflect: false
|
||||
meta_learning_rate: 4 # max edits per epoch-level meta-reflect
|
||||
use_slow_update: true
|
||||
slow_update_samples: 20
|
||||
longitudinal_pair_policy: mixed # mixed / changed / unchanged
|
||||
use_meta_skill: true
|
||||
|
||||
evaluation:
|
||||
use_gate: true
|
||||
sel_env_num: 0
|
||||
test_env_num: 0
|
||||
eval_test: true
|
||||
|
||||
env:
|
||||
name: ""
|
||||
skill_init: ""
|
||||
split_mode: ratio # ratio = build deterministic split from data_path; split_dir = use pre-split train/val/test
|
||||
split_ratio: "2:1:7" # explicit default for dataset-backed benchmarks: train:val:test
|
||||
split_seed: 42
|
||||
split_dir: ""
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
exec_timeout: 120 # per student model/code-agent call timeout in seconds
|
||||
out_root: ""
|
||||
@@ -0,0 +1,305 @@
|
||||
# Ablation Study Configuration Manifest
|
||||
|
||||
This folder records the final, reproducible settings for the ablation runs used
|
||||
in `docs/ablation_paper_tables.md`.
|
||||
|
||||
It is intentionally separate from the benchmark default configs. The benchmark
|
||||
configs under `configs/<benchmark>/default.yaml` remain the source task configs;
|
||||
this folder records the exact matrix-level overrides, run roots, launch commands,
|
||||
and validation rules used for the paper ablations.
|
||||
|
||||
## Files
|
||||
|
||||
- `matrix.yaml`: canonical ablation matrix, common overrides, benchmark splits,
|
||||
token/output caps, and invalid-run rules.
|
||||
- `launch_commands.sh`: exact launcher commands for the valid run roots.
|
||||
- `validation.md`: monitoring, result extraction, and invalidation checklist.
|
||||
|
||||
## Source Of Truth
|
||||
|
||||
Use the matrix launcher:
|
||||
|
||||
```bash
|
||||
/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python scripts/run_ablation_matrix.py
|
||||
```
|
||||
|
||||
The launcher builds runs from the same defaults and values recorded in
|
||||
`matrix.yaml`. It skips completed runs by checking `summary.json` and skips
|
||||
active runs by checking `env.out_root` in active `scripts/train.py` processes.
|
||||
|
||||
Do not manually rerun a completed run into the same `env.out_root`. If a run is
|
||||
invalid, archive or remove its output directory first, then let the launcher
|
||||
start it cleanly.
|
||||
|
||||
## Current Correct Run Roots
|
||||
|
||||
- SearchQA / SpreadsheetBench original ablations:
|
||||
`outputs/ablation_20260502_040604_unique48`
|
||||
- SearchQA / SpreadsheetBench batch-size ablations:
|
||||
`outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run`
|
||||
- LiveMathBench / ALFWorld clean ablations:
|
||||
`outputs/ablation_livemath_alfworld_clean_20260503_155155_run`
|
||||
- DocVQA ablations:
|
||||
`outputs/ablation_docvqa_20260503_160225_run`
|
||||
|
||||
Archived, superseded, misaligned, dry-run, or pre-fix directories must not be
|
||||
used for paper tables.
|
||||
|
||||
## End-To-End Runbook
|
||||
|
||||
### Environment
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
```bash
|
||||
cd /home/azureuser/workspace-gzy/SkillReflection
|
||||
```
|
||||
|
||||
Always use:
|
||||
|
||||
```bash
|
||||
PY=/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python
|
||||
export ALFWORLD_DATA=/home/azureuser/.cache/alfworld
|
||||
```
|
||||
|
||||
Default model/auth settings are generated by `scripts/run_ablation_matrix.py`:
|
||||
|
||||
```text
|
||||
teacher=gpt-5.5
|
||||
student=gpt-5.5
|
||||
teacher_backend=openai_chat
|
||||
student_backend=openai_chat
|
||||
reasoning_effort=medium
|
||||
teacher/student endpoint=https://t2vgoaigpt4o3.openai.azure.com/
|
||||
teacher/student api_version=2024-12-01-preview
|
||||
teacher/student auth_mode=azure_cli
|
||||
```
|
||||
|
||||
Core training settings:
|
||||
|
||||
```text
|
||||
train.num_epochs=4
|
||||
train.train_size=0
|
||||
train.batch_size=40
|
||||
train.accumulation=1
|
||||
train.seed=42
|
||||
gradient.minibatch_size=8
|
||||
gradient.merge_batch_size=8
|
||||
gradient.analyst_workers=16
|
||||
gradient.use_deep_reflect=false
|
||||
optimizer.learning_rate=4
|
||||
optimizer.min_learning_rate=2
|
||||
optimizer.lr_scheduler=cosine
|
||||
optimizer.lr_control_mode=fixed
|
||||
optimizer.use_slow_update=true
|
||||
optimizer.slow_update_samples=20
|
||||
optimizer.use_meta_skill=true
|
||||
optimizer.use_meta_reflect=false
|
||||
optimizer.longitudinal_pair_policy=mixed
|
||||
evaluation.use_gate=true
|
||||
evaluation.eval_test=true
|
||||
env.split_mode=split_dir
|
||||
```
|
||||
|
||||
`train.train_size=0` is intentional. The dataloader derives the train size from
|
||||
the fixed split. Batch-size ablations rely on the default `ceil(train_size /
|
||||
batch_size)` behavior; the last batch can be smaller than `train.batch_size`.
|
||||
|
||||
### Fixed Splits
|
||||
|
||||
Default split directories:
|
||||
|
||||
```text
|
||||
searchqa: data/ablation_splits/searchqa/2-1-7_seed42
|
||||
spreadsheetbench: data/ablation_splits/spreadsheetbench/2-1-7_seed42
|
||||
livemathematicianbench: data/ablation_splits/livemathematicianbench/2-1-7_seed42
|
||||
alfworld: data/ablation_splits/alfworld/2-1-7_seed42
|
||||
docvqa: /home/azureuser/zisu/SkillReflection/data/docvqa/splits
|
||||
```
|
||||
|
||||
Default train/val/test sizes:
|
||||
|
||||
| Benchmark | Train | Val | Test |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| SearchQA | 400 | 200 | 1400 |
|
||||
| SpreadsheetBench | 80 | 40 | 280 |
|
||||
| LiveMathBench | 35 | 18 | 124 |
|
||||
| ALFWorld | 39 | 18 | 134 |
|
||||
| DocVQA | 1070 | 535 | 3744 |
|
||||
|
||||
DocVQA images are not copied. The valid setup uses:
|
||||
|
||||
```text
|
||||
data/docvqa_images -> /home/azureuser/zisu/SkillReflection/data/docvqa_images
|
||||
```
|
||||
|
||||
2026-05-05 DocVQA data correction: all DocVQA final reruns should use the zisu
|
||||
10% split above and a fresh output root such as
|
||||
`outputs/ablation_docvqa_zisu10pct_20260505_run`. The older local
|
||||
`data/ablation_splits/docvqa/2-1-7_seed42` contains the same 5349 questionId
|
||||
pool but a different train/val/test assignment, so its completed summaries are
|
||||
historical only.
|
||||
|
||||
### Matrix Groups
|
||||
|
||||
Use these group names with `scripts/run_ablation_matrix.py`:
|
||||
|
||||
```text
|
||||
default split batch mbs lr sched slown mod smodel longpair lrctrl
|
||||
```
|
||||
|
||||
`longpair` is the slow-update/meta-skill comparison-example ablation. It keeps
|
||||
all prompts and training settings unchanged and only overrides:
|
||||
|
||||
```text
|
||||
optimizer.longitudinal_pair_policy=changed
|
||||
optimizer.longitudinal_pair_policy=unchanged
|
||||
```
|
||||
|
||||
The default paper setting remains `mixed`.
|
||||
|
||||
`lrctrl` contains the two learning-rate-control baselines:
|
||||
|
||||
```text
|
||||
optimizer.lr_control_mode=autonomous
|
||||
optimizer.lr_control_mode=none + optimizer.skill_update_mode=full_rewrite_minibatch
|
||||
```
|
||||
|
||||
The autonomous run logs the chosen integer per step in `lr_decision.json` and
|
||||
`lr_history.jsonl`. The full-rewrite run removes the LR/edit-selection concept:
|
||||
each minibatch analyst produces a complete skill candidate, and aggregate/merge
|
||||
produces the candidate skill directly.
|
||||
|
||||
Batch-size values are:
|
||||
|
||||
```text
|
||||
8 / 24 / 40 / 56 / full
|
||||
```
|
||||
|
||||
`40` is the default point. `full` expands to the benchmark train size.
|
||||
|
||||
### Launch Commands Used In This Session
|
||||
|
||||
The exact commands are recorded in `launch_commands.sh` and in
|
||||
`docs/ablation_plan.md`. The important current policy is:
|
||||
|
||||
- SearchQA / SpreadsheetBench batch-only matrix can run at `--max-parallel 8`.
|
||||
- DocVQA matrix can run with its launcher at `--max-parallel 8`; later top-up used `--max-parallel 16` only because completed runs were skipped and active roots were checked.
|
||||
- LiveMathBench is safe as API-only benchmark after the token cap fix.
|
||||
- ALFWorld must not be mixed into a 24-way run on this shared machine. Use `--bench alfworld --max-parallel 1` only after memory is available.
|
||||
|
||||
### Token And Timeout Fixes
|
||||
|
||||
LiveMathBench must use a large student completion cap:
|
||||
|
||||
```text
|
||||
max_completion_tokens=16384
|
||||
timeout=300
|
||||
```
|
||||
|
||||
The old 768/512 cap produced many empty visible responses because hidden
|
||||
reasoning consumed the budget.
|
||||
|
||||
ALFWorld must use:
|
||||
|
||||
```text
|
||||
max_completion_tokens=2048
|
||||
empty response fallback -> <action>look</action>
|
||||
missing action fallback -> <action>look</action>
|
||||
```
|
||||
|
||||
### Invalid Runs
|
||||
|
||||
Never fill paper tables from these archive directories:
|
||||
|
||||
```text
|
||||
outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_livemath_token768_20260504_022258/
|
||||
outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_token512_20260504_021417/
|
||||
outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_empty_action_20260504_025311/
|
||||
outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_prefallback_20260504_025402/
|
||||
outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_oom_partial_20260504_050517/
|
||||
outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_serial_lowmem_20260504_1300/
|
||||
```
|
||||
|
||||
### Current Resource Notes
|
||||
|
||||
ALFWorld model calls are API calls. The ablation branch now creates local
|
||||
ALFWorld/TextWorld environments through multiprocessing workers ported from
|
||||
`skillopt_final_zzw`, not through Ray actors. Old Ray-based archived runs are
|
||||
not valid for table fill. The observed historical failure mode in this session
|
||||
was system RAM pressure and Ray OOM prevention, not model GPU memory.
|
||||
|
||||
GPU memory currently shown by `nvidia-smi` came from unrelated Ray Serve visual
|
||||
models under:
|
||||
|
||||
```text
|
||||
/home/azureuser/workspace-gzy/zyf/gca-skill
|
||||
```
|
||||
|
||||
Those processes are `GroundingDINOModel` / `DA3Model`, not the
|
||||
SkillReflection ablation ALFWorld run.
|
||||
|
||||
There are also unrelated ALFWorld jobs under:
|
||||
|
||||
```text
|
||||
/home/azureuser/zisu/skill_distill
|
||||
```
|
||||
|
||||
Do not confuse those with this repository's ablation outputs.
|
||||
|
||||
### Monitoring
|
||||
|
||||
Active run and duplicate output-root check:
|
||||
|
||||
```bash
|
||||
$PY - <<'PY'
|
||||
import subprocess, re, collections, time
|
||||
try:
|
||||
raw = subprocess.check_output(["pgrep", "-af", "scripts/train.py"], text=True)
|
||||
except subprocess.CalledProcessError:
|
||||
raw = ""
|
||||
roots = []
|
||||
for line in raw.splitlines():
|
||||
m = re.search(r"env\.out_root=([^\s]+)", line)
|
||||
if m:
|
||||
roots.append(m.group(1))
|
||||
ctr = collections.Counter(roots)
|
||||
print("time", time.strftime("%F %T"))
|
||||
print("active_count", len(roots))
|
||||
print("duplicates", [r.rsplit("/", 1)[-1] for r, c in ctr.items() if c > 1])
|
||||
for root in sorted(roots):
|
||||
print(root.rsplit("/", 1)[-1])
|
||||
PY
|
||||
```
|
||||
|
||||
Error scan:
|
||||
|
||||
```bash
|
||||
rg -n "Traceback|ERROR|Error code|AuthenticationError|BadRequest|RateLimit|content_filter|Killed|OutOfMemory|CUDA out of memory|\\[FAIL\\]|LLM call failed" \
|
||||
outputs/ablation_docvqa_20260503_160225_run/logs \
|
||||
outputs/ablation_livemath_alfworld_clean_20260503_155155_run/logs \
|
||||
outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run/logs \
|
||||
-g '*.log' | tail -100 || true
|
||||
```
|
||||
|
||||
Resource checks:
|
||||
|
||||
```bash
|
||||
free -h | sed -n '1,3p'
|
||||
df -h /tmp
|
||||
du -sh /tmp/ray 2>/dev/null || true
|
||||
nvidia-smi
|
||||
```
|
||||
|
||||
### Filling Tables
|
||||
|
||||
Only use top-level `summary.json` from valid run roots. Fill
|
||||
`docs/ablation_paper_tables.md` from:
|
||||
|
||||
```text
|
||||
best_selection_hard
|
||||
baseline_test_hard
|
||||
test_hard
|
||||
test_delta_hard
|
||||
token_summary._total.total_tokens
|
||||
```
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd /home/azureuser/workspace-gzy/SkillReflection
|
||||
|
||||
PY=/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python
|
||||
export ALFWORLD_DATA=/home/azureuser/.cache/alfworld
|
||||
|
||||
# Original SearchQA / SpreadsheetBench full matrix reproduction command.
|
||||
# Do not run this into the existing root unless intentionally reproducing from
|
||||
# scratch; the current valid root is already populated:
|
||||
# outputs/ablation_20260502_040604_unique48
|
||||
#
|
||||
# setsid "$PY" scripts/run_ablation_matrix.py \
|
||||
# --groups default split mbs lr sched slown mod smodel \
|
||||
# --bench searchqa spreadsheetbench \
|
||||
# --run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_20260502_040604_unique48 \
|
||||
# --max-parallel 24 \
|
||||
# --execute \
|
||||
# > /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_20260502_040604_unique48/launcher_reproduce_full_matrix.log 2>&1 < /dev/null &
|
||||
#
|
||||
# SearchQA / SpreadsheetBench batch-size ablations only.
|
||||
# Original non-batch SearchQA/SpreadsheetBench ablations live in:
|
||||
# outputs/ablation_20260502_040604_unique48
|
||||
setsid "$PY" scripts/run_ablation_matrix.py \
|
||||
--groups batch \
|
||||
--bench searchqa spreadsheetbench \
|
||||
--run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run \
|
||||
--max-parallel 8 \
|
||||
--execute \
|
||||
> /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run/launcher_parallel8.log 2>&1 < /dev/null &
|
||||
|
||||
# DocVQA full matrix.
|
||||
setsid "$PY" scripts/run_ablation_matrix.py \
|
||||
--groups default split batch mbs lr sched slown mod smodel \
|
||||
--bench docvqa \
|
||||
--run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_docvqa_20260503_160225_run \
|
||||
--max-parallel 8 \
|
||||
--execute \
|
||||
> /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_docvqa_20260503_160225_run/launcher_parallel8.log 2>&1 < /dev/null &
|
||||
|
||||
# LiveMathBench clean matrix. ALFWorld should be launched separately at lower
|
||||
# concurrency because Ray OOM occurred when many ALFWorld runs were mixed into a
|
||||
# 24-way run.
|
||||
setsid "$PY" scripts/run_ablation_matrix.py \
|
||||
--groups default split batch mbs lr sched slown mod smodel \
|
||||
--bench livemathematicianbench \
|
||||
--run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_livemath_alfworld_clean_20260503_155155_run \
|
||||
--max-parallel 8 \
|
||||
--execute \
|
||||
> /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_livemath_alfworld_clean_20260503_155155_run/launcher_livemath_parallel8.log 2>&1 < /dev/null &
|
||||
|
||||
# ALFWorld clean matrix. Increase to 2 only after checking memory, /tmp/ray,
|
||||
# and that no other ALFWorld run is active. Do not use 8/16/24 for ALFWorld on
|
||||
# the current shared machine unless resources are explicitly reserved.
|
||||
setsid "$PY" scripts/run_ablation_matrix.py \
|
||||
--groups default split batch mbs lr sched slown mod smodel \
|
||||
--bench alfworld \
|
||||
--run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_livemath_alfworld_clean_20260503_155155_run \
|
||||
--max-parallel 1 \
|
||||
--execute \
|
||||
> /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_livemath_alfworld_clean_20260503_155155_run/launcher_alfworld_parallel1.log 2>&1 < /dev/null &
|
||||
|
||||
# Longitudinal comparison-example policy ablations. This intentionally excludes
|
||||
# ALFWorld. The only varied setting is optimizer.longitudinal_pair_policy.
|
||||
setsid "$PY" scripts/run_ablation_matrix.py \
|
||||
--groups longpair \
|
||||
--bench searchqa spreadsheetbench livemathematicianbench docvqa \
|
||||
--run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_longpair_20260504_run \
|
||||
--max-parallel 8 \
|
||||
--execute \
|
||||
> /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_longpair_20260504_run/launcher_longpair_parallel8.log 2>&1 < /dev/null &
|
||||
|
||||
# Learning-rate-control baselines. This intentionally excludes ALFWorld.
|
||||
setsid "$PY" scripts/run_ablation_matrix.py \
|
||||
--groups lrctrl \
|
||||
--bench searchqa spreadsheetbench livemathematicianbench docvqa \
|
||||
--run-root /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_lrctrl_20260504_run \
|
||||
--max-parallel 8 \
|
||||
--execute \
|
||||
> /home/azureuser/workspace-gzy/SkillReflection/outputs/ablation_lrctrl_20260504_run/launcher_lrctrl_parallel8.log 2>&1 < /dev/null &
|
||||
@@ -0,0 +1,257 @@
|
||||
version: 2026-05-04
|
||||
purpose: "Canonical paper ablation settings matching the valid current runs."
|
||||
|
||||
launcher:
|
||||
script: scripts/run_ablation_matrix.py
|
||||
python: /home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python
|
||||
skip_completed_by: summary.json
|
||||
skip_active_by: "active scripts/train.py env.out_root"
|
||||
|
||||
environment:
|
||||
working_directory: /home/azureuser/workspace-gzy/SkillReflection
|
||||
required_env:
|
||||
ALFWORLD_DATA: /home/azureuser/.cache/alfworld
|
||||
docvqa_images_symlink: "data/docvqa_images -> /home/azureuser/zisu/SkillReflection/data/docvqa_images"
|
||||
|
||||
common_overrides:
|
||||
model.teacher_backend: openai_chat
|
||||
model.student_backend: openai_chat
|
||||
model.teacher: gpt-5.5
|
||||
model.student: gpt-5.5
|
||||
model.teacher_azure_openai_endpoint: https://t2vgoaigpt4o3.openai.azure.com/
|
||||
model.teacher_azure_openai_api_version: 2024-12-01-preview
|
||||
model.teacher_azure_openai_auth_mode: azure_cli
|
||||
model.student_azure_openai_endpoint: https://t2vgoaigpt4o3.openai.azure.com/
|
||||
model.student_azure_openai_api_version: 2024-12-01-preview
|
||||
model.student_azure_openai_auth_mode: azure_cli
|
||||
model.reasoning_effort: medium
|
||||
train.num_epochs: 4
|
||||
train.train_size: 0
|
||||
train.batch_size: 40
|
||||
train.accumulation: 1
|
||||
train.seed: 42
|
||||
gradient.minibatch_size: 8
|
||||
gradient.merge_batch_size: 8
|
||||
gradient.analyst_workers: 16
|
||||
gradient.use_deep_reflect: false
|
||||
optimizer.learning_rate: 4
|
||||
optimizer.min_learning_rate: 2
|
||||
optimizer.lr_scheduler: cosine
|
||||
optimizer.skill_update_mode: patch
|
||||
optimizer.use_slow_update: true
|
||||
optimizer.slow_update_samples: 20
|
||||
optimizer.use_meta_skill: true
|
||||
optimizer.use_meta_reflect: false
|
||||
evaluation.use_gate: true
|
||||
evaluation.eval_test: true
|
||||
env.split_mode: split_dir
|
||||
|
||||
benchmarks:
|
||||
searchqa:
|
||||
config: configs/searchqa/default.yaml
|
||||
run_roots:
|
||||
original_matrix: outputs/ablation_20260502_040604_unique48
|
||||
batch_matrix: outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run
|
||||
default_split: data/ablation_splits/searchqa/2-1-7_seed42
|
||||
train: 400
|
||||
val: 200
|
||||
test: 1400
|
||||
student_rollout:
|
||||
function: reflact/envs/searchqa/rollout.py::chat_student
|
||||
max_completion_tokens:
|
||||
first_turn: 512
|
||||
refinement: 512
|
||||
rationale: "Short-answer QA; sampled empties are low and not LiveMath-like."
|
||||
|
||||
spreadsheetbench:
|
||||
config: configs/spreadsheetbench/default.yaml
|
||||
run_roots:
|
||||
original_matrix: outputs/ablation_20260502_040604_unique48
|
||||
batch_matrix: outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run
|
||||
default_split: data/ablation_splits/spreadsheetbench/2-1-7_seed42
|
||||
train: 80
|
||||
val: 40
|
||||
test: 280
|
||||
student_rollout:
|
||||
function: reflact/envs/spreadsheetbench/codegen_agent.py::run_multi
|
||||
max_output_tokens: 16384
|
||||
result_note: "results.jsonl stores execution fields, not a response field."
|
||||
|
||||
livemathematicianbench:
|
||||
config: configs/livemathematicianbench/default.yaml
|
||||
run_roots:
|
||||
clean_matrix: outputs/ablation_livemath_alfworld_clean_20260503_155155_run
|
||||
default_split: data/ablation_splits/livemathematicianbench/2-1-7_seed42
|
||||
train: 35
|
||||
val: 18
|
||||
test: 124
|
||||
student_rollout:
|
||||
function: reflact/envs/livemathematicianbench/rollout.py::chat_student
|
||||
max_completion_tokens:
|
||||
first_turn: 16384
|
||||
refinement: 16384
|
||||
timeout_seconds: 300
|
||||
invalid_old_caps:
|
||||
first_turn: 768
|
||||
refinement: 512
|
||||
invalid_archive: outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_livemath_token768_20260504_022258
|
||||
rationale: "GPT-5 reasoning consumed small budgets and produced many empty visible responses."
|
||||
|
||||
alfworld:
|
||||
config: configs/alfworld/default.yaml
|
||||
run_roots:
|
||||
clean_matrix: outputs/ablation_livemath_alfworld_clean_20260503_155155_run
|
||||
default_split: data/ablation_splits/alfworld/2-1-7_seed42
|
||||
train: 39
|
||||
val: 18
|
||||
test: 134
|
||||
student_rollout:
|
||||
function: reflact/envs/alfworld/rollout.py::chat_student
|
||||
max_completion_tokens: 2048
|
||||
timeout_seconds: 120
|
||||
max_steps: 50
|
||||
fallback_action: look
|
||||
invalid_old_cap: 512
|
||||
invalid_archives:
|
||||
- outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_token512_20260504_021417
|
||||
- outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_empty_action_20260504_025311
|
||||
- outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_prefallback_20260504_025402
|
||||
- outputs/ablation_livemath_alfworld_clean_20260503_155155_run/archive_alfworld_oom_partial_20260504_050517
|
||||
concurrency_note: "Do not mix many ALFWorld runs into 24-way total concurrency; Ray OOM occurred. Prefer 1-2 ALFWorld runs at a time unless resources are clearly free."
|
||||
|
||||
docvqa:
|
||||
config: configs/docvqa/default.yaml
|
||||
run_roots:
|
||||
matrix: outputs/ablation_docvqa_zisu10pct_20260505_run
|
||||
default_split: /home/azureuser/zisu/SkillReflection/data/docvqa/splits
|
||||
train: 1070
|
||||
val: 535
|
||||
test: 3744
|
||||
data_note: "2026-05-05: use zisu-provided 5349-item DocVQA split directly; previous local 2-1-7_seed42 used the same item pool but a different train/val/test assignment and must not be used for final DocVQA reruns."
|
||||
student_rollout:
|
||||
function: reflact/envs/docvqa/rollout.py::chat_student_messages
|
||||
max_completion_tokens:
|
||||
first_turn: 768
|
||||
refinement: 512
|
||||
rationale: "Short-answer VQA output; preserve current setting for alignment unless explicitly rerunning all affected DocVQA."
|
||||
|
||||
splits:
|
||||
tags:
|
||||
1shot:
|
||||
extra_overrides:
|
||||
optimizer.slow_update_samples: 1
|
||||
1-1-8: {}
|
||||
2-1-7:
|
||||
default: true
|
||||
4-1-5: {}
|
||||
paths:
|
||||
searchqa:
|
||||
1shot: data/ablation_splits/searchqa/1shot_seed42
|
||||
1-1-8: data/ablation_splits/searchqa/1-1-8_seed42
|
||||
2-1-7: data/ablation_splits/searchqa/2-1-7_seed42
|
||||
4-1-5: data/ablation_splits/searchqa/4-1-5_seed42
|
||||
spreadsheetbench:
|
||||
1shot: data/ablation_splits/spreadsheetbench/1shot_seed42
|
||||
1-1-8: data/ablation_splits/spreadsheetbench/1-1-8_seed42
|
||||
2-1-7: data/ablation_splits/spreadsheetbench/2-1-7_seed42
|
||||
4-1-5: data/ablation_splits/spreadsheetbench/4-1-5_seed42
|
||||
livemathematicianbench:
|
||||
1shot: data/ablation_splits/livemathematicianbench/1shot_seed42
|
||||
1-1-8: data/ablation_splits/livemathematicianbench/1-1-8_seed42
|
||||
2-1-7: data/ablation_splits/livemathematicianbench/2-1-7_seed42
|
||||
4-1-5: data/ablation_splits/livemathematicianbench/4-1-5_seed42
|
||||
alfworld:
|
||||
1shot: data/ablation_splits/alfworld/1shot_seed42
|
||||
1-1-8: data/ablation_splits/alfworld/1-1-8_seed42
|
||||
2-1-7: data/ablation_splits/alfworld/2-1-7_seed42
|
||||
4-1-5: data/ablation_splits/alfworld/4-1-5_seed42
|
||||
docvqa:
|
||||
1shot: data/ablation_splits/docvqa/1shot_seed42
|
||||
1-1-8: data/ablation_splits/docvqa/1-1-8_seed42
|
||||
2-1-7: /home/azureuser/zisu/SkillReflection/data/docvqa/splits
|
||||
4-1-5: data/ablation_splits/docvqa/4-1-5_seed42
|
||||
|
||||
groups:
|
||||
default:
|
||||
run_id: "DEFAULT-{benchmark}-5.5"
|
||||
overrides: {}
|
||||
split:
|
||||
values: [1shot, 1-1-8, 4-1-5]
|
||||
skip_default_2_1_7: true
|
||||
override_template: "env.split_dir={split_path}"
|
||||
batch:
|
||||
values: [8, 24, 56, full]
|
||||
default_value_reused: 40
|
||||
full_values:
|
||||
searchqa: 400
|
||||
spreadsheetbench: 80
|
||||
livemathematicianbench: 35
|
||||
alfworld: 39
|
||||
docvqa: 1070
|
||||
fixed_overrides:
|
||||
gradient.minibatch_size: 8
|
||||
mbs:
|
||||
values: [1, 2, 4, 16, 32]
|
||||
default_value_reused: 8
|
||||
override_template: "gradient.minibatch_size={value}"
|
||||
lr:
|
||||
values: [1, 2, 4, 8, 16]
|
||||
fixed_overrides:
|
||||
optimizer.lr_scheduler: constant
|
||||
optimizer.min_learning_rate: 1
|
||||
override_template: "optimizer.learning_rate={value}"
|
||||
sched:
|
||||
values: [constant, linear]
|
||||
default_value_reused: cosine
|
||||
override_template: "optimizer.lr_scheduler={value}"
|
||||
slown:
|
||||
values: [5, 10, 40]
|
||||
default_value_reused: 20
|
||||
override_template: "optimizer.slow_update_samples={value}"
|
||||
mod:
|
||||
values:
|
||||
slow-only:
|
||||
optimizer.use_slow_update: true
|
||||
optimizer.use_meta_skill: false
|
||||
meta-only:
|
||||
optimizer.use_slow_update: false
|
||||
optimizer.use_meta_skill: true
|
||||
none:
|
||||
optimizer.use_slow_update: false
|
||||
optimizer.use_meta_skill: false
|
||||
default_value_reused: slow-meta
|
||||
longpair:
|
||||
values: [changed, unchanged]
|
||||
default_value_reused: mixed
|
||||
override_template: "optimizer.longitudinal_pair_policy={value}"
|
||||
note: "Only changes slow-update/meta-skill comparison examples; prompts and other settings remain unchanged."
|
||||
lrctrl:
|
||||
values:
|
||||
autonomous:
|
||||
optimizer.lr_control_mode: autonomous
|
||||
full-rewrite:
|
||||
optimizer.lr_control_mode: none
|
||||
optimizer.skill_update_mode: full_rewrite_minibatch
|
||||
default_value_reused: "fixed patch learning_rate=4"
|
||||
note: "autonomous records lr_decision.json/lr_history.jsonl; full-rewrite removes LR/select/apply-edit and uses full skill candidates."
|
||||
smodel:
|
||||
values:
|
||||
"5.4":
|
||||
model.student: gpt-5.4-pro
|
||||
model.student_azure_openai_endpoint: https://t2vgoaigpt4o3.openai.azure.com/
|
||||
model.student_azure_openai_api_version: 2025-03-01-preview
|
||||
model.student_azure_openai_auth_mode: azure_cli
|
||||
"5.4-mini":
|
||||
model.student: gpt-5.4-mini
|
||||
model.student_azure_openai_endpoint: https://searchagent5.cognitiveservices.azure.com/
|
||||
model.student_azure_openai_api_version: 2024-12-01-preview
|
||||
model.student_azure_openai_auth_mode: azure_cli
|
||||
default_value_reused: "5.5"
|
||||
|
||||
validity_rules:
|
||||
use_for_tables:
|
||||
- "Only runs with summary.json in valid run roots."
|
||||
- "Do not use archive, archived, MISALIGNED, SUPERSEDED, dryrun, smoke, or debug directories."
|
||||
- "Do not use ALFWorld runs started before empty/missing-action fallback."
|
||||
- "Do not use old LiveMath runs with 768/512 token caps."
|
||||
rerun_rule: "Archive or remove invalid out_root before relaunch; never write a rerun into a polluted output directory."
|
||||
@@ -0,0 +1,141 @@
|
||||
# Ablation Validation Checklist
|
||||
|
||||
Use this checklist before launch, during monitoring, and before filling
|
||||
`docs/ablation_paper_tables.md`.
|
||||
|
||||
## Before Launch
|
||||
|
||||
Run from repo root:
|
||||
|
||||
```bash
|
||||
cd /home/azureuser/workspace-gzy/SkillReflection
|
||||
export ALFWORLD_DATA=/home/azureuser/.cache/alfworld
|
||||
```
|
||||
|
||||
Verify syntax for edited files:
|
||||
|
||||
```bash
|
||||
/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python -m py_compile \
|
||||
scripts/run_ablation_matrix.py \
|
||||
scripts/train.py \
|
||||
reflact/model/azure_openai.py \
|
||||
reflact/envs/searchqa/rollout.py \
|
||||
reflact/envs/spreadsheetbench/rollout.py \
|
||||
reflact/envs/livemathematicianbench/rollout.py \
|
||||
reflact/envs/alfworld/rollout.py \
|
||||
reflact/envs/docvqa/rollout.py
|
||||
```
|
||||
|
||||
Check active runs and duplicate `env.out_root` before starting more:
|
||||
|
||||
```bash
|
||||
/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python - <<'PY'
|
||||
import subprocess, re, collections
|
||||
try:
|
||||
raw = subprocess.check_output(["pgrep", "-af", "scripts/train.py"], text=True)
|
||||
except subprocess.CalledProcessError:
|
||||
raw = ""
|
||||
roots = []
|
||||
for line in raw.splitlines():
|
||||
m = re.search(r"env\.out_root=([^\s]+)", line)
|
||||
if m:
|
||||
roots.append(m.group(1))
|
||||
ctr = collections.Counter(roots)
|
||||
print("train_count", len(roots))
|
||||
print("duplicate_roots", [r.rsplit("/", 1)[-1] for r, c in ctr.items() if c > 1])
|
||||
for root in sorted(roots):
|
||||
print(root.rsplit("/", 1)[-1])
|
||||
PY
|
||||
```
|
||||
|
||||
## During Monitoring
|
||||
|
||||
Check launchers:
|
||||
|
||||
```bash
|
||||
pgrep -af 'scripts/run_ablation_matrix.py' || true
|
||||
tail -80 outputs/ablation_docvqa_20260503_160225_run/launcher_parallel8.log 2>/dev/null || true
|
||||
tail -80 outputs/ablation_livemath_alfworld_clean_20260503_155155_run/launcher_livemath_parallel8.log 2>/dev/null || true
|
||||
tail -80 outputs/ablation_livemath_alfworld_clean_20260503_155155_run/launcher_alfworld_parallel1.log 2>/dev/null || true
|
||||
```
|
||||
|
||||
Scan current logs for new hard failures:
|
||||
|
||||
```bash
|
||||
rg -n "Traceback|ERROR|Error code|AuthenticationError|BadRequest|RateLimit|content_filter|Killed|OutOfMemory|\\[FAIL\\]|\\[RETRY\\]" \
|
||||
outputs/ablation_docvqa_20260503_160225_run/logs \
|
||||
outputs/ablation_livemath_alfworld_clean_20260503_155155_run/logs \
|
||||
outputs/ablation_batch_searchqa_spreadsheet_20260503_153902_run/logs \
|
||||
-g '*.log' | tail -160 || true
|
||||
```
|
||||
|
||||
Check resource pressure:
|
||||
|
||||
```bash
|
||||
df -h /tmp
|
||||
du -sh /tmp/ray 2>/dev/null || true
|
||||
free -h | sed -n '1,3p'
|
||||
```
|
||||
|
||||
## Quality Checks
|
||||
|
||||
LiveMathBench current valid runs should not look like old 768/512 runs:
|
||||
|
||||
```bash
|
||||
/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python - <<'PY'
|
||||
import json, pathlib
|
||||
root = pathlib.Path("outputs/ablation_livemath_alfworld_clean_20260503_155155_run")
|
||||
for run in sorted(root.glob("*livemathematicianbench*")):
|
||||
if not run.is_dir() or "archive" in str(run):
|
||||
continue
|
||||
for rel in ["test_eval_baseline/results.jsonl", "test_eval/results.jsonl"]:
|
||||
p = run / rel
|
||||
if not p.exists():
|
||||
continue
|
||||
rows = [json.loads(l) for l in p.open(errors="ignore") if l.strip()]
|
||||
empty = sum(1 for r in rows if not str(r.get("response", "")).strip())
|
||||
answer = sum(1 for r in rows if "<answer>" in str(r.get("response", "")).lower())
|
||||
if empty:
|
||||
print(run.name, rel, "empty", empty, "answer", answer, "n", len(rows))
|
||||
PY
|
||||
```
|
||||
|
||||
ALFWorld valid runs must not contain empty action or missing action:
|
||||
|
||||
```bash
|
||||
/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python - <<'PY'
|
||||
import json, pathlib
|
||||
root = pathlib.Path("outputs/ablation_livemath_alfworld_clean_20260503_155155_run")
|
||||
for run in sorted(root.glob("*alfworld*")):
|
||||
if not run.is_dir() or "archive" in str(run):
|
||||
continue
|
||||
bad = []
|
||||
fallback = 0
|
||||
for c in run.glob("**/conversation.json"):
|
||||
data = json.load(c.open(errors="ignore"))
|
||||
for step in data:
|
||||
if step.get("step") is None:
|
||||
continue
|
||||
if not step.get("action"):
|
||||
bad.append(str(c.relative_to(run)))
|
||||
break
|
||||
mr = str(step.get("model_response", ""))
|
||||
if "empty model response" in mr or "missing action tag" in mr:
|
||||
fallback += 1
|
||||
print(run.name, "bad_action_files", len(bad), "fallback", fallback)
|
||||
PY
|
||||
```
|
||||
|
||||
## Filling Tables
|
||||
|
||||
Use only `summary.json` fields:
|
||||
|
||||
- `best_selection_hard` -> Best Sel
|
||||
- `baseline_test_hard` -> Base Test
|
||||
- `test_hard` -> Best Test
|
||||
- `test_delta_hard` -> Delta
|
||||
- `total_accepts` -> Accept
|
||||
- `total_rejects` -> Reject
|
||||
- `token_summary._total.total_tokens` -> Tokens
|
||||
|
||||
Do not fill table rows from logs alone.
|
||||
@@ -0,0 +1,30 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
train:
|
||||
train_size: 0
|
||||
accumulation: 1
|
||||
|
||||
gradient:
|
||||
minibatch_size: 8
|
||||
merge_batch_size: 8
|
||||
|
||||
optimizer:
|
||||
learning_rate: 4
|
||||
use_meta_reflect: false
|
||||
|
||||
evaluation:
|
||||
sel_env_num: 0
|
||||
test_env_num: 0
|
||||
|
||||
env:
|
||||
name: alfworld
|
||||
skill_init: reflact/envs/alfworld/skills/initial.md
|
||||
split_mode: split_dir
|
||||
split_ratio: "2:1:7"
|
||||
split_dir: data/ablation_splits/alfworld/2-1-7_seed42
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
max_steps: 50
|
||||
workers: 8
|
||||
max_api_workers: 8
|
||||
limit: 0
|
||||
@@ -0,0 +1,4 @@
|
||||
_base_: default.yaml
|
||||
|
||||
optimizer:
|
||||
use_meta_reflect: true
|
||||
@@ -0,0 +1,21 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
train:
|
||||
batch_size: 64
|
||||
accumulation: 1
|
||||
|
||||
env:
|
||||
name: babyvision
|
||||
skill_init: reflact/envs/babyvision/skills/initial.md
|
||||
split_mode: ratio
|
||||
split_ratio: "2:1:7"
|
||||
split_dir: ""
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
max_turns: 1
|
||||
workers: 16
|
||||
limit: 0
|
||||
image_detail: auto
|
||||
judge_model: gpt-5.4
|
||||
judge_max_completion_tokens: 256
|
||||
judge_retries: 5
|
||||
@@ -0,0 +1,28 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
model:
|
||||
reasoning_effort: medium
|
||||
|
||||
train:
|
||||
batch_size: 40
|
||||
accumulation: 1
|
||||
|
||||
gradient:
|
||||
minibatch_size: 8
|
||||
merge_batch_size: 8
|
||||
|
||||
optimizer:
|
||||
learning_rate: 4
|
||||
|
||||
env:
|
||||
name: docvqa
|
||||
skill_init: reflact/envs/docvqa/skills/initial.md
|
||||
split_mode: split_dir
|
||||
split_ratio: "2:1:7"
|
||||
split_dir: /home/azureuser/zisu/SkillReflection/data/docvqa/splits
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
max_turns: 1
|
||||
workers: 16
|
||||
image_detail: auto
|
||||
limit: 0
|
||||
@@ -0,0 +1,22 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
train:
|
||||
train_size: 0
|
||||
batch_size: 40
|
||||
accumulation: 1
|
||||
|
||||
env:
|
||||
name: livemathematicianbench
|
||||
skill_init: reflact/envs/livemathematicianbench/skills/initial.md
|
||||
split_mode: split_dir
|
||||
split_ratio: "2:1:7"
|
||||
split_dir: data/ablation_splits/livemathematicianbench/2-1-7_seed42
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
max_turns: 1
|
||||
exec_timeout: 300
|
||||
workers: 64
|
||||
limit: 0
|
||||
shuffle_choices: true
|
||||
use_theorem: false
|
||||
use_sketch: false
|
||||
@@ -0,0 +1,23 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
model:
|
||||
codex_exec_sandbox: danger-full-access
|
||||
|
||||
train:
|
||||
batch_size: 64
|
||||
accumulation: 1
|
||||
|
||||
env:
|
||||
name: mathverse
|
||||
skill_init: reflact/envs/mathverse/skills/initial.md
|
||||
split_dir: ""
|
||||
data_root: data/MathVerse
|
||||
problem_version: Text Lite
|
||||
use_text_dominant_reference: false
|
||||
max_turns: 1
|
||||
workers: 16
|
||||
limit: 0
|
||||
image_detail: auto
|
||||
judge_model: gpt-5.4
|
||||
judge_max_completion_tokens: 256
|
||||
judge_retries: 5
|
||||
@@ -0,0 +1,18 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
train:
|
||||
batch_size: 128
|
||||
accumulation: 1
|
||||
|
||||
env:
|
||||
name: mmrb
|
||||
skill_init: reflact/envs/mmrb/skills/initial.md
|
||||
split_mode: ratio
|
||||
split_ratio: "2:1:7"
|
||||
split_dir: ""
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
max_turns: 1
|
||||
workers: 16
|
||||
limit: 0
|
||||
image_detail: auto
|
||||
@@ -0,0 +1,25 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
model:
|
||||
reasoning_effort: medium
|
||||
|
||||
train:
|
||||
batch_size: 40
|
||||
accumulation: 1
|
||||
|
||||
gradient:
|
||||
minibatch_size: 8
|
||||
merge_batch_size: 8
|
||||
|
||||
optimizer:
|
||||
learning_rate: 4
|
||||
|
||||
env:
|
||||
name: officeqa
|
||||
skill_init: reflact/envs/officeqa/skills/initial.md
|
||||
split_dir: data/officeqa_split
|
||||
data_dirs:
|
||||
- data/officeqa_docs_official
|
||||
workers: 4
|
||||
max_tool_turns: 24
|
||||
limit: 0
|
||||
@@ -0,0 +1,23 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
model:
|
||||
reasoning_effort: medium
|
||||
|
||||
train:
|
||||
batch_size: 10
|
||||
accumulation: 1
|
||||
|
||||
gradient:
|
||||
minibatch_size: 8
|
||||
merge_batch_size: 8
|
||||
|
||||
optimizer:
|
||||
learning_rate: 4
|
||||
|
||||
env:
|
||||
name: sealqa
|
||||
skill_init: reflact/envs/sealqa/skills/initial.md
|
||||
split_dir: data/sealqa_split
|
||||
workers: 4
|
||||
max_tool_turns: 12
|
||||
limit: 0
|
||||
@@ -0,0 +1,32 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
model:
|
||||
reasoning_effort: medium
|
||||
|
||||
train:
|
||||
train_size: 400
|
||||
batch_size: 40
|
||||
accumulation: 1
|
||||
|
||||
gradient:
|
||||
minibatch_size: 8
|
||||
merge_batch_size: 8
|
||||
|
||||
optimizer:
|
||||
learning_rate: 4
|
||||
|
||||
evaluation:
|
||||
sel_env_num: 0
|
||||
test_env_num: 0
|
||||
|
||||
env:
|
||||
name: searchqa
|
||||
skill_init: reflact/envs/searchqa/skills/initial.md
|
||||
split_mode: split_dir
|
||||
split_ratio: "2:1:7"
|
||||
split_dir: data/searchqa_split
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
max_turns: 1
|
||||
workers: 24
|
||||
limit: 0
|
||||
@@ -0,0 +1,34 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
model:
|
||||
reasoning_effort: medium
|
||||
|
||||
train:
|
||||
train_size: 80
|
||||
batch_size: 40
|
||||
accumulation: 1
|
||||
|
||||
gradient:
|
||||
minibatch_size: 8
|
||||
merge_batch_size: 8
|
||||
|
||||
optimizer:
|
||||
learning_rate: 4
|
||||
|
||||
evaluation:
|
||||
sel_env_num: 0
|
||||
test_env_num: 0
|
||||
|
||||
env:
|
||||
name: spreadsheetbench
|
||||
skill_init: reflact/envs/spreadsheetbench/skills/initial.md
|
||||
split_mode: split_dir
|
||||
split_ratio: "2:1:7"
|
||||
split_dir: data/spreadsheetbench_split
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
data_root: data/spreadsheetbench_verified_400
|
||||
mode: multi
|
||||
max_turns: 30
|
||||
exec_timeout: 600
|
||||
workers: 24
|
||||
@@ -0,0 +1,36 @@
|
||||
_base_: ../_base_/default.yaml
|
||||
|
||||
model:
|
||||
reasoning_effort: medium
|
||||
|
||||
train:
|
||||
batch_size: 20
|
||||
accumulation: 1
|
||||
|
||||
gradient:
|
||||
minibatch_size: 4
|
||||
merge_batch_size: 8
|
||||
|
||||
optimizer:
|
||||
learning_rate: 4
|
||||
|
||||
evaluation:
|
||||
sel_env_num: 0
|
||||
test_env_num: 0
|
||||
|
||||
env:
|
||||
name: swebench
|
||||
skill_init: reflact/envs/swebench/skills/initial.md
|
||||
split_mode: ratio
|
||||
split_ratio: "2:1:7"
|
||||
split_dir: ""
|
||||
data_path: ""
|
||||
split_output_dir: ""
|
||||
dataset_name: lite
|
||||
hf_split: test
|
||||
workers: 8
|
||||
eval_workers: 8
|
||||
step_limit: 50
|
||||
cost_limit: 3.0
|
||||
timeout_per_instance: 600
|
||||
limit: 0
|
||||
Reference in New Issue
Block a user