SkillOpt v0.1.0: initial release

- Skill optimization framework with training loop analogy - 11 benchmarks, 4 model backends (Azure OpenAI, Claude, Codex, Qwen) - WebUI for browser-based training control - Pluggable architecture for extending benchmarks and backends
2026-05-21 17:22:04 +00:00
commit 244e346b83
237 changed files with 30248 additions and 0 deletions
@@ -0,0 +1,93 @@
+# SkillOpt default configuration — base for all environments.
+# Environment configs should inherit via: _base_: default.yaml
+
+model:
+  backend: azure_openai
+  teacher: gpt-5.5
+  student: gpt-5.5
+  teacher_backend: openai_chat
+  student_backend: openai_chat
+  reasoning_effort: medium
+  rewrite_reasoning_effort: ""
+  rewrite_max_completion_tokens: 64000
+  codex_exec_path: codex
+  codex_exec_sandbox: workspace-write
+  codex_exec_profile: ""
+  codex_exec_full_auto: false
+  codex_exec_reasoning_effort: none
+  codex_exec_use_sdk: auto
+  codex_exec_network_access: false
+  codex_exec_web_search: false
+  codex_exec_approval_policy: never
+  claude_code_exec_path: claude
+  claude_code_exec_profile: ""
+  claude_code_exec_use_sdk: auto
+  claude_code_exec_effort: medium
+  claude_code_exec_max_thinking_tokens: 16384
+  codex_trace_to_teacher: true
+  azure_openai_endpoint: ""           # e.g. "https://your-resource.openai.azure.com/"
+  azure_openai_api_version: "2024-12-01-preview"
+  azure_openai_api_key: ""       # Fill locally if you do not export AZURE_OPENAI_API_KEY
+  azure_openai_auth_mode: azure_cli
+  azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
+  azure_openai_managed_identity_client_id: ""
+  teacher_azure_openai_endpoint: ""   # e.g. "https://your-resource.openai.azure.com/"
+  teacher_azure_openai_api_version: "2024-12-01-preview"
+  teacher_azure_openai_api_key: ""
+  teacher_azure_openai_auth_mode: azure_cli
+  teacher_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
+  teacher_azure_openai_managed_identity_client_id: ""
+  student_azure_openai_endpoint: ""   # e.g. "https://your-resource.openai.azure.com/"
+  student_azure_openai_api_version: "2024-12-01-preview"
+  student_azure_openai_api_key: ""
+  student_azure_openai_auth_mode: azure_cli
+  student_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default"
+  student_azure_openai_managed_identity_client_id: ""
+
+train:
+  num_epochs: 4
+  train_size: 0          # 0 = derive from dataset split when available
+  batch_size: 40
+  accumulation: 1
+  seed: 42
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+  analyst_workers: 16
+  max_analyst_rounds: 3
+  failure_only: false
+  use_deep_reflect: false
+  deep_reflect_failures: 4
+  deep_reflect_successes: 2
+
+optimizer:
+  learning_rate: 4          # max edits per step (edit_budget)
+  min_learning_rate: 2      # min edits for decay schedulers
+  lr_scheduler: cosine      # constant / linear / cosine / autonomous
+  lr_control_mode: fixed    # fixed / autonomous / none
+  skill_update_mode: patch  # patch / rewrite_from_suggestions / full_rewrite_minibatch
+  use_meta_reflect: false
+  meta_learning_rate: 4     # max edits per epoch-level meta-reflect
+  use_slow_update: true
+  slow_update_samples: 20
+  longitudinal_pair_policy: mixed  # mixed / changed / unchanged
+  use_meta_skill: true
+
+evaluation:
+  use_gate: true
+  sel_env_num: 0
+  test_env_num: 0
+  eval_test: true
+
+env:
+  name: ""
+  skill_init: ""
+  split_mode: ratio       # ratio = build deterministic split from data_path; split_dir = use pre-split train/val/test
+  split_ratio: "2:1:7"    # explicit default for dataset-backed benchmarks: train:val:test
+  split_seed: 42
+  split_dir: ""
+  data_path: ""
+  split_output_dir: ""
+  exec_timeout: 120      # per student model/code-agent call timeout in seconds
+  out_root: ""
@@ -0,0 +1,30 @@
+_base_: ../_base_/default.yaml
+
+train:
+  train_size: 0
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+  use_meta_reflect: false
+
+evaluation:
+  sel_env_num: 0
+  test_env_num: 0
+
+env:
+  name: alfworld
+  skill_init: skillopt/envs/alfworld/skills/initial.md
+  split_mode: split_dir
+  split_ratio: "2:1:7"
+  split_dir: data/ablation_splits/alfworld/2-1-7_seed42
+  data_path: ""
+  split_output_dir: ""
+  max_steps: 50
+  workers: 8
+  max_api_workers: 8
+  limit: 0
@@ -0,0 +1,21 @@
+_base_: ../_base_/default.yaml
+
+train:
+  batch_size: 64
+  accumulation: 1
+
+env:
+  name: babyvision
+  skill_init: skillopt/envs/babyvision/skills/initial.md
+  split_mode: ratio
+  split_ratio: "2:1:7"
+  split_dir: ""
+  data_path: ""
+  split_output_dir: ""
+  max_turns: 1
+  workers: 16
+  limit: 0
+  image_detail: auto
+  judge_model: gpt-5.4
+  judge_max_completion_tokens: 256
+  judge_retries: 5
@@ -0,0 +1,28 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  batch_size: 40
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+env:
+  name: docvqa
+  skill_init: skillopt/envs/docvqa/skills/initial.md
+  split_mode: split_dir
+  split_ratio: "2:1:7"
+  split_dir: data/docvqa/splits
+  data_path: ""
+  split_output_dir: ""
+  max_turns: 1
+  workers: 16
+  image_detail: auto
+  limit: 0
@@ -0,0 +1,22 @@
+_base_: ../_base_/default.yaml
+
+train:
+  train_size: 0
+  batch_size: 40
+  accumulation: 1
+
+env:
+  name: livemathematicianbench
+  skill_init: skillopt/envs/livemathematicianbench/skills/initial.md
+  split_mode: split_dir
+  split_ratio: "2:1:7"
+  split_dir: data/ablation_splits/livemathematicianbench/2-1-7_seed42
+  data_path: ""
+  split_output_dir: ""
+  max_turns: 1
+  exec_timeout: 300
+  workers: 64
+  limit: 0
+  shuffle_choices: true
+  use_theorem: false
+  use_sketch: false
@@ -0,0 +1,23 @@
+_base_: ../_base_/default.yaml
+
+model:
+  codex_exec_sandbox: danger-full-access
+
+train:
+  batch_size: 64
+  accumulation: 1
+
+env:
+  name: mathverse
+  skill_init: skillopt/envs/mathverse/skills/initial.md
+  split_dir: ""
+  data_root: data/MathVerse
+  problem_version: Text Lite
+  use_text_dominant_reference: false
+  max_turns: 1
+  workers: 16
+  limit: 0
+  image_detail: auto
+  judge_model: gpt-5.4
+  judge_max_completion_tokens: 256
+  judge_retries: 5
@@ -0,0 +1,18 @@
+_base_: ../_base_/default.yaml
+
+train:
+  batch_size: 128
+  accumulation: 1
+
+env:
+  name: mmrb
+  skill_init: skillopt/envs/mmrb/skills/initial.md
+  split_mode: ratio
+  split_ratio: "2:1:7"
+  split_dir: ""
+  data_path: ""
+  split_output_dir: ""
+  max_turns: 1
+  workers: 16
+  limit: 0
+  image_detail: auto
@@ -0,0 +1,34 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  batch_size: 40
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+env:
+  name: officeqa
+  skill_init: skillopt/envs/officeqa/skills/initial.md
+  split_mode: split_dir
+  split_dir: data/officeqa_split
+  data_dirs:
+    - data/officeqa_docs_official
+  workers: 4
+  max_tool_turns: 24
+  max_completion_tokens: 10000
+  search_mode: offline
+  max_queries_per_turn: 4
+  search_api_url: http://apisix.westus2.cloudapp.azure.com/search_tool/search
+  search_auth_env: OFFICEQA_CUSTOM_SEARCH_AUTH
+  search_provider: duckduckgo
+  search_max_num_results: 4
+  search_timeout_seconds: 20
+  limit: 0
@@ -0,0 +1,23 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  batch_size: 10
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+env:
+  name: sealqa
+  skill_init: skillopt/envs/sealqa/skills/initial.md
+  split_dir: data/sealqa_split
+  workers: 4
+  max_tool_turns: 12
+  limit: 0
@@ -0,0 +1,32 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  train_size: 400
+  batch_size: 40
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+evaluation:
+  sel_env_num: 0
+  test_env_num: 0
+
+env:
+  name: searchqa
+  skill_init: skillopt/envs/searchqa/skills/initial.md
+  split_mode: split_dir
+  split_ratio: "2:1:7"
+  split_dir: data/searchqa_split
+  data_path: ""
+  split_output_dir: ""
+  max_turns: 1
+  workers: 24
+  limit: 0
@@ -0,0 +1,34 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  train_size: 80
+  batch_size: 40
+  accumulation: 1
+
+gradient:
+  minibatch_size: 8
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+evaluation:
+  sel_env_num: 0
+  test_env_num: 0
+
+env:
+  name: spreadsheetbench
+  skill_init: skillopt/envs/spreadsheetbench/skills/initial.md
+  split_mode: split_dir
+  split_ratio: "2:1:7"
+  split_dir: data/spreadsheetbench_split
+  data_path: ""
+  split_output_dir: ""
+  data_root: data/spreadsheetbench_verified_400
+  mode: multi
+  max_turns: 30
+  exec_timeout: 600
+  workers: 24
@@ -0,0 +1,36 @@
+_base_: ../_base_/default.yaml
+
+model:
+  reasoning_effort: medium
+
+train:
+  batch_size: 20
+  accumulation: 1
+
+gradient:
+  minibatch_size: 4
+  merge_batch_size: 8
+
+optimizer:
+  learning_rate: 4
+
+evaluation:
+  sel_env_num: 0
+  test_env_num: 0
+
+env:
+  name: swebench
+  skill_init: skillopt/envs/swebench/skills/initial.md
+  split_mode: ratio
+  split_ratio: "2:1:7"
+  split_dir: ""
+  data_path: ""
+  split_output_dir: ""
+  dataset_name: lite
+  hf_split: test
+  workers: 8
+  eval_workers: 8
+  step_limit: 50
+  cost_limit: 3.0
+  timeout_per_instance: 600
+  limit: 0