docs(new-backend): document the built-in openai_compatible backend
This commit is contained in:
@@ -2,6 +2,59 @@
|
|||||||
|
|
||||||
SkillOpt supports multiple LLM backends. This guide shows how to add your own.
|
SkillOpt supports multiple LLM backends. This guide shows how to add your own.
|
||||||
|
|
||||||
|
## Built-in: the generic OpenAI-compatible backend
|
||||||
|
|
||||||
|
Before writing a new backend, check whether your provider already speaks the
|
||||||
|
OpenAI Chat Completions protocol. Most do — in which case you can use the
|
||||||
|
built-in **`openai_compatible`** backend
|
||||||
|
(`skillopt/model/openai_compatible_backend.py`) with no code changes.
|
||||||
|
|
||||||
|
A single `base_url` + `api_key` pair lets you point SkillOpt at, for example:
|
||||||
|
|
||||||
|
| Provider | `base_url` | Example model |
|
||||||
|
|---|---|---|
|
||||||
|
| DeepSeek | `https://api.deepseek.com/v1` | `deepseek-chat` |
|
||||||
|
| Groq | `https://api.groq.com/openai/v1` | `llama-3.3-70b-versatile` |
|
||||||
|
| Together AI | `https://api.together.xyz/v1` | `meta-llama/Llama-3.3-70B-Instruct-Turbo` |
|
||||||
|
| Ollama (local) | `http://localhost:11434/v1` | `qwen2.5:7b` |
|
||||||
|
| vLLM / SGLang / TGI | `http://localhost:8000/v1` | your served model |
|
||||||
|
| LiteLLM proxy | `http://localhost:4000` | any proxied model |
|
||||||
|
| OpenRouter / Fireworks / xAI / … | provider base URL | provider model id |
|
||||||
|
|
||||||
|
Select it as the optimizer and/or target backend:
|
||||||
|
|
||||||
|
```python
|
||||||
|
import skillopt.model as model
|
||||||
|
|
||||||
|
# Shorthand: use it for both optimizer and target.
|
||||||
|
model.set_backend("openai_compatible")
|
||||||
|
|
||||||
|
# Point it at a provider (shared, or per-role with optimizer_*/target_*).
|
||||||
|
model.configure_openai_compatible(
|
||||||
|
base_url="https://api.deepseek.com/v1",
|
||||||
|
api_key="sk-...",
|
||||||
|
model="deepseek-chat",
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Or configure it entirely through environment variables (role-specific
|
||||||
|
`OPTIMIZER_*` / `TARGET_*` variants override the shared ones):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export TARGET_BACKEND=openai_compatible
|
||||||
|
export OPENAI_COMPATIBLE_BASE_URL="https://api.groq.com/openai/v1"
|
||||||
|
export OPENAI_COMPATIBLE_API_KEY="gsk_..."
|
||||||
|
export OPENAI_COMPATIBLE_MODEL="llama-3.3-70b-versatile"
|
||||||
|
# Optional: OPENAI_COMPATIBLE_TEMPERATURE, _MAX_TOKENS, _TIMEOUT_SECONDS
|
||||||
|
```
|
||||||
|
|
||||||
|
The backend uses the official `openai` SDK, records token usage through the
|
||||||
|
shared tracker, supports tool/function calling via
|
||||||
|
`chat_target_messages(..., tools=...)`, and exposes
|
||||||
|
`count_tokens()` (tiktoken with a character-based fallback for non-OpenAI
|
||||||
|
models). Only write a brand-new backend if your provider is *not*
|
||||||
|
OpenAI-compatible.
|
||||||
|
|
||||||
## Backend Architecture
|
## Backend Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
+22
-10
@@ -112,13 +112,18 @@ def chat_optimizer(
|
|||||||
reasoning_effort=reasoning_effort,
|
reasoning_effort=reasoning_effort,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
<<<<<<< HEAD
|
|
||||||
if get_optimizer_backend() == "minimax_chat":
|
if get_optimizer_backend() == "minimax_chat":
|
||||||
return _minimax.chat_optimizer(
|
return _minimax.chat_optimizer(
|
||||||
=======
|
system=system,
|
||||||
|
user=user,
|
||||||
|
max_completion_tokens=max_completion_tokens,
|
||||||
|
retries=retries,
|
||||||
|
stage=stage,
|
||||||
|
reasoning_effort=reasoning_effort,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
if get_optimizer_backend() == "openai_compatible":
|
if get_optimizer_backend() == "openai_compatible":
|
||||||
return _openai_compat.chat_optimizer(
|
return _openai_compat.chat_optimizer(
|
||||||
>>>>>>> 3a5fa59 (feat(model): route optimizer/target calls to the openai_compatible backend)
|
|
||||||
system=system,
|
system=system,
|
||||||
user=user,
|
user=user,
|
||||||
max_completion_tokens=max_completion_tokens,
|
max_completion_tokens=max_completion_tokens,
|
||||||
@@ -187,8 +192,8 @@ def chat_target(
|
|||||||
)
|
)
|
||||||
if not is_target_chat_backend():
|
if not is_target_chat_backend():
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"chat_target is only supported with target_backend=openai_chat, claude_chat, qwen_chat, or minimax_chat. "
|
"chat_target is only supported with target_backend=openai_chat, claude_chat, qwen_chat, minimax_chat, "
|
||||||
"Exec backends are handled in environment-specific rollout code."
|
"or openai_compatible. Exec backends are handled in environment-specific rollout code."
|
||||||
)
|
)
|
||||||
return _openai.chat_target(
|
return _openai.chat_target(
|
||||||
system=system,
|
system=system,
|
||||||
@@ -236,13 +241,20 @@ def chat_optimizer_messages(
|
|||||||
return_message=return_message,
|
return_message=return_message,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
<<<<<<< HEAD
|
|
||||||
if get_optimizer_backend() == "minimax_chat":
|
if get_optimizer_backend() == "minimax_chat":
|
||||||
return _minimax.chat_target_messages(
|
return _minimax.chat_target_messages(
|
||||||
=======
|
messages=messages,
|
||||||
|
max_completion_tokens=max_completion_tokens,
|
||||||
|
retries=retries,
|
||||||
|
stage=stage,
|
||||||
|
reasoning_effort=reasoning_effort,
|
||||||
|
tools=tools,
|
||||||
|
tool_choice=tool_choice,
|
||||||
|
return_message=return_message,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
if get_optimizer_backend() == "openai_compatible":
|
if get_optimizer_backend() == "openai_compatible":
|
||||||
return _openai_compat.chat_optimizer_messages(
|
return _openai_compat.chat_optimizer_messages(
|
||||||
>>>>>>> 3a5fa59 (feat(model): route optimizer/target calls to the openai_compatible backend)
|
|
||||||
messages=messages,
|
messages=messages,
|
||||||
max_completion_tokens=max_completion_tokens,
|
max_completion_tokens=max_completion_tokens,
|
||||||
retries=retries,
|
retries=retries,
|
||||||
@@ -326,8 +338,8 @@ def chat_target_messages(
|
|||||||
)
|
)
|
||||||
if not is_target_chat_backend():
|
if not is_target_chat_backend():
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"chat_target_messages is only supported with target_backend=openai_chat, claude_chat, qwen_chat, or minimax_chat. "
|
"chat_target_messages is only supported with target_backend=openai_chat, claude_chat, qwen_chat, "
|
||||||
"Exec backends are handled in environment-specific rollout code."
|
"minimax_chat, or openai_compatible. Exec backends are handled in environment-specific rollout code."
|
||||||
)
|
)
|
||||||
return _openai.chat_target_messages(
|
return _openai.chat_target_messages(
|
||||||
messages=messages,
|
messages=messages,
|
||||||
|
|||||||
Reference in New Issue
Block a user