fix(skillopt-sleep): redact secrets before persisting cycle diagnostics
PR #92 added a per-cycle diagnostics.json that surfaces backend stderr, optimizer replies, and task responses so a 0.0 night is self-diagnosing. Those free-text fields can carry credentials (e.g. a codex 401 stderr dump containing an auth token), so persisting them verbatim was a new on-disk leak surface. - Add a shared redact_secrets() in staging.py and route diagnostics.json's call_error / reflect_raw_head / holdout_detail through it before writing. - Redact the codex and Claude auth-error log lines too (a secondary sink when a file log handler is attached); last_call_error stays raw in memory so _AUTH_MARKERS matching is unaffected. - Centralize _SECRET_PATTERNS in staging.py (harvest_codex now reuses them) and extend coverage to AWS / GitHub / Slack / Google / JWT token shapes. - Tests: secret-shape coverage, private-key blocks, recursive/scalar passthrough, no over-redaction of plain prose, fail-fast auth-error log redaction, and an end-to-end check that diagnostics.json has no secret. Observability-only; the gate and learning algorithm are unchanged. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -9,12 +9,68 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from typing import List, Optional
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from skillopt_sleep.types import SleepReport
|
||||
|
||||
# Secret patterns scrubbed from any free-text we persist to the staging dir
|
||||
# (diagnostics, reports). Kept here so every on-disk artifact shares one
|
||||
# redaction pass; harvest_codex reuses these for session text too.
|
||||
_SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"sk-[A-Za-z0-9_-]{10,}"), "[REDACTED_OPENAI_KEY]"),
|
||||
# Distinctive vendor token prefixes (low false-positive: these prefixes do
|
||||
# not occur in normal diagnostic prose).
|
||||
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[REDACTED_AWS_KEY]"),
|
||||
(re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b"), "[REDACTED_GITHUB_TOKEN]"),
|
||||
(re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"), "[REDACTED_SLACK_TOKEN]"),
|
||||
(re.compile(r"\bAIza[0-9A-Za-z_-]{20,}\b"), "[REDACTED_GOOGLE_KEY]"),
|
||||
# Bare JWT (three base64url segments) — e.g. a leaked bearer body without
|
||||
# the "Authorization:" prefix.
|
||||
(re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"),
|
||||
"[REDACTED_JWT]"),
|
||||
(re.compile(r"(?i)(Authorization:\s*Bearer\s+)[^\s\"']+"), r"\1[REDACTED]"),
|
||||
(re.compile(r"(?i)(Authorization:\s*Basic\s+)[^\s\"']+"), r"\1[REDACTED]"),
|
||||
(
|
||||
re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s*[:=]\s*)[^\s\"']+"),
|
||||
r"\1\2[REDACTED]",
|
||||
),
|
||||
(
|
||||
re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s+)[^\s\"']+"),
|
||||
r"\1\2[REDACTED]",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
|
||||
re.DOTALL,
|
||||
),
|
||||
"[REDACTED_PRIVATE_KEY]",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def redact_secrets(value: Any) -> Any:
|
||||
"""Scrub secret-looking substrings (API keys, bearer tokens, private keys)
|
||||
from a string, or recursively from the string leaves of a list/dict.
|
||||
|
||||
Used before writing backend stderr / optimizer replies / task responses to
|
||||
on-disk diagnostics: those are surfaced for debugging, but the underlying
|
||||
text (e.g. a codex 401 stderr dump) can carry credentials. Non-string
|
||||
scalars pass through unchanged.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
out = value
|
||||
for pattern, replacement in _SECRET_PATTERNS:
|
||||
out = pattern.sub(replacement, out)
|
||||
return out
|
||||
if isinstance(value, list):
|
||||
return [redact_secrets(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: redact_secrets(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def _ts_dir() -> str:
|
||||
return time.strftime("%Y%m%d-%H%M%S", time.localtime())
|
||||
|
||||
Reference in New Issue
Block a user