feat: v2.5.0 — sliding-window circuit breaker, graduated backoff, increased timeouts

Emergency fix for multi-agent (ClawTeam) burst scenario that caused cascading
failures: when multiple Opus agents ran concurrently, the old consecutive-count
breaker (threshold=3) tripped within seconds, blocking ALL subsequent requests
globally — including non-ClawTeam agents and new sessions.

Changes:
- Circuit breaker now uses a sliding time window (default 5min) instead of
  consecutive failure count. 6 failures in 5min triggers open, vs old 3-in-a-row.
- Half-open state allows 2 concurrent probe requests (was 1), improving recovery
  speed when API comes back.
- Graduated backoff: cooldown doubles on each re-open (120s→240s→300s cap),
  resets fully on first success.
- Increased default timeouts: Opus first-byte 60s→90s, Sonnet 45s→60s,
  overall 120s→300s, max concurrent 5→8.
- /health endpoint now exposes per-model breaker state, window stats, and
  marks status as "degraded" when any breaker is open.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

package.json

@@ -1,7 +1,7 @@
 {
   "name": "openclaw-claude-proxy",
-  "version": "2.4.1",
+  "version": "2.5.0",
-  "description": "OpenAI-compatible proxy for Claude CLI v2 — per-model circuit breaker, adaptive first-byte timeout, structured logging",
+  "description": "OpenAI-compatible proxy for Claude CLI v2 — sliding-window circuit breaker, adaptive first-byte timeout, structured logging",
   "type": "module",
   "bin": {
     "openclaw-claude-proxy": "./server.mjs"

server.mjs

@@ -1,10 +1,19 @@
 #!/usr/bin/env node
 /**
- * openclaw-claude-proxy v2.4.0 — OpenAI-compatible proxy for Claude CLI
+ * openclaw-claude-proxy v2.5.0 — OpenAI-compatible proxy for Claude CLI
  *
  * Translates OpenAI chat/completions requests into `claude -p` CLI calls,
  * letting you use your Claude Pro/Max subscription as an OpenClaw model provider.
  *
+ * v2.5.0:
+ *   - Sliding-window circuit breaker: uses time-windowed failure rate instead of
+ *     consecutive-count, preventing multi-agent burst scenarios from tripping the
+ *     breaker too aggressively. Half-open state allows configurable probe requests.
+ *   - Graduated backoff: cooldown doubles on each re-open (capped at 5min),
+ *     resets fully on success.
+ *   - Health endpoint now exposes per-model breaker state and sliding window stats.
+ *   - Increased default timeout tiers for Opus/Sonnet to handle large agent prompts.
+ *
  * v2.4.0:
  *   - Per-model circuit breaker: consecutive timeouts temporarily mark a model as degraded
  *   - Adaptive first-byte timeout: scales by model tier + prompt size
@@ -14,16 +23,18 @@
  * Env vars:
  *   CLAUDE_PROXY_PORT            — listen port (default: 3456)
  *   CLAUDE_BIN                   — path to claude binary (default: auto-detect)
- *   CLAUDE_TIMEOUT           — per-request timeout in ms (default: 120000)
+ *   CLAUDE_TIMEOUT               — per-request timeout in ms (default: 300000)
- *   CLAUDE_FIRST_BYTE_TIMEOUT — base first-byte timeout in ms (default: 45000)
+ *   CLAUDE_FIRST_BYTE_TIMEOUT    — base first-byte timeout in ms (default: 90000)
  *   CLAUDE_ALLOWED_TOOLS         — comma-separated tools to allow (default: expanded set)
  *   CLAUDE_SKIP_PERMISSIONS      — "true" to bypass all permission checks (default: false)
  *   CLAUDE_SYSTEM_PROMPT         — system prompt appended to all requests
  *   CLAUDE_MCP_CONFIG            — path to MCP server config JSON file
  *   CLAUDE_SESSION_TTL           — session TTL in ms (default: 3600000 = 1h)
- *   CLAUDE_MAX_CONCURRENT    — max concurrent claude processes (default: 5)
+ *   CLAUDE_MAX_CONCURRENT        — max concurrent claude processes (default: 8)
- *   CLAUDE_BREAKER_THRESHOLD — consecutive timeouts before circuit opens (default: 3)
+ *   CLAUDE_BREAKER_THRESHOLD     — failures in window before circuit opens (default: 6)
- *   CLAUDE_BREAKER_COOLDOWN  — ms to wait before retrying after circuit opens (default: 60000)
+ *   CLAUDE_BREAKER_COOLDOWN      — base ms to wait before retrying after circuit opens (default: 120000)
+ *   CLAUDE_BREAKER_WINDOW        — sliding window duration in ms (default: 300000 = 5min)
+ *   CLAUDE_BREAKER_HALF_OPEN_MAX — max concurrent probes in half-open state (default: 2)
  *   PROXY_API_KEY                — Bearer token for API auth (optional)
  */
 import { createServer } from "node:http";
@@ -76,8 +87,8 @@ function resolveClaude() {
 // ── Configuration ───────────────────────────────────────────────────────
 const PORT = parseInt(process.env.CLAUDE_PROXY_PORT || "3456", 10);
 const CLAUDE = resolveClaude();
-const TIMEOUT = parseInt(process.env.CLAUDE_TIMEOUT || "120000", 10);
+const TIMEOUT = parseInt(process.env.CLAUDE_TIMEOUT || "300000", 10);
-const BASE_FIRST_BYTE_TIMEOUT = parseInt(process.env.CLAUDE_FIRST_BYTE_TIMEOUT || "45000", 10);
+const BASE_FIRST_BYTE_TIMEOUT = parseInt(process.env.CLAUDE_FIRST_BYTE_TIMEOUT || "90000", 10);
 const PROXY_API_KEY = process.env.PROXY_API_KEY || "";
 const SKIP_PERMISSIONS = process.env.CLAUDE_SKIP_PERMISSIONS === "true";
 const ALLOWED_TOOLS = (process.env.CLAUDE_ALLOWED_TOOLS ||
@@ -86,9 +97,11 @@ const ALLOWED_TOOLS = (process.env.CLAUDE_ALLOWED_TOOLS ||
 const SYSTEM_PROMPT = process.env.CLAUDE_SYSTEM_PROMPT || "";
 const MCP_CONFIG = process.env.CLAUDE_MCP_CONFIG || "";
 const SESSION_TTL = parseInt(process.env.CLAUDE_SESSION_TTL || "3600000", 10);
-const MAX_CONCURRENT = parseInt(process.env.CLAUDE_MAX_CONCURRENT || "5", 10);
+const MAX_CONCURRENT = parseInt(process.env.CLAUDE_MAX_CONCURRENT || "8", 10);
-const BREAKER_THRESHOLD = parseInt(process.env.CLAUDE_BREAKER_THRESHOLD || "3", 10);
+const BREAKER_THRESHOLD = parseInt(process.env.CLAUDE_BREAKER_THRESHOLD || "6", 10);
-const BREAKER_COOLDOWN = parseInt(process.env.CLAUDE_BREAKER_COOLDOWN || "60000", 10);
+const BREAKER_COOLDOWN = parseInt(process.env.CLAUDE_BREAKER_COOLDOWN || "120000", 10);
+const BREAKER_WINDOW = parseInt(process.env.CLAUDE_BREAKER_WINDOW || "300000", 10);
+const BREAKER_HALF_OPEN_MAX = parseInt(process.env.CLAUDE_BREAKER_HALF_OPEN_MAX || "2", 10);
 
 const VERSION = _pkg.version;
 const START_TIME = Date.now();
@@ -103,47 +116,128 @@ function logEvent(level, event, data = {}) {
   }
 }
 
-// ── Per-model circuit breaker ───────────────────────────────────────────
+// ── Per-model sliding-window circuit breaker ─────────────────────────────
-// Tracks consecutive timeouts per model. When threshold is reached, the
+// Uses a time-windowed failure rate instead of consecutive-count. This prevents
-// model is marked "open" (degraded) for BREAKER_COOLDOWN ms. During that
+// multi-agent burst scenarios (e.g. ClawTeam spawning 5+ Opus agents) from
-// window, requests for this model fail fast with a clear error instead of
+// tripping the breaker after just 3 quick timeouts.
-// waiting for yet another timeout that would block the gateway.
+//
-const breakers = new Map(); // cliModel → { failures, state, openedAt }
+// States: closed → open → half-open → closed (on success) or open (on failure)
+// Half-open allows up to BREAKER_HALF_OPEN_MAX concurrent probes (not just 1).
+// Cooldown uses graduated backoff: doubles on each re-open, resets on success.
+const breakers = new Map(); // cliModel → BreakerState
+
+function newBreakerState() {
+  return {
+    state: "closed",        // closed | open | half-open
+    failureTimestamps: [],  // timestamps of failures within the sliding window
+    successCount: 0,        // successes within window (for rate calculation)
+    openedAt: 0,
+    currentCooldown: BREAKER_COOLDOWN, // graduates on repeated opens
+    reopenCount: 0,         // how many times breaker has re-opened without a full reset
+    halfOpenProbes: 0,      // active probe requests in half-open state
+  };
+}
+
+function pruneWindow(b) {
+  const cutoff = Date.now() - BREAKER_WINDOW;
+  b.failureTimestamps = b.failureTimestamps.filter(ts => ts > cutoff);
+}
 
 function getBreakerState(cliModel) {
   if (!breakers.has(cliModel)) {
-    breakers.set(cliModel, { failures: 0, state: "closed", openedAt: 0 });
+    breakers.set(cliModel, newBreakerState());
   }
   const b = breakers.get(cliModel);
 
   // Auto-recover: if cooldown has elapsed, transition to half-open
-  if (b.state === "open" && Date.now() - b.openedAt >= BREAKER_COOLDOWN) {
+  if (b.state === "open" && Date.now() - b.openedAt >= b.currentCooldown) {
     b.state = "half-open";
-    logEvent("info", "breaker_half_open", { model: cliModel, cooldownMs: BREAKER_COOLDOWN });
+    b.halfOpenProbes = 0;
+    logEvent("info", "breaker_half_open", { model: cliModel, cooldownMs: b.currentCooldown, reopenCount: b.reopenCount });
   }
   return b;
 }
 
 function breakerRecordSuccess(cliModel) {
   const b = getBreakerState(cliModel);
-  if (b.failures > 0 || b.state !== "closed") {
+  b.successCount++;
-    logEvent("info", "breaker_reset", { model: cliModel, previousFailures: b.failures, previousState: b.state });
+
+  if (b.state === "half-open") {
+    b.halfOpenProbes = Math.max(0, b.halfOpenProbes - 1);
   }
-  b.failures = 0;
+
+  if (b.state !== "closed") {
+    logEvent("info", "breaker_reset", {
+      model: cliModel,
+      previousFailures: b.failureTimestamps.length,
+      previousState: b.state,
+      reopenCount: b.reopenCount,
+    });
+    // Full reset on success — graduated backoff resets too
     b.state = "closed";
     b.openedAt = 0;
+    b.currentCooldown = BREAKER_COOLDOWN;
+    b.reopenCount = 0;
+    b.halfOpenProbes = 0;
+    b.failureTimestamps = [];
+    b.successCount = 0;
+  }
 }
 
 function breakerRecordTimeout(cliModel) {
   const b = getBreakerState(cliModel);
-  b.failures++;
+  const now = Date.now();
-  logEvent("warn", "breaker_failure", { model: cliModel, consecutiveFailures: b.failures, threshold: BREAKER_THRESHOLD });
+  b.failureTimestamps.push(now);
+  pruneWindow(b);
 
-  if (b.failures >= BREAKER_THRESHOLD && b.state !== "open") {
+  if (b.state === "half-open") {
-    b.state = "open";
+    b.halfOpenProbes = Math.max(0, b.halfOpenProbes - 1);
-    b.openedAt = Date.now();
-    logEvent("error", "breaker_open", { model: cliModel, failures: b.failures, cooldownMs: BREAKER_COOLDOWN });
   }
+
+  const windowFailures = b.failureTimestamps.length;
+  logEvent("warn", "breaker_failure", {
+    model: cliModel,
+    windowFailures,
+    threshold: BREAKER_THRESHOLD,
+    windowMs: BREAKER_WINDOW,
+    state: b.state,
+  });
+
+  if (windowFailures >= BREAKER_THRESHOLD && b.state !== "open") {
+    b.state = "open";
+    b.openedAt = now;
+    b.halfOpenProbes = 0;
+    // Graduated backoff: double cooldown on each re-open, cap at 5 min
+    if (b.reopenCount > 0) {
+      b.currentCooldown = Math.min(b.currentCooldown * 2, 300000);
+    }
+    b.reopenCount++;
+    logEvent("error", "breaker_open", {
+      model: cliModel,
+      windowFailures,
+      cooldownMs: b.currentCooldown,
+      reopenCount: b.reopenCount,
+    });
+  }
+}
+
+// Expose breaker snapshot for /health endpoint
+function getBreakerSnapshot() {
+  const snapshot = {};
+  for (const [model, b] of breakers) {
+    pruneWindow(b);
+    snapshot[model] = {
+      state: b.state,
+      windowFailures: b.failureTimestamps.length,
+      threshold: BREAKER_THRESHOLD,
+      windowMs: BREAKER_WINDOW,
+      currentCooldown: b.currentCooldown,
+      reopenCount: b.reopenCount,
+      halfOpenProbes: b.halfOpenProbes,
+      ...(b.openedAt ? { openedAt: new Date(b.openedAt).toISOString() } : {}),
+    };
+  }
+  return snapshot;
 }
 
 // ── Model mapping ───────────────────────────────────────────────────────
@@ -271,8 +365,8 @@ function messagesToPrompt(messages) {
 // Model tier multipliers for first-byte timeout.
 // Opus is much slower to produce first token, especially with large contexts.
 const MODEL_TIMEOUT_TIERS = {
-  "opus": { base: 60000, perPromptChar: 0.00015 },    // 60s base + ~15s per 100k chars
+  "opus": { base: 90000, perPromptChar: 0.00020 },    // 90s base + ~20s per 100k chars
-  "sonnet": { base: 45000, perPromptChar: 0.00008 },  // 45s base + ~8s per 100k chars
+  "sonnet": { base: 60000, perPromptChar: 0.00010 },  // 60s base + ~10s per 100k chars
   "haiku": { base: 30000, perPromptChar: 0.00005 },   // 30s base + ~5s per 100k chars
 };
 
@@ -301,9 +395,18 @@ function spawnClaudeProcess(model, messages, conversationId) {
   // Circuit breaker check: fail fast if model is in open state
   const breaker = getBreakerState(cliModel);
   if (breaker.state === "open") {
-    const remainingMs = BREAKER_COOLDOWN - (Date.now() - breaker.openedAt);
+    const remainingMs = breaker.currentCooldown - (Date.now() - breaker.openedAt);
-    logEvent("warn", "breaker_rejected", { model: cliModel, remainingCooldownMs: remainingMs });
+    logEvent("warn", "breaker_rejected", { model: cliModel, remainingCooldownMs: remainingMs, reopenCount: breaker.reopenCount });
-    throw new Error(`circuit breaker open for ${cliModel}: ${breaker.failures} consecutive timeouts, retry in ${Math.ceil(remainingMs / 1000)}s`);
+    throw new Error(`circuit breaker open for ${cliModel}: ${breaker.failureTimestamps.length} timeouts in window, retry in ${Math.ceil(remainingMs / 1000)}s`);
+  }
+  // Half-open: allow limited probe requests
+  if (breaker.state === "half-open" && breaker.halfOpenProbes >= BREAKER_HALF_OPEN_MAX) {
+    logEvent("warn", "breaker_half_open_full", { model: cliModel, activeProbes: breaker.halfOpenProbes, max: BREAKER_HALF_OPEN_MAX });
+    throw new Error(`circuit breaker half-open for ${cliModel}: ${breaker.halfOpenProbes}/${BREAKER_HALF_OPEN_MAX} probe slots in use, wait for probe result`);
+  }
+  if (breaker.state === "half-open") {
+    breaker.halfOpenProbes++;
+    logEvent("info", "breaker_probe", { model: cliModel, activeProbes: breaker.halfOpenProbes, max: BREAKER_HALF_OPEN_MAX });
   }
 
   stats.activeRequests++;
@@ -696,8 +799,11 @@ const server = createServer(async (req, res) => {
       });
     }
 
+    const breakerState = getBreakerSnapshot();
+    const anyBreakerOpen = Object.values(breakerState).some(b => b.state === "open");
+
     return jsonResponse(res, 200, {
-      status: binaryOk && authStatus.ok !== false ? "ok" : "degraded",
+      status: binaryOk && authStatus.ok !== false && !anyBreakerOpen ? "ok" : "degraded",
       version: VERSION,
       architecture: "on-demand (v2)",
       uptime: uptimeMs,
@@ -710,11 +816,16 @@ const server = createServer(async (req, res) => {
         firstByteTimeout: BASE_FIRST_BYTE_TIMEOUT,
         maxConcurrent: MAX_CONCURRENT,
         sessionTTL: SESSION_TTL,
+        breakerThreshold: BREAKER_THRESHOLD,
+        breakerCooldown: BREAKER_COOLDOWN,
+        breakerWindow: BREAKER_WINDOW,
+        breakerHalfOpenMax: BREAKER_HALF_OPEN_MAX,
         allowedTools: SKIP_PERMISSIONS ? "all (skip-permissions)" : ALLOWED_TOOLS,
         systemPrompt: SYSTEM_PROMPT ? `${SYSTEM_PROMPT.slice(0, 50)}...` : "(none)",
         mcpConfig: MCP_CONFIG || "(none)",
       },
       stats,
+      breakers: breakerState,
       sessions: sessionList,
       recentErrors: recentErrors.slice(-5),
     });
@@ -799,6 +910,7 @@ server.listen(PORT, "127.0.0.1", () => {
   console.log(`Models: ${MODELS.map((m) => m.id).join(", ")}`);
   console.log(`Claude binary: ${CLAUDE}`);
   console.log(`Timeout: ${TIMEOUT}ms (base first-byte: ${BASE_FIRST_BYTE_TIMEOUT}ms, adaptive by model/prompt) | Max concurrent: ${MAX_CONCURRENT}`);
+  console.log(`Circuit breaker: threshold=${BREAKER_THRESHOLD} in ${BREAKER_WINDOW/1000}s window, cooldown=${BREAKER_COOLDOWN/1000}s (graduated), half-open probes=${BREAKER_HALF_OPEN_MAX}`);
   console.log(`Tools: ${SKIP_PERMISSIONS ? "all (skip-permissions)" : ALLOWED_TOOLS.join(", ")}`);
   console.log(`Sessions: TTL=${SESSION_TTL / 1000}s`);
   if (SYSTEM_PROMPT) console.log(`System prompt: "${SYSTEM_PROMPT.slice(0, 80)}..."`);