mirror of
https://github.com/dtzp555-max/olp.git
synced 2026-07-21 21:15:10 +00:00
taodengandClaude Opus 4.7 (noreply@anthropic.com)
5960486542
Phase 1 Day 6 — the architectural milestone of Phase 1. OLP is no longer
a single-provider proxy with cache; it is now a multi-provider router
with idempotent-failure-safe fallback. ADR 0004 ratified in v0.1 governance
is fully implemented for D9 scope. Configuration-driven chains activate
when a user populates ~/.olp/config.json routing.chains; at v0.1 default
(empty chains), behaviour is single-hop pass-through identical to D5.
Files:
NEW: lib/fallback/engine.mjs (~470 lines) — Trigger taxonomy, chain
advancement, first-chunk safety, observability annotation.
MOD: server.mjs (+80 lines net) — wires executeWithFallback between IR
construction and provider.spawn. Per-hop cache key isolation
(ADR 0005 § Cross-provider fallback). Test seams added:
__setFallbackConfig, __resetFallbackConfig, __clearCache.
MOD: test-features.mjs (+845 lines, 6 new suites = +55 tests).
Authority citations:
ADR 0004 § Decision § Trigger taxonomy — Hard / Soft / Deterministic
(deferred) / Cost-aware (deferred) implemented exactly per spec.
ADR 0004 § Decision § Fallback safety — first-chunk rule satisfied at
D9 by buffering composition (executeHopFn = collectAllChunks; server
writes to res only after engine returns).
ADR 0004 § Decision § Chain advancement — one-at-a-time iteration;
originalError preserved from FIRST hop (not last) on exhaustion.
ADR 0004 § Decision § No fallback for client-side errors — HTTP 400/
401/403/404/422 stop the chain immediately; AUTH_MISSING also stops
immediately (user-config failure, not provider-quota).
ADR 0004 § Decision § Observability headers — X-OLP-Fallback-Hops set
from engine return value; X-OLP-Fallback-Exhausted lists tried
providers on exhaustion.
ADR 0005 § Cross-provider fallback cache behavior — each fallback hop
computes a fresh cache key with the hop's (provider, model) tuple;
primary's cache entries cannot leak to secondary's hop.
Reviewer chain (Iron Rule 10):
Implementer: sonnet (general-purpose).
Fresh-context reviewer: opus. Verdict APPROVE_WITH_MINOR.
Reviewer ran npm test (277/277 pass), read ADR 0004 + ADR 0005 end-
to-end, verified first-chunk-safety composition, checked all
trigger taxonomy mappings.
Reviewer non-blocking findings folded in this commit:
1. Test label at test-features.mjs:3756 clarified. Original title
"client error (400) → does NOT fall back" was misleading because
the test actually exercises both-fail SPAWN_FAILED → exhausted
path (400-no-fallback semantic is covered at unit level instead).
Renamed + commented to match actual behaviour.
2. triedProviders semantic (includes soft-skipped) documented in
engine.mjs comment near the soft-trigger branch.
3. SOFT_TRIGGER synthetic code documented in engine.mjs as engine-
internal, NOT a member of base.mjs PROVIDER_ERROR_CODES.
Reviewer non-blocking suggestions deferred (open D-later questions):
- Q1 cacheStatus on fallback hops: conservative 'miss', acceptable
per ADR 0005 § Per-model isolation.
- Q2 X-OLP-Fallback-Exhausted only when triedProviders > 1:
acceptable per ADR 0004 spec.
- Q3 soft-trigger quotaSnapshot always null at D9: acceptable per
ADR 0004 § Consequences/Negative graceful-degrade clause. Phase 2
will add quota poll-worker.
- Q4 loadFallbackConfigSync sync-only: acceptable for bounded
startup I/O.
Test count: 222 (D8) → 277 (D9). 6 new test suites cover trigger
taxonomy, engine chain advancement (including the load-bearing
originalError-from-FIRST-hop property), HTTP integration through
real server.mjs, soft trigger pre-fetch path, exhausted-chain
header emission.
Verification:
node --check on all touched files: clean.
npm test on Node 25.8.0: 277/277 pass in 266ms.
Hygiene grep: zero personal-name/path/token hits.
No new external npm deps.
Co-Authored-By: Claude Opus 4.7 (noreply@anthropic.com)
3847 lines
157 KiB
JavaScript
3847 lines
157 KiB
JavaScript
/**
|
|
* test-features.mjs — OLP D5 test suite (extends D4)
|
|
*
|
|
* Uses Node's built-in node:test runner. No external dependencies.
|
|
* Run: node test-features.mjs (or: npm test)
|
|
*
|
|
* Authority: ADR 0002 (provider contract), ADR 0003 (IR v1.0), ADR 0005 (cache layer)
|
|
* D4 adds: Anthropic plugin conformance, IR translation, mock-spawn behaviour.
|
|
* D5 adds: Suite 9 (cache layer unit + HTTP integration), Suite 10 (Anthropic E2E gated)
|
|
*/
|
|
|
|
import { describe, it, before, after } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { request as httpRequest } from 'node:http';
|
|
import { EventEmitter } from 'node:events';
|
|
import { homedir } from 'node:os';
|
|
import { computeCacheKey, extractCacheControlMarkers, hasCacheControl } from './lib/cache/keys.mjs';
|
|
import { CacheStore } from './lib/cache/store.mjs';
|
|
|
|
// ── Modules under test ────────────────────────────────────────────────────
|
|
|
|
import { validateIRRequest, validateIRMessage, VALID_ROLES, IR_VERSION } from './lib/ir/types.mjs';
|
|
import { openAIToIR, BadRequestError } from './lib/ir/openai-to-ir.mjs';
|
|
import {
|
|
irChunkToOpenAISSE,
|
|
irResponseToOpenAINonStream,
|
|
generateRequestId,
|
|
SSE_DONE,
|
|
} from './lib/ir/ir-to-openai.mjs';
|
|
import { validateProvider, ProviderError, withTimeout } from './lib/providers/base.mjs';
|
|
import { loadProviders, getProviderForModel, getProviderByName, listAllProviderNames } from './lib/providers/index.mjs';
|
|
import anthropic, {
|
|
irToAnthropic,
|
|
anthropicChunkToIR,
|
|
anthropicStopToIR,
|
|
readAuthArtifact,
|
|
estimateCost as anthropicEstimateCost,
|
|
quotaStatus as anthropicQuotaStatus,
|
|
healthCheck as anthropicHealthCheck,
|
|
__setSpawnImpl,
|
|
__resetSpawnImpl,
|
|
} from './lib/providers/anthropic.mjs';
|
|
import codex, {
|
|
irToCodex,
|
|
codexChunkToIR,
|
|
readAuthArtifact as codexReadAuthArtifact,
|
|
estimateCost as codexEstimateCost,
|
|
quotaStatus as codexQuotaStatus,
|
|
healthCheck as codexHealthCheck,
|
|
__setSpawnImpl as codexSetSpawnImpl,
|
|
__resetSpawnImpl as codexResetSpawnImpl,
|
|
} from './lib/providers/codex.mjs';
|
|
import mistral, {
|
|
irToMistral,
|
|
mistralChunkToIR,
|
|
readAuthArtifact as mistralReadAuthArtifact,
|
|
estimateCost as mistralEstimateCost,
|
|
quotaStatus as mistralQuotaStatus,
|
|
healthCheck as mistralHealthCheck,
|
|
__setSpawnImpl as mistralSetSpawnImpl,
|
|
__resetSpawnImpl as mistralResetSpawnImpl,
|
|
} from './lib/providers/mistral.mjs';
|
|
import modelsRegistry from './models-registry.json' with { type: 'json' };
|
|
|
|
// ── Helpers ───────────────────────────────────────────────────────────────
|
|
|
|
/** Minimal valid IR request for use in tests */
|
|
function makeIR(overrides = {}) {
|
|
return {
|
|
irVersion: IR_VERSION,
|
|
model: 'test-model',
|
|
stream: false,
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
/** Minimal valid provider stub that satisfies the v1.0 contract (including D4 contractVersion) */
|
|
function makeProvider(overrides = {}) {
|
|
return {
|
|
name: 'stub',
|
|
displayName: 'Stub Provider',
|
|
contractVersion: '1.0',
|
|
models: ['stub-model-v1'],
|
|
auth: { type: 'none', storage: 'none', path: '', refresh: null },
|
|
spawn: async function* () { yield { type: 'stop', finish_reason: 'stop' }; },
|
|
estimateCost: () => null,
|
|
quotaStatus: async () => null,
|
|
healthCheck: async () => ({ ok: true, latencyMs: 0 }),
|
|
hints: { requiresTTY: false, concurrentSpawnSafe: true, maxConcurrent: 4 },
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
// ── HTTP helper ───────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Makes an HTTP request to the test server.
|
|
* @param {{ port, method, path, headers?, body? }} opts
|
|
* @returns {Promise<{ status: number, headers: object, body: string }>}
|
|
*/
|
|
function fetch(opts) {
|
|
return new Promise((resolve, reject) => {
|
|
const bodyStr = opts.body ? JSON.stringify(opts.body) : undefined;
|
|
const req = httpRequest({
|
|
hostname: '127.0.0.1',
|
|
port: opts.port,
|
|
method: opts.method ?? 'GET',
|
|
path: opts.path,
|
|
headers: {
|
|
...(bodyStr && { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(bodyStr) }),
|
|
...(opts.headers ?? {}),
|
|
},
|
|
}, res => {
|
|
let data = '';
|
|
res.on('data', c => { data += c; });
|
|
res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: data }));
|
|
});
|
|
req.on('error', reject);
|
|
if (bodyStr) req.write(bodyStr);
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
// ── Suite 1: IR validation ────────────────────────────────────────────────
|
|
|
|
describe('IR validation — validateIRRequest', () => {
|
|
it('accepts a minimal valid IR request', () => {
|
|
const r = validateIRRequest(makeIR());
|
|
assert.equal(r.valid, true);
|
|
assert.deepEqual(r.errors, []);
|
|
});
|
|
|
|
it('accepts an IR request with all optional fields', () => {
|
|
const r = validateIRRequest(makeIR({
|
|
max_tokens: 256,
|
|
temperature: 0.7,
|
|
top_p: 0.9,
|
|
stop: ['\n'],
|
|
tools: [],
|
|
tool_choice: 'auto',
|
|
response_format: { type: 'text' },
|
|
}));
|
|
assert.equal(r.valid, true);
|
|
});
|
|
|
|
it('rejects when messages is missing', () => {
|
|
const ir = makeIR();
|
|
delete ir.messages;
|
|
const r = validateIRRequest(ir);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('messages')));
|
|
});
|
|
|
|
it('rejects when messages is empty', () => {
|
|
const r = validateIRRequest(makeIR({ messages: [] }));
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('messages must not be empty')));
|
|
});
|
|
|
|
it('rejects when model is missing', () => {
|
|
const ir = makeIR();
|
|
delete ir.model;
|
|
const r = validateIRRequest(ir);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('model')));
|
|
});
|
|
|
|
it('rejects when stream is not boolean', () => {
|
|
const r = validateIRRequest(makeIR({ stream: 'true' }));
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('stream')));
|
|
});
|
|
|
|
it('rejects temperature out of range', () => {
|
|
const r = validateIRRequest(makeIR({ temperature: 3 }));
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('temperature')));
|
|
});
|
|
|
|
it('rejects top_p out of range', () => {
|
|
const r = validateIRRequest(makeIR({ top_p: -0.1 }));
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('top_p')));
|
|
});
|
|
|
|
it('rejects non-object input', () => {
|
|
const r = validateIRRequest(null);
|
|
assert.equal(r.valid, false);
|
|
});
|
|
});
|
|
|
|
describe('IR validation — validateIRMessage', () => {
|
|
for (const role of VALID_ROLES) {
|
|
it(`accepts role="${role}"`, () => {
|
|
const r = validateIRMessage({ role, content: 'test' });
|
|
assert.equal(r.valid, true);
|
|
});
|
|
}
|
|
|
|
it('rejects invalid role', () => {
|
|
const r = validateIRMessage({ role: 'admin', content: 'x' });
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('role')));
|
|
});
|
|
|
|
it('rejects missing content', () => {
|
|
const r = validateIRMessage({ role: 'user' });
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('content')));
|
|
});
|
|
|
|
it('accepts array content (multi-part)', () => {
|
|
const r = validateIRMessage({ role: 'user', content: [{ type: 'text', text: 'hi' }] });
|
|
assert.equal(r.valid, true);
|
|
});
|
|
|
|
it('rejects non-object input', () => {
|
|
const r = validateIRMessage('not an object');
|
|
assert.equal(r.valid, false);
|
|
});
|
|
});
|
|
|
|
// ── Suite 2: openAIToIR translation ──────────────────────────────────────
|
|
|
|
describe('openAIToIR translation', () => {
|
|
it('translates a minimal request', () => {
|
|
const ir = openAIToIR({ model: 'gpt-4o', messages: [{ role: 'user', content: 'Hi' }] });
|
|
assert.equal(ir.irVersion, IR_VERSION);
|
|
assert.equal(ir.model, 'gpt-4o');
|
|
assert.equal(ir.stream, false);
|
|
assert.equal(ir.messages.length, 1);
|
|
assert.equal(ir.messages[0].role, 'user');
|
|
assert.equal(ir.messages[0].content, 'Hi');
|
|
});
|
|
|
|
it('defaults stream to false when absent', () => {
|
|
const ir = openAIToIR({ model: 'm', messages: [{ role: 'user', content: 'x' }] });
|
|
assert.equal(ir.stream, false);
|
|
});
|
|
|
|
it('passes stream=true through', () => {
|
|
const ir = openAIToIR({ model: 'm', messages: [{ role: 'user', content: 'x' }], stream: true });
|
|
assert.equal(ir.stream, true);
|
|
});
|
|
|
|
it('translates multi-turn with system message', () => {
|
|
const ir = openAIToIR({
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [
|
|
{ role: 'system', content: 'You are helpful.' },
|
|
{ role: 'user', content: 'What is 2+2?' },
|
|
{ role: 'assistant', content: '4' },
|
|
{ role: 'user', content: 'Thanks!' },
|
|
],
|
|
});
|
|
assert.equal(ir.messages.length, 4);
|
|
assert.equal(ir.messages[0].role, 'system');
|
|
assert.equal(ir.messages[1].role, 'user');
|
|
assert.equal(ir.messages[2].role, 'assistant');
|
|
});
|
|
|
|
it('maps deprecated role=function to role=tool', () => {
|
|
const ir = openAIToIR({
|
|
model: 'm',
|
|
messages: [{ role: 'function', name: 'my_fn', content: '{"result":1}' }],
|
|
});
|
|
assert.equal(ir.messages[0].role, 'tool');
|
|
assert.equal(ir.messages[0].name, 'my_fn');
|
|
});
|
|
|
|
it('translates request with tools', () => {
|
|
const ir = openAIToIR({
|
|
model: 'm',
|
|
messages: [{ role: 'user', content: 'Search for X' }],
|
|
tools: [{
|
|
type: 'function',
|
|
function: { name: 'search', description: 'Web search', parameters: { type: 'object', properties: {} } },
|
|
}],
|
|
tool_choice: 'auto',
|
|
});
|
|
assert.equal(ir.tools.length, 1);
|
|
assert.equal(ir.tools[0].function.name, 'search');
|
|
assert.equal(ir.tool_choice, 'auto');
|
|
});
|
|
|
|
it('translates request with response_format', () => {
|
|
const ir = openAIToIR({
|
|
model: 'm',
|
|
messages: [{ role: 'user', content: 'Give JSON' }],
|
|
response_format: { type: 'json_object' },
|
|
});
|
|
assert.deepEqual(ir.response_format, { type: 'json_object' });
|
|
});
|
|
|
|
it('translates optional numeric fields', () => {
|
|
const ir = openAIToIR({
|
|
model: 'm',
|
|
messages: [{ role: 'user', content: 'x' }],
|
|
max_tokens: 100,
|
|
temperature: 0.5,
|
|
top_p: 0.95,
|
|
});
|
|
assert.equal(ir.max_tokens, 100);
|
|
assert.equal(ir.temperature, 0.5);
|
|
assert.equal(ir.top_p, 0.95);
|
|
});
|
|
|
|
it('throws BadRequestError when model is missing', () => {
|
|
assert.throws(
|
|
() => openAIToIR({ messages: [{ role: 'user', content: 'x' }] }),
|
|
BadRequestError,
|
|
);
|
|
});
|
|
|
|
it('throws BadRequestError when messages is empty', () => {
|
|
assert.throws(
|
|
() => openAIToIR({ model: 'm', messages: [] }),
|
|
BadRequestError,
|
|
);
|
|
});
|
|
|
|
it('throws BadRequestError when body is not an object', () => {
|
|
assert.throws(() => openAIToIR(null), BadRequestError);
|
|
});
|
|
});
|
|
|
|
// ── Suite 3: irChunkToOpenAISSE format ────────────────────────────────────
|
|
|
|
describe('irChunkToOpenAISSE format', () => {
|
|
const ID = 'chatcmpl-test123';
|
|
const MODEL = 'test-model';
|
|
|
|
it('generates request IDs with chatcmpl- prefix', () => {
|
|
const id = generateRequestId();
|
|
assert.ok(id.startsWith('chatcmpl-'));
|
|
assert.ok(id.length > 12);
|
|
});
|
|
|
|
it('formats a delta chunk as SSE event', () => {
|
|
const sse = irChunkToOpenAISSE({ type: 'delta', role: 'assistant', content: 'Hello' }, ID, MODEL);
|
|
assert.ok(sse.startsWith('data: '));
|
|
assert.ok(sse.endsWith('\n\n'));
|
|
const payload = JSON.parse(sse.slice(6).trim());
|
|
assert.equal(payload.object, 'chat.completion.chunk');
|
|
assert.equal(payload.id, ID);
|
|
assert.equal(payload.model, MODEL);
|
|
assert.equal(payload.choices[0].delta.content, 'Hello');
|
|
assert.equal(payload.choices[0].delta.role, 'assistant');
|
|
assert.equal(payload.choices[0].finish_reason, null);
|
|
});
|
|
|
|
it('formats a stop chunk with finish_reason', () => {
|
|
const sse = irChunkToOpenAISSE({ type: 'stop', finish_reason: 'stop' }, ID, MODEL);
|
|
const payload = JSON.parse(sse.slice(6).trim());
|
|
assert.equal(payload.choices[0].finish_reason, 'stop');
|
|
assert.deepEqual(payload.choices[0].delta, {});
|
|
});
|
|
|
|
it('formats an error chunk with finish_reason within the OpenAI enum', () => {
|
|
// ALIGNMENT.md Rule 2 (b): finish_reason must stay within the OpenAI spec
|
|
// enum (stop|length|tool_calls|content_filter|function_call|null).
|
|
// Provider errors surface via the top-level `error` object, not via an
|
|
// invented finish_reason value.
|
|
const sse = irChunkToOpenAISSE({ type: 'error', error: 'spawn failed' }, ID, MODEL);
|
|
const payload = JSON.parse(sse.slice(6).trim());
|
|
assert.ok(payload.error);
|
|
assert.equal(payload.error.type, 'provider_error');
|
|
assert.ok(['stop', 'length', 'tool_calls', 'content_filter', 'function_call', null].includes(payload.choices[0].finish_reason));
|
|
assert.equal(payload.choices[0].finish_reason, 'stop');
|
|
});
|
|
|
|
it('SSE_DONE is the [DONE] terminator', () => {
|
|
assert.equal(SSE_DONE, 'data: [DONE]\n\n');
|
|
});
|
|
|
|
it('irResponseToOpenAINonStream assembles a complete response', () => {
|
|
const chunks = [
|
|
{ type: 'delta', role: 'assistant', content: 'Hello' },
|
|
{ type: 'delta', content: ' world' },
|
|
{ type: 'stop', finish_reason: 'stop', usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } },
|
|
];
|
|
const resp = irResponseToOpenAINonStream(chunks, ID, MODEL);
|
|
assert.equal(resp.object, 'chat.completion');
|
|
assert.equal(resp.id, ID);
|
|
assert.equal(resp.model, MODEL);
|
|
assert.equal(resp.choices[0].message.content, 'Hello world');
|
|
assert.equal(resp.choices[0].message.role, 'assistant');
|
|
assert.equal(resp.choices[0].finish_reason, 'stop');
|
|
assert.equal(resp.usage.total_tokens, 7);
|
|
});
|
|
});
|
|
|
|
// ── Suite 4: Provider contract validation ─────────────────────────────────
|
|
|
|
describe('Provider contract validation', () => {
|
|
it('accepts a fully valid provider stub', () => {
|
|
const r = validateProvider(makeProvider());
|
|
assert.equal(r.valid, true);
|
|
assert.deepEqual(r.errors, []);
|
|
});
|
|
|
|
it('rejects provider with missing name', () => {
|
|
const p = makeProvider();
|
|
delete p.name;
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('name')));
|
|
});
|
|
|
|
it('rejects provider with non-lowercase name', () => {
|
|
const r = validateProvider(makeProvider({ name: 'MyProvider' }));
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('name')));
|
|
});
|
|
|
|
it('rejects provider with missing displayName', () => {
|
|
const p = makeProvider();
|
|
delete p.displayName;
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('displayName')));
|
|
});
|
|
|
|
it('rejects provider with missing models array', () => {
|
|
const p = makeProvider();
|
|
delete p.models;
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('models')));
|
|
});
|
|
|
|
it('rejects provider with missing spawn', () => {
|
|
const p = makeProvider();
|
|
delete p.spawn;
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('spawn')));
|
|
});
|
|
|
|
it('rejects provider with missing estimateCost', () => {
|
|
const p = makeProvider();
|
|
delete p.estimateCost;
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('estimateCost')));
|
|
});
|
|
|
|
it('rejects provider with missing healthCheck', () => {
|
|
const p = makeProvider();
|
|
delete p.healthCheck;
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('healthCheck')));
|
|
});
|
|
|
|
it('rejects provider with missing hints', () => {
|
|
const p = makeProvider();
|
|
delete p.hints;
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('hints')));
|
|
});
|
|
|
|
it('rejects provider with invalid hints.maxConcurrent', () => {
|
|
const r = validateProvider(makeProvider({ hints: { requiresTTY: false, concurrentSpawnSafe: true, maxConcurrent: -1 } }));
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('maxConcurrent')));
|
|
});
|
|
|
|
it('rejects non-object input', () => {
|
|
const r = validateProvider(null);
|
|
assert.equal(r.valid, false);
|
|
});
|
|
|
|
it('ProviderError carries code field', () => {
|
|
const e = new ProviderError('auth missing', 'AUTH_MISSING');
|
|
assert.equal(e.code, 'AUTH_MISSING');
|
|
assert.equal(e.name, 'ProviderError');
|
|
assert.ok(e instanceof Error);
|
|
});
|
|
|
|
it('withTimeout rejects after deadline', async () => {
|
|
const p = new Promise(r => setTimeout(() => r('late'), 200));
|
|
await assert.rejects(
|
|
() => withTimeout(p, 50, 'SPAWN_FAILED'),
|
|
err => err instanceof ProviderError && err.code === 'SPAWN_FAILED',
|
|
);
|
|
});
|
|
|
|
it('withTimeout resolves when promise is fast', async () => {
|
|
const p = Promise.resolve(42);
|
|
const v = await withTimeout(p, 1000, 'SPAWN_FAILED');
|
|
assert.equal(v, 42);
|
|
});
|
|
});
|
|
|
|
// ── Suite 5: Plugin registry ──────────────────────────────────────────────
|
|
|
|
describe('Plugin registry', () => {
|
|
it('STATIC_REGISTRY has 3 entries (anthropic + openai + mistral candidates) at D8', () => {
|
|
// D4 added anthropic; D6 adds openai; D8 adds mistral. Default config has all enabled:false.
|
|
assert.equal(listAllProviderNames().length, 3);
|
|
});
|
|
|
|
it('loadProviders with empty config → empty Map (anthropic not enabled)', () => {
|
|
const m = loadProviders({});
|
|
assert.equal(m.size, 0);
|
|
});
|
|
|
|
it('loadProviders with no config → empty Map', () => {
|
|
const m = loadProviders();
|
|
assert.equal(m.size, 0);
|
|
});
|
|
|
|
it('listAllProviderNames returns [anthropic, openai, mistral] at D8', () => {
|
|
// D4: ['anthropic']. D6 adds openai. D8 adds mistral.
|
|
assert.deepEqual(listAllProviderNames(), ['anthropic', 'openai', 'mistral']);
|
|
});
|
|
|
|
it('getProviderForModel returns null when no providers loaded', () => {
|
|
const m = loadProviders({});
|
|
const r = getProviderForModel(m, 'gpt-4o');
|
|
assert.equal(r, null);
|
|
});
|
|
|
|
it('getProviderForModel finds provider by exact model string', () => {
|
|
// Build a synthetic loaded map to test the function without touching STATIC_REGISTRY
|
|
const p = makeProvider({ name: 'alpha', models: ['alpha-v1', 'alpha-v2'] });
|
|
const m = new Map([['alpha', p]]);
|
|
const r = getProviderForModel(m, 'alpha-v1');
|
|
assert.ok(r !== null);
|
|
assert.equal(r.name, 'alpha');
|
|
});
|
|
|
|
it('getProviderForModel returns null for unknown model', () => {
|
|
const p = makeProvider({ name: 'alpha', models: ['alpha-v1'] });
|
|
const m = new Map([['alpha', p]]);
|
|
assert.equal(getProviderForModel(m, 'beta-v1'), null);
|
|
});
|
|
|
|
it('getProviderByName returns null for empty loaded map', () => {
|
|
const m = new Map();
|
|
assert.equal(getProviderByName(m, 'anthropic'), null);
|
|
});
|
|
|
|
it('getProviderByName returns provider when found', () => {
|
|
const p = makeProvider({ name: 'alpha', models: ['alpha-v1'] });
|
|
const m = new Map([['alpha', p]]);
|
|
assert.ok(getProviderByName(m, 'alpha') !== null);
|
|
assert.equal(getProviderByName(m, 'alpha').name, 'alpha');
|
|
});
|
|
|
|
it('anthropic provider passes contract validation (STATIC_REGISTRY entry)', () => {
|
|
// Even though anthropic is Candidate (not enabled by default), it must
|
|
// pass contract validation at module load — loadProviders() validates all
|
|
// registry entries regardless of enabled flag.
|
|
const { valid, errors } = validateProvider(anthropic);
|
|
assert.equal(valid, true, `Validation errors: ${errors.join('; ')}`);
|
|
});
|
|
});
|
|
|
|
// ── Suite 6: Anthropic plugin (D4) ───────────────────────────────────────
|
|
//
|
|
// All tests in this suite are UNIT tests. No real `claude` binary is invoked.
|
|
// Mock spawn is injected via __setSpawnImpl / __resetSpawnImpl.
|
|
// Tests verify: contract conformance, contractVersion enforcement, registry
|
|
// consistency, IR translation, mock-spawn stream, healthCheck, estimateCost.
|
|
|
|
/**
|
|
* Creates a fake spawn that emits canned stdout chunks then exits cleanly.
|
|
* Returns a fake ChildProcess-like EventEmitter with stdin, stdout, stderr.
|
|
* @param {string[]} stdoutChunks — text chunks emitted in order
|
|
* @param {number} [exitCode=0]
|
|
*/
|
|
function makeMockSpawn(stdoutChunks, exitCode = 0) {
|
|
return function mockSpawnImpl(_bin, _args, _opts) {
|
|
const proc = new EventEmitter();
|
|
proc.stdout = new EventEmitter();
|
|
proc.stderr = new EventEmitter();
|
|
proc.stdin = {
|
|
write: () => {},
|
|
end: () => {
|
|
// Emit stdout chunks and close asynchronously
|
|
setImmediate(async () => {
|
|
for (const chunk of stdoutChunks) {
|
|
proc.stdout.emit('data', Buffer.from(chunk));
|
|
}
|
|
proc.stdout.emit('end');
|
|
proc.stderr.emit('end');
|
|
proc.emit('close', exitCode, null);
|
|
});
|
|
},
|
|
};
|
|
proc.killed = false;
|
|
proc.kill = () => {};
|
|
return proc;
|
|
};
|
|
}
|
|
|
|
describe('Anthropic plugin (D4)', () => {
|
|
|
|
// ── Test 1: Contract conformance ──────────────────────────────────────
|
|
it('anthropic module satisfies validateProvider() — all 10 fields present', () => {
|
|
const { valid, errors } = validateProvider(anthropic);
|
|
assert.equal(valid, true, `Validation errors: ${errors.join('; ')}`);
|
|
// Verify all 10 contract fields explicitly
|
|
assert.ok('name' in anthropic, 'missing: name');
|
|
assert.ok('displayName' in anthropic, 'missing: displayName');
|
|
assert.ok('contractVersion' in anthropic, 'missing: contractVersion');
|
|
assert.ok('models' in anthropic, 'missing: models');
|
|
assert.ok('auth' in anthropic, 'missing: auth');
|
|
assert.ok(typeof anthropic.spawn === 'function', 'missing: spawn');
|
|
assert.ok(typeof anthropic.estimateCost === 'function', 'missing: estimateCost');
|
|
assert.ok(typeof anthropic.quotaStatus === 'function', 'missing: quotaStatus');
|
|
assert.ok(typeof anthropic.healthCheck === 'function', 'missing: healthCheck');
|
|
assert.ok('hints' in anthropic, 'missing: hints');
|
|
});
|
|
|
|
// ── Test 2: contractVersion enforced ─────────────────────────────────
|
|
it('validateProvider rejects provider missing contractVersion', () => {
|
|
const p = makeProvider();
|
|
delete p.contractVersion;
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('contractVersion')));
|
|
});
|
|
|
|
it('validateProvider rejects contractVersion: "0.9"', () => {
|
|
const p = makeProvider({ contractVersion: '0.9' });
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('contractVersion')));
|
|
});
|
|
|
|
it('validateProvider accepts contractVersion: "1.0"', () => {
|
|
const p = makeProvider({ contractVersion: '1.0' });
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, true);
|
|
});
|
|
|
|
it('validateProvider rejects contractVersion: undefined', () => {
|
|
const p = makeProvider({ contractVersion: undefined });
|
|
const r = validateProvider(p);
|
|
assert.equal(r.valid, false);
|
|
assert.ok(r.errors.some(e => e.includes('contractVersion')));
|
|
});
|
|
|
|
// ── Test 3: models match registry ────────────────────────────────────
|
|
it('anthropic.models matches models-registry.json providers.anthropic.models', () => {
|
|
const registryIds = modelsRegistry.providers.anthropic.models.map(m => m.id);
|
|
assert.deepEqual(anthropic.models, registryIds);
|
|
});
|
|
|
|
it('anthropic.models contains the three expected model IDs', () => {
|
|
assert.ok(anthropic.models.includes('claude-opus-4-7'));
|
|
assert.ok(anthropic.models.includes('claude-sonnet-4-6'));
|
|
assert.ok(anthropic.models.includes('claude-haiku-4-5'));
|
|
assert.equal(anthropic.models.length, 3);
|
|
});
|
|
|
|
// ── Test 4: getProviderForModel finds anthropic for each model ────────
|
|
it('getProviderForModel finds anthropic for claude-sonnet-4-6 when enabled', () => {
|
|
const loaded = new Map([['anthropic', anthropic]]);
|
|
const result = getProviderForModel(loaded, 'claude-sonnet-4-6');
|
|
assert.ok(result !== null);
|
|
assert.equal(result.name, 'anthropic');
|
|
});
|
|
|
|
it('getProviderForModel finds anthropic for claude-opus-4-7 when enabled', () => {
|
|
const loaded = new Map([['anthropic', anthropic]]);
|
|
const result = getProviderForModel(loaded, 'claude-opus-4-7');
|
|
assert.ok(result !== null);
|
|
assert.equal(result.name, 'anthropic');
|
|
});
|
|
|
|
it('getProviderForModel finds anthropic for claude-haiku-4-5 when enabled', () => {
|
|
const loaded = new Map([['anthropic', anthropic]]);
|
|
const result = getProviderForModel(loaded, 'claude-haiku-4-5');
|
|
assert.ok(result !== null);
|
|
assert.equal(result.name, 'anthropic');
|
|
});
|
|
|
|
// ── Test 5: irToAnthropic translation ────────────────────────────────
|
|
it('irToAnthropic: user message → plain text', () => {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [{ role: 'user', content: 'Hello world' }],
|
|
});
|
|
const prompt = irToAnthropic(ir);
|
|
assert.ok(typeof prompt === 'string');
|
|
assert.ok(prompt.includes('Hello world'));
|
|
assert.ok(!prompt.includes('[System]'));
|
|
assert.ok(!prompt.includes('[Assistant]'));
|
|
});
|
|
|
|
it('irToAnthropic: system + user → system annotation + user text', () => {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [
|
|
{ role: 'system', content: 'You are a helper.' },
|
|
{ role: 'user', content: 'What is 2+2?' },
|
|
],
|
|
});
|
|
const prompt = irToAnthropic(ir);
|
|
assert.ok(prompt.includes('[System] You are a helper.'));
|
|
assert.ok(prompt.includes('What is 2+2?'));
|
|
});
|
|
|
|
it('irToAnthropic: assistant turn → [Assistant] annotation', () => {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [
|
|
{ role: 'user', content: 'Hi' },
|
|
{ role: 'assistant', content: 'Hello!' },
|
|
{ role: 'user', content: 'Bye' },
|
|
],
|
|
});
|
|
const prompt = irToAnthropic(ir);
|
|
assert.ok(prompt.includes('[Assistant] Hello!'));
|
|
});
|
|
|
|
it('irToAnthropic: response_format json_object injects system prompt', () => {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [{ role: 'user', content: 'Give JSON' }],
|
|
response_format: { type: 'json_object' },
|
|
});
|
|
const prompt = irToAnthropic(ir);
|
|
assert.ok(prompt.includes('Reply with valid JSON only'));
|
|
});
|
|
|
|
it('irToAnthropic: tool result turn → [Tool Result] annotation', () => {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [
|
|
{ role: 'user', content: 'Search for something' },
|
|
{ role: 'tool', content: '{"results": []}', name: 'search' },
|
|
],
|
|
});
|
|
const prompt = irToAnthropic(ir);
|
|
assert.ok(prompt.includes('[Tool Result'));
|
|
assert.ok(prompt.includes('search'));
|
|
});
|
|
|
|
it('irToAnthropic: array content is JSON-stringified', () => {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }],
|
|
});
|
|
const prompt = irToAnthropic(ir);
|
|
assert.ok(typeof prompt === 'string');
|
|
assert.ok(prompt.includes('text'));
|
|
});
|
|
|
|
// ── Test 6: anthropicChunkToIR and anthropicStopToIR ─────────────────
|
|
it('anthropicChunkToIR: produces delta chunk with content', () => {
|
|
const chunk = anthropicChunkToIR('Hello ', false);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.content, 'Hello ');
|
|
assert.ok(!('role' in chunk));
|
|
});
|
|
|
|
it('anthropicChunkToIR: first chunk includes role=assistant', () => {
|
|
const chunk = anthropicChunkToIR('Hello', true);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.role, 'assistant');
|
|
assert.equal(chunk.content, 'Hello');
|
|
});
|
|
|
|
it('anthropicStopToIR: produces stop chunk with finish_reason', () => {
|
|
const chunk = anthropicStopToIR('stop');
|
|
assert.equal(chunk.type, 'stop');
|
|
assert.equal(chunk.finish_reason, 'stop');
|
|
});
|
|
|
|
// ── Test 7: mock spawn — AsyncIterator yields correct IR chunks ───────
|
|
it('spawn with mock: yields delta chunks then stop chunk', async () => {
|
|
const fakeSpawn = makeMockSpawn(['Hello', ' world']);
|
|
__setSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-oauth-token>' };
|
|
const chunks = [];
|
|
for await (const chunk of anthropic.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
// Should have 2 delta chunks + 1 stop chunk
|
|
const deltas = chunks.filter(c => c.type === 'delta');
|
|
const stops = chunks.filter(c => c.type === 'stop');
|
|
assert.ok(deltas.length >= 1, `Expected at least 1 delta, got ${deltas.length}`);
|
|
assert.equal(stops.length, 1, `Expected 1 stop, got ${stops.length}`);
|
|
const allContent = deltas.map(c => c.content).join('');
|
|
assert.equal(allContent, 'Hello world');
|
|
} finally {
|
|
__resetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: first delta chunk has role=assistant', async () => {
|
|
const fakeSpawn = makeMockSpawn(['Test output']);
|
|
__setSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-oauth-token>' };
|
|
const chunks = [];
|
|
for await (const chunk of anthropic.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const firstDelta = chunks.find(c => c.type === 'delta');
|
|
assert.ok(firstDelta, 'No delta chunk found');
|
|
assert.equal(firstDelta.role, 'assistant');
|
|
} finally {
|
|
__resetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: non-zero exit code throws ProviderError', async () => {
|
|
const fakeSpawn = makeMockSpawn([], 1);
|
|
__setSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-oauth-token>' };
|
|
let caught = null;
|
|
try {
|
|
// eslint-disable-next-line no-unused-vars
|
|
for await (const _chunk of anthropic.spawn(ir, authCtx)) {
|
|
// drain
|
|
}
|
|
} catch (e) {
|
|
caught = e;
|
|
}
|
|
assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`);
|
|
assert.equal(caught.code, 'SPAWN_FAILED');
|
|
} finally {
|
|
__resetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn throws ProviderError(AUTH_MISSING) when no auth context and no env/file', async () => {
|
|
const fakeSpawn = makeMockSpawn(['output']);
|
|
__setSpawnImpl(fakeSpawn);
|
|
// Temporarily clear the env var if set
|
|
const savedToken = process.env.CLAUDE_CODE_OAUTH_TOKEN;
|
|
delete process.env.CLAUDE_CODE_OAUTH_TOKEN;
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
stream: false,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
let caught = null;
|
|
try {
|
|
// Pass null as authContext to force re-read
|
|
for await (const _chunk of anthropic.spawn(ir, null)) { // eslint-disable-line no-unused-vars
|
|
// may or may not throw depending on whether credentials exist on this machine
|
|
}
|
|
} catch (e) {
|
|
caught = e;
|
|
}
|
|
// If credentials.json or keychain exists on the test machine, this won't throw.
|
|
// We only assert that IF it throws, it's AUTH_MISSING.
|
|
if (caught !== null) {
|
|
assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`);
|
|
assert.equal(caught.code, 'AUTH_MISSING');
|
|
}
|
|
} finally {
|
|
if (savedToken !== undefined) process.env.CLAUDE_CODE_OAUTH_TOKEN = savedToken;
|
|
__resetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
// ── Test 8: healthCheck — binary not found ────────────────────────────
|
|
it('healthCheck returns {ok: false, error: "claude binary not found"} when binary absent', async () => {
|
|
const result = await anthropicHealthCheck({
|
|
_binaryExistsFn: () => false,
|
|
_authReadFn: () => ({ accessToken: '<fake-token>' }),
|
|
});
|
|
assert.equal(result.ok, false);
|
|
assert.equal(result.error, 'claude binary not found');
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
// ── Test 9: healthCheck — auth artifact missing ───────────────────────
|
|
it('healthCheck returns {ok: false, error: "auth artifact missing"} when auth missing', async () => {
|
|
const result = await anthropicHealthCheck({
|
|
_binaryExistsFn: () => true,
|
|
_authReadFn: () => null,
|
|
});
|
|
assert.equal(result.ok, false);
|
|
assert.equal(result.error, 'auth artifact missing');
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
it('healthCheck returns {ok: true} when binary and auth present', async () => {
|
|
const result = await anthropicHealthCheck({
|
|
_binaryExistsFn: () => true,
|
|
_authReadFn: () => ({ accessToken: '<fake-token>' }),
|
|
});
|
|
assert.equal(result.ok, true);
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
// ── Test 10: estimateCost shape ───────────────────────────────────────
|
|
it('estimateCost returns object with four fields for a valid request', () => {
|
|
const request = makeIR({
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [
|
|
{ role: 'system', content: 'You are a helper.' },
|
|
{ role: 'user', content: 'Count to ten.' },
|
|
],
|
|
});
|
|
const result = anthropicEstimateCost(request);
|
|
assert.ok(result !== null, 'estimateCost returned null');
|
|
assert.ok('inputTokens' in result, 'missing inputTokens');
|
|
assert.ok('outputTokensEstimate' in result, 'missing outputTokensEstimate');
|
|
assert.ok('currency' in result, 'missing currency');
|
|
assert.ok('usd' in result, 'missing usd');
|
|
assert.equal(result.currency, 'USD');
|
|
assert.equal(result.usd, null); // not pinned at D4
|
|
assert.ok(result.inputTokens > 0, 'inputTokens should be > 0');
|
|
assert.ok(result.outputTokensEstimate >= 0, 'outputTokensEstimate should be >= 0');
|
|
});
|
|
|
|
it('estimateCost returns null for null/missing request', () => {
|
|
assert.equal(anthropicEstimateCost(null), null);
|
|
assert.equal(anthropicEstimateCost({}), null);
|
|
});
|
|
|
|
// ── Test 11: quotaStatus ──────────────────────────────────────────────
|
|
it('quotaStatus returns null at D4', async () => {
|
|
const result = await anthropicQuotaStatus({});
|
|
assert.equal(result, null);
|
|
});
|
|
|
|
// ── Test 12: auth object shape ────────────────────────────────────────
|
|
it('anthropic.auth has correct shape', () => {
|
|
assert.equal(typeof anthropic.auth.type, 'string');
|
|
assert.equal(typeof anthropic.auth.storage, 'string');
|
|
assert.equal(typeof anthropic.auth.path, 'string');
|
|
assert.ok(anthropic.auth.path.includes('.claude'), 'auth.path should reference .claude directory');
|
|
// Portability check: path must start with the runtime homedir() value (set at module load)
|
|
// rather than a hardcoded literal. Since path.join(homedir(), ...) produces the home dir
|
|
// as a prefix, we verify it matches what homedir() returns at test time.
|
|
assert.ok(
|
|
anthropic.auth.path.startsWith(homedir()),
|
|
`auth.path "${anthropic.auth.path}" should start with homedir() "${homedir()}"`,
|
|
);
|
|
assert.ok(typeof anthropic.auth.refresh === 'string' || anthropic.auth.refresh === null);
|
|
});
|
|
|
|
// ── Test 13: hints shape ──────────────────────────────────────────────
|
|
it('anthropic.hints has correct shape', () => {
|
|
assert.equal(typeof anthropic.hints.requiresTTY, 'boolean');
|
|
assert.equal(typeof anthropic.hints.concurrentSpawnSafe, 'boolean');
|
|
assert.ok(Number.isInteger(anthropic.hints.maxConcurrent) && anthropic.hints.maxConcurrent > 0);
|
|
assert.equal(anthropic.hints.requiresTTY, false);
|
|
});
|
|
|
|
// ── Test 14: loadProviders with anthropic enabled ─────────────────────
|
|
it('loadProviders with {enabled: {anthropic: true}} returns Map of size 1', () => {
|
|
const loaded = loadProviders({ enabled: { anthropic: true } });
|
|
assert.equal(loaded.size, 1);
|
|
assert.ok(loaded.has('anthropic'));
|
|
});
|
|
|
|
it('anthropic loaded via loadProviders passes contract and has correct models', () => {
|
|
const loaded = loadProviders({ enabled: { anthropic: true } });
|
|
const p = loaded.get('anthropic');
|
|
const { valid, errors } = validateProvider(p);
|
|
assert.equal(valid, true, `Contract errors: ${errors.join('; ')}`);
|
|
assert.ok(p.models.includes('claude-sonnet-4-6'));
|
|
});
|
|
|
|
});
|
|
|
|
// ── Suite 7: HTTP integration tests ──────────────────────────────────────
|
|
|
|
describe('HTTP integration', () => {
|
|
let serverInstance;
|
|
let port;
|
|
|
|
before(async () => {
|
|
// Use the REAL server module via its createOlpServer() factory. The
|
|
// main guard in server.mjs prevents auto-listen on import; we call
|
|
// .listen() ourselves on a test port. This means every HTTP test below
|
|
// exercises the real router code — there is no parallel implementation
|
|
// to drift.
|
|
const { createOlpServer } = await import('./server.mjs');
|
|
|
|
// Pick a port: env OLP_TEST_PORT or random high port
|
|
port = parseInt(process.env.OLP_TEST_PORT ?? String(13456 + Math.floor(Math.random() * 1000)), 10);
|
|
|
|
serverInstance = createOlpServer();
|
|
|
|
// Retry once on port-in-use
|
|
await new Promise((resolve, reject) => {
|
|
serverInstance.listen(port, '127.0.0.1', resolve);
|
|
serverInstance.once('error', async (e) => {
|
|
if (e.code === 'EADDRINUSE') {
|
|
port++;
|
|
serverInstance.listen(port, '127.0.0.1', resolve);
|
|
serverInstance.once('error', reject);
|
|
} else {
|
|
reject(e);
|
|
}
|
|
});
|
|
});
|
|
});
|
|
|
|
after(() => new Promise(r => serverInstance.close(r)));
|
|
|
|
it('GET /health returns 200 with expected shape', async () => {
|
|
const r = await fetch({ port, method: 'GET', path: '/health' });
|
|
assert.equal(r.status, 200);
|
|
const body = JSON.parse(r.body);
|
|
assert.equal(body.ok, true);
|
|
assert.ok(typeof body.version === 'string');
|
|
assert.ok(typeof body.providers.enabled === 'number');
|
|
assert.ok(typeof body.providers.available === 'number');
|
|
});
|
|
|
|
it('GET /v1/models returns 200 with empty data array', async () => {
|
|
const r = await fetch({ port, method: 'GET', path: '/v1/models' });
|
|
assert.equal(r.status, 200);
|
|
const body = JSON.parse(r.body);
|
|
assert.equal(body.object, 'list');
|
|
assert.deepEqual(body.data, []);
|
|
});
|
|
|
|
it('POST /v1/chat/completions with no providers → 503 with no_enabled_provider', async () => {
|
|
const r = await fetch({
|
|
port,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: { model: 'gpt-4o', messages: [{ role: 'user', content: 'Hi' }] },
|
|
});
|
|
assert.equal(r.status, 503);
|
|
const body = JSON.parse(r.body);
|
|
assert.equal(body.error.type, 'no_enabled_provider');
|
|
assert.ok(body.error.message.includes('gpt-4o'));
|
|
});
|
|
|
|
it('POST /v1/chat/completions with invalid JSON body → 400', async () => {
|
|
const req = httpRequest({
|
|
hostname: '127.0.0.1',
|
|
port,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
headers: { 'Content-Type': 'application/json', 'Content-Length': '5' },
|
|
});
|
|
const result = await new Promise((resolve, reject) => {
|
|
req.on('error', reject);
|
|
let body = '';
|
|
req.on('response', res => {
|
|
res.on('data', c => body += c);
|
|
res.on('end', () => resolve({ status: res.statusCode, body }));
|
|
});
|
|
req.write('{bad}');
|
|
req.end();
|
|
});
|
|
assert.equal(result.status, 400);
|
|
});
|
|
|
|
it('POST /v1/chat/completions with missing model → 400', async () => {
|
|
const r = await fetch({
|
|
port,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: { messages: [{ role: 'user', content: 'x' }] },
|
|
});
|
|
assert.equal(r.status, 400);
|
|
const body = JSON.parse(r.body);
|
|
assert.equal(body.error.type, 'invalid_request_error');
|
|
});
|
|
|
|
it('GET /unknown → 404', async () => {
|
|
const r = await fetch({ port, method: 'GET', path: '/unknown/route' });
|
|
assert.equal(r.status, 404);
|
|
const body = JSON.parse(r.body);
|
|
assert.equal(body.error.type, 'not_found');
|
|
});
|
|
|
|
it('POST /v1/chat/completions without Content-Type → 415', async () => {
|
|
// Our fetch helper sets Content-Type to application/json when body is truthy;
|
|
// send text/plain directly to verify the 415 path.
|
|
const result = await new Promise((resolve, reject) => {
|
|
const req = httpRequest({
|
|
hostname: '127.0.0.1',
|
|
port,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
headers: { 'Content-Type': 'text/plain', 'Content-Length': '2' },
|
|
}, res => {
|
|
let data = '';
|
|
res.on('data', c => data += c);
|
|
res.on('end', () => resolve({ status: res.statusCode, body: data }));
|
|
});
|
|
req.on('error', reject);
|
|
req.write('{}');
|
|
req.end();
|
|
});
|
|
assert.equal(result.status, 415);
|
|
});
|
|
});
|
|
|
|
// ── Suite 8: Suite 8 formerly counted as D3/D4 base; suites renumber here ──
|
|
// (No new Suite 8 — the numbering skips from 7 to 9 to match D5 spec.)
|
|
|
|
// ── Suite 9: Cache layer ──────────────────────────────────────────────────
|
|
//
|
|
// Unit tests + HTTP integration tests for ADR 0005 (D1 + D4).
|
|
// No real `claude` binary invoked. Mock spawn injected via __setSpawnImpl.
|
|
// Authority: ADR 0005 § Cache key composition, D1 per-key isolation, D4 singleflight.
|
|
|
|
describe('Cache layer — computeCacheKey (Suite 9)', () => {
|
|
|
|
// ── Test 1: Determinism ───────────────────────────────────────────────
|
|
it('computeCacheKey is deterministic: same inputs → same key', () => {
|
|
const ir = makeIR({ model: 'claude-haiku-4-5', messages: [{ role: 'user', content: 'hello' }] });
|
|
const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir);
|
|
const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir);
|
|
assert.equal(k1, k2);
|
|
assert.equal(typeof k1, 'string');
|
|
assert.equal(k1.length, 64); // SHA-256 hex
|
|
});
|
|
|
|
// ── Test 2: Provider distinguishes ───────────────────────────────────
|
|
it('computeCacheKey distinguishes different providers', () => {
|
|
const ir = makeIR({ model: 'model-x', messages: [{ role: 'user', content: 'hi' }] });
|
|
const k1 = computeCacheKey('anthropic', 'model-x', ir);
|
|
const k2 = computeCacheKey('openai', 'model-x', ir);
|
|
assert.notEqual(k1, k2);
|
|
});
|
|
|
|
// ── Test 3: Model distinguishes ───────────────────────────────────────
|
|
it('computeCacheKey distinguishes different models', () => {
|
|
const ir = makeIR({ messages: [{ role: 'user', content: 'hi' }] });
|
|
const k1 = computeCacheKey('anthropic', 'claude-sonnet-4-6', ir);
|
|
const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir);
|
|
assert.notEqual(k1, k2);
|
|
});
|
|
|
|
// ── Test 4: Messages distinguishes ───────────────────────────────────
|
|
it('computeCacheKey distinguishes different messages', () => {
|
|
const ir1 = makeIR({ messages: [{ role: 'user', content: 'hello' }] });
|
|
const ir2 = makeIR({ messages: [{ role: 'user', content: 'world' }] });
|
|
const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir1);
|
|
const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir2);
|
|
assert.notEqual(k1, k2);
|
|
});
|
|
|
|
// ── Test 5: Tools distinguishes ───────────────────────────────────────
|
|
it('computeCacheKey distinguishes requests with vs without tools', () => {
|
|
const base = makeIR({ messages: [{ role: 'user', content: 'search' }] });
|
|
const withTools = makeIR({
|
|
messages: [{ role: 'user', content: 'search' }],
|
|
tools: [{ type: 'function', function: { name: 'search', description: 'web search', parameters: {} } }],
|
|
});
|
|
const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', base);
|
|
const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', withTools);
|
|
assert.notEqual(k1, k2);
|
|
});
|
|
|
|
// ── Test 6: Temperature distinguishes ────────────────────────────────
|
|
it('computeCacheKey distinguishes different temperature values', () => {
|
|
const ir1 = makeIR({ messages: [{ role: 'user', content: 'x' }], temperature: 0.0 });
|
|
const ir2 = makeIR({ messages: [{ role: 'user', content: 'x' }], temperature: 1.0 });
|
|
const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir1);
|
|
const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir2);
|
|
assert.notEqual(k1, k2);
|
|
});
|
|
|
|
// ── Test 7: response_format distinguishes ────────────────────────────
|
|
it('computeCacheKey distinguishes different response_format values', () => {
|
|
const ir1 = makeIR({ messages: [{ role: 'user', content: 'x' }] });
|
|
const ir2 = makeIR({ messages: [{ role: 'user', content: 'x' }], response_format: { type: 'json_object' } });
|
|
const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir1);
|
|
const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir2);
|
|
assert.notEqual(k1, k2);
|
|
});
|
|
|
|
// ── Test 8: Content array property order stability ───────────────────
|
|
it('computeCacheKey is stable for content arrays with same properties in different insertion order', () => {
|
|
const ir1 = makeIR({
|
|
messages: [{ role: 'user', content: [{ type: 'text', text: 'hi', extra: 1 }] }],
|
|
});
|
|
const ir2 = makeIR({
|
|
messages: [{ role: 'user', content: [{ extra: 1, text: 'hi', type: 'text' }] }],
|
|
});
|
|
const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir1);
|
|
const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir2);
|
|
assert.equal(k1, k2);
|
|
});
|
|
});
|
|
|
|
describe('Cache layer — extractCacheControlMarkers + hasCacheControl (Suite 9 cont.)', () => {
|
|
|
|
// ── Test 9: extractCacheControlMarkers — top-level ───────────────────
|
|
it('extractCacheControlMarkers finds cache_control at message top level', () => {
|
|
const messages = [
|
|
{ role: 'user', content: 'hi', cache_control: { type: 'ephemeral' } },
|
|
];
|
|
const markers = extractCacheControlMarkers(messages);
|
|
assert.equal(markers.length, 1);
|
|
assert.deepEqual(markers[0], { type: 'ephemeral' });
|
|
});
|
|
|
|
// ── Test 10: extractCacheControlMarkers — nested in content array ─────
|
|
it('extractCacheControlMarkers finds cache_control nested in content array', () => {
|
|
const messages = [
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'text', text: 'hello', cache_control: { type: 'ephemeral' } },
|
|
{ type: 'text', text: 'world' },
|
|
],
|
|
},
|
|
];
|
|
const markers = extractCacheControlMarkers(messages);
|
|
assert.equal(markers.length, 1);
|
|
assert.deepEqual(markers[0], { type: 'ephemeral' });
|
|
});
|
|
|
|
// ── Test 11: extractCacheControlMarkers — no markers ─────────────────
|
|
it('extractCacheControlMarkers returns [] when no cache_control markers present', () => {
|
|
const messages = [
|
|
{ role: 'user', content: 'hello' },
|
|
{ role: 'assistant', content: 'world' },
|
|
];
|
|
assert.deepEqual(extractCacheControlMarkers(messages), []);
|
|
});
|
|
|
|
// ── Test 12: hasCacheControl — true ──────────────────────────────────
|
|
it('hasCacheControl returns true when cache_control markers present', () => {
|
|
const ir = makeIR({
|
|
messages: [{ role: 'user', content: 'hi', cache_control: { type: 'ephemeral' } }],
|
|
});
|
|
assert.equal(hasCacheControl(ir), true);
|
|
});
|
|
|
|
// ── Test 13: hasCacheControl — false ─────────────────────────────────
|
|
it('hasCacheControl returns false when no cache_control markers', () => {
|
|
const ir = makeIR({ messages: [{ role: 'user', content: 'hi' }] });
|
|
assert.equal(hasCacheControl(ir), false);
|
|
});
|
|
|
|
// ── Test 14: hasCacheControl — null/undefined safety ─────────────────
|
|
it('hasCacheControl returns false for null/undefined ir', () => {
|
|
assert.equal(hasCacheControl(null), false);
|
|
assert.equal(hasCacheControl(undefined), false);
|
|
assert.equal(hasCacheControl({}), false);
|
|
});
|
|
});
|
|
|
|
describe('Cache layer — CacheStore unit tests (Suite 9 cont.)', () => {
|
|
|
|
// ── Test 15: set/get round-trip ───────────────────────────────────────
|
|
it('CacheStore.set/get round-trips a value', async () => {
|
|
const store = new CacheStore();
|
|
await store.set('keyA', 'hash1', [{ type: 'stop', finish_reason: 'stop' }]);
|
|
const entry = await store.get('keyA', 'hash1');
|
|
assert.ok(entry !== null);
|
|
assert.deepEqual(entry.value, [{ type: 'stop', finish_reason: 'stop' }]);
|
|
});
|
|
|
|
// ── Test 16: get returns null for missing key ─────────────────────────
|
|
it('CacheStore.get returns null for missing (keyId, cacheKey)', async () => {
|
|
const store = new CacheStore();
|
|
const entry = await store.get('keyA', 'nonexistent-hash');
|
|
assert.equal(entry, null);
|
|
});
|
|
|
|
// ── Test 17: Per-key isolation ────────────────────────────────────────
|
|
it('CacheStore per-key isolation: keyId1 entries invisible to keyId2', async () => {
|
|
const store = new CacheStore();
|
|
await store.set('keyA', 'hash1', 'value-for-keyA');
|
|
const fromKeyA = await store.get('keyA', 'hash1');
|
|
const fromKeyB = await store.get('keyB', 'hash1');
|
|
assert.ok(fromKeyA !== null);
|
|
assert.equal(fromKeyA.value, 'value-for-keyA');
|
|
assert.equal(fromKeyB, null);
|
|
});
|
|
|
|
// ── Test 18: has ─────────────────────────────────────────────────────
|
|
it('CacheStore.has returns true for existing entry, false for missing', async () => {
|
|
const store = new CacheStore();
|
|
await store.set('keyA', 'hash1', 'val');
|
|
assert.equal(await store.has('keyA', 'hash1'), true);
|
|
assert.equal(await store.has('keyA', 'hash-missing'), false);
|
|
assert.equal(await store.has('keyB', 'hash1'), false);
|
|
});
|
|
|
|
// ── Test 19: TTL expiry ───────────────────────────────────────────────
|
|
it('CacheStore respects TTL: expired entries return null', async () => {
|
|
// Inject a _nowFn to control time without sleeping.
|
|
// First call: "now" = 0 (entry creation time).
|
|
// Second call: "now" = 2000 (2 seconds later; entry has 1s TTL → expired).
|
|
let fakeNow = 0;
|
|
const store = new CacheStore({ _nowFn: () => fakeNow });
|
|
await store.set('keyA', 'hash1', 'val', 1000); // 1000ms TTL
|
|
// Entry should be alive at t=0
|
|
const entry1 = await store.get('keyA', 'hash1');
|
|
assert.ok(entry1 !== null, 'Expected entry to be alive at t=0');
|
|
// Advance time past TTL
|
|
fakeNow = 2000;
|
|
const entry2 = await store.get('keyA', 'hash1');
|
|
assert.equal(entry2, null, 'Expected entry to be expired at t=2000');
|
|
});
|
|
|
|
// ── Test 20: stats reports hits/misses ────────────────────────────────
|
|
it('CacheStore.stats reports hits and misses', async () => {
|
|
const store = new CacheStore();
|
|
await store.set('keyA', 'hash1', 'val');
|
|
await store.get('keyA', 'hash1'); // hit
|
|
await store.get('keyA', 'hash1'); // hit
|
|
await store.get('keyA', 'missing'); // miss
|
|
const s = store.stats('keyA');
|
|
assert.ok(s.hits >= 2, `Expected hits >= 2, got ${s.hits}`);
|
|
assert.ok(s.misses >= 1, `Expected misses >= 1, got ${s.misses}`);
|
|
assert.ok(typeof s.size === 'number');
|
|
assert.ok(typeof s.inflightCount === 'number');
|
|
});
|
|
|
|
// ── Test 21: getOrCompute returns computed value on miss ──────────────
|
|
it('getOrCompute returns computed value on miss and caches it', async () => {
|
|
const store = new CacheStore();
|
|
let callCount = 0;
|
|
const computeFn = async () => { callCount++; return [{ type: 'delta', content: 'hello' }]; };
|
|
const v1 = await store.getOrCompute('keyA', 'hash1', computeFn);
|
|
assert.deepEqual(v1, [{ type: 'delta', content: 'hello' }]);
|
|
assert.equal(callCount, 1);
|
|
});
|
|
|
|
// ── Test 22: getOrCompute returns cached value on hit (no recomputation) ──
|
|
it('getOrCompute returns cached value on hit — computeFn not called again', async () => {
|
|
const store = new CacheStore();
|
|
let callCount = 0;
|
|
const computeFn = async () => { callCount++; return 'computed-value'; };
|
|
await store.getOrCompute('keyA', 'hash1', computeFn);
|
|
const v2 = await store.getOrCompute('keyA', 'hash1', computeFn);
|
|
assert.equal(v2, 'computed-value');
|
|
assert.equal(callCount, 1, 'computeFn should only be called once on cache hit');
|
|
});
|
|
|
|
// ── Test 23: Singleflight — the key D4 test ───────────────────────────
|
|
// 5 concurrent getOrCompute calls with same key + slow computeFn (50ms).
|
|
// Verifies: computeFn called exactly once; all 5 callers receive same value;
|
|
// all 5 return within a tight time window (singleflight working).
|
|
it('getOrCompute singleflight: 5 concurrent callers → computeFn called exactly once', async () => {
|
|
const store = new CacheStore();
|
|
let callCount = 0;
|
|
|
|
const slowCompute = async () => {
|
|
callCount++;
|
|
// Simulate a slow provider spawn (50ms)
|
|
await new Promise(r => setTimeout(r, 50));
|
|
return [{ type: 'delta', content: 'singleflight-result' }, { type: 'stop', finish_reason: 'stop' }];
|
|
};
|
|
|
|
const t0 = Date.now();
|
|
// Launch 5 concurrent callers simultaneously
|
|
const results = await Promise.all([
|
|
store.getOrCompute('keyA', 'sf-hash', slowCompute),
|
|
store.getOrCompute('keyA', 'sf-hash', slowCompute),
|
|
store.getOrCompute('keyA', 'sf-hash', slowCompute),
|
|
store.getOrCompute('keyA', 'sf-hash', slowCompute),
|
|
store.getOrCompute('keyA', 'sf-hash', slowCompute),
|
|
]);
|
|
const elapsed = Date.now() - t0;
|
|
|
|
// Singleflight invariant: computeFn called exactly once
|
|
assert.equal(callCount, 1, `Expected computeFn called 1 time, got ${callCount}`);
|
|
|
|
// All 5 callers receive the same result
|
|
for (const result of results) {
|
|
assert.deepEqual(result, [{ type: 'delta', content: 'singleflight-result' }, { type: 'stop', finish_reason: 'stop' }]);
|
|
}
|
|
|
|
// All 5 callers return within a tight window (not 5 * 50ms = 250ms).
|
|
// Allow generous margin (3x the slow compute time) for CI variance.
|
|
assert.ok(elapsed < 250, `Expected singleflight to complete in < 250ms, took ${elapsed}ms`);
|
|
});
|
|
|
|
// ── Test 24: getOrCompute releases inflight on completion ────────────
|
|
it('getOrCompute: subsequent calls after completion hit cache (no re-compute)', async () => {
|
|
const store = new CacheStore();
|
|
let callCount = 0;
|
|
const computeFn = async () => { callCount++; return 'done'; };
|
|
// First call — computes and caches
|
|
await store.getOrCompute('keyA', 'hash1', computeFn);
|
|
assert.equal(callCount, 1);
|
|
// Verify inflight is released: calling again should hit cache
|
|
await store.getOrCompute('keyA', 'hash1', computeFn);
|
|
assert.equal(callCount, 1, 'computeFn should not be called again after completion');
|
|
assert.equal(store.stats('keyA').inflightCount, 0, 'No inflight entries after completion');
|
|
});
|
|
|
|
// ── Test 25: getOrCompute releases inflight on error ─────────────────
|
|
it('getOrCompute: inflight released when computeFn throws; subsequent calls retry', async () => {
|
|
const store = new CacheStore();
|
|
let callCount = 0;
|
|
let shouldFail = true;
|
|
const computeFn = async () => {
|
|
callCount++;
|
|
if (shouldFail) throw new Error('provider error');
|
|
return 'success';
|
|
};
|
|
|
|
// First call — throws
|
|
await assert.rejects(() => store.getOrCompute('keyA', 'hash1', computeFn), /provider error/);
|
|
assert.equal(callCount, 1);
|
|
// Inflight must be released even after error
|
|
assert.equal(store.stats('keyA').inflightCount, 0, 'Inflight must be released after error');
|
|
|
|
// Second call — now succeeds (verify re-try works)
|
|
shouldFail = false;
|
|
const v = await store.getOrCompute('keyA', 'hash1', computeFn);
|
|
assert.equal(v, 'success');
|
|
assert.equal(callCount, 2, 'computeFn should be retried after error');
|
|
});
|
|
|
|
// ── Test 26: clear(keyId) clears only that namespace ─────────────────
|
|
it('CacheStore.clear(keyId) clears only that namespace', async () => {
|
|
const store = new CacheStore();
|
|
await store.set('keyA', 'hash1', 'val-a');
|
|
await store.set('keyB', 'hash1', 'val-b');
|
|
store.clear('keyA');
|
|
assert.equal(await store.has('keyA', 'hash1'), false, 'keyA should be cleared');
|
|
assert.equal(await store.has('keyB', 'hash1'), true, 'keyB should remain');
|
|
});
|
|
|
|
// ── Test 27: clear() with no args clears all ──────────────────────────
|
|
it('CacheStore.clear() with no argument clears all namespaces', async () => {
|
|
const store = new CacheStore();
|
|
await store.set('keyA', 'hash1', 'val-a');
|
|
await store.set('keyB', 'hash1', 'val-b');
|
|
store.clear();
|
|
assert.equal(await store.has('keyA', 'hash1'), false, 'keyA should be cleared');
|
|
assert.equal(await store.has('keyB', 'hash1'), false, 'keyB should be cleared');
|
|
assert.equal(store.stats().size, 0);
|
|
});
|
|
});
|
|
|
|
// ── Suite 9 (HTTP integration — cache) ───────────────────────────────────
|
|
//
|
|
// Tests cache miss / hit / bypass paths via HTTP integration with a mock
|
|
// provider. Anthropic provider is enabled with:
|
|
// 1. CLAUDE_CODE_OAUTH_TOKEN set to a fake value to bypass auth check
|
|
// (the mock spawn never actually uses the token)
|
|
// 2. Mock spawn injected via __setSpawnImpl so no real claude binary runs
|
|
//
|
|
// This tests the full HTTP → server.mjs → cache layer → provider dispatch
|
|
// path end-to-end, with the spawn binary call itself mocked out.
|
|
|
|
describe('Cache layer — HTTP integration (Suite 9 cont.)', () => {
|
|
let serverInstance9;
|
|
let port9;
|
|
let savedOAuthToken;
|
|
let serverMod9;
|
|
|
|
before(async () => {
|
|
// Inject a fake OAuth token so auth check passes without a real token.
|
|
// The mock spawn ignores this value entirely.
|
|
savedOAuthToken = process.env.CLAUDE_CODE_OAUTH_TOKEN;
|
|
process.env.CLAUDE_CODE_OAUTH_TOKEN = 'test-fake-oauth-token-for-cache-tests';
|
|
|
|
// Set mock spawn that returns a proper response (delta + stop chunks via
|
|
// raw text output — anthropic.mjs treats each stdout data chunk as raw text).
|
|
__setSpawnImpl(makeMockSpawn(['mock-cache-content']));
|
|
|
|
// Import the server module (already cached by Node module system — same instance
|
|
// as Suite 7). Mutate the loadedProviders map to add anthropic.
|
|
serverMod9 = await import('./server.mjs');
|
|
const { createOlpServer, loadedProviders: lp } = serverMod9;
|
|
|
|
// Wire anthropic into the loaded providers map
|
|
const testProviders = loadProviders({ enabled: { anthropic: true } });
|
|
for (const [name, p] of testProviders) {
|
|
lp.set(name, p);
|
|
}
|
|
|
|
port9 = parseInt(
|
|
process.env.OLP_TEST_PORT
|
|
? String(parseInt(process.env.OLP_TEST_PORT) + 5000)
|
|
: String(18456 + Math.floor(Math.random() * 1000)),
|
|
10,
|
|
);
|
|
|
|
serverInstance9 = createOlpServer();
|
|
await new Promise((resolve, reject) => {
|
|
serverInstance9.listen(port9, '127.0.0.1', resolve);
|
|
serverInstance9.once('error', async (e) => {
|
|
if (e.code === 'EADDRINUSE') {
|
|
port9++;
|
|
serverInstance9.listen(port9, '127.0.0.1', resolve);
|
|
serverInstance9.once('error', reject);
|
|
} else reject(e);
|
|
});
|
|
});
|
|
});
|
|
|
|
after(() => {
|
|
// Restore OAuth token
|
|
if (savedOAuthToken !== undefined) {
|
|
process.env.CLAUDE_CODE_OAUTH_TOKEN = savedOAuthToken;
|
|
} else {
|
|
delete process.env.CLAUDE_CODE_OAUTH_TOKEN;
|
|
}
|
|
__resetSpawnImpl();
|
|
return new Promise(r => serverInstance9.close(r));
|
|
});
|
|
|
|
// ── Test 28: cache miss path ──────────────────────────────────────────
|
|
// First request with a unique message → cache miss (not yet in cache).
|
|
it('HTTP: first request returns X-OLP-Cache: miss', async () => {
|
|
// Unique content ensures this test doesn't collide with other tests' cached entries
|
|
const testMsg = `http-cache-miss-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
__setSpawnImpl(makeMockSpawn([`response-for-${testMsg}`]));
|
|
|
|
const r = await fetch({
|
|
port: port9,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: {
|
|
model: 'claude-haiku-4-5',
|
|
messages: [{ role: 'user', content: testMsg }],
|
|
},
|
|
});
|
|
|
|
assert.equal(r.status, 200, `Expected 200, got ${r.status}: ${r.body.slice(0, 200)}`);
|
|
assert.equal(r.headers['x-olp-cache'], 'miss', `Expected miss, got: ${r.headers['x-olp-cache']}`);
|
|
assert.equal(r.headers['x-olp-provider-used'], 'anthropic');
|
|
assert.equal(r.headers['x-olp-model-used'], 'claude-haiku-4-5');
|
|
});
|
|
|
|
// ── Test 29: cache hit path ───────────────────────────────────────────
|
|
// Two identical requests: first → miss, second → hit (same content served from cache).
|
|
it('HTTP: second identical request returns X-OLP-Cache: hit with same content', async () => {
|
|
const testMsg = `http-cache-hit-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
const mockResponse = `hit-response-${testMsg}`;
|
|
__setSpawnImpl(makeMockSpawn([mockResponse]));
|
|
|
|
const reqParams = {
|
|
port: port9,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: {
|
|
model: 'claude-haiku-4-5',
|
|
messages: [{ role: 'user', content: testMsg }],
|
|
},
|
|
};
|
|
|
|
// First request — miss, spawns real (mock) provider
|
|
const r1 = await fetch(reqParams);
|
|
assert.equal(r1.status, 200, `First request failed: ${r1.status} ${r1.body.slice(0, 200)}`);
|
|
assert.equal(r1.headers['x-olp-cache'], 'miss', `First request should be miss`);
|
|
const body1 = JSON.parse(r1.body);
|
|
const content1 = body1?.choices?.[0]?.message?.content ?? '';
|
|
|
|
// Second request — replace spawn with a failing mock to prove spawn is NOT called
|
|
// (if spawn were called, this would produce a 502 error)
|
|
__setSpawnImpl(makeMockSpawn([], 1)); // exit code 1 = ProviderError on spawn
|
|
|
|
const r2 = await fetch(reqParams);
|
|
assert.equal(r2.status, 200, `Second request (cache hit) should be 200, got ${r2.status}: ${r2.body.slice(0, 200)}`);
|
|
assert.equal(r2.headers['x-olp-cache'], 'hit', `Second request should be cache hit`);
|
|
|
|
// Content should be identical (replayed from cache)
|
|
const body2 = JSON.parse(r2.body);
|
|
const content2 = body2?.choices?.[0]?.message?.content ?? '';
|
|
assert.equal(content2, content1, `Cache hit content should match original`);
|
|
});
|
|
|
|
// ── Test 30: cache bypass path (cache_control marker) ────────────────
|
|
// Request with cache_control marker → X-OLP-Cache: bypass (no OLP caching).
|
|
it('HTTP: request with cache_control marker returns X-OLP-Cache: bypass', async () => {
|
|
const testMsg = `http-cache-bypass-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
__setSpawnImpl(makeMockSpawn([`bypass-response-${testMsg}`]));
|
|
|
|
const r = await fetch({
|
|
port: port9,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: {
|
|
model: 'claude-haiku-4-5',
|
|
messages: [
|
|
{
|
|
role: 'user',
|
|
content: testMsg,
|
|
cache_control: { type: 'ephemeral' },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
assert.equal(r.status, 200, `Expected 200, got ${r.status}: ${r.body.slice(0, 200)}`);
|
|
assert.equal(r.headers['x-olp-cache'], 'bypass', `Expected bypass header, got: ${r.headers['x-olp-cache']}`);
|
|
});
|
|
});
|
|
|
|
// ── Suite 11: Codex plugin (D6) ──────────────────────────────────────────
|
|
//
|
|
// All tests are UNIT tests. No real `codex` binary is invoked.
|
|
// Mock spawn is injected via codexSetSpawnImpl / codexResetSpawnImpl.
|
|
//
|
|
// Authority: Codex CLI reference https://developers.openai.com/codex/cli/reference
|
|
// § "codex exec [flags] PROMPT" — exec subcommand syntax
|
|
// § "--json" — NDJSON output format
|
|
// § "--model, -m" — model override
|
|
// § "$CODEX_HOME/auth.json" — auth artifact location
|
|
//
|
|
// Lossy-translation acknowledgements per ADR 0003 (documented in codex.mjs header):
|
|
// top_p, temperature, stop, max_tokens, tools[], tool_calls → all dropped.
|
|
|
|
/**
|
|
* Creates a fake NDJSON-emitting mock spawn for Codex.
|
|
* Lines are emitted as they would arrive from `codex exec --json`.
|
|
*
|
|
* @param {string[]} ndjsonLines — raw NDJSON lines emitted in order (no trailing \n needed)
|
|
* @param {number} [exitCode=0]
|
|
*/
|
|
function makeMockCodexSpawn(ndjsonLines, exitCode = 0) {
|
|
return function mockCodexSpawnImpl(_bin, _args, _opts) {
|
|
const proc = new EventEmitter();
|
|
proc.stdout = new EventEmitter();
|
|
proc.stderr = new EventEmitter();
|
|
proc.stdin = {
|
|
write: () => {},
|
|
end: () => {
|
|
setImmediate(async () => {
|
|
for (const line of ndjsonLines) {
|
|
proc.stdout.emit('data', Buffer.from(line + '\n'));
|
|
}
|
|
proc.stdout.emit('end');
|
|
proc.stderr.emit('end');
|
|
proc.emit('close', exitCode, null);
|
|
});
|
|
},
|
|
};
|
|
proc.killed = false;
|
|
proc.kill = () => {};
|
|
return proc;
|
|
};
|
|
}
|
|
|
|
describe('Codex plugin (D6)', () => {
|
|
|
|
// ── Test 1: Contract conformance ─────────────────────────────────────
|
|
it('codex module satisfies validateProvider() — all 10 fields present', () => {
|
|
const { valid, errors } = validateProvider(codex);
|
|
assert.equal(valid, true, `Validation errors: ${errors.join('; ')}`);
|
|
assert.ok('name' in codex, 'missing: name');
|
|
assert.ok('displayName' in codex, 'missing: displayName');
|
|
assert.ok('contractVersion' in codex, 'missing: contractVersion');
|
|
assert.ok('models' in codex, 'missing: models');
|
|
assert.ok('auth' in codex, 'missing: auth');
|
|
assert.ok(typeof codex.spawn === 'function', 'missing: spawn');
|
|
assert.ok(typeof codex.estimateCost === 'function', 'missing: estimateCost');
|
|
assert.ok(typeof codex.quotaStatus === 'function', 'missing: quotaStatus');
|
|
assert.ok(typeof codex.healthCheck === 'function', 'missing: healthCheck');
|
|
assert.ok('hints' in codex, 'missing: hints');
|
|
});
|
|
|
|
// ── Test 2: contractVersion === '1.0' ────────────────────────────────
|
|
it('codex declares contractVersion === "1.0"', () => {
|
|
assert.equal(codex.contractVersion, '1.0');
|
|
});
|
|
|
|
// ── Test 3: name and displayName ─────────────────────────────────────
|
|
it('codex.name === "openai" and displayName is set', () => {
|
|
assert.equal(codex.name, 'openai');
|
|
assert.equal(typeof codex.displayName, 'string');
|
|
assert.ok(codex.displayName.length > 0);
|
|
});
|
|
|
|
// ── Test 4: models match registry ───────────────────────────────────
|
|
it('codex.models matches models-registry.json providers.openai.models[].id', () => {
|
|
const registryIds = modelsRegistry.providers.openai.models.map(m => m.id);
|
|
assert.deepEqual(codex.models, registryIds);
|
|
});
|
|
|
|
it('codex.models contains all 5 docs-listed model IDs', () => {
|
|
// Per https://developers.openai.com/codex/models — each id has a
|
|
// `codex -m <id>` example on that page. D6 review-2 expanded the
|
|
// registry to include gpt-5.4-mini and gpt-5.3-codex-spark which the
|
|
// original sonnet draft missed.
|
|
assert.ok(codex.models.includes('gpt-5.5'), 'missing gpt-5.5');
|
|
assert.ok(codex.models.includes('gpt-5.4'), 'missing gpt-5.4');
|
|
assert.ok(codex.models.includes('gpt-5.4-mini'), 'missing gpt-5.4-mini');
|
|
assert.ok(codex.models.includes('gpt-5.3-codex'), 'missing gpt-5.3-codex');
|
|
assert.ok(codex.models.includes('gpt-5.3-codex-spark'), 'missing gpt-5.3-codex-spark');
|
|
assert.equal(codex.models.length, 5, `Expected 5 models, got ${codex.models.length}`);
|
|
});
|
|
|
|
// ── Test 5: getProviderForModel finds codex for each model ──────────
|
|
it('getProviderForModel finds openai provider for gpt-5.5', () => {
|
|
const loaded = new Map([['openai', codex]]);
|
|
const result = getProviderForModel(loaded, 'gpt-5.5');
|
|
assert.ok(result !== null);
|
|
assert.equal(result.name, 'openai');
|
|
});
|
|
|
|
it('getProviderForModel finds openai provider for gpt-5.4', () => {
|
|
const loaded = new Map([['openai', codex]]);
|
|
const result = getProviderForModel(loaded, 'gpt-5.4');
|
|
assert.ok(result !== null);
|
|
assert.equal(result.name, 'openai');
|
|
});
|
|
|
|
it('getProviderForModel finds openai provider for gpt-5.3-codex', () => {
|
|
const loaded = new Map([['openai', codex]]);
|
|
const result = getProviderForModel(loaded, 'gpt-5.3-codex');
|
|
assert.ok(result !== null);
|
|
assert.equal(result.name, 'openai');
|
|
});
|
|
|
|
// ── Test 6: irToCodex translation ────────────────────────────────────
|
|
it('irToCodex: user message → args with exec --json --model, prompt as positional', () => {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
messages: [{ role: 'user', content: 'Hello world' }],
|
|
});
|
|
const { args, prompt, useStdin } = irToCodex(ir);
|
|
// Authority: Codex CLI reference § "codex exec [flags] PROMPT"
|
|
assert.ok(args.includes('exec'), 'args must include "exec"');
|
|
assert.ok(args.includes('--json'), 'args must include "--json"');
|
|
assert.ok(args.includes('--model'), 'args must include "--model"');
|
|
assert.ok(args.includes('gpt-5.5'), 'args must include model value');
|
|
assert.ok(typeof prompt === 'string', 'prompt must be a string');
|
|
assert.ok(prompt.includes('Hello world'), 'prompt must contain user text');
|
|
assert.equal(useStdin, false, 'single-line prompt should use argv, not stdin');
|
|
});
|
|
|
|
it('irToCodex: system + user → system annotation + user text in prompt', () => {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.4',
|
|
messages: [
|
|
{ role: 'system', content: 'You are a coder.' },
|
|
{ role: 'user', content: 'Write a function.' },
|
|
],
|
|
});
|
|
const { prompt } = irToCodex(ir);
|
|
assert.ok(prompt.includes('[System] You are a coder.'));
|
|
assert.ok(prompt.includes('Write a function.'));
|
|
});
|
|
|
|
it('irToCodex: assistant prior turn → [Assistant] annotation', () => {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
messages: [
|
|
{ role: 'user', content: 'Hi' },
|
|
{ role: 'assistant', content: 'Hello!' },
|
|
{ role: 'user', content: 'Thanks' },
|
|
],
|
|
});
|
|
const { prompt } = irToCodex(ir);
|
|
assert.ok(prompt.includes('[Assistant] Hello!'));
|
|
assert.ok(prompt.includes('Thanks'));
|
|
});
|
|
|
|
it('irToCodex: tool_calls in assistant message — content preserved, metadata dropped (lossy)', () => {
|
|
// ADR 0003 § Lossy: structured tool_calls dropped; textual content preserved.
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
messages: [
|
|
{ role: 'user', content: 'Search for X' },
|
|
{
|
|
role: 'assistant',
|
|
content: 'Searching...',
|
|
tool_calls: [{ id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"X"}' } }],
|
|
},
|
|
],
|
|
});
|
|
const { prompt } = irToCodex(ir);
|
|
// Content preserved
|
|
assert.ok(prompt.includes('Searching...'), 'assistant text content must be preserved');
|
|
// tool_calls metadata not directly in prompt (dropped per lossy spec)
|
|
// (We do NOT assert the metadata IS there — it is documented as dropped.)
|
|
});
|
|
|
|
it('irToCodex: response_format json_object injects system prompt', () => {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
messages: [{ role: 'user', content: 'Give JSON' }],
|
|
response_format: { type: 'json_object' },
|
|
});
|
|
const { prompt } = irToCodex(ir);
|
|
assert.ok(prompt.includes('Reply with valid JSON only'));
|
|
});
|
|
|
|
it('irToCodex: multiline prompt uses stdin path (useStdin=true) with `-` positional', () => {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
messages: [
|
|
{ role: 'system', content: 'Line one.' },
|
|
{ role: 'user', content: 'Line two.' },
|
|
],
|
|
});
|
|
const { useStdin, args } = irToCodex(ir);
|
|
// System + user messages join with '\n\n' → contains newline → stdin
|
|
assert.equal(useStdin, true, 'multi-section prompt should use stdin');
|
|
// Per Codex CLI reference, stdin requires the literal `-` positional.
|
|
// (D6 review-2 finding: original draft omitted positional entirely;
|
|
// docs explicitly state stdin requires `-`.)
|
|
assert.ok(args.includes('-'), 'stdin path must pass `-` as positional');
|
|
assert.ok(!args.includes('Line one.'), 'literal prompt must not appear in args when useStdin');
|
|
});
|
|
|
|
it('irToCodex: tool result turn → [Tool Result] annotation', () => {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
messages: [
|
|
{ role: 'user', content: 'Search for X' },
|
|
{ role: 'tool', content: '{"results":[]}', name: 'search' },
|
|
],
|
|
});
|
|
const { prompt } = irToCodex(ir);
|
|
assert.ok(prompt.includes('[Tool Result'));
|
|
assert.ok(prompt.includes('search'));
|
|
});
|
|
|
|
// ── Test 7: codexChunkToIR translation ────────────────────────────────
|
|
it('codexChunkToIR: content field → delta chunk', () => {
|
|
const chunk = codexChunkToIR('{"content":"Hello world"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.content, 'Hello world');
|
|
});
|
|
|
|
it('codexChunkToIR: delta field → delta chunk', () => {
|
|
const chunk = codexChunkToIR('{"type":"delta","delta":"token text"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.content, 'token text');
|
|
});
|
|
|
|
it('codexChunkToIR: text field → delta chunk', () => {
|
|
// Possible alternative event shape with "text" field
|
|
const chunk = codexChunkToIR('{"type":"output_text","text":"output here"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.content, 'output here');
|
|
});
|
|
|
|
it('codexChunkToIR: type === "stop" → stop chunk', () => {
|
|
const chunk = codexChunkToIR('{"type":"stop"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'stop');
|
|
assert.equal(chunk.finish_reason, 'stop');
|
|
});
|
|
|
|
it('codexChunkToIR: done === true → stop chunk', () => {
|
|
const chunk = codexChunkToIR('{"done":true,"id":"run_123"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'stop');
|
|
assert.equal(chunk.finish_reason, 'stop');
|
|
});
|
|
|
|
it('codexChunkToIR: type === "error" → error chunk', () => {
|
|
const chunk = codexChunkToIR('{"type":"error","error":"quota exceeded"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'error');
|
|
assert.equal(chunk.error, 'quota exceeded');
|
|
});
|
|
|
|
it('codexChunkToIR: error field present → error chunk', () => {
|
|
const chunk = codexChunkToIR('{"error":"something went wrong","code":500}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'error');
|
|
});
|
|
|
|
it('codexChunkToIR: unknown/progress event type → null (silently ignored)', () => {
|
|
// Events like {"type":"progress","step":1} are ignored per D6 spec
|
|
const chunk = codexChunkToIR('{"type":"progress","step":1}');
|
|
assert.equal(chunk, null);
|
|
});
|
|
|
|
it('codexChunkToIR: empty line → null', () => {
|
|
assert.equal(codexChunkToIR(''), null);
|
|
assert.equal(codexChunkToIR(' '), null);
|
|
});
|
|
|
|
it('codexChunkToIR: malformed JSON → null (no throw)', () => {
|
|
assert.doesNotThrow(() => {
|
|
const result = codexChunkToIR('{bad json');
|
|
assert.equal(result, null);
|
|
});
|
|
});
|
|
|
|
// ── Test 8: mock spawn — NDJSON stream yields correct IR chunks ───────
|
|
it('spawn with mock: NDJSON lines → delta chunks then stop chunk', async () => {
|
|
const fakeSpawn = makeMockCodexSpawn([
|
|
'{"content":"Hello"}',
|
|
'{"content":" world"}',
|
|
'{"type":"stop"}',
|
|
]);
|
|
codexSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-openai-token>' };
|
|
const chunks = [];
|
|
for await (const chunk of codex.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const deltas = chunks.filter(c => c.type === 'delta');
|
|
const stops = chunks.filter(c => c.type === 'stop');
|
|
assert.ok(deltas.length >= 1, `Expected at least 1 delta, got ${deltas.length}`);
|
|
assert.equal(stops.length, 1, `Expected 1 stop, got ${stops.length}`);
|
|
const allContent = deltas.map(c => c.content).join('');
|
|
assert.equal(allContent, 'Hello world');
|
|
} finally {
|
|
codexResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: first delta chunk has role=assistant', async () => {
|
|
const fakeSpawn = makeMockCodexSpawn(['{"content":"Test output"}']);
|
|
codexSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-openai-token>' };
|
|
const chunks = [];
|
|
for await (const chunk of codex.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const firstDelta = chunks.find(c => c.type === 'delta');
|
|
assert.ok(firstDelta, 'No delta chunk found');
|
|
assert.equal(firstDelta.role, 'assistant');
|
|
} finally {
|
|
codexResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: NDJSON stop event present → no extra synthetic stop appended', async () => {
|
|
// If NDJSON stream already contains a stop event, we should NOT double-emit
|
|
const fakeSpawn = makeMockCodexSpawn([
|
|
'{"content":"done"}',
|
|
'{"type":"stop"}',
|
|
]);
|
|
codexSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
stream: false,
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-openai-token>' };
|
|
const chunks = [];
|
|
for await (const chunk of codex.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const stops = chunks.filter(c => c.type === 'stop');
|
|
assert.equal(stops.length, 1, `Expected exactly 1 stop chunk, got ${stops.length}`);
|
|
} finally {
|
|
codexResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: non-zero exit code throws ProviderError(SPAWN_FAILED)', async () => {
|
|
const fakeSpawn = makeMockCodexSpawn([], 1);
|
|
codexSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-openai-token>' };
|
|
let caught = null;
|
|
try {
|
|
for await (const _chunk of codex.spawn(ir, authCtx)) {
|
|
// drain
|
|
}
|
|
} catch (e) {
|
|
caught = e;
|
|
}
|
|
assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`);
|
|
assert.equal(caught.code, 'SPAWN_FAILED');
|
|
} finally {
|
|
codexResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn throws ProviderError(AUTH_MISSING) when no auth context and no auth file', async () => {
|
|
const fakeSpawn = makeMockCodexSpawn(['{"content":"test"}']);
|
|
codexSetSpawnImpl(fakeSpawn);
|
|
// Override auth path to a nonexistent file to guarantee missing auth
|
|
const savedAuthPath = process.env.OPENAI_CODEX_AUTH_PATH;
|
|
process.env.OPENAI_CODEX_AUTH_PATH = '/nonexistent/path/auth.json';
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
stream: false,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
let caught = null;
|
|
try {
|
|
for await (const _chunk of codex.spawn(ir, null)) { // eslint-disable-line no-unused-vars
|
|
// drain
|
|
}
|
|
} catch (e) {
|
|
caught = e;
|
|
}
|
|
assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`);
|
|
assert.equal(caught.code, 'AUTH_MISSING');
|
|
} finally {
|
|
if (savedAuthPath !== undefined) {
|
|
process.env.OPENAI_CODEX_AUTH_PATH = savedAuthPath;
|
|
} else {
|
|
delete process.env.OPENAI_CODEX_AUTH_PATH;
|
|
}
|
|
codexResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
// ── Test 9: healthCheck ───────────────────────────────────────────────
|
|
it('healthCheck returns {ok: false, error: "codex binary not found"} when binary absent', async () => {
|
|
const result = await codexHealthCheck({
|
|
_binaryExistsFn: () => false,
|
|
_authReadFn: () => ({ accessToken: '<fake-token>' }),
|
|
});
|
|
assert.equal(result.ok, false);
|
|
assert.equal(result.error, 'codex binary not found');
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
it('healthCheck returns {ok: false, error: "auth artifact missing"} when auth missing', async () => {
|
|
const result = await codexHealthCheck({
|
|
_binaryExistsFn: () => true,
|
|
_authReadFn: () => null,
|
|
});
|
|
assert.equal(result.ok, false);
|
|
assert.equal(result.error, 'auth artifact missing');
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
it('healthCheck returns {ok: true} when binary and auth both present', async () => {
|
|
const result = await codexHealthCheck({
|
|
_binaryExistsFn: () => true,
|
|
_authReadFn: () => ({ accessToken: '<fake-token>' }),
|
|
});
|
|
assert.equal(result.ok, true);
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
// ── Test 10: estimateCost ─────────────────────────────────────────────
|
|
it('estimateCost returns shape with currency USD', () => {
|
|
const request = makeIR({
|
|
model: 'gpt-5.5',
|
|
messages: [
|
|
{ role: 'system', content: 'You are a coding assistant.' },
|
|
{ role: 'user', content: 'Write hello world in Python.' },
|
|
],
|
|
});
|
|
const result = codexEstimateCost(request);
|
|
assert.ok(result !== null, 'estimateCost returned null');
|
|
assert.ok('inputTokens' in result, 'missing inputTokens');
|
|
assert.ok('outputTokensEstimate' in result, 'missing outputTokensEstimate');
|
|
assert.ok('currency' in result, 'missing currency');
|
|
assert.ok('usd' in result, 'missing usd');
|
|
assert.equal(result.currency, 'USD');
|
|
assert.equal(result.usd, null); // not pinned at D6
|
|
assert.ok(result.inputTokens > 0, 'inputTokens should be > 0');
|
|
assert.ok(result.outputTokensEstimate >= 0);
|
|
});
|
|
|
|
it('estimateCost returns null for null/missing request', () => {
|
|
assert.equal(codexEstimateCost(null), null);
|
|
assert.equal(codexEstimateCost({}), null);
|
|
});
|
|
|
|
// ── Test 11: quotaStatus ──────────────────────────────────────────────
|
|
it('quotaStatus returns null at D6', async () => {
|
|
const result = await codexQuotaStatus({});
|
|
assert.equal(result, null);
|
|
});
|
|
|
|
// ── Test 12: auth artifact path uses os.homedir(), not hardcoded ──────
|
|
it('codex.auth.path uses homedir() and references .codex directory', () => {
|
|
assert.equal(typeof codex.auth.path, 'string');
|
|
assert.ok(codex.auth.path.includes('.codex'), 'auth.path should reference .codex directory');
|
|
// Portability check: path must start with homedir() value at runtime
|
|
assert.ok(
|
|
codex.auth.path.startsWith(homedir()),
|
|
`auth.path "${codex.auth.path}" should start with homedir() "${homedir()}"`,
|
|
);
|
|
});
|
|
|
|
it('codex.auth.type === "oauth" and storage === "file"', () => {
|
|
assert.equal(codex.auth.type, 'oauth');
|
|
assert.equal(codex.auth.storage, 'file');
|
|
});
|
|
|
|
// ── Test 13: readAuthArtifact reads OPENAI_CODEX_AUTH_PATH override ──
|
|
it('readAuthArtifact: OPENAI_CODEX_AUTH_PATH pointing to nonexistent file → null', () => {
|
|
const saved = process.env.OPENAI_CODEX_AUTH_PATH;
|
|
process.env.OPENAI_CODEX_AUTH_PATH = '/definitely/not/a/real/path/auth.json';
|
|
try {
|
|
const result = codexReadAuthArtifact();
|
|
assert.equal(result, null);
|
|
} finally {
|
|
if (saved !== undefined) process.env.OPENAI_CODEX_AUTH_PATH = saved;
|
|
else delete process.env.OPENAI_CODEX_AUTH_PATH;
|
|
}
|
|
});
|
|
|
|
// ── Test 14: hints shape ──────────────────────────────────────────────
|
|
it('codex.hints has correct shape', () => {
|
|
assert.equal(typeof codex.hints.requiresTTY, 'boolean');
|
|
assert.equal(typeof codex.hints.concurrentSpawnSafe, 'boolean');
|
|
assert.ok(Number.isInteger(codex.hints.maxConcurrent) && codex.hints.maxConcurrent > 0);
|
|
assert.equal(codex.hints.requiresTTY, false);
|
|
assert.equal(codex.hints.concurrentSpawnSafe, true);
|
|
});
|
|
|
|
// ── Test 15: STATIC_REGISTRY length after D8 still includes openai ────
|
|
it('STATIC_REGISTRY includes openai (D6) after D8 (length >= 2)', () => {
|
|
// D8 adds mistral; anthropic + openai must still be present.
|
|
assert.ok(listAllProviderNames().length >= 2);
|
|
assert.ok(listAllProviderNames().includes('anthropic'));
|
|
assert.ok(listAllProviderNames().includes('openai'));
|
|
});
|
|
|
|
// ── Test 16: loadProviders with openai enabled ────────────────────────
|
|
it('loadProviders with {enabled: {openai: true}} returns Map of size 1 with openai', () => {
|
|
const loaded = loadProviders({ enabled: { openai: true } });
|
|
assert.equal(loaded.size, 1);
|
|
assert.ok(loaded.has('openai'));
|
|
});
|
|
|
|
it('loadProviders with both anthropic and openai enabled returns Map of size 2', () => {
|
|
const loaded = loadProviders({ enabled: { anthropic: true, openai: true } });
|
|
assert.equal(loaded.size, 2);
|
|
assert.ok(loaded.has('anthropic'));
|
|
assert.ok(loaded.has('openai'));
|
|
});
|
|
|
|
it('openai loaded via loadProviders passes contract validation', () => {
|
|
const loaded = loadProviders({ enabled: { openai: true } });
|
|
const p = loaded.get('openai');
|
|
const { valid, errors } = validateProvider(p);
|
|
assert.equal(valid, true, `Contract errors: ${errors.join('; ')}`);
|
|
assert.ok(p.models.includes('gpt-5.5'));
|
|
});
|
|
|
|
// ── Test 17: spawn progress events silently ignored ──────────────────
|
|
it('spawn with mock: progress events are silently ignored, only content emitted', async () => {
|
|
const fakeSpawn = makeMockCodexSpawn([
|
|
'{"type":"progress","step":1}',
|
|
'{"type":"progress","step":2}',
|
|
'{"content":"actual response"}',
|
|
'{"type":"stop"}',
|
|
]);
|
|
codexSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Do something' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-openai-token>' };
|
|
const chunks = [];
|
|
for await (const chunk of codex.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const deltas = chunks.filter(c => c.type === 'delta');
|
|
assert.equal(deltas.length, 1);
|
|
assert.equal(deltas[0].content, 'actual response');
|
|
} finally {
|
|
codexResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
// ── Test 18: spawn synthesizes stop when NDJSON stream has no stop event ──
|
|
it('spawn with mock: synthetic stop emitted when NDJSON has no stop event', async () => {
|
|
const fakeSpawn = makeMockCodexSpawn([
|
|
'{"content":"only content, no stop"}',
|
|
// No stop event in the stream
|
|
]);
|
|
codexSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'gpt-5.5',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
});
|
|
const authCtx = { accessToken: '<fake-openai-token>' };
|
|
const chunks = [];
|
|
for await (const chunk of codex.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const stops = chunks.filter(c => c.type === 'stop');
|
|
assert.equal(stops.length, 1, 'Should have exactly 1 synthetic stop chunk');
|
|
assert.equal(stops[0].finish_reason, 'stop');
|
|
} finally {
|
|
codexResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
});
|
|
|
|
// ── Suite 10: Anthropic E2E (GATED) ──────────────────────────────────────
|
|
//
|
|
// Run with: OLP_RUN_E2E=1 npm test
|
|
// Skipped by default; consumes real Anthropic tokens (~200 tokens per run,
|
|
// est. <$0.001 at haiku rates). Requires `claude` binary + keychain OAuth
|
|
// token. The orchestrator runs this once per D5 verification.
|
|
//
|
|
// This suite is NOT run in CI (CI does not set OLP_RUN_E2E). The skip notice
|
|
// is emitted as a console message so CI logs show the gated test exists.
|
|
//
|
|
// Tests:
|
|
// 1. Start OLP server with anthropic enabled.
|
|
// 2. POST minimal request to claude-haiku-4-5.
|
|
// 3. Assert 200, response contains "OK", correct provider/model headers.
|
|
// 4. Send same request again → assert X-OLP-Cache: hit, identical content.
|
|
// 5. Assert X-OLP-Fallback-Hops: 0.
|
|
//
|
|
// Model: claude-haiku-4-5 (cheapest). Prompt: "Reply with exactly the word OK
|
|
// and nothing else." max_tokens: 10. Target: < 200 tokens per run.
|
|
//
|
|
// Do NOT include any real OAuth tokens or API keys in this test. Auth is read
|
|
// from keychain / CLAUDE_CODE_OAUTH_TOKEN env / ~/.claude/.credentials.json
|
|
// by readAuthArtifact() inside the anthropic plugin at spawn time.
|
|
|
|
const RUN_E2E = process.env.OLP_RUN_E2E === '1';
|
|
|
|
if (!RUN_E2E) {
|
|
// Emit a skip notice to CI logs without failing the suite
|
|
process.stdout.write('::notice::Suite 10 (Anthropic E2E) skipped — set OLP_RUN_E2E=1 to run. Requires claude binary + keychain OAuth token. ~200 tokens per run.\n');
|
|
}
|
|
|
|
describe('Anthropic E2E — real claude spawn (Suite 10)', { skip: !RUN_E2E }, () => {
|
|
let e2eServer;
|
|
let e2ePort;
|
|
let e2eLoadedProviders;
|
|
let e2eCacheStore;
|
|
|
|
before(async () => {
|
|
if (!RUN_E2E) return;
|
|
|
|
__resetSpawnImpl(); // ensure real spawn is active
|
|
|
|
const { createOlpServer, loadedProviders: lp, cacheStore: cs } = await import('./server.mjs');
|
|
e2eLoadedProviders = lp;
|
|
e2eCacheStore = cs;
|
|
|
|
// Enable anthropic provider for E2E
|
|
const testProviders = loadProviders({ enabled: { anthropic: true } });
|
|
for (const [name, p] of testProviders) {
|
|
lp.set(name, p);
|
|
}
|
|
|
|
e2ePort = parseInt(process.env.OLP_E2E_PORT ?? String(19456 + Math.floor(Math.random() * 500)), 10);
|
|
e2eServer = createOlpServer();
|
|
await new Promise((resolve, reject) => {
|
|
e2eServer.listen(e2ePort, '127.0.0.1', resolve);
|
|
e2eServer.once('error', async (e) => {
|
|
if (e.code === 'EADDRINUSE') {
|
|
e2ePort++;
|
|
e2eServer.listen(e2ePort, '127.0.0.1', resolve);
|
|
e2eServer.once('error', reject);
|
|
} else reject(e);
|
|
});
|
|
});
|
|
});
|
|
|
|
after(() => {
|
|
if (!e2eServer) return;
|
|
return new Promise(r => e2eServer.close(r));
|
|
});
|
|
|
|
it('E2E: POST claude-haiku-4-5 with minimal prompt → 200 + "OK" content + correct headers', async () => {
|
|
const body = {
|
|
model: 'claude-haiku-4-5',
|
|
messages: [{ role: 'user', content: 'Reply with exactly the word OK and nothing else.' }],
|
|
max_tokens: 10,
|
|
};
|
|
|
|
const r = await fetch({ port: e2ePort, method: 'POST', path: '/v1/chat/completions', body });
|
|
assert.equal(r.status, 200, `Expected 200, got ${r.status}: ${r.body}`);
|
|
|
|
const respBody = JSON.parse(r.body);
|
|
const content = respBody?.choices?.[0]?.message?.content ?? '';
|
|
assert.ok(
|
|
content.toLowerCase().includes('ok'),
|
|
`Expected response to contain "OK", got: ${content}`,
|
|
);
|
|
|
|
// Provider/model headers
|
|
assert.equal(r.headers['x-olp-provider-used'], 'anthropic', `Expected anthropic provider`);
|
|
assert.equal(r.headers['x-olp-model-used'], 'claude-haiku-4-5', `Expected haiku model`);
|
|
|
|
// First request is always a miss
|
|
assert.equal(r.headers['x-olp-cache'], 'miss', `First request should be cache miss`);
|
|
|
|
// Fallback hops
|
|
assert.equal(r.headers['x-olp-fallback-hops'], '0', `Expected 0 fallback hops`);
|
|
|
|
// Second request — should hit cache, content identical
|
|
const r2 = await fetch({ port: e2ePort, method: 'POST', path: '/v1/chat/completions', body });
|
|
assert.equal(r2.status, 200, `Cache hit request should be 200`);
|
|
assert.equal(r2.headers['x-olp-cache'], 'hit', `Second request should be cache hit`);
|
|
|
|
const resp2Body = JSON.parse(r2.body);
|
|
const content2 = resp2Body?.choices?.[0]?.message?.content ?? '';
|
|
assert.ok(
|
|
content2.toLowerCase().includes('ok'),
|
|
`Cache hit response should also contain "OK", got: ${content2}`,
|
|
);
|
|
});
|
|
});
|
|
|
|
// ── Suite 12: Mistral Vibe plugin (D8) ───────────────────────────────────
|
|
//
|
|
// All tests are UNIT tests. No real `vibe` binary is invoked.
|
|
// Mock spawn is injected via mistralSetSpawnImpl / mistralResetSpawnImpl.
|
|
//
|
|
// Authority: Mistral Vibe docs (WebFetched 2026-05-23):
|
|
// DOCS-1: https://docs.mistral.ai/mistral-vibe/terminal/quickstart
|
|
// § "--prompt TEXT" + "--output FORMAT" — programmatic mode syntax
|
|
// DOCS-2: https://docs.mistral.ai/mistral-vibe/terminal/configuration
|
|
// § "~/.vibe/.env" — auth file; MISTRAL_API_KEY env var
|
|
// DOCS-5: https://mistral.ai/news/devstral-2-vibe-cli
|
|
// § "Devstral 2", "Devstral Small 2" — model names
|
|
// DOCS-6: https://help.mistral.ai/en/articles/347532
|
|
// § "Mistral Vibe is included in every Le Chat Pro subscription"
|
|
// DOCS-7: https://legal.mistral.ai/terms/usage-policy
|
|
// § No anti-third-party clauses (Tier D confirmed for ADR 0006)
|
|
//
|
|
// Spec assumption acknowledgements (see mistral.mjs header):
|
|
// A1 CONFIRMED: `vibe --prompt "PROMPT" --output json` spawn shape
|
|
// A2 CONFIRMED: MISTRAL_API_KEY env var is the auth mechanism
|
|
// A3 CONFIRMED: ~/.vibe/.env is the auth file path
|
|
// A4 UNPINNED: JSON output event schema — defensive 4-shape parser used
|
|
// A5 UNPINNED: --model flag existence not confirmed by docs
|
|
// A6 UNPINNED: exact model IDs (devstral-2, devstral-small-2) best-effort
|
|
// A7 UNPINNED: --output json is NDJSON not single blob
|
|
// A8 UNPINNED: multi-line prompt handling via --prompt flag
|
|
//
|
|
// Lossy-translation acknowledgements per ADR 0003 (documented in mistral.mjs header):
|
|
// top_p, temperature, stop, max_tokens, tools[], tool_calls → all dropped.
|
|
|
|
/**
|
|
* Creates a fake JSON-line-emitting mock spawn for Mistral Vibe.
|
|
* Lines are emitted as they would arrive from `vibe --output json`
|
|
* (D8 assumption A7: NDJSON lines — D-later will pin the real format).
|
|
*
|
|
* @param {string[]} jsonLines — raw JSON lines emitted in order (no trailing \n needed)
|
|
* @param {number} [exitCode=0]
|
|
*/
|
|
function makeMockMistralSpawn(jsonLines, exitCode = 0) {
|
|
return function mockMistralSpawnImpl(_bin, _args, _opts) {
|
|
const proc = new EventEmitter();
|
|
proc.stdout = new EventEmitter();
|
|
proc.stderr = new EventEmitter();
|
|
proc.stdin = {
|
|
write: () => {},
|
|
end: () => {
|
|
setImmediate(async () => {
|
|
for (const line of jsonLines) {
|
|
proc.stdout.emit('data', Buffer.from(line + '\n'));
|
|
}
|
|
proc.stdout.emit('end');
|
|
proc.stderr.emit('end');
|
|
proc.emit('close', exitCode, null);
|
|
});
|
|
},
|
|
};
|
|
proc.killed = false;
|
|
proc.kill = () => {};
|
|
return proc;
|
|
};
|
|
}
|
|
|
|
describe('Mistral Vibe plugin (D8)', () => {
|
|
|
|
// ── Test 1: Contract conformance ──────────────────────────────────────
|
|
it('mistral module satisfies validateProvider() — all 10 fields present', () => {
|
|
const { valid, errors } = validateProvider(mistral);
|
|
assert.equal(valid, true, `Validation errors: ${errors.join('; ')}`);
|
|
assert.ok('name' in mistral, 'missing: name');
|
|
assert.ok('displayName' in mistral, 'missing: displayName');
|
|
assert.ok('contractVersion' in mistral, 'missing: contractVersion');
|
|
assert.ok('models' in mistral, 'missing: models');
|
|
assert.ok('auth' in mistral, 'missing: auth');
|
|
assert.ok(typeof mistral.spawn === 'function', 'missing: spawn');
|
|
assert.ok(typeof mistral.estimateCost === 'function', 'missing: estimateCost');
|
|
assert.ok(typeof mistral.quotaStatus === 'function', 'missing: quotaStatus');
|
|
assert.ok(typeof mistral.healthCheck === 'function', 'missing: healthCheck');
|
|
assert.ok('hints' in mistral, 'missing: hints');
|
|
});
|
|
|
|
// ── Test 2: contractVersion === '1.0' ────────────────────────────────
|
|
it('mistral declares contractVersion === "1.0"', () => {
|
|
assert.equal(mistral.contractVersion, '1.0');
|
|
});
|
|
|
|
// ── Test 3: name and displayName ─────────────────────────────────────
|
|
it('mistral.name === "mistral" and displayName is set', () => {
|
|
assert.equal(mistral.name, 'mistral');
|
|
assert.equal(typeof mistral.displayName, 'string');
|
|
assert.ok(mistral.displayName.length > 0);
|
|
assert.ok(mistral.displayName.toLowerCase().includes('mistral'));
|
|
});
|
|
|
|
// ── Test 4: models include all registry IDs + aliases ────────────────
|
|
it('mistral.models includes every canonical registry id AND every alias', () => {
|
|
// The plugin merges canonical IDs and alias keys into models[] so that
|
|
// getProviderForModel routes either form. Test that the registry's
|
|
// canonical IDs are a subset, and the aliases are all included too.
|
|
const registryIds = modelsRegistry.providers.mistral.models.map(m => m.id);
|
|
const registryAliases = Object.keys(modelsRegistry.providers.mistral.aliases ?? {});
|
|
for (const id of registryIds) {
|
|
assert.ok(mistral.models.includes(id), `canonical id ${id} missing from mistral.models`);
|
|
}
|
|
for (const alias of registryAliases) {
|
|
assert.ok(mistral.models.includes(alias), `alias ${alias} missing from mistral.models`);
|
|
}
|
|
assert.equal(mistral.models.length, registryIds.length + registryAliases.length);
|
|
});
|
|
|
|
it('mistral.models contains canonical IDs + short-form aliases', () => {
|
|
// Per canonical models registry (docs.mistral.ai/getting-started/models/
|
|
// models_overview, D8 review-2 finding): canonical IDs are date-stamped
|
|
// (devstral-2-25-12, devstral-small-2-25-12). Vibe config.toml uses short
|
|
// forms (devstral-2, devstral-small-2). Plugin exposes BOTH so
|
|
// getProviderForModel routes either form.
|
|
assert.ok(mistral.models.includes('devstral-2-25-12'), 'missing canonical devstral-2-25-12');
|
|
assert.ok(mistral.models.includes('devstral-small-2-25-12'), 'missing canonical devstral-small-2-25-12');
|
|
assert.ok(mistral.models.includes('devstral-2'), 'missing short-form alias devstral-2');
|
|
assert.ok(mistral.models.includes('devstral-small-2'), 'missing short-form alias devstral-small-2');
|
|
assert.ok(mistral.models.includes('devstral'), 'missing alias devstral');
|
|
assert.ok(mistral.models.includes('devstral-small'), 'missing alias devstral-small');
|
|
// Length: 2 canonical + 4 aliases = 6 total
|
|
assert.equal(mistral.models.length, 6, `Expected 6 entries (2 canonical + 4 aliases), got ${mistral.models.length}`);
|
|
});
|
|
|
|
// ── Test 5: getProviderForModel finds mistral for each model ──────────
|
|
it('getProviderForModel finds mistral for devstral-2', () => {
|
|
const loaded = new Map([['mistral', mistral]]);
|
|
const result = getProviderForModel(loaded, 'devstral-2');
|
|
assert.ok(result !== null);
|
|
assert.equal(result.name, 'mistral');
|
|
});
|
|
|
|
it('getProviderForModel finds mistral for devstral-small-2', () => {
|
|
const loaded = new Map([['mistral', mistral]]);
|
|
const result = getProviderForModel(loaded, 'devstral-small-2');
|
|
assert.ok(result !== null);
|
|
assert.equal(result.name, 'mistral');
|
|
});
|
|
|
|
// ── Test 6: irToMistral translation ──────────────────────────────────
|
|
it('irToMistral: user message → args with --prompt and --output streaming', () => {
|
|
// Authority: DOCS-1 § "Output Format Options" — `streaming` is the
|
|
// newline-delimited JSON per message mode. `json` emits a single blob
|
|
// at the end (D8 review-2 finding: original draft used `json`,
|
|
// incompatible with the line-buffered stdout parser; corrected to
|
|
// `streaming` per docs verbatim).
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
messages: [{ role: 'user', content: 'Hello world' }],
|
|
});
|
|
const { args, prompt } = irToMistral(ir);
|
|
assert.ok(args.includes('--prompt'), 'args must include "--prompt"');
|
|
assert.ok(args.includes('--output'), 'args must include "--output"');
|
|
assert.ok(args.includes('streaming'), 'args must include "streaming" output format (NDJSON per docs)');
|
|
assert.ok(!args.includes('json'), 'args must NOT include "json" (single-blob mode, incompatible with line-buffered parser)');
|
|
// D8 assumption A5: --model NOT in args (flag existence unconfirmed from docs)
|
|
assert.ok(!args.includes('--model'), 'args must NOT include "--model" at D8 (A5 UNPINNED)');
|
|
assert.ok(typeof prompt === 'string', 'prompt must be a string');
|
|
assert.ok(prompt.includes('Hello world'), 'prompt must contain user text');
|
|
});
|
|
|
|
it('irToMistral: system + user → system annotation + user text in prompt', () => {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
messages: [
|
|
{ role: 'system', content: 'You are a coding assistant.' },
|
|
{ role: 'user', content: 'Write a function.' },
|
|
],
|
|
});
|
|
const { prompt } = irToMistral(ir);
|
|
assert.ok(prompt.includes('[System] You are a coding assistant.'));
|
|
assert.ok(prompt.includes('Write a function.'));
|
|
});
|
|
|
|
it('irToMistral: assistant prior turn → [Assistant] annotation', () => {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
messages: [
|
|
{ role: 'user', content: 'Hi' },
|
|
{ role: 'assistant', content: 'Hello!' },
|
|
{ role: 'user', content: 'Thanks' },
|
|
],
|
|
});
|
|
const { prompt } = irToMistral(ir);
|
|
assert.ok(prompt.includes('[Assistant] Hello!'));
|
|
assert.ok(prompt.includes('Thanks'));
|
|
});
|
|
|
|
it('irToMistral: tool result turn → [Tool Result] annotation', () => {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
messages: [
|
|
{ role: 'user', content: 'Search for X' },
|
|
{ role: 'tool', content: '{"results":[]}', name: 'search' },
|
|
],
|
|
});
|
|
const { prompt } = irToMistral(ir);
|
|
assert.ok(prompt.includes('[Tool Result'));
|
|
assert.ok(prompt.includes('search'));
|
|
});
|
|
|
|
it('irToMistral: response_format json_object injects system prompt (lossy)', () => {
|
|
// ADR 0003 § Lossy: Vibe CLI does not honor response_format natively.
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
messages: [{ role: 'user', content: 'Give JSON' }],
|
|
response_format: { type: 'json_object' },
|
|
});
|
|
const { prompt } = irToMistral(ir);
|
|
assert.ok(prompt.includes('Reply with valid JSON only'));
|
|
});
|
|
|
|
it('irToMistral: tool_calls in assistant message — content preserved, metadata dropped (lossy)', () => {
|
|
// ADR 0003 § Lossy: structured tool_calls dropped; textual content preserved.
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
messages: [
|
|
{ role: 'user', content: 'Search for X' },
|
|
{
|
|
role: 'assistant',
|
|
content: 'Searching...',
|
|
tool_calls: [{ id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"X"}' } }],
|
|
},
|
|
],
|
|
});
|
|
const { prompt } = irToMistral(ir);
|
|
assert.ok(prompt.includes('Searching...'), 'assistant text content must be preserved');
|
|
// tool_calls metadata documented as dropped (lossy) — do NOT assert it's present
|
|
});
|
|
|
|
it('irToMistral: array content is JSON-stringified', () => {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }],
|
|
});
|
|
const { prompt } = irToMistral(ir);
|
|
assert.ok(typeof prompt === 'string');
|
|
assert.ok(prompt.includes('text'));
|
|
});
|
|
|
|
// ── Test 7: mistralChunkToIR translation ──────────────────────────────
|
|
it('mistralChunkToIR: text field → delta chunk (Mistral La Plateforme shape)', () => {
|
|
// D8 assumption A4: "text" is the preferred field name.
|
|
const chunk = mistralChunkToIR('{"text":"Hello world"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.content, 'Hello world');
|
|
});
|
|
|
|
it('mistralChunkToIR: content field → delta chunk (OpenAI-compat shape)', () => {
|
|
// D8 assumption A4: "content" as fallback field name.
|
|
const chunk = mistralChunkToIR('{"content":"Hello world"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.content, 'Hello world');
|
|
});
|
|
|
|
it('mistralChunkToIR: delta field shape → delta chunk', () => {
|
|
// D8 assumption A4: { type: 'delta', delta: '...' } shape.
|
|
const chunk = mistralChunkToIR('{"type":"delta","delta":"token text"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.content, 'token text');
|
|
});
|
|
|
|
it('mistralChunkToIR: OpenAI streaming choices[0].delta.content shape → delta', () => {
|
|
// D8 assumption A4: Vibe may use OpenAI streaming event shape.
|
|
const chunk = mistralChunkToIR('{"choices":[{"delta":{"content":"output"},"finish_reason":null}]}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'delta');
|
|
assert.equal(chunk.content, 'output');
|
|
});
|
|
|
|
it('mistralChunkToIR: type === "stop" → stop chunk', () => {
|
|
const chunk = mistralChunkToIR('{"type":"stop"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'stop');
|
|
assert.equal(chunk.finish_reason, 'stop');
|
|
});
|
|
|
|
it('mistralChunkToIR: done === true → stop chunk', () => {
|
|
const chunk = mistralChunkToIR('{"done":true}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'stop');
|
|
assert.equal(chunk.finish_reason, 'stop');
|
|
});
|
|
|
|
it('mistralChunkToIR: OpenAI streaming stop via choices[0].finish_reason === "stop"', () => {
|
|
// D8 assumption A4: OpenAI streaming stop shape.
|
|
const chunk = mistralChunkToIR('{"choices":[{"delta":{},"finish_reason":"stop"}]}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'stop');
|
|
assert.equal(chunk.finish_reason, 'stop');
|
|
});
|
|
|
|
it('mistralChunkToIR: type === "error" → error chunk', () => {
|
|
const chunk = mistralChunkToIR('{"type":"error","error":"quota exceeded"}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'error');
|
|
assert.equal(chunk.error, 'quota exceeded');
|
|
});
|
|
|
|
it('mistralChunkToIR: error field present → error chunk', () => {
|
|
const chunk = mistralChunkToIR('{"error":"something went wrong","code":429}');
|
|
assert.ok(chunk !== null);
|
|
assert.equal(chunk.type, 'error');
|
|
});
|
|
|
|
it('mistralChunkToIR: unknown/progress event type → null (silently ignored)', () => {
|
|
const chunk = mistralChunkToIR('{"type":"progress","step":1}');
|
|
assert.equal(chunk, null);
|
|
});
|
|
|
|
it('mistralChunkToIR: empty line → null', () => {
|
|
assert.equal(mistralChunkToIR(''), null);
|
|
assert.equal(mistralChunkToIR(' '), null);
|
|
});
|
|
|
|
it('mistralChunkToIR: malformed JSON → null (no throw)', () => {
|
|
assert.doesNotThrow(() => {
|
|
const result = mistralChunkToIR('{bad json');
|
|
assert.equal(result, null);
|
|
});
|
|
});
|
|
|
|
// ── Test 8: mock spawn — JSON stream yields correct IR chunks ──────────
|
|
it('spawn with mock: JSON lines → delta chunks then stop chunk', async () => {
|
|
const fakeSpawn = makeMockMistralSpawn([
|
|
'{"text":"Hello"}',
|
|
'{"text":" world"}',
|
|
'{"type":"stop"}',
|
|
]);
|
|
mistralSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
const authCtx = { apiKey: '<fake-mistral-api-key>' };
|
|
const chunks = [];
|
|
for await (const chunk of mistral.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const deltas = chunks.filter(c => c.type === 'delta');
|
|
const stops = chunks.filter(c => c.type === 'stop');
|
|
assert.ok(deltas.length >= 1, `Expected at least 1 delta, got ${deltas.length}`);
|
|
assert.equal(stops.length, 1, `Expected 1 stop, got ${stops.length}`);
|
|
const allContent = deltas.map(c => c.content).join('');
|
|
assert.equal(allContent, 'Hello world');
|
|
} finally {
|
|
mistralResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: first delta chunk has role=assistant', async () => {
|
|
const fakeSpawn = makeMockMistralSpawn(['{"text":"Test output"}']);
|
|
mistralSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
});
|
|
const authCtx = { apiKey: '<fake-mistral-api-key>' };
|
|
const chunks = [];
|
|
for await (const chunk of mistral.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const firstDelta = chunks.find(c => c.type === 'delta');
|
|
assert.ok(firstDelta, 'No delta chunk found');
|
|
assert.equal(firstDelta.role, 'assistant');
|
|
} finally {
|
|
mistralResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: stop event present → no extra synthetic stop appended', async () => {
|
|
const fakeSpawn = makeMockMistralSpawn([
|
|
'{"text":"done"}',
|
|
'{"type":"stop"}',
|
|
]);
|
|
mistralSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
stream: false,
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
});
|
|
const authCtx = { apiKey: '<fake-mistral-api-key>' };
|
|
const chunks = [];
|
|
for await (const chunk of mistral.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const stops = chunks.filter(c => c.type === 'stop');
|
|
assert.equal(stops.length, 1, `Expected exactly 1 stop chunk, got ${stops.length}`);
|
|
} finally {
|
|
mistralResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: synthetic stop emitted when JSON stream has no stop event', async () => {
|
|
const fakeSpawn = makeMockMistralSpawn([
|
|
'{"text":"only content, no stop"}',
|
|
]);
|
|
mistralSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hello' }],
|
|
});
|
|
const authCtx = { apiKey: '<fake-mistral-api-key>' };
|
|
const chunks = [];
|
|
for await (const chunk of mistral.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const stops = chunks.filter(c => c.type === 'stop');
|
|
assert.equal(stops.length, 1, 'Should have exactly 1 synthetic stop chunk');
|
|
assert.equal(stops[0].finish_reason, 'stop');
|
|
} finally {
|
|
mistralResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: non-zero exit code throws ProviderError(SPAWN_FAILED)', async () => {
|
|
const fakeSpawn = makeMockMistralSpawn([], 1);
|
|
mistralSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
const authCtx = { apiKey: '<fake-mistral-api-key>' };
|
|
let caught = null;
|
|
try {
|
|
for await (const _chunk of mistral.spawn(ir, authCtx)) { // eslint-disable-line no-unused-vars
|
|
// drain
|
|
}
|
|
} catch (e) {
|
|
caught = e;
|
|
}
|
|
assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`);
|
|
assert.equal(caught.code, 'SPAWN_FAILED');
|
|
} finally {
|
|
mistralResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn throws ProviderError(AUTH_MISSING) when no auth context and no env/file', async () => {
|
|
const fakeSpawn = makeMockMistralSpawn(['{"text":"test"}']);
|
|
mistralSetSpawnImpl(fakeSpawn);
|
|
// Override auth path to nonexistent + clear MISTRAL_API_KEY env to guarantee missing auth
|
|
const savedApiKey = process.env.MISTRAL_API_KEY;
|
|
const savedAuthPath = process.env.MISTRAL_VIBE_AUTH_PATH;
|
|
delete process.env.MISTRAL_API_KEY;
|
|
process.env.MISTRAL_VIBE_AUTH_PATH = '/nonexistent/path/.env';
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
stream: false,
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
});
|
|
let caught = null;
|
|
try {
|
|
for await (const _chunk of mistral.spawn(ir, null)) { // eslint-disable-line no-unused-vars
|
|
// drain
|
|
}
|
|
} catch (e) {
|
|
caught = e;
|
|
}
|
|
assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`);
|
|
assert.equal(caught.code, 'AUTH_MISSING');
|
|
} finally {
|
|
if (savedApiKey !== undefined) process.env.MISTRAL_API_KEY = savedApiKey;
|
|
else delete process.env.MISTRAL_API_KEY;
|
|
if (savedAuthPath !== undefined) process.env.MISTRAL_VIBE_AUTH_PATH = savedAuthPath;
|
|
else delete process.env.MISTRAL_VIBE_AUTH_PATH;
|
|
mistralResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('spawn with mock: progress events silently ignored, only content emitted', async () => {
|
|
const fakeSpawn = makeMockMistralSpawn([
|
|
'{"type":"progress","step":1}',
|
|
'{"type":"progress","step":2}',
|
|
'{"text":"actual response"}',
|
|
'{"type":"stop"}',
|
|
]);
|
|
mistralSetSpawnImpl(fakeSpawn);
|
|
try {
|
|
const ir = makeIR({
|
|
model: 'devstral-2',
|
|
stream: true,
|
|
messages: [{ role: 'user', content: 'Do something' }],
|
|
});
|
|
const authCtx = { apiKey: '<fake-mistral-api-key>' };
|
|
const chunks = [];
|
|
for await (const chunk of mistral.spawn(ir, authCtx)) {
|
|
chunks.push(chunk);
|
|
}
|
|
const deltas = chunks.filter(c => c.type === 'delta');
|
|
assert.equal(deltas.length, 1);
|
|
assert.equal(deltas[0].content, 'actual response');
|
|
} finally {
|
|
mistralResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
// ── Test 9: healthCheck ───────────────────────────────────────────────
|
|
it('healthCheck returns {ok: false, error: "vibe binary not found"} when binary absent', async () => {
|
|
const result = await mistralHealthCheck({
|
|
_binaryExistsFn: () => false,
|
|
_authReadFn: () => ({ apiKey: '<fake-key>' }),
|
|
});
|
|
assert.equal(result.ok, false);
|
|
assert.equal(result.error, 'vibe binary not found');
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
it('healthCheck returns {ok: false, error: "auth artifact missing"} when auth missing', async () => {
|
|
const result = await mistralHealthCheck({
|
|
_binaryExistsFn: () => true,
|
|
_authReadFn: () => null,
|
|
});
|
|
assert.equal(result.ok, false);
|
|
assert.equal(result.error, 'auth artifact missing');
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
it('healthCheck returns {ok: true} when binary and auth both present', async () => {
|
|
const result = await mistralHealthCheck({
|
|
_binaryExistsFn: () => true,
|
|
_authReadFn: () => ({ apiKey: '<fake-key>' }),
|
|
});
|
|
assert.equal(result.ok, true);
|
|
assert.ok(typeof result.latencyMs === 'number');
|
|
});
|
|
|
|
// ── Test 10: estimateCost ─────────────────────────────────────────────
|
|
it('estimateCost returns shape with currency USD', () => {
|
|
const request = makeIR({
|
|
model: 'devstral-2',
|
|
messages: [
|
|
{ role: 'system', content: 'You are a coding assistant.' },
|
|
{ role: 'user', content: 'Write hello world in Python.' },
|
|
],
|
|
});
|
|
const result = mistralEstimateCost(request);
|
|
assert.ok(result !== null, 'estimateCost returned null');
|
|
assert.ok('inputTokens' in result, 'missing inputTokens');
|
|
assert.ok('outputTokensEstimate' in result, 'missing outputTokensEstimate');
|
|
assert.ok('currency' in result, 'missing currency');
|
|
assert.ok('usd' in result, 'missing usd');
|
|
assert.equal(result.currency, 'USD');
|
|
assert.equal(result.usd, null); // not pinned at D8
|
|
assert.ok(result.inputTokens > 0, 'inputTokens should be > 0');
|
|
assert.ok(result.outputTokensEstimate >= 0);
|
|
});
|
|
|
|
it('estimateCost returns null for null/missing request', () => {
|
|
assert.equal(mistralEstimateCost(null), null);
|
|
assert.equal(mistralEstimateCost({}), null);
|
|
});
|
|
|
|
// ── Test 11: quotaStatus ──────────────────────────────────────────────
|
|
it('quotaStatus returns null at D8 (Le Chat Pro budget not exposed via API)', async () => {
|
|
const result = await mistralQuotaStatus({});
|
|
assert.equal(result, null);
|
|
});
|
|
|
|
// ── Test 12: auth object shape ────────────────────────────────────────
|
|
it('mistral.auth has correct shape', () => {
|
|
// Authority: DOCS-2 § auth type is api-key, path is ~/.vibe/.env
|
|
assert.equal(mistral.auth.type, 'api-key');
|
|
assert.equal(mistral.auth.storage, 'file');
|
|
assert.equal(typeof mistral.auth.path, 'string');
|
|
assert.ok(mistral.auth.path.includes('.vibe'), 'auth.path should reference .vibe directory');
|
|
assert.ok(mistral.auth.path.includes('.env'), 'auth.path should reference .env file');
|
|
// Portability check: path must start with homedir() (no hardcoded literal)
|
|
assert.ok(
|
|
mistral.auth.path.startsWith(homedir()),
|
|
`auth.path "${mistral.auth.path}" should start with homedir() "${homedir()}"`,
|
|
);
|
|
});
|
|
|
|
// ── Test 13: hints shape ──────────────────────────────────────────────
|
|
it('mistral.hints has correct shape', () => {
|
|
assert.equal(typeof mistral.hints.requiresTTY, 'boolean');
|
|
assert.equal(typeof mistral.hints.concurrentSpawnSafe, 'boolean');
|
|
assert.ok(Number.isInteger(mistral.hints.maxConcurrent) && mistral.hints.maxConcurrent > 0);
|
|
assert.equal(mistral.hints.requiresTTY, false);
|
|
assert.equal(mistral.hints.concurrentSpawnSafe, true);
|
|
});
|
|
|
|
// ── Test 14: STATIC_REGISTRY length after D8 ─────────────────────────
|
|
it('STATIC_REGISTRY.length === 3 after D8 (anthropic + openai + mistral)', () => {
|
|
assert.equal(listAllProviderNames().length, 3);
|
|
assert.ok(listAllProviderNames().includes('anthropic'));
|
|
assert.ok(listAllProviderNames().includes('openai'));
|
|
assert.ok(listAllProviderNames().includes('mistral'));
|
|
});
|
|
|
|
// ── Test 15: loadProviders with mistral enabled ───────────────────────
|
|
it('loadProviders with {enabled: {mistral: true}} returns Map of size 1 with mistral', () => {
|
|
const loaded = loadProviders({ enabled: { mistral: true } });
|
|
assert.equal(loaded.size, 1);
|
|
assert.ok(loaded.has('mistral'));
|
|
});
|
|
|
|
it('loadProviders with all 3 providers enabled returns Map of size 3', () => {
|
|
const loaded = loadProviders({ enabled: { anthropic: true, openai: true, mistral: true } });
|
|
assert.equal(loaded.size, 3);
|
|
assert.ok(loaded.has('anthropic'));
|
|
assert.ok(loaded.has('openai'));
|
|
assert.ok(loaded.has('mistral'));
|
|
});
|
|
|
|
it('mistral loaded via loadProviders passes contract validation', () => {
|
|
const loaded = loadProviders({ enabled: { mistral: true } });
|
|
const p = loaded.get('mistral');
|
|
const { valid, errors } = validateProvider(p);
|
|
assert.equal(valid, true, `Contract errors: ${errors.join('; ')}`);
|
|
assert.ok(p.models.includes('devstral-2'));
|
|
});
|
|
|
|
// ── Test 16: auth artifact reading helpers ────────────────────────────
|
|
it('readAuthArtifact: MISTRAL_API_KEY env var → returns {apiKey}', () => {
|
|
const saved = process.env.MISTRAL_API_KEY;
|
|
process.env.MISTRAL_API_KEY = '<fake-mistral-api-key>';
|
|
try {
|
|
const result = mistralReadAuthArtifact();
|
|
assert.ok(result !== null, 'Expected auth result');
|
|
assert.equal(result.apiKey, '<fake-mistral-api-key>');
|
|
} finally {
|
|
if (saved !== undefined) process.env.MISTRAL_API_KEY = saved;
|
|
else delete process.env.MISTRAL_API_KEY;
|
|
}
|
|
});
|
|
|
|
it('readAuthArtifact: MISTRAL_VIBE_AUTH_PATH pointing to nonexistent file → null', () => {
|
|
const savedApiKey = process.env.MISTRAL_API_KEY;
|
|
const savedAuthPath = process.env.MISTRAL_VIBE_AUTH_PATH;
|
|
delete process.env.MISTRAL_API_KEY;
|
|
process.env.MISTRAL_VIBE_AUTH_PATH = '/definitely/not/a/real/path/.env';
|
|
try {
|
|
const result = mistralReadAuthArtifact();
|
|
assert.equal(result, null);
|
|
} finally {
|
|
if (savedApiKey !== undefined) process.env.MISTRAL_API_KEY = savedApiKey;
|
|
else delete process.env.MISTRAL_API_KEY;
|
|
if (savedAuthPath !== undefined) process.env.MISTRAL_VIBE_AUTH_PATH = savedAuthPath;
|
|
else delete process.env.MISTRAL_VIBE_AUTH_PATH;
|
|
}
|
|
});
|
|
|
|
// ── Test 17: Suite 5 registry test updated for D8 ─────────────────────
|
|
// (This re-tests the registry length assertion which now expects 3.)
|
|
it('listAllProviderNames() now returns 3-element array with mistral included', () => {
|
|
const names = listAllProviderNames();
|
|
assert.equal(names.length, 3);
|
|
assert.deepEqual(names, ['anthropic', 'openai', 'mistral']);
|
|
});
|
|
|
|
});
|
|
|
|
// ── Suite 13: Fallback engine (D9) ───────────────────────────────────────
|
|
//
|
|
// All tests are unit tests. No real provider CLIs are invoked.
|
|
// Tests cover:
|
|
// 13a: Trigger taxonomy (evaluateHardTriggers / evaluateSoftTriggers)
|
|
// 13b: executeWithFallback engine behaviour
|
|
// 13c: First-chunk safety (buffering semantics at D9)
|
|
// 13d: Soft trigger skipping spawn
|
|
// 13e: Header annotation (providerUsed / modelUsed / fallbackHops / originalError)
|
|
// 13f: HTTP integration with __setFallbackConfig test seam
|
|
//
|
|
// Authority: ADR 0004 — Fallback Engine Semantics and Safety
|
|
|
|
import {
|
|
evaluateHardTriggers,
|
|
evaluateSoftTriggers,
|
|
executeWithFallback,
|
|
buildDefaultChain,
|
|
loadFallbackConfigSync,
|
|
isClientError,
|
|
} from './lib/fallback/engine.mjs';
|
|
|
|
import {
|
|
createOlpServer,
|
|
__setFallbackConfig,
|
|
__resetFallbackConfig,
|
|
__clearCache,
|
|
} from './server.mjs';
|
|
|
|
// ── 13a: Trigger taxonomy ────────────────────────────────────────────────
|
|
|
|
describe('Fallback engine — trigger taxonomy (D9)', () => {
|
|
|
|
// ── evaluateHardTriggers ─────────────────────────────────────────────
|
|
|
|
it('evaluateHardTriggers: HTTP 500 → fires (5xx hard trigger)', () => {
|
|
const err = Object.assign(new Error('Server error'), { statusCode: 500 });
|
|
assert.equal(evaluateHardTriggers(err), true);
|
|
});
|
|
|
|
it('evaluateHardTriggers: HTTP 503 → fires (5xx hard trigger)', () => {
|
|
const err = Object.assign(new Error('Service unavailable'), { statusCode: 503 });
|
|
assert.equal(evaluateHardTriggers(err), true);
|
|
});
|
|
|
|
it('evaluateHardTriggers: HTTP 400 → does NOT fire (client error)', () => {
|
|
const err = Object.assign(new Error('Bad request'), { statusCode: 400 });
|
|
assert.equal(evaluateHardTriggers(err), false);
|
|
});
|
|
|
|
it('evaluateHardTriggers: HTTP 401 → does NOT fire (client error)', () => {
|
|
const err = Object.assign(new Error('Unauthorized'), { statusCode: 401 });
|
|
assert.equal(evaluateHardTriggers(err), false);
|
|
});
|
|
|
|
it('evaluateHardTriggers: HTTP 403 → does NOT fire (client error)', () => {
|
|
const err = Object.assign(new Error('Forbidden'), { statusCode: 403 });
|
|
assert.equal(evaluateHardTriggers(err), false);
|
|
});
|
|
|
|
it('evaluateHardTriggers: HTTP 404 → does NOT fire (client error)', () => {
|
|
const err = Object.assign(new Error('Not found'), { statusCode: 404 });
|
|
assert.equal(evaluateHardTriggers(err), false);
|
|
});
|
|
|
|
it('evaluateHardTriggers: HTTP 422 → does NOT fire (client error)', () => {
|
|
const err = Object.assign(new Error('Unprocessable'), { statusCode: 422 });
|
|
assert.equal(evaluateHardTriggers(err), false);
|
|
});
|
|
|
|
it('evaluateHardTriggers: HTTP 429 → fires (quota-like 4xx not in client-error set)', () => {
|
|
const err = Object.assign(new Error('Too Many Requests'), { statusCode: 429 });
|
|
assert.equal(evaluateHardTriggers(err), true);
|
|
});
|
|
|
|
it('evaluateHardTriggers: ProviderError QUOTA_EXHAUSTED → fires', () => {
|
|
const err = new ProviderError('Quota exhausted', 'QUOTA_EXHAUSTED');
|
|
assert.equal(evaluateHardTriggers(err), true);
|
|
});
|
|
|
|
it('evaluateHardTriggers: ProviderError RATE_LIMITED → fires', () => {
|
|
const err = new ProviderError('Rate limited', 'RATE_LIMITED');
|
|
assert.equal(evaluateHardTriggers(err), true);
|
|
});
|
|
|
|
it('evaluateHardTriggers: ProviderError SPAWN_FAILED → fires', () => {
|
|
const err = new ProviderError('Spawn failed', 'SPAWN_FAILED');
|
|
assert.equal(evaluateHardTriggers(err), true);
|
|
});
|
|
|
|
it('evaluateHardTriggers: ProviderError CLI_NOT_FOUND → fires', () => {
|
|
const err = new ProviderError('CLI not found', 'CLI_NOT_FOUND');
|
|
assert.equal(evaluateHardTriggers(err), true);
|
|
});
|
|
|
|
it('evaluateHardTriggers: ProviderError AUTH_MISSING → does NOT fire (user must fix)', () => {
|
|
const err = new ProviderError('Auth missing', 'AUTH_MISSING');
|
|
assert.equal(evaluateHardTriggers(err), false);
|
|
});
|
|
|
|
it('evaluateHardTriggers: ProviderError OUTPUT_PARSE_ERROR → fires', () => {
|
|
const err = new ProviderError('Parse error', 'OUTPUT_PARSE_ERROR');
|
|
assert.equal(evaluateHardTriggers(err), true);
|
|
});
|
|
|
|
it('evaluateHardTriggers: generic Error with no statusCode → does NOT fire', () => {
|
|
const err = new Error('Something went wrong');
|
|
assert.equal(evaluateHardTriggers(err), false);
|
|
});
|
|
|
|
it('evaluateHardTriggers: null error → does NOT fire', () => {
|
|
assert.equal(evaluateHardTriggers(null), false);
|
|
});
|
|
|
|
// ── evaluateSoftTriggers ─────────────────────────────────────────────
|
|
|
|
it('evaluateSoftTriggers: credit_pool_percent_threshold at exactly threshold → fires', () => {
|
|
const cfg = { credit_pool_percent_threshold: 90 };
|
|
const quota = { percentUsed: 90 };
|
|
assert.equal(evaluateSoftTriggers(cfg, quota), true);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: credit_pool_percent_threshold above threshold → fires', () => {
|
|
const cfg = { credit_pool_percent_threshold: 90 };
|
|
const quota = { percentUsed: 95 };
|
|
assert.equal(evaluateSoftTriggers(cfg, quota), true);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: credit_pool_percent_threshold below threshold → does NOT fire', () => {
|
|
const cfg = { credit_pool_percent_threshold: 90 };
|
|
const quota = { percentUsed: 89 };
|
|
assert.equal(evaluateSoftTriggers(cfg, quota), false);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: daily_request_count_threshold at threshold → fires', () => {
|
|
const cfg = { daily_request_count_threshold: 500 };
|
|
const quota = { dailyCount: 500 };
|
|
assert.equal(evaluateSoftTriggers(cfg, quota), true);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: daily_request_count_threshold below threshold → does NOT fire', () => {
|
|
const cfg = { daily_request_count_threshold: 500 };
|
|
const quota = { dailyCount: 499 };
|
|
assert.equal(evaluateSoftTriggers(cfg, quota), false);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: five_hour_window_percent_threshold at threshold → fires', () => {
|
|
const cfg = { five_hour_window_percent_threshold: 85 };
|
|
const quota = { fiveHourWindowPercent: 85 };
|
|
assert.equal(evaluateSoftTriggers(cfg, quota), true);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: null quotaSnapshot → does NOT fire (graceful degrade)', () => {
|
|
const cfg = { credit_pool_percent_threshold: 90 };
|
|
assert.equal(evaluateSoftTriggers(cfg, null), false);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: undefined quotaSnapshot → does NOT fire (graceful degrade)', () => {
|
|
const cfg = { credit_pool_percent_threshold: 90 };
|
|
assert.equal(evaluateSoftTriggers(cfg, undefined), false);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: null triggerConfig → does NOT fire', () => {
|
|
const quota = { percentUsed: 95 };
|
|
assert.equal(evaluateSoftTriggers(null, quota), false);
|
|
});
|
|
|
|
it('evaluateSoftTriggers: empty triggerConfig → does NOT fire', () => {
|
|
const quota = { percentUsed: 95 };
|
|
assert.equal(evaluateSoftTriggers({}, quota), false);
|
|
});
|
|
|
|
// ── isClientError ────────────────────────────────────────────────────
|
|
|
|
it('isClientError: 400 → true', () => { assert.equal(isClientError(400), true); });
|
|
it('isClientError: 401 → true', () => { assert.equal(isClientError(401), true); });
|
|
it('isClientError: 403 → true', () => { assert.equal(isClientError(403), true); });
|
|
it('isClientError: 404 → true', () => { assert.equal(isClientError(404), true); });
|
|
it('isClientError: 422 → true', () => { assert.equal(isClientError(422), true); });
|
|
it('isClientError: 429 → false (quota, not client error)', () => { assert.equal(isClientError(429), false); });
|
|
it('isClientError: 500 → false', () => { assert.equal(isClientError(500), false); });
|
|
|
|
});
|
|
|
|
// ── 13b: executeWithFallback engine ─────────────────────────────────────
|
|
|
|
describe('Fallback engine — executeWithFallback (D9)', () => {
|
|
|
|
// helper: build a mock executeHopFn that succeeds or throws per provider name
|
|
function makeHopFn(outcomes) {
|
|
// outcomes: { [provider]: 'success' | Error }
|
|
return async function (provider, model, _ir) {
|
|
const outcome = outcomes[provider];
|
|
if (!outcome || outcome === 'success') {
|
|
return [{ type: 'delta', role: 'assistant', content: `response from ${provider}` },
|
|
{ type: 'stop', finish_reason: 'stop' }];
|
|
}
|
|
throw outcome;
|
|
};
|
|
}
|
|
|
|
const dummyIR = makeIR({ model: 'test-model' });
|
|
|
|
it('empty chain → throws Error', async () => {
|
|
let caught = null;
|
|
try {
|
|
await executeWithFallback([], dummyIR, makeHopFn({}));
|
|
} catch (e) {
|
|
caught = e;
|
|
}
|
|
assert.ok(caught instanceof Error, 'Expected Error for empty chain');
|
|
assert.ok(caught.message.includes('chain'));
|
|
});
|
|
|
|
it('single-hop chain with success → returns chunks + fallbackHops=0', async () => {
|
|
const chain = [{ provider: 'anthropic', model: 'claude-sonnet-4-6' }];
|
|
const result = await executeWithFallback(chain, dummyIR, makeHopFn({ anthropic: 'success' }));
|
|
assert.ok(Array.isArray(result.chunks), 'Expected chunks array');
|
|
assert.equal(result.fallbackHops, 0);
|
|
assert.equal(result.providerUsed, 'anthropic');
|
|
assert.equal(result.originalError, null);
|
|
assert.deepEqual(result.triedProviders, ['anthropic']);
|
|
});
|
|
|
|
it('single-hop chain with hard-triggered error → exhausted, returns originalError', async () => {
|
|
const err = new ProviderError('Quota exhausted', 'QUOTA_EXHAUSTED');
|
|
const chain = [{ provider: 'anthropic', model: 'claude-sonnet-4-6' }];
|
|
const result = await executeWithFallback(chain, dummyIR, makeHopFn({ anthropic: err }));
|
|
assert.equal(result.chunks, null, 'Expected null chunks on exhausted chain');
|
|
assert.equal(result.originalError, err);
|
|
assert.equal(result.fallbackHops, 1); // chain.length = 1, all exhausted
|
|
assert.deepEqual(result.triedProviders, ['anthropic']);
|
|
});
|
|
|
|
it('two-hop chain, primary fails with hard trigger → falls back to secondary, fallbackHops=1', async () => {
|
|
const err = new ProviderError('Spawn failed', 'SPAWN_FAILED');
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
];
|
|
const result = await executeWithFallback(chain, dummyIR, makeHopFn({ anthropic: err, openai: 'success' }));
|
|
assert.ok(Array.isArray(result.chunks), 'Expected chunks from secondary');
|
|
assert.equal(result.fallbackHops, 1);
|
|
assert.equal(result.providerUsed, 'openai');
|
|
assert.equal(result.originalError, null);
|
|
assert.deepEqual(result.triedProviders, ['anthropic', 'openai']);
|
|
});
|
|
|
|
it('three-hop chain, primary + secondary fail → tertiary returns chunks, fallbackHops=2', async () => {
|
|
const errA = new ProviderError('Rate limited', 'RATE_LIMITED');
|
|
const errB = Object.assign(new Error('Service unavailable'), { statusCode: 503 });
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
{ provider: 'mistral', model: 'devstral-2' },
|
|
];
|
|
const result = await executeWithFallback(
|
|
chain, dummyIR,
|
|
makeHopFn({ anthropic: errA, openai: errB, mistral: 'success' }),
|
|
);
|
|
assert.ok(Array.isArray(result.chunks));
|
|
assert.equal(result.fallbackHops, 2);
|
|
assert.equal(result.providerUsed, 'mistral');
|
|
assert.equal(result.originalError, null);
|
|
assert.deepEqual(result.triedProviders, ['anthropic', 'openai', 'mistral']);
|
|
});
|
|
|
|
it('three-hop chain, all fail → exhausted, originalError is from FIRST hop (not last)', async () => {
|
|
const errA = new ProviderError('Rate limited', 'RATE_LIMITED');
|
|
const errB = new ProviderError('Spawn failed', 'SPAWN_FAILED');
|
|
const errC = new ProviderError('CLI not found', 'CLI_NOT_FOUND');
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
{ provider: 'mistral', model: 'devstral-2' },
|
|
];
|
|
const result = await executeWithFallback(
|
|
chain, dummyIR,
|
|
makeHopFn({ anthropic: errA, openai: errB, mistral: errC }),
|
|
);
|
|
assert.equal(result.chunks, null);
|
|
assert.equal(result.originalError, errA, 'Should be first hop error, not last');
|
|
assert.equal(result.fallbackHops, 3); // chain.length = 3
|
|
assert.deepEqual(result.triedProviders, ['anthropic', 'openai', 'mistral']);
|
|
});
|
|
|
|
it('client error (400) on primary → does NOT fall back, surfaces error immediately', async () => {
|
|
const err = Object.assign(new Error('Bad request'), { statusCode: 400 });
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
];
|
|
let openaiCalled = false;
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'anthropic') throw err;
|
|
openaiCalled = true;
|
|
return [{ type: 'stop', finish_reason: 'stop' }];
|
|
};
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.equal(result.chunks, null);
|
|
assert.equal(result.originalError, err);
|
|
assert.equal(openaiCalled, false, 'Second provider must NOT be called on client error');
|
|
assert.deepEqual(result.triedProviders, ['anthropic']);
|
|
});
|
|
|
|
it('AUTH_MISSING on primary → does NOT fall back (user must fix config)', async () => {
|
|
const err = new ProviderError('Auth missing', 'AUTH_MISSING');
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
];
|
|
let openaiCalled = false;
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'anthropic') throw err;
|
|
openaiCalled = true;
|
|
return [{ type: 'stop', finish_reason: 'stop' }];
|
|
};
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.equal(result.chunks, null);
|
|
assert.equal(result.originalError, err);
|
|
assert.equal(openaiCalled, false, 'Second provider must NOT be called on AUTH_MISSING');
|
|
});
|
|
|
|
it('non-trigger error (generic) on primary → does NOT fall back', async () => {
|
|
const err = new Error('Unexpected internal error');
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
];
|
|
let openaiCalled = false;
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'anthropic') throw err;
|
|
openaiCalled = true;
|
|
return [{ type: 'stop', finish_reason: 'stop' }];
|
|
};
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.equal(result.chunks, null);
|
|
assert.equal(openaiCalled, false, 'Must not fall back on non-trigger error');
|
|
});
|
|
|
|
});
|
|
|
|
// ── 13c: First-chunk safety ──────────────────────────────────────────────
|
|
|
|
describe('Fallback engine — first-chunk safety (D9)', () => {
|
|
|
|
// At D9, executeHopFn = collectAllChunks() which is fully buffered.
|
|
// Safety is enforced by construction: if executeHopFn returns successfully,
|
|
// chunks are already in memory (none written to res yet). Once chunks are
|
|
// returned, fallback is no longer attempted — we return immediately.
|
|
// If executeHopFn throws, zero bytes went to the client.
|
|
|
|
it('successful executeHopFn returns chunks immediately without retrying secondary', async () => {
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
];
|
|
let callCount = 0;
|
|
const hopFn = async (_provider) => {
|
|
callCount++;
|
|
return [{ type: 'stop', finish_reason: 'stop' }];
|
|
};
|
|
const result = await executeWithFallback(chain, makeIR({ model: 'test' }), hopFn);
|
|
assert.equal(result.fallbackHops, 0, 'Primary served — no fallback');
|
|
assert.equal(callCount, 1, 'executeHopFn called exactly once (first-chunk safety)');
|
|
});
|
|
|
|
it('error from executeHopFn (hard trigger) means zero chunks emitted — advance is safe', async () => {
|
|
// Simulate: anthropic throws (no bytes emitted), openai succeeds
|
|
const err = new ProviderError('Rate limited', 'RATE_LIMITED');
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
];
|
|
let writtenToClient = false;
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'anthropic') {
|
|
// Throw BEFORE any "write" — simulates first-chunk rule
|
|
throw err;
|
|
}
|
|
// openai succeeds — simulate "writtenToClient" only AFTER returning
|
|
// (in real server, chunks are written after executeWithFallback returns)
|
|
writtenToClient = true;
|
|
return [{ type: 'stop', finish_reason: 'stop' }];
|
|
};
|
|
const result = await executeWithFallback(chain, makeIR({ model: 'test' }), hopFn);
|
|
// openai served, which means writtenToClient is set (simulates post-return write)
|
|
assert.equal(result.fallbackHops, 1, 'Fell back to openai');
|
|
// Crucially: the "write" happened AFTER executeHopFn returned (not during chain iteration)
|
|
assert.equal(writtenToClient, true, 'Client write happens post-return, not during chain iteration');
|
|
});
|
|
|
|
});
|
|
|
|
// ── 13d: Soft triggers skipping spawn ────────────────────────────────────
|
|
|
|
describe('Fallback engine — soft trigger skipping (D9)', () => {
|
|
|
|
const dummyIR = makeIR({ model: 'test-model' });
|
|
|
|
it('chain [A, B], A soft trigger fires → engine skips A entirely, calls B; fallbackHops=1', async () => {
|
|
let aCalled = false;
|
|
let bCalled = false;
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'a') { aCalled = true; return [{ type: 'stop', finish_reason: 'stop' }]; }
|
|
if (provider === 'b') { bCalled = true; return [{ type: 'stop', finish_reason: 'stop' }]; }
|
|
throw new Error('Unexpected provider');
|
|
};
|
|
const chain = [
|
|
{
|
|
provider: 'a',
|
|
model: 'model-a',
|
|
softTriggers: { credit_pool_percent_threshold: 90 },
|
|
quotaSnapshot: { percentUsed: 95 }, // fires the trigger
|
|
},
|
|
{ provider: 'b', model: 'model-b' },
|
|
];
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.equal(aCalled, false, 'A must NOT be called (soft trigger skipped it)');
|
|
assert.equal(bCalled, true, 'B must be called');
|
|
assert.equal(result.fallbackHops, 1, 'B served as fallback hop 1');
|
|
assert.equal(result.providerUsed, 'b');
|
|
});
|
|
|
|
it('chain [A, B], neither soft trigger fires → A is called; if A succeeds, B never called', async () => {
|
|
let aCalled = false;
|
|
let bCalled = false;
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'a') { aCalled = true; return [{ type: 'stop', finish_reason: 'stop' }]; }
|
|
if (provider === 'b') { bCalled = true; return [{ type: 'stop', finish_reason: 'stop' }]; }
|
|
throw new Error('Unexpected provider');
|
|
};
|
|
const chain = [
|
|
{
|
|
provider: 'a',
|
|
model: 'model-a',
|
|
softTriggers: { credit_pool_percent_threshold: 90 },
|
|
quotaSnapshot: { percentUsed: 85 }, // below threshold, trigger does NOT fire
|
|
},
|
|
{ provider: 'b', model: 'model-b' },
|
|
];
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.equal(aCalled, true, 'A must be called');
|
|
assert.equal(bCalled, false, 'B must NOT be called (A succeeded)');
|
|
assert.equal(result.fallbackHops, 0);
|
|
assert.equal(result.providerUsed, 'a');
|
|
});
|
|
|
|
it('chain [A], A soft trigger fires (null quotaSnapshot) → trigger does NOT fire, A called', async () => {
|
|
// Per ADR 0004: null quotaStatus → treat as "don't fire"
|
|
let aCalled = false;
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'a') { aCalled = true; return [{ type: 'stop', finish_reason: 'stop' }]; }
|
|
throw new Error('Unexpected provider');
|
|
};
|
|
const chain = [
|
|
{
|
|
provider: 'a',
|
|
model: 'model-a',
|
|
softTriggers: { credit_pool_percent_threshold: 90 },
|
|
quotaSnapshot: null, // null → trigger never fires
|
|
},
|
|
];
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.equal(aCalled, true, 'A must be called (null quota → trigger does not fire)');
|
|
assert.equal(result.fallbackHops, 0);
|
|
});
|
|
|
|
});
|
|
|
|
// ── 13e: Header annotation ───────────────────────────────────────────────
|
|
|
|
describe('Fallback engine — observability / header annotation (D9)', () => {
|
|
|
|
const dummyIR = makeIR({ model: 'test-model' });
|
|
|
|
it('success on primary: providerUsed + modelUsed match primary hop', async () => {
|
|
const chain = [{ provider: 'anthropic', model: 'claude-sonnet-4-6' }];
|
|
const result = await executeWithFallback(chain, dummyIR, async () => [{ type: 'stop', finish_reason: 'stop' }]);
|
|
assert.equal(result.providerUsed, 'anthropic');
|
|
assert.equal(result.modelUsed, 'claude-sonnet-4-6');
|
|
});
|
|
|
|
it('success on fallback: providerUsed + modelUsed match the serving hop', async () => {
|
|
const err = new ProviderError('Rate limited', 'RATE_LIMITED');
|
|
const chain = [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
];
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'anthropic') throw err;
|
|
return [{ type: 'stop', finish_reason: 'stop' }];
|
|
};
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.equal(result.providerUsed, 'openai');
|
|
assert.equal(result.modelUsed, 'gpt-5.5');
|
|
assert.equal(result.fallbackHops, 1);
|
|
});
|
|
|
|
it('chain exhausted: originalError is from FIRST hop, not second or third', async () => {
|
|
const errA = new ProviderError('Rate limited', 'RATE_LIMITED');
|
|
const errB = new ProviderError('Spawn failed', 'SPAWN_FAILED');
|
|
const errC = new ProviderError('CLI not found', 'CLI_NOT_FOUND');
|
|
const chain = [
|
|
{ provider: 'a', model: 'model-a' },
|
|
{ provider: 'b', model: 'model-b' },
|
|
{ provider: 'c', model: 'model-c' },
|
|
];
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'a') throw errA;
|
|
if (provider === 'b') throw errB;
|
|
if (provider === 'c') throw errC;
|
|
throw new Error('Unexpected');
|
|
};
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.equal(result.originalError, errA, 'originalError must be first-hop error');
|
|
assert.notEqual(result.originalError, errB, 'Must NOT be second-hop error');
|
|
assert.notEqual(result.originalError, errC, 'Must NOT be third-hop error');
|
|
});
|
|
|
|
it('triedProviders lists all attempted hops in chain order', async () => {
|
|
const err = new ProviderError('Rate limited', 'RATE_LIMITED');
|
|
const chain = [
|
|
{ provider: 'a', model: 'model-a' },
|
|
{ provider: 'b', model: 'model-b' },
|
|
{ provider: 'c', model: 'model-c' },
|
|
];
|
|
const hopFn = async (provider) => {
|
|
if (provider === 'a') throw err;
|
|
if (provider === 'b') throw err;
|
|
return [{ type: 'stop', finish_reason: 'stop' }];
|
|
};
|
|
const result = await executeWithFallback(chain, dummyIR, hopFn);
|
|
assert.deepEqual(result.triedProviders, ['a', 'b', 'c']);
|
|
assert.equal(result.fallbackHops, 2);
|
|
assert.equal(result.providerUsed, 'c');
|
|
});
|
|
|
|
});
|
|
|
|
// ── 13f: HTTP integration ────────────────────────────────────────────────
|
|
|
|
describe('Fallback engine — HTTP integration (D9)', () => {
|
|
let server;
|
|
let port;
|
|
|
|
before(async () => {
|
|
__resetSpawnImpl();
|
|
codexResetSpawnImpl();
|
|
mistralResetSpawnImpl();
|
|
__resetFallbackConfig();
|
|
|
|
// Start test server with all 3 providers enabled
|
|
const testProviders = loadProviders({ enabled: { anthropic: true, openai: true, mistral: true } });
|
|
const { loadedProviders: lp, cacheStore: cs } = await import('./server.mjs');
|
|
// Inject test providers
|
|
for (const [name, p] of testProviders) {
|
|
lp.set(name, p);
|
|
}
|
|
cs.clear();
|
|
|
|
server = createOlpServer();
|
|
port = 20456 + Math.floor(Math.random() * 500);
|
|
await new Promise((resolve, reject) => {
|
|
server.listen(port, '127.0.0.1', resolve);
|
|
server.once('error', (e) => {
|
|
if (e.code === 'EADDRINUSE') {
|
|
port++;
|
|
server.listen(port, '127.0.0.1', resolve);
|
|
server.once('error', reject);
|
|
} else reject(e);
|
|
});
|
|
});
|
|
});
|
|
|
|
after(() => {
|
|
__resetFallbackConfig();
|
|
__resetSpawnImpl();
|
|
codexResetSpawnImpl();
|
|
mistralResetSpawnImpl();
|
|
if (!server) return;
|
|
return new Promise(r => server.close(r));
|
|
});
|
|
|
|
// Shared mock spawn builder that returns a successful response
|
|
function makeSuccessSpawn(content = 'Hello from mock') {
|
|
return function mockSpawnImpl(_bin, _args, _opts) {
|
|
const proc = new EventEmitter();
|
|
proc.stdout = new EventEmitter();
|
|
proc.stderr = new EventEmitter();
|
|
proc.stdin = {
|
|
write: () => {},
|
|
end: () => {
|
|
setImmediate(() => {
|
|
// Emit a minimal SSE-like response that anthropic plugin parses
|
|
// (Anthropic mock format: JSON lines per the real anthropic.mjs parser)
|
|
const lines = [
|
|
JSON.stringify({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: content } }),
|
|
JSON.stringify({ type: 'message_stop' }),
|
|
];
|
|
for (const line of lines) {
|
|
proc.stdout.emit('data', Buffer.from(`data: ${line}\n\n`));
|
|
}
|
|
proc.stdout.emit('end');
|
|
proc.stderr.emit('end');
|
|
proc.emit('close', 0, null);
|
|
});
|
|
},
|
|
};
|
|
proc.killed = false;
|
|
proc.kill = () => {};
|
|
return proc;
|
|
};
|
|
}
|
|
|
|
it('no fallback config: POST with no enabled provider → 503', async () => {
|
|
// Remove all providers temporarily
|
|
const { loadedProviders: lp } = await import('./server.mjs');
|
|
const savedMap = new Map(lp);
|
|
lp.clear();
|
|
try {
|
|
const r = await fetch({
|
|
port,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: { model: 'unknown-model', messages: [{ role: 'user', content: 'Hi' }] },
|
|
});
|
|
assert.equal(r.status, 503);
|
|
} finally {
|
|
for (const [name, p] of savedMap) lp.set(name, p);
|
|
}
|
|
});
|
|
|
|
it('no fallback config + mock anthropic: POST claude-sonnet-4-6 → 200 + X-OLP-Fallback-Hops: 0', async () => {
|
|
__setSpawnImpl(makeSuccessSpawn('Test response'));
|
|
try {
|
|
const r = await fetch({
|
|
port,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: {
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
max_tokens: 10,
|
|
},
|
|
});
|
|
assert.equal(r.status, 200, `Expected 200, got ${r.status}: ${r.body.slice(0, 200)}`);
|
|
assert.equal(r.headers['x-olp-fallback-hops'], '0');
|
|
assert.equal(r.headers['x-olp-provider-used'], 'anthropic');
|
|
} finally {
|
|
__resetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
it('fallback config: mock anthropic fails (SPAWN_FAILED), chain [anthropic→openai], openai mock succeeds → 200 + X-OLP-Fallback-Hops: 1 + X-OLP-Provider-Used: openai', async () => {
|
|
// Clear cache to prevent cache-hit from previous test polluting this one.
|
|
// ADR 0005: each (provider, model) pair is independently cached; a hit
|
|
// from the previous test would cause executeHopFn to skip the spawn and
|
|
// return cached chunks, masking the SPAWN_FAILED mock.
|
|
__clearCache();
|
|
|
|
// Provide fake codex auth so AUTH_MISSING doesn't stop the chain before
|
|
// the codex mock can respond. Codex checks readAuthArtifact() before spawnImpl.
|
|
// Write a temp file and set OPENAI_CODEX_AUTH_PATH to point at it.
|
|
const { writeFileSync, unlinkSync } = await import('node:fs');
|
|
const { tmpdir } = await import('node:os');
|
|
const { join: pathJoin } = await import('node:path');
|
|
const tmpAuthFile = pathJoin(tmpdir(), `olp-test-codex-auth-${Date.now()}.json`);
|
|
writeFileSync(tmpAuthFile, JSON.stringify({ accessToken: 'fake-test-token-for-codex' }), 'utf8');
|
|
const savedCodexAuthPath = process.env.OPENAI_CODEX_AUTH_PATH;
|
|
process.env.OPENAI_CODEX_AUTH_PATH = tmpAuthFile;
|
|
|
|
// Set up a 2-hop chain: anthropic → openai for claude-sonnet-4-6
|
|
__setFallbackConfig({
|
|
chains: {
|
|
'claude-sonnet-4-6': [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
],
|
|
},
|
|
soft_triggers: {},
|
|
});
|
|
|
|
// Anthropic mock: always exits with code 1 → SPAWN_FAILED (hard trigger)
|
|
__setSpawnImpl(function (_bin, _args, _opts) {
|
|
const proc = new EventEmitter();
|
|
proc.stdout = new EventEmitter();
|
|
proc.stderr = new EventEmitter();
|
|
proc.stdin = {
|
|
write: () => {},
|
|
end: () => {
|
|
setImmediate(() => {
|
|
proc.stderr.emit('data', Buffer.from('spawn failed\n'));
|
|
proc.stdout.emit('end');
|
|
proc.stderr.emit('end');
|
|
proc.emit('close', 1, null); // non-zero exit → SPAWN_FAILED
|
|
});
|
|
},
|
|
};
|
|
proc.killed = false;
|
|
proc.kill = () => {};
|
|
return proc;
|
|
});
|
|
|
|
// Codex (openai) mock: succeeds with a delta+stop response
|
|
codexSetSpawnImpl(function (_bin, _args, _opts) {
|
|
const proc = new EventEmitter();
|
|
proc.stdout = new EventEmitter();
|
|
proc.stderr = new EventEmitter();
|
|
proc.stdin = {
|
|
write: () => {},
|
|
end: () => {
|
|
setImmediate(() => {
|
|
proc.stdout.emit('data', Buffer.from('{"content":"Fallback response from openai"}\n'));
|
|
proc.stdout.emit('data', Buffer.from('{"type":"stop"}\n'));
|
|
proc.stdout.emit('end');
|
|
proc.stderr.emit('end');
|
|
proc.emit('close', 0, null);
|
|
});
|
|
},
|
|
};
|
|
proc.killed = false;
|
|
proc.kill = () => {};
|
|
return proc;
|
|
});
|
|
|
|
try {
|
|
const r = await fetch({
|
|
port,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: {
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
max_tokens: 10,
|
|
},
|
|
});
|
|
assert.equal(r.status, 200, `Expected 200 on fallback, got ${r.status}: ${r.body.slice(0, 300)}`);
|
|
assert.equal(r.headers['x-olp-fallback-hops'], '1', `Expected fallback-hops=1, got: ${r.headers['x-olp-fallback-hops']}`);
|
|
assert.equal(r.headers['x-olp-provider-used'], 'openai', `Expected provider-used=openai, got: ${r.headers['x-olp-provider-used']}`);
|
|
} finally {
|
|
__resetFallbackConfig();
|
|
__resetSpawnImpl();
|
|
codexResetSpawnImpl();
|
|
if (savedCodexAuthPath !== undefined) {
|
|
process.env.OPENAI_CODEX_AUTH_PATH = savedCodexAuthPath;
|
|
} else {
|
|
delete process.env.OPENAI_CODEX_AUTH_PATH;
|
|
}
|
|
try { unlinkSync(tmpAuthFile); } catch { /* ignore */ }
|
|
}
|
|
});
|
|
|
|
it('fallback config: two-hop chain with both providers failing SPAWN_FAILED → exhausted-chain path → error response with X-OLP-Fallback-Exhausted header', async () => {
|
|
// NOTE on this test's renaming (D9 review-2): the original sonnet draft
|
|
// titled this as a "client error (400) → no fallback" HTTP test. But the
|
|
// anthropic plugin surfaces non-zero exit codes as ProviderError
|
|
// SPAWN_FAILED (a HARD trigger), not as a typed-400 client error. So at
|
|
// the HTTP integration layer we can't easily inject a synthetic 400
|
|
// without monkey-patching executeHopFn. The "client error stops chain
|
|
// immediately" semantic IS covered at the UNIT level in
|
|
// "Engine: two-hop chain primary client error 400 → secondary NOT
|
|
// called" (line 3289+ in this file). This HTTP test now exercises the
|
|
// complementary path: both hops fail with SPAWN_FAILED → chain
|
|
// exhausted → originalError surfaces with X-OLP-Fallback-Exhausted
|
|
// header listing both providers.
|
|
__clearCache();
|
|
|
|
// Set up a 2-hop chain
|
|
__setFallbackConfig({
|
|
chains: {
|
|
'claude-sonnet-4-6': [
|
|
{ provider: 'anthropic', model: 'claude-sonnet-4-6' },
|
|
{ provider: 'openai', model: 'gpt-5.5' },
|
|
],
|
|
},
|
|
soft_triggers: {},
|
|
});
|
|
|
|
// Both providers fail with SPAWN_FAILED
|
|
__setSpawnImpl(function (_bin, _args, _opts) {
|
|
const proc = new EventEmitter();
|
|
proc.stdout = new EventEmitter();
|
|
proc.stderr = new EventEmitter();
|
|
proc.stdin = {
|
|
write: () => {},
|
|
end: () => {
|
|
setImmediate(() => {
|
|
proc.stdout.emit('end');
|
|
proc.stderr.emit('end');
|
|
proc.emit('close', 1, null);
|
|
});
|
|
},
|
|
};
|
|
proc.killed = false;
|
|
proc.kill = () => {};
|
|
return proc;
|
|
});
|
|
|
|
codexSetSpawnImpl(function (_bin, _args, _opts) {
|
|
const proc = new EventEmitter();
|
|
proc.stdout = new EventEmitter();
|
|
proc.stderr = new EventEmitter();
|
|
proc.stdin = {
|
|
write: () => {},
|
|
end: () => {
|
|
setImmediate(() => {
|
|
proc.stdout.emit('end');
|
|
proc.stderr.emit('end');
|
|
proc.emit('close', 1, null);
|
|
});
|
|
},
|
|
};
|
|
proc.killed = false;
|
|
proc.kill = () => {};
|
|
return proc;
|
|
});
|
|
|
|
try {
|
|
const r = await fetch({
|
|
port,
|
|
method: 'POST',
|
|
path: '/v1/chat/completions',
|
|
body: {
|
|
model: 'claude-sonnet-4-6',
|
|
messages: [{ role: 'user', content: 'Hi' }],
|
|
max_tokens: 10,
|
|
},
|
|
});
|
|
// Both providers fail → exhausted chain → 502 error response with exhausted header
|
|
assert.ok(r.status >= 400 && r.status < 600, `Expected error status, got ${r.status}`);
|
|
// X-OLP-Fallback-Exhausted header should be present since both providers tried
|
|
assert.ok(
|
|
r.headers['x-olp-fallback-exhausted'] !== undefined,
|
|
`Expected X-OLP-Fallback-Exhausted header, got headers: ${JSON.stringify(r.headers)}`,
|
|
);
|
|
} finally {
|
|
__resetFallbackConfig();
|
|
__resetSpawnImpl();
|
|
codexResetSpawnImpl();
|
|
}
|
|
});
|
|
|
|
});
|