/** * test-features.mjs — OLP D5 test suite (extends D4) * * Uses Node's built-in node:test runner. No external dependencies. * Run: node test-features.mjs (or: npm test) * * Authority: ADR 0002 (provider contract), ADR 0003 (IR v1.0), ADR 0005 (cache layer) * D4 adds: Anthropic plugin conformance, IR translation, mock-spawn behaviour. * D5 adds: Suite 9 (cache layer unit + HTTP integration), Suite 10 (Anthropic E2E gated) */ import { describe, it, before, after } from 'node:test'; import assert from 'node:assert/strict'; import { request as httpRequest } from 'node:http'; import { EventEmitter } from 'node:events'; import { homedir } from 'node:os'; import { computeCacheKey, extractCacheControlMarkers, hasCacheControl } from './lib/cache/keys.mjs'; import { CacheStore } from './lib/cache/store.mjs'; // ── Modules under test ──────────────────────────────────────────────────── import { validateIRRequest, validateIRMessage, VALID_ROLES, IR_VERSION } from './lib/ir/types.mjs'; import { openAIToIR, BadRequestError } from './lib/ir/openai-to-ir.mjs'; import { irChunkToOpenAISSE, irResponseToOpenAINonStream, generateRequestId, SSE_DONE, } from './lib/ir/ir-to-openai.mjs'; import { validateProvider, ProviderError, withTimeout } from './lib/providers/base.mjs'; import { loadProviders, getProviderForModel, getProviderByName, listAllProviderNames } from './lib/providers/index.mjs'; import anthropic, { irToAnthropic, anthropicChunkToIR, anthropicStopToIR, readAuthArtifact, estimateCost as anthropicEstimateCost, quotaStatus as anthropicQuotaStatus, healthCheck as anthropicHealthCheck, __setSpawnImpl, __resetSpawnImpl, } from './lib/providers/anthropic.mjs'; import codex, { irToCodex, codexChunkToIR, readAuthArtifact as codexReadAuthArtifact, estimateCost as codexEstimateCost, quotaStatus as codexQuotaStatus, healthCheck as codexHealthCheck, __setSpawnImpl as codexSetSpawnImpl, __resetSpawnImpl as codexResetSpawnImpl, } from './lib/providers/codex.mjs'; import mistral, { irToMistral, mistralChunkToIR, readAuthArtifact as mistralReadAuthArtifact, estimateCost as mistralEstimateCost, quotaStatus as mistralQuotaStatus, healthCheck as mistralHealthCheck, __setSpawnImpl as mistralSetSpawnImpl, __resetSpawnImpl as mistralResetSpawnImpl, } from './lib/providers/mistral.mjs'; import modelsRegistry from './models-registry.json' with { type: 'json' }; // ── Helpers ─────────────────────────────────────────────────────────────── /** Minimal valid IR request for use in tests */ function makeIR(overrides = {}) { return { irVersion: IR_VERSION, model: 'test-model', stream: false, messages: [{ role: 'user', content: 'Hello' }], ...overrides, }; } /** Minimal valid provider stub that satisfies the v1.0 contract (including D4 contractVersion) */ function makeProvider(overrides = {}) { return { name: 'stub', displayName: 'Stub Provider', contractVersion: '1.0', models: ['stub-model-v1'], auth: { type: 'none', storage: 'none', path: '', refresh: null }, spawn: async function* () { yield { type: 'stop', finish_reason: 'stop' }; }, estimateCost: () => null, quotaStatus: async () => null, healthCheck: async () => ({ ok: true, latencyMs: 0 }), hints: { requiresTTY: false, concurrentSpawnSafe: true, maxConcurrent: 4 }, ...overrides, }; } // ── HTTP helper ─────────────────────────────────────────────────────────── /** * Makes an HTTP request to the test server. * @param {{ port, method, path, headers?, body? }} opts * @returns {Promise<{ status: number, headers: object, body: string }>} */ function fetch(opts) { return new Promise((resolve, reject) => { const bodyStr = opts.body ? JSON.stringify(opts.body) : undefined; const req = httpRequest({ hostname: '127.0.0.1', port: opts.port, method: opts.method ?? 'GET', path: opts.path, headers: { ...(bodyStr && { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(bodyStr) }), ...(opts.headers ?? {}), }, }, res => { let data = ''; res.on('data', c => { data += c; }); res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: data })); }); req.on('error', reject); if (bodyStr) req.write(bodyStr); req.end(); }); } // ── Suite 1: IR validation ──────────────────────────────────────────────── describe('IR validation — validateIRRequest', () => { it('accepts a minimal valid IR request', () => { const r = validateIRRequest(makeIR()); assert.equal(r.valid, true); assert.deepEqual(r.errors, []); }); it('accepts an IR request with all optional fields', () => { const r = validateIRRequest(makeIR({ max_tokens: 256, temperature: 0.7, top_p: 0.9, stop: ['\n'], tools: [], tool_choice: 'auto', response_format: { type: 'text' }, })); assert.equal(r.valid, true); }); it('rejects when messages is missing', () => { const ir = makeIR(); delete ir.messages; const r = validateIRRequest(ir); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('messages'))); }); it('rejects when messages is empty', () => { const r = validateIRRequest(makeIR({ messages: [] })); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('messages must not be empty'))); }); it('rejects when model is missing', () => { const ir = makeIR(); delete ir.model; const r = validateIRRequest(ir); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('model'))); }); it('rejects when stream is not boolean', () => { const r = validateIRRequest(makeIR({ stream: 'true' })); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('stream'))); }); it('rejects temperature out of range', () => { const r = validateIRRequest(makeIR({ temperature: 3 })); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('temperature'))); }); it('rejects top_p out of range', () => { const r = validateIRRequest(makeIR({ top_p: -0.1 })); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('top_p'))); }); it('rejects non-object input', () => { const r = validateIRRequest(null); assert.equal(r.valid, false); }); }); describe('IR validation — validateIRMessage', () => { for (const role of VALID_ROLES) { it(`accepts role="${role}"`, () => { const r = validateIRMessage({ role, content: 'test' }); assert.equal(r.valid, true); }); } it('rejects invalid role', () => { const r = validateIRMessage({ role: 'admin', content: 'x' }); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('role'))); }); it('rejects missing content', () => { const r = validateIRMessage({ role: 'user' }); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('content'))); }); it('accepts array content (multi-part)', () => { const r = validateIRMessage({ role: 'user', content: [{ type: 'text', text: 'hi' }] }); assert.equal(r.valid, true); }); it('rejects non-object input', () => { const r = validateIRMessage('not an object'); assert.equal(r.valid, false); }); }); // ── Suite 2: openAIToIR translation ────────────────────────────────────── describe('openAIToIR translation', () => { it('translates a minimal request', () => { const ir = openAIToIR({ model: 'gpt-4o', messages: [{ role: 'user', content: 'Hi' }] }); assert.equal(ir.irVersion, IR_VERSION); assert.equal(ir.model, 'gpt-4o'); assert.equal(ir.stream, false); assert.equal(ir.messages.length, 1); assert.equal(ir.messages[0].role, 'user'); assert.equal(ir.messages[0].content, 'Hi'); }); it('defaults stream to false when absent', () => { const ir = openAIToIR({ model: 'm', messages: [{ role: 'user', content: 'x' }] }); assert.equal(ir.stream, false); }); it('passes stream=true through', () => { const ir = openAIToIR({ model: 'm', messages: [{ role: 'user', content: 'x' }], stream: true }); assert.equal(ir.stream, true); }); it('translates multi-turn with system message', () => { const ir = openAIToIR({ model: 'claude-sonnet-4-6', messages: [ { role: 'system', content: 'You are helpful.' }, { role: 'user', content: 'What is 2+2?' }, { role: 'assistant', content: '4' }, { role: 'user', content: 'Thanks!' }, ], }); assert.equal(ir.messages.length, 4); assert.equal(ir.messages[0].role, 'system'); assert.equal(ir.messages[1].role, 'user'); assert.equal(ir.messages[2].role, 'assistant'); }); it('maps deprecated role=function to role=tool', () => { const ir = openAIToIR({ model: 'm', messages: [{ role: 'function', name: 'my_fn', content: '{"result":1}' }], }); assert.equal(ir.messages[0].role, 'tool'); assert.equal(ir.messages[0].name, 'my_fn'); }); it('translates request with tools', () => { const ir = openAIToIR({ model: 'm', messages: [{ role: 'user', content: 'Search for X' }], tools: [{ type: 'function', function: { name: 'search', description: 'Web search', parameters: { type: 'object', properties: {} } }, }], tool_choice: 'auto', }); assert.equal(ir.tools.length, 1); assert.equal(ir.tools[0].function.name, 'search'); assert.equal(ir.tool_choice, 'auto'); }); it('translates request with response_format', () => { const ir = openAIToIR({ model: 'm', messages: [{ role: 'user', content: 'Give JSON' }], response_format: { type: 'json_object' }, }); assert.deepEqual(ir.response_format, { type: 'json_object' }); }); it('translates optional numeric fields', () => { const ir = openAIToIR({ model: 'm', messages: [{ role: 'user', content: 'x' }], max_tokens: 100, temperature: 0.5, top_p: 0.95, }); assert.equal(ir.max_tokens, 100); assert.equal(ir.temperature, 0.5); assert.equal(ir.top_p, 0.95); }); it('throws BadRequestError when model is missing', () => { assert.throws( () => openAIToIR({ messages: [{ role: 'user', content: 'x' }] }), BadRequestError, ); }); it('throws BadRequestError when messages is empty', () => { assert.throws( () => openAIToIR({ model: 'm', messages: [] }), BadRequestError, ); }); it('throws BadRequestError when body is not an object', () => { assert.throws(() => openAIToIR(null), BadRequestError); }); }); // ── Suite 3: irChunkToOpenAISSE format ──────────────────────────────────── describe('irChunkToOpenAISSE format', () => { const ID = 'chatcmpl-test123'; const MODEL = 'test-model'; it('generates request IDs with chatcmpl- prefix', () => { const id = generateRequestId(); assert.ok(id.startsWith('chatcmpl-')); assert.ok(id.length > 12); }); it('formats a delta chunk as SSE event', () => { const sse = irChunkToOpenAISSE({ type: 'delta', role: 'assistant', content: 'Hello' }, ID, MODEL); assert.ok(sse.startsWith('data: ')); assert.ok(sse.endsWith('\n\n')); const payload = JSON.parse(sse.slice(6).trim()); assert.equal(payload.object, 'chat.completion.chunk'); assert.equal(payload.id, ID); assert.equal(payload.model, MODEL); assert.equal(payload.choices[0].delta.content, 'Hello'); assert.equal(payload.choices[0].delta.role, 'assistant'); assert.equal(payload.choices[0].finish_reason, null); }); it('formats a stop chunk with finish_reason', () => { const sse = irChunkToOpenAISSE({ type: 'stop', finish_reason: 'stop' }, ID, MODEL); const payload = JSON.parse(sse.slice(6).trim()); assert.equal(payload.choices[0].finish_reason, 'stop'); assert.deepEqual(payload.choices[0].delta, {}); }); it('formats an error chunk with finish_reason within the OpenAI enum', () => { // ALIGNMENT.md Rule 2 (b): finish_reason must stay within the OpenAI spec // enum (stop|length|tool_calls|content_filter|function_call|null). // Provider errors surface via the top-level `error` object, not via an // invented finish_reason value. const sse = irChunkToOpenAISSE({ type: 'error', error: 'spawn failed' }, ID, MODEL); const payload = JSON.parse(sse.slice(6).trim()); assert.ok(payload.error); assert.equal(payload.error.type, 'provider_error'); assert.ok(['stop', 'length', 'tool_calls', 'content_filter', 'function_call', null].includes(payload.choices[0].finish_reason)); assert.equal(payload.choices[0].finish_reason, 'stop'); }); it('SSE_DONE is the [DONE] terminator', () => { assert.equal(SSE_DONE, 'data: [DONE]\n\n'); }); it('irResponseToOpenAINonStream assembles a complete response', () => { const chunks = [ { type: 'delta', role: 'assistant', content: 'Hello' }, { type: 'delta', content: ' world' }, { type: 'stop', finish_reason: 'stop', usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 } }, ]; const resp = irResponseToOpenAINonStream(chunks, ID, MODEL); assert.equal(resp.object, 'chat.completion'); assert.equal(resp.id, ID); assert.equal(resp.model, MODEL); assert.equal(resp.choices[0].message.content, 'Hello world'); assert.equal(resp.choices[0].message.role, 'assistant'); assert.equal(resp.choices[0].finish_reason, 'stop'); assert.equal(resp.usage.total_tokens, 7); }); }); // ── Suite 4: Provider contract validation ───────────────────────────────── describe('Provider contract validation', () => { it('accepts a fully valid provider stub', () => { const r = validateProvider(makeProvider()); assert.equal(r.valid, true); assert.deepEqual(r.errors, []); }); it('rejects provider with missing name', () => { const p = makeProvider(); delete p.name; const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('name'))); }); it('rejects provider with non-lowercase name', () => { const r = validateProvider(makeProvider({ name: 'MyProvider' })); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('name'))); }); it('rejects provider with missing displayName', () => { const p = makeProvider(); delete p.displayName; const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('displayName'))); }); it('rejects provider with missing models array', () => { const p = makeProvider(); delete p.models; const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('models'))); }); it('rejects provider with missing spawn', () => { const p = makeProvider(); delete p.spawn; const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('spawn'))); }); it('rejects provider with missing estimateCost', () => { const p = makeProvider(); delete p.estimateCost; const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('estimateCost'))); }); it('rejects provider with missing healthCheck', () => { const p = makeProvider(); delete p.healthCheck; const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('healthCheck'))); }); it('rejects provider with missing hints', () => { const p = makeProvider(); delete p.hints; const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('hints'))); }); it('rejects provider with invalid hints.maxConcurrent', () => { const r = validateProvider(makeProvider({ hints: { requiresTTY: false, concurrentSpawnSafe: true, maxConcurrent: -1 } })); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('maxConcurrent'))); }); it('rejects non-object input', () => { const r = validateProvider(null); assert.equal(r.valid, false); }); it('ProviderError carries code field', () => { const e = new ProviderError('auth missing', 'AUTH_MISSING'); assert.equal(e.code, 'AUTH_MISSING'); assert.equal(e.name, 'ProviderError'); assert.ok(e instanceof Error); }); it('withTimeout rejects after deadline', async () => { const p = new Promise(r => setTimeout(() => r('late'), 200)); await assert.rejects( () => withTimeout(p, 50, 'SPAWN_FAILED'), err => err instanceof ProviderError && err.code === 'SPAWN_FAILED', ); }); it('withTimeout resolves when promise is fast', async () => { const p = Promise.resolve(42); const v = await withTimeout(p, 1000, 'SPAWN_FAILED'); assert.equal(v, 42); }); }); // ── Suite 5: Plugin registry ────────────────────────────────────────────── describe('Plugin registry', () => { it('STATIC_REGISTRY has 3 entries (anthropic + openai + mistral candidates) at D8', () => { // D4 added anthropic; D6 adds openai; D8 adds mistral. Default config has all enabled:false. assert.equal(listAllProviderNames().length, 3); }); it('loadProviders with empty config → empty Map (anthropic not enabled)', () => { const m = loadProviders({}); assert.equal(m.size, 0); }); it('loadProviders with no config → empty Map', () => { const m = loadProviders(); assert.equal(m.size, 0); }); it('listAllProviderNames returns [anthropic, openai, mistral] at D8', () => { // D4: ['anthropic']. D6 adds openai. D8 adds mistral. assert.deepEqual(listAllProviderNames(), ['anthropic', 'openai', 'mistral']); }); it('getProviderForModel returns null when no providers loaded', () => { const m = loadProviders({}); const r = getProviderForModel(m, 'gpt-4o'); assert.equal(r, null); }); it('getProviderForModel finds provider by exact model string', () => { // Build a synthetic loaded map to test the function without touching STATIC_REGISTRY const p = makeProvider({ name: 'alpha', models: ['alpha-v1', 'alpha-v2'] }); const m = new Map([['alpha', p]]); const r = getProviderForModel(m, 'alpha-v1'); assert.ok(r !== null); assert.equal(r.name, 'alpha'); }); it('getProviderForModel returns null for unknown model', () => { const p = makeProvider({ name: 'alpha', models: ['alpha-v1'] }); const m = new Map([['alpha', p]]); assert.equal(getProviderForModel(m, 'beta-v1'), null); }); it('getProviderByName returns null for empty loaded map', () => { const m = new Map(); assert.equal(getProviderByName(m, 'anthropic'), null); }); it('getProviderByName returns provider when found', () => { const p = makeProvider({ name: 'alpha', models: ['alpha-v1'] }); const m = new Map([['alpha', p]]); assert.ok(getProviderByName(m, 'alpha') !== null); assert.equal(getProviderByName(m, 'alpha').name, 'alpha'); }); it('anthropic provider passes contract validation (STATIC_REGISTRY entry)', () => { // Even though anthropic is Candidate (not enabled by default), it must // pass contract validation at module load — loadProviders() validates all // registry entries regardless of enabled flag. const { valid, errors } = validateProvider(anthropic); assert.equal(valid, true, `Validation errors: ${errors.join('; ')}`); }); }); // ── Suite 6: Anthropic plugin (D4) ─────────────────────────────────────── // // All tests in this suite are UNIT tests. No real `claude` binary is invoked. // Mock spawn is injected via __setSpawnImpl / __resetSpawnImpl. // Tests verify: contract conformance, contractVersion enforcement, registry // consistency, IR translation, mock-spawn stream, healthCheck, estimateCost. /** * Creates a fake spawn that emits canned stdout chunks then exits cleanly. * Returns a fake ChildProcess-like EventEmitter with stdin, stdout, stderr. * @param {string[]} stdoutChunks — text chunks emitted in order * @param {number} [exitCode=0] */ function makeMockSpawn(stdoutChunks, exitCode = 0) { return function mockSpawnImpl(_bin, _args, _opts) { const proc = new EventEmitter(); proc.stdout = new EventEmitter(); proc.stderr = new EventEmitter(); proc.stdin = { write: () => {}, end: () => { // Emit stdout chunks and close asynchronously setImmediate(async () => { for (const chunk of stdoutChunks) { proc.stdout.emit('data', Buffer.from(chunk)); } proc.stdout.emit('end'); proc.stderr.emit('end'); proc.emit('close', exitCode, null); }); }, }; proc.killed = false; proc.kill = () => {}; return proc; }; } describe('Anthropic plugin (D4)', () => { // ── Test 1: Contract conformance ────────────────────────────────────── it('anthropic module satisfies validateProvider() — all 10 fields present', () => { const { valid, errors } = validateProvider(anthropic); assert.equal(valid, true, `Validation errors: ${errors.join('; ')}`); // Verify all 10 contract fields explicitly assert.ok('name' in anthropic, 'missing: name'); assert.ok('displayName' in anthropic, 'missing: displayName'); assert.ok('contractVersion' in anthropic, 'missing: contractVersion'); assert.ok('models' in anthropic, 'missing: models'); assert.ok('auth' in anthropic, 'missing: auth'); assert.ok(typeof anthropic.spawn === 'function', 'missing: spawn'); assert.ok(typeof anthropic.estimateCost === 'function', 'missing: estimateCost'); assert.ok(typeof anthropic.quotaStatus === 'function', 'missing: quotaStatus'); assert.ok(typeof anthropic.healthCheck === 'function', 'missing: healthCheck'); assert.ok('hints' in anthropic, 'missing: hints'); }); // ── Test 2: contractVersion enforced ───────────────────────────────── it('validateProvider rejects provider missing contractVersion', () => { const p = makeProvider(); delete p.contractVersion; const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('contractVersion'))); }); it('validateProvider rejects contractVersion: "0.9"', () => { const p = makeProvider({ contractVersion: '0.9' }); const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('contractVersion'))); }); it('validateProvider accepts contractVersion: "1.0"', () => { const p = makeProvider({ contractVersion: '1.0' }); const r = validateProvider(p); assert.equal(r.valid, true); }); it('validateProvider rejects contractVersion: undefined', () => { const p = makeProvider({ contractVersion: undefined }); const r = validateProvider(p); assert.equal(r.valid, false); assert.ok(r.errors.some(e => e.includes('contractVersion'))); }); // ── Test 3: models match registry ──────────────────────────────────── it('anthropic.models matches models-registry.json providers.anthropic.models', () => { const registryIds = modelsRegistry.providers.anthropic.models.map(m => m.id); assert.deepEqual(anthropic.models, registryIds); }); it('anthropic.models contains the three expected model IDs', () => { assert.ok(anthropic.models.includes('claude-opus-4-7')); assert.ok(anthropic.models.includes('claude-sonnet-4-6')); assert.ok(anthropic.models.includes('claude-haiku-4-5')); assert.equal(anthropic.models.length, 3); }); // ── Test 4: getProviderForModel finds anthropic for each model ──────── it('getProviderForModel finds anthropic for claude-sonnet-4-6 when enabled', () => { const loaded = new Map([['anthropic', anthropic]]); const result = getProviderForModel(loaded, 'claude-sonnet-4-6'); assert.ok(result !== null); assert.equal(result.name, 'anthropic'); }); it('getProviderForModel finds anthropic for claude-opus-4-7 when enabled', () => { const loaded = new Map([['anthropic', anthropic]]); const result = getProviderForModel(loaded, 'claude-opus-4-7'); assert.ok(result !== null); assert.equal(result.name, 'anthropic'); }); it('getProviderForModel finds anthropic for claude-haiku-4-5 when enabled', () => { const loaded = new Map([['anthropic', anthropic]]); const result = getProviderForModel(loaded, 'claude-haiku-4-5'); assert.ok(result !== null); assert.equal(result.name, 'anthropic'); }); // ── Test 5: irToAnthropic translation ──────────────────────────────── it('irToAnthropic: user message → plain text', () => { const ir = makeIR({ model: 'claude-sonnet-4-6', messages: [{ role: 'user', content: 'Hello world' }], }); const prompt = irToAnthropic(ir); assert.ok(typeof prompt === 'string'); assert.ok(prompt.includes('Hello world')); assert.ok(!prompt.includes('[System]')); assert.ok(!prompt.includes('[Assistant]')); }); it('irToAnthropic: system + user → system annotation + user text', () => { const ir = makeIR({ model: 'claude-sonnet-4-6', messages: [ { role: 'system', content: 'You are a helper.' }, { role: 'user', content: 'What is 2+2?' }, ], }); const prompt = irToAnthropic(ir); assert.ok(prompt.includes('[System] You are a helper.')); assert.ok(prompt.includes('What is 2+2?')); }); it('irToAnthropic: assistant turn → [Assistant] annotation', () => { const ir = makeIR({ model: 'claude-sonnet-4-6', messages: [ { role: 'user', content: 'Hi' }, { role: 'assistant', content: 'Hello!' }, { role: 'user', content: 'Bye' }, ], }); const prompt = irToAnthropic(ir); assert.ok(prompt.includes('[Assistant] Hello!')); }); it('irToAnthropic: response_format json_object injects system prompt', () => { const ir = makeIR({ model: 'claude-sonnet-4-6', messages: [{ role: 'user', content: 'Give JSON' }], response_format: { type: 'json_object' }, }); const prompt = irToAnthropic(ir); assert.ok(prompt.includes('Reply with valid JSON only')); }); it('irToAnthropic: tool result turn → [Tool Result] annotation', () => { const ir = makeIR({ model: 'claude-sonnet-4-6', messages: [ { role: 'user', content: 'Search for something' }, { role: 'tool', content: '{"results": []}', name: 'search' }, ], }); const prompt = irToAnthropic(ir); assert.ok(prompt.includes('[Tool Result')); assert.ok(prompt.includes('search')); }); it('irToAnthropic: array content is JSON-stringified', () => { const ir = makeIR({ model: 'claude-sonnet-4-6', messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], }); const prompt = irToAnthropic(ir); assert.ok(typeof prompt === 'string'); assert.ok(prompt.includes('text')); }); // ── Test 6: anthropicChunkToIR and anthropicStopToIR ───────────────── it('anthropicChunkToIR: produces delta chunk with content', () => { const chunk = anthropicChunkToIR('Hello ', false); assert.equal(chunk.type, 'delta'); assert.equal(chunk.content, 'Hello '); assert.ok(!('role' in chunk)); }); it('anthropicChunkToIR: first chunk includes role=assistant', () => { const chunk = anthropicChunkToIR('Hello', true); assert.equal(chunk.type, 'delta'); assert.equal(chunk.role, 'assistant'); assert.equal(chunk.content, 'Hello'); }); it('anthropicStopToIR: produces stop chunk with finish_reason', () => { const chunk = anthropicStopToIR('stop'); assert.equal(chunk.type, 'stop'); assert.equal(chunk.finish_reason, 'stop'); }); // ── Test 7: mock spawn — AsyncIterator yields correct IR chunks ─────── it('spawn with mock: yields delta chunks then stop chunk', async () => { const fakeSpawn = makeMockSpawn(['Hello', ' world']); __setSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'claude-sonnet-4-6', stream: true, messages: [{ role: 'user', content: 'Hi' }], }); const authCtx = { accessToken: '' }; const chunks = []; for await (const chunk of anthropic.spawn(ir, authCtx)) { chunks.push(chunk); } // Should have 2 delta chunks + 1 stop chunk const deltas = chunks.filter(c => c.type === 'delta'); const stops = chunks.filter(c => c.type === 'stop'); assert.ok(deltas.length >= 1, `Expected at least 1 delta, got ${deltas.length}`); assert.equal(stops.length, 1, `Expected 1 stop, got ${stops.length}`); const allContent = deltas.map(c => c.content).join(''); assert.equal(allContent, 'Hello world'); } finally { __resetSpawnImpl(); } }); it('spawn with mock: first delta chunk has role=assistant', async () => { const fakeSpawn = makeMockSpawn(['Test output']); __setSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'claude-sonnet-4-6', stream: true, messages: [{ role: 'user', content: 'Hello' }], }); const authCtx = { accessToken: '' }; const chunks = []; for await (const chunk of anthropic.spawn(ir, authCtx)) { chunks.push(chunk); } const firstDelta = chunks.find(c => c.type === 'delta'); assert.ok(firstDelta, 'No delta chunk found'); assert.equal(firstDelta.role, 'assistant'); } finally { __resetSpawnImpl(); } }); it('spawn with mock: non-zero exit code throws ProviderError', async () => { const fakeSpawn = makeMockSpawn([], 1); __setSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'claude-sonnet-4-6', stream: true, messages: [{ role: 'user', content: 'Hi' }], }); const authCtx = { accessToken: '' }; let caught = null; try { // eslint-disable-next-line no-unused-vars for await (const _chunk of anthropic.spawn(ir, authCtx)) { // drain } } catch (e) { caught = e; } assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`); assert.equal(caught.code, 'SPAWN_FAILED'); } finally { __resetSpawnImpl(); } }); it('spawn throws ProviderError(AUTH_MISSING) when no auth context and no env/file', async () => { const fakeSpawn = makeMockSpawn(['output']); __setSpawnImpl(fakeSpawn); // Temporarily clear the env var if set const savedToken = process.env.CLAUDE_CODE_OAUTH_TOKEN; delete process.env.CLAUDE_CODE_OAUTH_TOKEN; try { const ir = makeIR({ model: 'claude-sonnet-4-6', stream: false, messages: [{ role: 'user', content: 'Hi' }], }); let caught = null; try { // Pass null as authContext to force re-read for await (const _chunk of anthropic.spawn(ir, null)) { // eslint-disable-line no-unused-vars // may or may not throw depending on whether credentials exist on this machine } } catch (e) { caught = e; } // If credentials.json or keychain exists on the test machine, this won't throw. // We only assert that IF it throws, it's AUTH_MISSING. if (caught !== null) { assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`); assert.equal(caught.code, 'AUTH_MISSING'); } } finally { if (savedToken !== undefined) process.env.CLAUDE_CODE_OAUTH_TOKEN = savedToken; __resetSpawnImpl(); } }); // ── Test 8: healthCheck — binary not found ──────────────────────────── it('healthCheck returns {ok: false, error: "claude binary not found"} when binary absent', async () => { const result = await anthropicHealthCheck({ _binaryExistsFn: () => false, _authReadFn: () => ({ accessToken: '' }), }); assert.equal(result.ok, false); assert.equal(result.error, 'claude binary not found'); assert.ok(typeof result.latencyMs === 'number'); }); // ── Test 9: healthCheck — auth artifact missing ─────────────────────── it('healthCheck returns {ok: false, error: "auth artifact missing"} when auth missing', async () => { const result = await anthropicHealthCheck({ _binaryExistsFn: () => true, _authReadFn: () => null, }); assert.equal(result.ok, false); assert.equal(result.error, 'auth artifact missing'); assert.ok(typeof result.latencyMs === 'number'); }); it('healthCheck returns {ok: true} when binary and auth present', async () => { const result = await anthropicHealthCheck({ _binaryExistsFn: () => true, _authReadFn: () => ({ accessToken: '' }), }); assert.equal(result.ok, true); assert.ok(typeof result.latencyMs === 'number'); }); // ── Test 10: estimateCost shape ─────────────────────────────────────── it('estimateCost returns object with four fields for a valid request', () => { const request = makeIR({ model: 'claude-sonnet-4-6', messages: [ { role: 'system', content: 'You are a helper.' }, { role: 'user', content: 'Count to ten.' }, ], }); const result = anthropicEstimateCost(request); assert.ok(result !== null, 'estimateCost returned null'); assert.ok('inputTokens' in result, 'missing inputTokens'); assert.ok('outputTokensEstimate' in result, 'missing outputTokensEstimate'); assert.ok('currency' in result, 'missing currency'); assert.ok('usd' in result, 'missing usd'); assert.equal(result.currency, 'USD'); assert.equal(result.usd, null); // not pinned at D4 assert.ok(result.inputTokens > 0, 'inputTokens should be > 0'); assert.ok(result.outputTokensEstimate >= 0, 'outputTokensEstimate should be >= 0'); }); it('estimateCost returns null for null/missing request', () => { assert.equal(anthropicEstimateCost(null), null); assert.equal(anthropicEstimateCost({}), null); }); // ── Test 11: quotaStatus ────────────────────────────────────────────── it('quotaStatus returns null at D4', async () => { const result = await anthropicQuotaStatus({}); assert.equal(result, null); }); // ── Test 12: auth object shape ──────────────────────────────────────── it('anthropic.auth has correct shape', () => { assert.equal(typeof anthropic.auth.type, 'string'); assert.equal(typeof anthropic.auth.storage, 'string'); assert.equal(typeof anthropic.auth.path, 'string'); assert.ok(anthropic.auth.path.includes('.claude'), 'auth.path should reference .claude directory'); // Portability check: path must start with the runtime homedir() value (set at module load) // rather than a hardcoded literal. Since path.join(homedir(), ...) produces the home dir // as a prefix, we verify it matches what homedir() returns at test time. assert.ok( anthropic.auth.path.startsWith(homedir()), `auth.path "${anthropic.auth.path}" should start with homedir() "${homedir()}"`, ); assert.ok(typeof anthropic.auth.refresh === 'string' || anthropic.auth.refresh === null); }); // ── Test 13: hints shape ────────────────────────────────────────────── it('anthropic.hints has correct shape', () => { assert.equal(typeof anthropic.hints.requiresTTY, 'boolean'); assert.equal(typeof anthropic.hints.concurrentSpawnSafe, 'boolean'); assert.ok(Number.isInteger(anthropic.hints.maxConcurrent) && anthropic.hints.maxConcurrent > 0); assert.equal(anthropic.hints.requiresTTY, false); }); // ── Test 14: loadProviders with anthropic enabled ───────────────────── it('loadProviders with {enabled: {anthropic: true}} returns Map of size 1', () => { const loaded = loadProviders({ enabled: { anthropic: true } }); assert.equal(loaded.size, 1); assert.ok(loaded.has('anthropic')); }); it('anthropic loaded via loadProviders passes contract and has correct models', () => { const loaded = loadProviders({ enabled: { anthropic: true } }); const p = loaded.get('anthropic'); const { valid, errors } = validateProvider(p); assert.equal(valid, true, `Contract errors: ${errors.join('; ')}`); assert.ok(p.models.includes('claude-sonnet-4-6')); }); }); // ── Suite 7: HTTP integration tests ────────────────────────────────────── describe('HTTP integration', () => { let serverInstance; let port; before(async () => { // Use the REAL server module via its createOlpServer() factory. The // main guard in server.mjs prevents auto-listen on import; we call // .listen() ourselves on a test port. This means every HTTP test below // exercises the real router code — there is no parallel implementation // to drift. const { createOlpServer } = await import('./server.mjs'); // Pick a port: env OLP_TEST_PORT or random high port port = parseInt(process.env.OLP_TEST_PORT ?? String(13456 + Math.floor(Math.random() * 1000)), 10); serverInstance = createOlpServer(); // Retry once on port-in-use await new Promise((resolve, reject) => { serverInstance.listen(port, '127.0.0.1', resolve); serverInstance.once('error', async (e) => { if (e.code === 'EADDRINUSE') { port++; serverInstance.listen(port, '127.0.0.1', resolve); serverInstance.once('error', reject); } else { reject(e); } }); }); }); after(() => new Promise(r => serverInstance.close(r))); it('GET /health returns 200 with expected shape', async () => { const r = await fetch({ port, method: 'GET', path: '/health' }); assert.equal(r.status, 200); const body = JSON.parse(r.body); assert.equal(body.ok, true); assert.ok(typeof body.version === 'string'); assert.ok(typeof body.providers.enabled === 'number'); assert.ok(typeof body.providers.available === 'number'); }); it('GET /v1/models returns 200 with empty data array', async () => { const r = await fetch({ port, method: 'GET', path: '/v1/models' }); assert.equal(r.status, 200); const body = JSON.parse(r.body); assert.equal(body.object, 'list'); assert.deepEqual(body.data, []); }); it('POST /v1/chat/completions with no providers → 503 with no_enabled_provider', async () => { const r = await fetch({ port, method: 'POST', path: '/v1/chat/completions', body: { model: 'gpt-4o', messages: [{ role: 'user', content: 'Hi' }] }, }); assert.equal(r.status, 503); const body = JSON.parse(r.body); assert.equal(body.error.type, 'no_enabled_provider'); assert.ok(body.error.message.includes('gpt-4o')); }); it('POST /v1/chat/completions with invalid JSON body → 400', async () => { const req = httpRequest({ hostname: '127.0.0.1', port, method: 'POST', path: '/v1/chat/completions', headers: { 'Content-Type': 'application/json', 'Content-Length': '5' }, }); const result = await new Promise((resolve, reject) => { req.on('error', reject); let body = ''; req.on('response', res => { res.on('data', c => body += c); res.on('end', () => resolve({ status: res.statusCode, body })); }); req.write('{bad}'); req.end(); }); assert.equal(result.status, 400); }); it('POST /v1/chat/completions with missing model → 400', async () => { const r = await fetch({ port, method: 'POST', path: '/v1/chat/completions', body: { messages: [{ role: 'user', content: 'x' }] }, }); assert.equal(r.status, 400); const body = JSON.parse(r.body); assert.equal(body.error.type, 'invalid_request_error'); }); it('GET /unknown → 404', async () => { const r = await fetch({ port, method: 'GET', path: '/unknown/route' }); assert.equal(r.status, 404); const body = JSON.parse(r.body); assert.equal(body.error.type, 'not_found'); }); it('POST /v1/chat/completions without Content-Type → 415', async () => { // Our fetch helper sets Content-Type to application/json when body is truthy; // send text/plain directly to verify the 415 path. const result = await new Promise((resolve, reject) => { const req = httpRequest({ hostname: '127.0.0.1', port, method: 'POST', path: '/v1/chat/completions', headers: { 'Content-Type': 'text/plain', 'Content-Length': '2' }, }, res => { let data = ''; res.on('data', c => data += c); res.on('end', () => resolve({ status: res.statusCode, body: data })); }); req.on('error', reject); req.write('{}'); req.end(); }); assert.equal(result.status, 415); }); }); // ── Suite 8: Suite 8 formerly counted as D3/D4 base; suites renumber here ── // (No new Suite 8 — the numbering skips from 7 to 9 to match D5 spec.) // ── Suite 9: Cache layer ────────────────────────────────────────────────── // // Unit tests + HTTP integration tests for ADR 0005 (D1 + D4). // No real `claude` binary invoked. Mock spawn injected via __setSpawnImpl. // Authority: ADR 0005 § Cache key composition, D1 per-key isolation, D4 singleflight. describe('Cache layer — computeCacheKey (Suite 9)', () => { // ── Test 1: Determinism ─────────────────────────────────────────────── it('computeCacheKey is deterministic: same inputs → same key', () => { const ir = makeIR({ model: 'claude-haiku-4-5', messages: [{ role: 'user', content: 'hello' }] }); const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir); const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir); assert.equal(k1, k2); assert.equal(typeof k1, 'string'); assert.equal(k1.length, 64); // SHA-256 hex }); // ── Test 2: Provider distinguishes ─────────────────────────────────── it('computeCacheKey distinguishes different providers', () => { const ir = makeIR({ model: 'model-x', messages: [{ role: 'user', content: 'hi' }] }); const k1 = computeCacheKey('anthropic', 'model-x', ir); const k2 = computeCacheKey('openai', 'model-x', ir); assert.notEqual(k1, k2); }); // ── Test 3: Model distinguishes ─────────────────────────────────────── it('computeCacheKey distinguishes different models', () => { const ir = makeIR({ messages: [{ role: 'user', content: 'hi' }] }); const k1 = computeCacheKey('anthropic', 'claude-sonnet-4-6', ir); const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir); assert.notEqual(k1, k2); }); // ── Test 4: Messages distinguishes ─────────────────────────────────── it('computeCacheKey distinguishes different messages', () => { const ir1 = makeIR({ messages: [{ role: 'user', content: 'hello' }] }); const ir2 = makeIR({ messages: [{ role: 'user', content: 'world' }] }); const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir1); const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir2); assert.notEqual(k1, k2); }); // ── Test 5: Tools distinguishes ─────────────────────────────────────── it('computeCacheKey distinguishes requests with vs without tools', () => { const base = makeIR({ messages: [{ role: 'user', content: 'search' }] }); const withTools = makeIR({ messages: [{ role: 'user', content: 'search' }], tools: [{ type: 'function', function: { name: 'search', description: 'web search', parameters: {} } }], }); const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', base); const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', withTools); assert.notEqual(k1, k2); }); // ── Test 6: Temperature distinguishes ──────────────────────────────── it('computeCacheKey distinguishes different temperature values', () => { const ir1 = makeIR({ messages: [{ role: 'user', content: 'x' }], temperature: 0.0 }); const ir2 = makeIR({ messages: [{ role: 'user', content: 'x' }], temperature: 1.0 }); const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir1); const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir2); assert.notEqual(k1, k2); }); // ── Test 7: response_format distinguishes ──────────────────────────── it('computeCacheKey distinguishes different response_format values', () => { const ir1 = makeIR({ messages: [{ role: 'user', content: 'x' }] }); const ir2 = makeIR({ messages: [{ role: 'user', content: 'x' }], response_format: { type: 'json_object' } }); const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir1); const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir2); assert.notEqual(k1, k2); }); // ── Test 8: Content array property order stability ─────────────────── it('computeCacheKey is stable for content arrays with same properties in different insertion order', () => { const ir1 = makeIR({ messages: [{ role: 'user', content: [{ type: 'text', text: 'hi', extra: 1 }] }], }); const ir2 = makeIR({ messages: [{ role: 'user', content: [{ extra: 1, text: 'hi', type: 'text' }] }], }); const k1 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir1); const k2 = computeCacheKey('anthropic', 'claude-haiku-4-5', ir2); assert.equal(k1, k2); }); }); describe('Cache layer — extractCacheControlMarkers + hasCacheControl (Suite 9 cont.)', () => { // ── Test 9: extractCacheControlMarkers — top-level ─────────────────── it('extractCacheControlMarkers finds cache_control at message top level', () => { const messages = [ { role: 'user', content: 'hi', cache_control: { type: 'ephemeral' } }, ]; const markers = extractCacheControlMarkers(messages); assert.equal(markers.length, 1); assert.deepEqual(markers[0], { type: 'ephemeral' }); }); // ── Test 10: extractCacheControlMarkers — nested in content array ───── it('extractCacheControlMarkers finds cache_control nested in content array', () => { const messages = [ { role: 'user', content: [ { type: 'text', text: 'hello', cache_control: { type: 'ephemeral' } }, { type: 'text', text: 'world' }, ], }, ]; const markers = extractCacheControlMarkers(messages); assert.equal(markers.length, 1); assert.deepEqual(markers[0], { type: 'ephemeral' }); }); // ── Test 11: extractCacheControlMarkers — no markers ───────────────── it('extractCacheControlMarkers returns [] when no cache_control markers present', () => { const messages = [ { role: 'user', content: 'hello' }, { role: 'assistant', content: 'world' }, ]; assert.deepEqual(extractCacheControlMarkers(messages), []); }); // ── Test 12: hasCacheControl — true ────────────────────────────────── it('hasCacheControl returns true when cache_control markers present', () => { const ir = makeIR({ messages: [{ role: 'user', content: 'hi', cache_control: { type: 'ephemeral' } }], }); assert.equal(hasCacheControl(ir), true); }); // ── Test 13: hasCacheControl — false ───────────────────────────────── it('hasCacheControl returns false when no cache_control markers', () => { const ir = makeIR({ messages: [{ role: 'user', content: 'hi' }] }); assert.equal(hasCacheControl(ir), false); }); // ── Test 14: hasCacheControl — null/undefined safety ───────────────── it('hasCacheControl returns false for null/undefined ir', () => { assert.equal(hasCacheControl(null), false); assert.equal(hasCacheControl(undefined), false); assert.equal(hasCacheControl({}), false); }); }); describe('Cache layer — CacheStore unit tests (Suite 9 cont.)', () => { // ── Test 15: set/get round-trip ─────────────────────────────────────── it('CacheStore.set/get round-trips a value', async () => { const store = new CacheStore(); await store.set('keyA', 'hash1', [{ type: 'stop', finish_reason: 'stop' }]); const entry = await store.get('keyA', 'hash1'); assert.ok(entry !== null); assert.deepEqual(entry.value, [{ type: 'stop', finish_reason: 'stop' }]); }); // ── Test 16: get returns null for missing key ───────────────────────── it('CacheStore.get returns null for missing (keyId, cacheKey)', async () => { const store = new CacheStore(); const entry = await store.get('keyA', 'nonexistent-hash'); assert.equal(entry, null); }); // ── Test 17: Per-key isolation ──────────────────────────────────────── it('CacheStore per-key isolation: keyId1 entries invisible to keyId2', async () => { const store = new CacheStore(); await store.set('keyA', 'hash1', 'value-for-keyA'); const fromKeyA = await store.get('keyA', 'hash1'); const fromKeyB = await store.get('keyB', 'hash1'); assert.ok(fromKeyA !== null); assert.equal(fromKeyA.value, 'value-for-keyA'); assert.equal(fromKeyB, null); }); // ── Test 18: has ───────────────────────────────────────────────────── it('CacheStore.has returns true for existing entry, false for missing', async () => { const store = new CacheStore(); await store.set('keyA', 'hash1', 'val'); assert.equal(await store.has('keyA', 'hash1'), true); assert.equal(await store.has('keyA', 'hash-missing'), false); assert.equal(await store.has('keyB', 'hash1'), false); }); // ── Test 19: TTL expiry ─────────────────────────────────────────────── it('CacheStore respects TTL: expired entries return null', async () => { // Inject a _nowFn to control time without sleeping. // First call: "now" = 0 (entry creation time). // Second call: "now" = 2000 (2 seconds later; entry has 1s TTL → expired). let fakeNow = 0; const store = new CacheStore({ _nowFn: () => fakeNow }); await store.set('keyA', 'hash1', 'val', 1000); // 1000ms TTL // Entry should be alive at t=0 const entry1 = await store.get('keyA', 'hash1'); assert.ok(entry1 !== null, 'Expected entry to be alive at t=0'); // Advance time past TTL fakeNow = 2000; const entry2 = await store.get('keyA', 'hash1'); assert.equal(entry2, null, 'Expected entry to be expired at t=2000'); }); // ── Test 20: stats reports hits/misses ──────────────────────────────── it('CacheStore.stats reports hits and misses', async () => { const store = new CacheStore(); await store.set('keyA', 'hash1', 'val'); await store.get('keyA', 'hash1'); // hit await store.get('keyA', 'hash1'); // hit await store.get('keyA', 'missing'); // miss const s = store.stats('keyA'); assert.ok(s.hits >= 2, `Expected hits >= 2, got ${s.hits}`); assert.ok(s.misses >= 1, `Expected misses >= 1, got ${s.misses}`); assert.ok(typeof s.size === 'number'); assert.ok(typeof s.inflightCount === 'number'); }); // ── Test 21: getOrCompute returns computed value on miss ────────────── it('getOrCompute returns computed value on miss and caches it', async () => { const store = new CacheStore(); let callCount = 0; const computeFn = async () => { callCount++; return [{ type: 'delta', content: 'hello' }]; }; const v1 = await store.getOrCompute('keyA', 'hash1', computeFn); assert.deepEqual(v1, [{ type: 'delta', content: 'hello' }]); assert.equal(callCount, 1); }); // ── Test 22: getOrCompute returns cached value on hit (no recomputation) ── it('getOrCompute returns cached value on hit — computeFn not called again', async () => { const store = new CacheStore(); let callCount = 0; const computeFn = async () => { callCount++; return 'computed-value'; }; await store.getOrCompute('keyA', 'hash1', computeFn); const v2 = await store.getOrCompute('keyA', 'hash1', computeFn); assert.equal(v2, 'computed-value'); assert.equal(callCount, 1, 'computeFn should only be called once on cache hit'); }); // ── Test 23: Singleflight — the key D4 test ─────────────────────────── // 5 concurrent getOrCompute calls with same key + slow computeFn (50ms). // Verifies: computeFn called exactly once; all 5 callers receive same value; // all 5 return within a tight time window (singleflight working). it('getOrCompute singleflight: 5 concurrent callers → computeFn called exactly once', async () => { const store = new CacheStore(); let callCount = 0; const slowCompute = async () => { callCount++; // Simulate a slow provider spawn (50ms) await new Promise(r => setTimeout(r, 50)); return [{ type: 'delta', content: 'singleflight-result' }, { type: 'stop', finish_reason: 'stop' }]; }; const t0 = Date.now(); // Launch 5 concurrent callers simultaneously const results = await Promise.all([ store.getOrCompute('keyA', 'sf-hash', slowCompute), store.getOrCompute('keyA', 'sf-hash', slowCompute), store.getOrCompute('keyA', 'sf-hash', slowCompute), store.getOrCompute('keyA', 'sf-hash', slowCompute), store.getOrCompute('keyA', 'sf-hash', slowCompute), ]); const elapsed = Date.now() - t0; // Singleflight invariant: computeFn called exactly once assert.equal(callCount, 1, `Expected computeFn called 1 time, got ${callCount}`); // All 5 callers receive the same result for (const result of results) { assert.deepEqual(result, [{ type: 'delta', content: 'singleflight-result' }, { type: 'stop', finish_reason: 'stop' }]); } // All 5 callers return within a tight window (not 5 * 50ms = 250ms). // Allow generous margin (3x the slow compute time) for CI variance. assert.ok(elapsed < 250, `Expected singleflight to complete in < 250ms, took ${elapsed}ms`); }); // ── Test 24: getOrCompute releases inflight on completion ──────────── it('getOrCompute: subsequent calls after completion hit cache (no re-compute)', async () => { const store = new CacheStore(); let callCount = 0; const computeFn = async () => { callCount++; return 'done'; }; // First call — computes and caches await store.getOrCompute('keyA', 'hash1', computeFn); assert.equal(callCount, 1); // Verify inflight is released: calling again should hit cache await store.getOrCompute('keyA', 'hash1', computeFn); assert.equal(callCount, 1, 'computeFn should not be called again after completion'); assert.equal(store.stats('keyA').inflightCount, 0, 'No inflight entries after completion'); }); // ── Test 25: getOrCompute releases inflight on error ───────────────── it('getOrCompute: inflight released when computeFn throws; subsequent calls retry', async () => { const store = new CacheStore(); let callCount = 0; let shouldFail = true; const computeFn = async () => { callCount++; if (shouldFail) throw new Error('provider error'); return 'success'; }; // First call — throws await assert.rejects(() => store.getOrCompute('keyA', 'hash1', computeFn), /provider error/); assert.equal(callCount, 1); // Inflight must be released even after error assert.equal(store.stats('keyA').inflightCount, 0, 'Inflight must be released after error'); // Second call — now succeeds (verify re-try works) shouldFail = false; const v = await store.getOrCompute('keyA', 'hash1', computeFn); assert.equal(v, 'success'); assert.equal(callCount, 2, 'computeFn should be retried after error'); }); // ── Test 26: clear(keyId) clears only that namespace ───────────────── it('CacheStore.clear(keyId) clears only that namespace', async () => { const store = new CacheStore(); await store.set('keyA', 'hash1', 'val-a'); await store.set('keyB', 'hash1', 'val-b'); store.clear('keyA'); assert.equal(await store.has('keyA', 'hash1'), false, 'keyA should be cleared'); assert.equal(await store.has('keyB', 'hash1'), true, 'keyB should remain'); }); // ── Test 27: clear() with no args clears all ────────────────────────── it('CacheStore.clear() with no argument clears all namespaces', async () => { const store = new CacheStore(); await store.set('keyA', 'hash1', 'val-a'); await store.set('keyB', 'hash1', 'val-b'); store.clear(); assert.equal(await store.has('keyA', 'hash1'), false, 'keyA should be cleared'); assert.equal(await store.has('keyB', 'hash1'), false, 'keyB should be cleared'); assert.equal(store.stats().size, 0); }); }); // ── Suite 9 (HTTP integration — cache) ─────────────────────────────────── // // Tests cache miss / hit / bypass paths via HTTP integration with a mock // provider. Anthropic provider is enabled with: // 1. CLAUDE_CODE_OAUTH_TOKEN set to a fake value to bypass auth check // (the mock spawn never actually uses the token) // 2. Mock spawn injected via __setSpawnImpl so no real claude binary runs // // This tests the full HTTP → server.mjs → cache layer → provider dispatch // path end-to-end, with the spawn binary call itself mocked out. describe('Cache layer — HTTP integration (Suite 9 cont.)', () => { let serverInstance9; let port9; let savedOAuthToken; let serverMod9; before(async () => { // Inject a fake OAuth token so auth check passes without a real token. // The mock spawn ignores this value entirely. savedOAuthToken = process.env.CLAUDE_CODE_OAUTH_TOKEN; process.env.CLAUDE_CODE_OAUTH_TOKEN = 'test-fake-oauth-token-for-cache-tests'; // Set mock spawn that returns a proper response (delta + stop chunks via // raw text output — anthropic.mjs treats each stdout data chunk as raw text). __setSpawnImpl(makeMockSpawn(['mock-cache-content'])); // Import the server module (already cached by Node module system — same instance // as Suite 7). Mutate the loadedProviders map to add anthropic. serverMod9 = await import('./server.mjs'); const { createOlpServer, loadedProviders: lp } = serverMod9; // Wire anthropic into the loaded providers map const testProviders = loadProviders({ enabled: { anthropic: true } }); for (const [name, p] of testProviders) { lp.set(name, p); } port9 = parseInt( process.env.OLP_TEST_PORT ? String(parseInt(process.env.OLP_TEST_PORT) + 5000) : String(18456 + Math.floor(Math.random() * 1000)), 10, ); serverInstance9 = createOlpServer(); await new Promise((resolve, reject) => { serverInstance9.listen(port9, '127.0.0.1', resolve); serverInstance9.once('error', async (e) => { if (e.code === 'EADDRINUSE') { port9++; serverInstance9.listen(port9, '127.0.0.1', resolve); serverInstance9.once('error', reject); } else reject(e); }); }); }); after(() => { // Restore OAuth token if (savedOAuthToken !== undefined) { process.env.CLAUDE_CODE_OAUTH_TOKEN = savedOAuthToken; } else { delete process.env.CLAUDE_CODE_OAUTH_TOKEN; } __resetSpawnImpl(); return new Promise(r => serverInstance9.close(r)); }); // ── Test 28: cache miss path ────────────────────────────────────────── // First request with a unique message → cache miss (not yet in cache). it('HTTP: first request returns X-OLP-Cache: miss', async () => { // Unique content ensures this test doesn't collide with other tests' cached entries const testMsg = `http-cache-miss-${Date.now()}-${Math.random().toString(36).slice(2)}`; __setSpawnImpl(makeMockSpawn([`response-for-${testMsg}`])); const r = await fetch({ port: port9, method: 'POST', path: '/v1/chat/completions', body: { model: 'claude-haiku-4-5', messages: [{ role: 'user', content: testMsg }], }, }); assert.equal(r.status, 200, `Expected 200, got ${r.status}: ${r.body.slice(0, 200)}`); assert.equal(r.headers['x-olp-cache'], 'miss', `Expected miss, got: ${r.headers['x-olp-cache']}`); assert.equal(r.headers['x-olp-provider-used'], 'anthropic'); assert.equal(r.headers['x-olp-model-used'], 'claude-haiku-4-5'); }); // ── Test 29: cache hit path ─────────────────────────────────────────── // Two identical requests: first → miss, second → hit (same content served from cache). it('HTTP: second identical request returns X-OLP-Cache: hit with same content', async () => { const testMsg = `http-cache-hit-${Date.now()}-${Math.random().toString(36).slice(2)}`; const mockResponse = `hit-response-${testMsg}`; __setSpawnImpl(makeMockSpawn([mockResponse])); const reqParams = { port: port9, method: 'POST', path: '/v1/chat/completions', body: { model: 'claude-haiku-4-5', messages: [{ role: 'user', content: testMsg }], }, }; // First request — miss, spawns real (mock) provider const r1 = await fetch(reqParams); assert.equal(r1.status, 200, `First request failed: ${r1.status} ${r1.body.slice(0, 200)}`); assert.equal(r1.headers['x-olp-cache'], 'miss', `First request should be miss`); const body1 = JSON.parse(r1.body); const content1 = body1?.choices?.[0]?.message?.content ?? ''; // Second request — replace spawn with a failing mock to prove spawn is NOT called // (if spawn were called, this would produce a 502 error) __setSpawnImpl(makeMockSpawn([], 1)); // exit code 1 = ProviderError on spawn const r2 = await fetch(reqParams); assert.equal(r2.status, 200, `Second request (cache hit) should be 200, got ${r2.status}: ${r2.body.slice(0, 200)}`); assert.equal(r2.headers['x-olp-cache'], 'hit', `Second request should be cache hit`); // Content should be identical (replayed from cache) const body2 = JSON.parse(r2.body); const content2 = body2?.choices?.[0]?.message?.content ?? ''; assert.equal(content2, content1, `Cache hit content should match original`); }); // ── Test 30: cache bypass path (cache_control marker) ──────────────── // Request with cache_control marker → X-OLP-Cache: bypass (no OLP caching). it('HTTP: request with cache_control marker returns X-OLP-Cache: bypass', async () => { const testMsg = `http-cache-bypass-${Date.now()}-${Math.random().toString(36).slice(2)}`; __setSpawnImpl(makeMockSpawn([`bypass-response-${testMsg}`])); const r = await fetch({ port: port9, method: 'POST', path: '/v1/chat/completions', body: { model: 'claude-haiku-4-5', messages: [ { role: 'user', content: testMsg, cache_control: { type: 'ephemeral' }, }, ], }, }); assert.equal(r.status, 200, `Expected 200, got ${r.status}: ${r.body.slice(0, 200)}`); assert.equal(r.headers['x-olp-cache'], 'bypass', `Expected bypass header, got: ${r.headers['x-olp-cache']}`); }); }); // ── Suite 11: Codex plugin (D6) ────────────────────────────────────────── // // All tests are UNIT tests. No real `codex` binary is invoked. // Mock spawn is injected via codexSetSpawnImpl / codexResetSpawnImpl. // // Authority: Codex CLI reference https://developers.openai.com/codex/cli/reference // § "codex exec [flags] PROMPT" — exec subcommand syntax // § "--json" — NDJSON output format // § "--model, -m" — model override // § "$CODEX_HOME/auth.json" — auth artifact location // // Lossy-translation acknowledgements per ADR 0003 (documented in codex.mjs header): // top_p, temperature, stop, max_tokens, tools[], tool_calls → all dropped. /** * Creates a fake NDJSON-emitting mock spawn for Codex. * Lines are emitted as they would arrive from `codex exec --json`. * * @param {string[]} ndjsonLines — raw NDJSON lines emitted in order (no trailing \n needed) * @param {number} [exitCode=0] */ function makeMockCodexSpawn(ndjsonLines, exitCode = 0) { return function mockCodexSpawnImpl(_bin, _args, _opts) { const proc = new EventEmitter(); proc.stdout = new EventEmitter(); proc.stderr = new EventEmitter(); proc.stdin = { write: () => {}, end: () => { setImmediate(async () => { for (const line of ndjsonLines) { proc.stdout.emit('data', Buffer.from(line + '\n')); } proc.stdout.emit('end'); proc.stderr.emit('end'); proc.emit('close', exitCode, null); }); }, }; proc.killed = false; proc.kill = () => {}; return proc; }; } describe('Codex plugin (D6)', () => { // ── Test 1: Contract conformance ───────────────────────────────────── it('codex module satisfies validateProvider() — all 10 fields present', () => { const { valid, errors } = validateProvider(codex); assert.equal(valid, true, `Validation errors: ${errors.join('; ')}`); assert.ok('name' in codex, 'missing: name'); assert.ok('displayName' in codex, 'missing: displayName'); assert.ok('contractVersion' in codex, 'missing: contractVersion'); assert.ok('models' in codex, 'missing: models'); assert.ok('auth' in codex, 'missing: auth'); assert.ok(typeof codex.spawn === 'function', 'missing: spawn'); assert.ok(typeof codex.estimateCost === 'function', 'missing: estimateCost'); assert.ok(typeof codex.quotaStatus === 'function', 'missing: quotaStatus'); assert.ok(typeof codex.healthCheck === 'function', 'missing: healthCheck'); assert.ok('hints' in codex, 'missing: hints'); }); // ── Test 2: contractVersion === '1.0' ──────────────────────────────── it('codex declares contractVersion === "1.0"', () => { assert.equal(codex.contractVersion, '1.0'); }); // ── Test 3: name and displayName ───────────────────────────────────── it('codex.name === "openai" and displayName is set', () => { assert.equal(codex.name, 'openai'); assert.equal(typeof codex.displayName, 'string'); assert.ok(codex.displayName.length > 0); }); // ── Test 4: models match registry ─────────────────────────────────── it('codex.models matches models-registry.json providers.openai.models[].id', () => { const registryIds = modelsRegistry.providers.openai.models.map(m => m.id); assert.deepEqual(codex.models, registryIds); }); it('codex.models contains all 5 docs-listed model IDs', () => { // Per https://developers.openai.com/codex/models — each id has a // `codex -m ` example on that page. D6 review-2 expanded the // registry to include gpt-5.4-mini and gpt-5.3-codex-spark which the // original sonnet draft missed. assert.ok(codex.models.includes('gpt-5.5'), 'missing gpt-5.5'); assert.ok(codex.models.includes('gpt-5.4'), 'missing gpt-5.4'); assert.ok(codex.models.includes('gpt-5.4-mini'), 'missing gpt-5.4-mini'); assert.ok(codex.models.includes('gpt-5.3-codex'), 'missing gpt-5.3-codex'); assert.ok(codex.models.includes('gpt-5.3-codex-spark'), 'missing gpt-5.3-codex-spark'); assert.equal(codex.models.length, 5, `Expected 5 models, got ${codex.models.length}`); }); // ── Test 5: getProviderForModel finds codex for each model ────────── it('getProviderForModel finds openai provider for gpt-5.5', () => { const loaded = new Map([['openai', codex]]); const result = getProviderForModel(loaded, 'gpt-5.5'); assert.ok(result !== null); assert.equal(result.name, 'openai'); }); it('getProviderForModel finds openai provider for gpt-5.4', () => { const loaded = new Map([['openai', codex]]); const result = getProviderForModel(loaded, 'gpt-5.4'); assert.ok(result !== null); assert.equal(result.name, 'openai'); }); it('getProviderForModel finds openai provider for gpt-5.3-codex', () => { const loaded = new Map([['openai', codex]]); const result = getProviderForModel(loaded, 'gpt-5.3-codex'); assert.ok(result !== null); assert.equal(result.name, 'openai'); }); // ── Test 6: irToCodex translation ──────────────────────────────────── it('irToCodex: user message → args with exec --json --model, prompt as positional', () => { const ir = makeIR({ model: 'gpt-5.5', messages: [{ role: 'user', content: 'Hello world' }], }); const { args, prompt, useStdin } = irToCodex(ir); // Authority: Codex CLI reference § "codex exec [flags] PROMPT" assert.ok(args.includes('exec'), 'args must include "exec"'); assert.ok(args.includes('--json'), 'args must include "--json"'); assert.ok(args.includes('--model'), 'args must include "--model"'); assert.ok(args.includes('gpt-5.5'), 'args must include model value'); assert.ok(typeof prompt === 'string', 'prompt must be a string'); assert.ok(prompt.includes('Hello world'), 'prompt must contain user text'); assert.equal(useStdin, false, 'single-line prompt should use argv, not stdin'); }); it('irToCodex: system + user → system annotation + user text in prompt', () => { const ir = makeIR({ model: 'gpt-5.4', messages: [ { role: 'system', content: 'You are a coder.' }, { role: 'user', content: 'Write a function.' }, ], }); const { prompt } = irToCodex(ir); assert.ok(prompt.includes('[System] You are a coder.')); assert.ok(prompt.includes('Write a function.')); }); it('irToCodex: assistant prior turn → [Assistant] annotation', () => { const ir = makeIR({ model: 'gpt-5.5', messages: [ { role: 'user', content: 'Hi' }, { role: 'assistant', content: 'Hello!' }, { role: 'user', content: 'Thanks' }, ], }); const { prompt } = irToCodex(ir); assert.ok(prompt.includes('[Assistant] Hello!')); assert.ok(prompt.includes('Thanks')); }); it('irToCodex: tool_calls in assistant message — content preserved, metadata dropped (lossy)', () => { // ADR 0003 § Lossy: structured tool_calls dropped; textual content preserved. const ir = makeIR({ model: 'gpt-5.5', messages: [ { role: 'user', content: 'Search for X' }, { role: 'assistant', content: 'Searching...', tool_calls: [{ id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"X"}' } }], }, ], }); const { prompt } = irToCodex(ir); // Content preserved assert.ok(prompt.includes('Searching...'), 'assistant text content must be preserved'); // tool_calls metadata not directly in prompt (dropped per lossy spec) // (We do NOT assert the metadata IS there — it is documented as dropped.) }); it('irToCodex: response_format json_object injects system prompt', () => { const ir = makeIR({ model: 'gpt-5.5', messages: [{ role: 'user', content: 'Give JSON' }], response_format: { type: 'json_object' }, }); const { prompt } = irToCodex(ir); assert.ok(prompt.includes('Reply with valid JSON only')); }); it('irToCodex: multiline prompt uses stdin path (useStdin=true) with `-` positional', () => { const ir = makeIR({ model: 'gpt-5.5', messages: [ { role: 'system', content: 'Line one.' }, { role: 'user', content: 'Line two.' }, ], }); const { useStdin, args } = irToCodex(ir); // System + user messages join with '\n\n' → contains newline → stdin assert.equal(useStdin, true, 'multi-section prompt should use stdin'); // Per Codex CLI reference, stdin requires the literal `-` positional. // (D6 review-2 finding: original draft omitted positional entirely; // docs explicitly state stdin requires `-`.) assert.ok(args.includes('-'), 'stdin path must pass `-` as positional'); assert.ok(!args.includes('Line one.'), 'literal prompt must not appear in args when useStdin'); }); it('irToCodex: tool result turn → [Tool Result] annotation', () => { const ir = makeIR({ model: 'gpt-5.5', messages: [ { role: 'user', content: 'Search for X' }, { role: 'tool', content: '{"results":[]}', name: 'search' }, ], }); const { prompt } = irToCodex(ir); assert.ok(prompt.includes('[Tool Result')); assert.ok(prompt.includes('search')); }); // ── Test 7: codexChunkToIR translation ──────────────────────────────── it('codexChunkToIR: content field → delta chunk', () => { const chunk = codexChunkToIR('{"content":"Hello world"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'delta'); assert.equal(chunk.content, 'Hello world'); }); it('codexChunkToIR: delta field → delta chunk', () => { const chunk = codexChunkToIR('{"type":"delta","delta":"token text"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'delta'); assert.equal(chunk.content, 'token text'); }); it('codexChunkToIR: text field → delta chunk', () => { // Possible alternative event shape with "text" field const chunk = codexChunkToIR('{"type":"output_text","text":"output here"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'delta'); assert.equal(chunk.content, 'output here'); }); it('codexChunkToIR: type === "stop" → stop chunk', () => { const chunk = codexChunkToIR('{"type":"stop"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'stop'); assert.equal(chunk.finish_reason, 'stop'); }); it('codexChunkToIR: done === true → stop chunk', () => { const chunk = codexChunkToIR('{"done":true,"id":"run_123"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'stop'); assert.equal(chunk.finish_reason, 'stop'); }); it('codexChunkToIR: type === "error" → error chunk', () => { const chunk = codexChunkToIR('{"type":"error","error":"quota exceeded"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'error'); assert.equal(chunk.error, 'quota exceeded'); }); it('codexChunkToIR: error field present → error chunk', () => { const chunk = codexChunkToIR('{"error":"something went wrong","code":500}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'error'); }); it('codexChunkToIR: unknown/progress event type → null (silently ignored)', () => { // Events like {"type":"progress","step":1} are ignored per D6 spec const chunk = codexChunkToIR('{"type":"progress","step":1}'); assert.equal(chunk, null); }); it('codexChunkToIR: empty line → null', () => { assert.equal(codexChunkToIR(''), null); assert.equal(codexChunkToIR(' '), null); }); it('codexChunkToIR: malformed JSON → null (no throw)', () => { assert.doesNotThrow(() => { const result = codexChunkToIR('{bad json'); assert.equal(result, null); }); }); // ── Test 8: mock spawn — NDJSON stream yields correct IR chunks ─────── it('spawn with mock: NDJSON lines → delta chunks then stop chunk', async () => { const fakeSpawn = makeMockCodexSpawn([ '{"content":"Hello"}', '{"content":" world"}', '{"type":"stop"}', ]); codexSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'gpt-5.5', stream: true, messages: [{ role: 'user', content: 'Hi' }], }); const authCtx = { accessToken: '' }; const chunks = []; for await (const chunk of codex.spawn(ir, authCtx)) { chunks.push(chunk); } const deltas = chunks.filter(c => c.type === 'delta'); const stops = chunks.filter(c => c.type === 'stop'); assert.ok(deltas.length >= 1, `Expected at least 1 delta, got ${deltas.length}`); assert.equal(stops.length, 1, `Expected 1 stop, got ${stops.length}`); const allContent = deltas.map(c => c.content).join(''); assert.equal(allContent, 'Hello world'); } finally { codexResetSpawnImpl(); } }); it('spawn with mock: first delta chunk has role=assistant', async () => { const fakeSpawn = makeMockCodexSpawn(['{"content":"Test output"}']); codexSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'gpt-5.5', stream: true, messages: [{ role: 'user', content: 'Hello' }], }); const authCtx = { accessToken: '' }; const chunks = []; for await (const chunk of codex.spawn(ir, authCtx)) { chunks.push(chunk); } const firstDelta = chunks.find(c => c.type === 'delta'); assert.ok(firstDelta, 'No delta chunk found'); assert.equal(firstDelta.role, 'assistant'); } finally { codexResetSpawnImpl(); } }); it('spawn with mock: NDJSON stop event present → no extra synthetic stop appended', async () => { // If NDJSON stream already contains a stop event, we should NOT double-emit const fakeSpawn = makeMockCodexSpawn([ '{"content":"done"}', '{"type":"stop"}', ]); codexSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'gpt-5.5', stream: false, messages: [{ role: 'user', content: 'Hello' }], }); const authCtx = { accessToken: '' }; const chunks = []; for await (const chunk of codex.spawn(ir, authCtx)) { chunks.push(chunk); } const stops = chunks.filter(c => c.type === 'stop'); assert.equal(stops.length, 1, `Expected exactly 1 stop chunk, got ${stops.length}`); } finally { codexResetSpawnImpl(); } }); it('spawn with mock: non-zero exit code throws ProviderError(SPAWN_FAILED)', async () => { const fakeSpawn = makeMockCodexSpawn([], 1); codexSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'gpt-5.5', stream: true, messages: [{ role: 'user', content: 'Hi' }], }); const authCtx = { accessToken: '' }; let caught = null; try { for await (const _chunk of codex.spawn(ir, authCtx)) { // drain } } catch (e) { caught = e; } assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`); assert.equal(caught.code, 'SPAWN_FAILED'); } finally { codexResetSpawnImpl(); } }); it('spawn throws ProviderError(AUTH_MISSING) when no auth context and no auth file', async () => { const fakeSpawn = makeMockCodexSpawn(['{"content":"test"}']); codexSetSpawnImpl(fakeSpawn); // Override auth path to a nonexistent file to guarantee missing auth const savedAuthPath = process.env.OPENAI_CODEX_AUTH_PATH; process.env.OPENAI_CODEX_AUTH_PATH = '/nonexistent/path/auth.json'; try { const ir = makeIR({ model: 'gpt-5.5', stream: false, messages: [{ role: 'user', content: 'Hi' }], }); let caught = null; try { for await (const _chunk of codex.spawn(ir, null)) { // eslint-disable-line no-unused-vars // drain } } catch (e) { caught = e; } assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`); assert.equal(caught.code, 'AUTH_MISSING'); } finally { if (savedAuthPath !== undefined) { process.env.OPENAI_CODEX_AUTH_PATH = savedAuthPath; } else { delete process.env.OPENAI_CODEX_AUTH_PATH; } codexResetSpawnImpl(); } }); // ── Test 9: healthCheck ─────────────────────────────────────────────── it('healthCheck returns {ok: false, error: "codex binary not found"} when binary absent', async () => { const result = await codexHealthCheck({ _binaryExistsFn: () => false, _authReadFn: () => ({ accessToken: '' }), }); assert.equal(result.ok, false); assert.equal(result.error, 'codex binary not found'); assert.ok(typeof result.latencyMs === 'number'); }); it('healthCheck returns {ok: false, error: "auth artifact missing"} when auth missing', async () => { const result = await codexHealthCheck({ _binaryExistsFn: () => true, _authReadFn: () => null, }); assert.equal(result.ok, false); assert.equal(result.error, 'auth artifact missing'); assert.ok(typeof result.latencyMs === 'number'); }); it('healthCheck returns {ok: true} when binary and auth both present', async () => { const result = await codexHealthCheck({ _binaryExistsFn: () => true, _authReadFn: () => ({ accessToken: '' }), }); assert.equal(result.ok, true); assert.ok(typeof result.latencyMs === 'number'); }); // ── Test 10: estimateCost ───────────────────────────────────────────── it('estimateCost returns shape with currency USD', () => { const request = makeIR({ model: 'gpt-5.5', messages: [ { role: 'system', content: 'You are a coding assistant.' }, { role: 'user', content: 'Write hello world in Python.' }, ], }); const result = codexEstimateCost(request); assert.ok(result !== null, 'estimateCost returned null'); assert.ok('inputTokens' in result, 'missing inputTokens'); assert.ok('outputTokensEstimate' in result, 'missing outputTokensEstimate'); assert.ok('currency' in result, 'missing currency'); assert.ok('usd' in result, 'missing usd'); assert.equal(result.currency, 'USD'); assert.equal(result.usd, null); // not pinned at D6 assert.ok(result.inputTokens > 0, 'inputTokens should be > 0'); assert.ok(result.outputTokensEstimate >= 0); }); it('estimateCost returns null for null/missing request', () => { assert.equal(codexEstimateCost(null), null); assert.equal(codexEstimateCost({}), null); }); // ── Test 11: quotaStatus ────────────────────────────────────────────── it('quotaStatus returns null at D6', async () => { const result = await codexQuotaStatus({}); assert.equal(result, null); }); // ── Test 12: auth artifact path uses os.homedir(), not hardcoded ────── it('codex.auth.path uses homedir() and references .codex directory', () => { assert.equal(typeof codex.auth.path, 'string'); assert.ok(codex.auth.path.includes('.codex'), 'auth.path should reference .codex directory'); // Portability check: path must start with homedir() value at runtime assert.ok( codex.auth.path.startsWith(homedir()), `auth.path "${codex.auth.path}" should start with homedir() "${homedir()}"`, ); }); it('codex.auth.type === "oauth" and storage === "file"', () => { assert.equal(codex.auth.type, 'oauth'); assert.equal(codex.auth.storage, 'file'); }); // ── Test 13: readAuthArtifact reads OPENAI_CODEX_AUTH_PATH override ── it('readAuthArtifact: OPENAI_CODEX_AUTH_PATH pointing to nonexistent file → null', () => { const saved = process.env.OPENAI_CODEX_AUTH_PATH; process.env.OPENAI_CODEX_AUTH_PATH = '/definitely/not/a/real/path/auth.json'; try { const result = codexReadAuthArtifact(); assert.equal(result, null); } finally { if (saved !== undefined) process.env.OPENAI_CODEX_AUTH_PATH = saved; else delete process.env.OPENAI_CODEX_AUTH_PATH; } }); // ── Test 14: hints shape ────────────────────────────────────────────── it('codex.hints has correct shape', () => { assert.equal(typeof codex.hints.requiresTTY, 'boolean'); assert.equal(typeof codex.hints.concurrentSpawnSafe, 'boolean'); assert.ok(Number.isInteger(codex.hints.maxConcurrent) && codex.hints.maxConcurrent > 0); assert.equal(codex.hints.requiresTTY, false); assert.equal(codex.hints.concurrentSpawnSafe, true); }); // ── Test 15: STATIC_REGISTRY length after D8 still includes openai ──── it('STATIC_REGISTRY includes openai (D6) after D8 (length >= 2)', () => { // D8 adds mistral; anthropic + openai must still be present. assert.ok(listAllProviderNames().length >= 2); assert.ok(listAllProviderNames().includes('anthropic')); assert.ok(listAllProviderNames().includes('openai')); }); // ── Test 16: loadProviders with openai enabled ──────────────────────── it('loadProviders with {enabled: {openai: true}} returns Map of size 1 with openai', () => { const loaded = loadProviders({ enabled: { openai: true } }); assert.equal(loaded.size, 1); assert.ok(loaded.has('openai')); }); it('loadProviders with both anthropic and openai enabled returns Map of size 2', () => { const loaded = loadProviders({ enabled: { anthropic: true, openai: true } }); assert.equal(loaded.size, 2); assert.ok(loaded.has('anthropic')); assert.ok(loaded.has('openai')); }); it('openai loaded via loadProviders passes contract validation', () => { const loaded = loadProviders({ enabled: { openai: true } }); const p = loaded.get('openai'); const { valid, errors } = validateProvider(p); assert.equal(valid, true, `Contract errors: ${errors.join('; ')}`); assert.ok(p.models.includes('gpt-5.5')); }); // ── Test 17: spawn progress events silently ignored ────────────────── it('spawn with mock: progress events are silently ignored, only content emitted', async () => { const fakeSpawn = makeMockCodexSpawn([ '{"type":"progress","step":1}', '{"type":"progress","step":2}', '{"content":"actual response"}', '{"type":"stop"}', ]); codexSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'gpt-5.5', stream: true, messages: [{ role: 'user', content: 'Do something' }], }); const authCtx = { accessToken: '' }; const chunks = []; for await (const chunk of codex.spawn(ir, authCtx)) { chunks.push(chunk); } const deltas = chunks.filter(c => c.type === 'delta'); assert.equal(deltas.length, 1); assert.equal(deltas[0].content, 'actual response'); } finally { codexResetSpawnImpl(); } }); // ── Test 18: spawn synthesizes stop when NDJSON stream has no stop event ── it('spawn with mock: synthetic stop emitted when NDJSON has no stop event', async () => { const fakeSpawn = makeMockCodexSpawn([ '{"content":"only content, no stop"}', // No stop event in the stream ]); codexSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'gpt-5.5', stream: true, messages: [{ role: 'user', content: 'Hello' }], }); const authCtx = { accessToken: '' }; const chunks = []; for await (const chunk of codex.spawn(ir, authCtx)) { chunks.push(chunk); } const stops = chunks.filter(c => c.type === 'stop'); assert.equal(stops.length, 1, 'Should have exactly 1 synthetic stop chunk'); assert.equal(stops[0].finish_reason, 'stop'); } finally { codexResetSpawnImpl(); } }); }); // ── Suite 10: Anthropic E2E (GATED) ────────────────────────────────────── // // Run with: OLP_RUN_E2E=1 npm test // Skipped by default; consumes real Anthropic tokens (~200 tokens per run, // est. <$0.001 at haiku rates). Requires `claude` binary + keychain OAuth // token. The orchestrator runs this once per D5 verification. // // This suite is NOT run in CI (CI does not set OLP_RUN_E2E). The skip notice // is emitted as a console message so CI logs show the gated test exists. // // Tests: // 1. Start OLP server with anthropic enabled. // 2. POST minimal request to claude-haiku-4-5. // 3. Assert 200, response contains "OK", correct provider/model headers. // 4. Send same request again → assert X-OLP-Cache: hit, identical content. // 5. Assert X-OLP-Fallback-Hops: 0. // // Model: claude-haiku-4-5 (cheapest). Prompt: "Reply with exactly the word OK // and nothing else." max_tokens: 10. Target: < 200 tokens per run. // // Do NOT include any real OAuth tokens or API keys in this test. Auth is read // from keychain / CLAUDE_CODE_OAUTH_TOKEN env / ~/.claude/.credentials.json // by readAuthArtifact() inside the anthropic plugin at spawn time. const RUN_E2E = process.env.OLP_RUN_E2E === '1'; if (!RUN_E2E) { // Emit a skip notice to CI logs without failing the suite process.stdout.write('::notice::Suite 10 (Anthropic E2E) skipped — set OLP_RUN_E2E=1 to run. Requires claude binary + keychain OAuth token. ~200 tokens per run.\n'); } describe('Anthropic E2E — real claude spawn (Suite 10)', { skip: !RUN_E2E }, () => { let e2eServer; let e2ePort; let e2eLoadedProviders; let e2eCacheStore; before(async () => { if (!RUN_E2E) return; __resetSpawnImpl(); // ensure real spawn is active const { createOlpServer, loadedProviders: lp, cacheStore: cs } = await import('./server.mjs'); e2eLoadedProviders = lp; e2eCacheStore = cs; // Enable anthropic provider for E2E const testProviders = loadProviders({ enabled: { anthropic: true } }); for (const [name, p] of testProviders) { lp.set(name, p); } e2ePort = parseInt(process.env.OLP_E2E_PORT ?? String(19456 + Math.floor(Math.random() * 500)), 10); e2eServer = createOlpServer(); await new Promise((resolve, reject) => { e2eServer.listen(e2ePort, '127.0.0.1', resolve); e2eServer.once('error', async (e) => { if (e.code === 'EADDRINUSE') { e2ePort++; e2eServer.listen(e2ePort, '127.0.0.1', resolve); e2eServer.once('error', reject); } else reject(e); }); }); }); after(() => { if (!e2eServer) return; return new Promise(r => e2eServer.close(r)); }); it('E2E: POST claude-haiku-4-5 with minimal prompt → 200 + "OK" content + correct headers', async () => { const body = { model: 'claude-haiku-4-5', messages: [{ role: 'user', content: 'Reply with exactly the word OK and nothing else.' }], max_tokens: 10, }; const r = await fetch({ port: e2ePort, method: 'POST', path: '/v1/chat/completions', body }); assert.equal(r.status, 200, `Expected 200, got ${r.status}: ${r.body}`); const respBody = JSON.parse(r.body); const content = respBody?.choices?.[0]?.message?.content ?? ''; assert.ok( content.toLowerCase().includes('ok'), `Expected response to contain "OK", got: ${content}`, ); // Provider/model headers assert.equal(r.headers['x-olp-provider-used'], 'anthropic', `Expected anthropic provider`); assert.equal(r.headers['x-olp-model-used'], 'claude-haiku-4-5', `Expected haiku model`); // First request is always a miss assert.equal(r.headers['x-olp-cache'], 'miss', `First request should be cache miss`); // Fallback hops assert.equal(r.headers['x-olp-fallback-hops'], '0', `Expected 0 fallback hops`); // Second request — should hit cache, content identical const r2 = await fetch({ port: e2ePort, method: 'POST', path: '/v1/chat/completions', body }); assert.equal(r2.status, 200, `Cache hit request should be 200`); assert.equal(r2.headers['x-olp-cache'], 'hit', `Second request should be cache hit`); const resp2Body = JSON.parse(r2.body); const content2 = resp2Body?.choices?.[0]?.message?.content ?? ''; assert.ok( content2.toLowerCase().includes('ok'), `Cache hit response should also contain "OK", got: ${content2}`, ); }); }); // ── Suite 12: Mistral Vibe plugin (D8) ─────────────────────────────────── // // All tests are UNIT tests. No real `vibe` binary is invoked. // Mock spawn is injected via mistralSetSpawnImpl / mistralResetSpawnImpl. // // Authority: Mistral Vibe docs (WebFetched 2026-05-23): // DOCS-1: https://docs.mistral.ai/mistral-vibe/terminal/quickstart // § "--prompt TEXT" + "--output FORMAT" — programmatic mode syntax // DOCS-2: https://docs.mistral.ai/mistral-vibe/terminal/configuration // § "~/.vibe/.env" — auth file; MISTRAL_API_KEY env var // DOCS-5: https://mistral.ai/news/devstral-2-vibe-cli // § "Devstral 2", "Devstral Small 2" — model names // DOCS-6: https://help.mistral.ai/en/articles/347532 // § "Mistral Vibe is included in every Le Chat Pro subscription" // DOCS-7: https://legal.mistral.ai/terms/usage-policy // § No anti-third-party clauses (Tier D confirmed for ADR 0006) // // Spec assumption acknowledgements (see mistral.mjs header): // A1 CONFIRMED: `vibe --prompt "PROMPT" --output json` spawn shape // A2 CONFIRMED: MISTRAL_API_KEY env var is the auth mechanism // A3 CONFIRMED: ~/.vibe/.env is the auth file path // A4 UNPINNED: JSON output event schema — defensive 4-shape parser used // A5 UNPINNED: --model flag existence not confirmed by docs // A6 UNPINNED: exact model IDs (devstral-2, devstral-small-2) best-effort // A7 UNPINNED: --output json is NDJSON not single blob // A8 UNPINNED: multi-line prompt handling via --prompt flag // // Lossy-translation acknowledgements per ADR 0003 (documented in mistral.mjs header): // top_p, temperature, stop, max_tokens, tools[], tool_calls → all dropped. /** * Creates a fake JSON-line-emitting mock spawn for Mistral Vibe. * Lines are emitted as they would arrive from `vibe --output json` * (D8 assumption A7: NDJSON lines — D-later will pin the real format). * * @param {string[]} jsonLines — raw JSON lines emitted in order (no trailing \n needed) * @param {number} [exitCode=0] */ function makeMockMistralSpawn(jsonLines, exitCode = 0) { return function mockMistralSpawnImpl(_bin, _args, _opts) { const proc = new EventEmitter(); proc.stdout = new EventEmitter(); proc.stderr = new EventEmitter(); proc.stdin = { write: () => {}, end: () => { setImmediate(async () => { for (const line of jsonLines) { proc.stdout.emit('data', Buffer.from(line + '\n')); } proc.stdout.emit('end'); proc.stderr.emit('end'); proc.emit('close', exitCode, null); }); }, }; proc.killed = false; proc.kill = () => {}; return proc; }; } describe('Mistral Vibe plugin (D8)', () => { // ── Test 1: Contract conformance ────────────────────────────────────── it('mistral module satisfies validateProvider() — all 10 fields present', () => { const { valid, errors } = validateProvider(mistral); assert.equal(valid, true, `Validation errors: ${errors.join('; ')}`); assert.ok('name' in mistral, 'missing: name'); assert.ok('displayName' in mistral, 'missing: displayName'); assert.ok('contractVersion' in mistral, 'missing: contractVersion'); assert.ok('models' in mistral, 'missing: models'); assert.ok('auth' in mistral, 'missing: auth'); assert.ok(typeof mistral.spawn === 'function', 'missing: spawn'); assert.ok(typeof mistral.estimateCost === 'function', 'missing: estimateCost'); assert.ok(typeof mistral.quotaStatus === 'function', 'missing: quotaStatus'); assert.ok(typeof mistral.healthCheck === 'function', 'missing: healthCheck'); assert.ok('hints' in mistral, 'missing: hints'); }); // ── Test 2: contractVersion === '1.0' ──────────────────────────────── it('mistral declares contractVersion === "1.0"', () => { assert.equal(mistral.contractVersion, '1.0'); }); // ── Test 3: name and displayName ───────────────────────────────────── it('mistral.name === "mistral" and displayName is set', () => { assert.equal(mistral.name, 'mistral'); assert.equal(typeof mistral.displayName, 'string'); assert.ok(mistral.displayName.length > 0); assert.ok(mistral.displayName.toLowerCase().includes('mistral')); }); // ── Test 4: models include all registry IDs + aliases ──────────────── it('mistral.models includes every canonical registry id AND every alias', () => { // The plugin merges canonical IDs and alias keys into models[] so that // getProviderForModel routes either form. Test that the registry's // canonical IDs are a subset, and the aliases are all included too. const registryIds = modelsRegistry.providers.mistral.models.map(m => m.id); const registryAliases = Object.keys(modelsRegistry.providers.mistral.aliases ?? {}); for (const id of registryIds) { assert.ok(mistral.models.includes(id), `canonical id ${id} missing from mistral.models`); } for (const alias of registryAliases) { assert.ok(mistral.models.includes(alias), `alias ${alias} missing from mistral.models`); } assert.equal(mistral.models.length, registryIds.length + registryAliases.length); }); it('mistral.models contains canonical IDs + short-form aliases', () => { // Per canonical models registry (docs.mistral.ai/getting-started/models/ // models_overview, D8 review-2 finding): canonical IDs are date-stamped // (devstral-2-25-12, devstral-small-2-25-12). Vibe config.toml uses short // forms (devstral-2, devstral-small-2). Plugin exposes BOTH so // getProviderForModel routes either form. assert.ok(mistral.models.includes('devstral-2-25-12'), 'missing canonical devstral-2-25-12'); assert.ok(mistral.models.includes('devstral-small-2-25-12'), 'missing canonical devstral-small-2-25-12'); assert.ok(mistral.models.includes('devstral-2'), 'missing short-form alias devstral-2'); assert.ok(mistral.models.includes('devstral-small-2'), 'missing short-form alias devstral-small-2'); assert.ok(mistral.models.includes('devstral'), 'missing alias devstral'); assert.ok(mistral.models.includes('devstral-small'), 'missing alias devstral-small'); // Length: 2 canonical + 4 aliases = 6 total assert.equal(mistral.models.length, 6, `Expected 6 entries (2 canonical + 4 aliases), got ${mistral.models.length}`); }); // ── Test 5: getProviderForModel finds mistral for each model ────────── it('getProviderForModel finds mistral for devstral-2', () => { const loaded = new Map([['mistral', mistral]]); const result = getProviderForModel(loaded, 'devstral-2'); assert.ok(result !== null); assert.equal(result.name, 'mistral'); }); it('getProviderForModel finds mistral for devstral-small-2', () => { const loaded = new Map([['mistral', mistral]]); const result = getProviderForModel(loaded, 'devstral-small-2'); assert.ok(result !== null); assert.equal(result.name, 'mistral'); }); // ── Test 6: irToMistral translation ────────────────────────────────── it('irToMistral: user message → args with --prompt and --output streaming', () => { // Authority: DOCS-1 § "Output Format Options" — `streaming` is the // newline-delimited JSON per message mode. `json` emits a single blob // at the end (D8 review-2 finding: original draft used `json`, // incompatible with the line-buffered stdout parser; corrected to // `streaming` per docs verbatim). const ir = makeIR({ model: 'devstral-2', messages: [{ role: 'user', content: 'Hello world' }], }); const { args, prompt } = irToMistral(ir); assert.ok(args.includes('--prompt'), 'args must include "--prompt"'); assert.ok(args.includes('--output'), 'args must include "--output"'); assert.ok(args.includes('streaming'), 'args must include "streaming" output format (NDJSON per docs)'); assert.ok(!args.includes('json'), 'args must NOT include "json" (single-blob mode, incompatible with line-buffered parser)'); // D8 assumption A5: --model NOT in args (flag existence unconfirmed from docs) assert.ok(!args.includes('--model'), 'args must NOT include "--model" at D8 (A5 UNPINNED)'); assert.ok(typeof prompt === 'string', 'prompt must be a string'); assert.ok(prompt.includes('Hello world'), 'prompt must contain user text'); }); it('irToMistral: system + user → system annotation + user text in prompt', () => { const ir = makeIR({ model: 'devstral-2', messages: [ { role: 'system', content: 'You are a coding assistant.' }, { role: 'user', content: 'Write a function.' }, ], }); const { prompt } = irToMistral(ir); assert.ok(prompt.includes('[System] You are a coding assistant.')); assert.ok(prompt.includes('Write a function.')); }); it('irToMistral: assistant prior turn → [Assistant] annotation', () => { const ir = makeIR({ model: 'devstral-2', messages: [ { role: 'user', content: 'Hi' }, { role: 'assistant', content: 'Hello!' }, { role: 'user', content: 'Thanks' }, ], }); const { prompt } = irToMistral(ir); assert.ok(prompt.includes('[Assistant] Hello!')); assert.ok(prompt.includes('Thanks')); }); it('irToMistral: tool result turn → [Tool Result] annotation', () => { const ir = makeIR({ model: 'devstral-2', messages: [ { role: 'user', content: 'Search for X' }, { role: 'tool', content: '{"results":[]}', name: 'search' }, ], }); const { prompt } = irToMistral(ir); assert.ok(prompt.includes('[Tool Result')); assert.ok(prompt.includes('search')); }); it('irToMistral: response_format json_object injects system prompt (lossy)', () => { // ADR 0003 § Lossy: Vibe CLI does not honor response_format natively. const ir = makeIR({ model: 'devstral-2', messages: [{ role: 'user', content: 'Give JSON' }], response_format: { type: 'json_object' }, }); const { prompt } = irToMistral(ir); assert.ok(prompt.includes('Reply with valid JSON only')); }); it('irToMistral: tool_calls in assistant message — content preserved, metadata dropped (lossy)', () => { // ADR 0003 § Lossy: structured tool_calls dropped; textual content preserved. const ir = makeIR({ model: 'devstral-2', messages: [ { role: 'user', content: 'Search for X' }, { role: 'assistant', content: 'Searching...', tool_calls: [{ id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"X"}' } }], }, ], }); const { prompt } = irToMistral(ir); assert.ok(prompt.includes('Searching...'), 'assistant text content must be preserved'); // tool_calls metadata documented as dropped (lossy) — do NOT assert it's present }); it('irToMistral: array content is JSON-stringified', () => { const ir = makeIR({ model: 'devstral-2', messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], }); const { prompt } = irToMistral(ir); assert.ok(typeof prompt === 'string'); assert.ok(prompt.includes('text')); }); // ── Test 7: mistralChunkToIR translation ────────────────────────────── it('mistralChunkToIR: text field → delta chunk (Mistral La Plateforme shape)', () => { // D8 assumption A4: "text" is the preferred field name. const chunk = mistralChunkToIR('{"text":"Hello world"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'delta'); assert.equal(chunk.content, 'Hello world'); }); it('mistralChunkToIR: content field → delta chunk (OpenAI-compat shape)', () => { // D8 assumption A4: "content" as fallback field name. const chunk = mistralChunkToIR('{"content":"Hello world"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'delta'); assert.equal(chunk.content, 'Hello world'); }); it('mistralChunkToIR: delta field shape → delta chunk', () => { // D8 assumption A4: { type: 'delta', delta: '...' } shape. const chunk = mistralChunkToIR('{"type":"delta","delta":"token text"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'delta'); assert.equal(chunk.content, 'token text'); }); it('mistralChunkToIR: OpenAI streaming choices[0].delta.content shape → delta', () => { // D8 assumption A4: Vibe may use OpenAI streaming event shape. const chunk = mistralChunkToIR('{"choices":[{"delta":{"content":"output"},"finish_reason":null}]}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'delta'); assert.equal(chunk.content, 'output'); }); it('mistralChunkToIR: type === "stop" → stop chunk', () => { const chunk = mistralChunkToIR('{"type":"stop"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'stop'); assert.equal(chunk.finish_reason, 'stop'); }); it('mistralChunkToIR: done === true → stop chunk', () => { const chunk = mistralChunkToIR('{"done":true}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'stop'); assert.equal(chunk.finish_reason, 'stop'); }); it('mistralChunkToIR: OpenAI streaming stop via choices[0].finish_reason === "stop"', () => { // D8 assumption A4: OpenAI streaming stop shape. const chunk = mistralChunkToIR('{"choices":[{"delta":{},"finish_reason":"stop"}]}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'stop'); assert.equal(chunk.finish_reason, 'stop'); }); it('mistralChunkToIR: type === "error" → error chunk', () => { const chunk = mistralChunkToIR('{"type":"error","error":"quota exceeded"}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'error'); assert.equal(chunk.error, 'quota exceeded'); }); it('mistralChunkToIR: error field present → error chunk', () => { const chunk = mistralChunkToIR('{"error":"something went wrong","code":429}'); assert.ok(chunk !== null); assert.equal(chunk.type, 'error'); }); it('mistralChunkToIR: unknown/progress event type → null (silently ignored)', () => { const chunk = mistralChunkToIR('{"type":"progress","step":1}'); assert.equal(chunk, null); }); it('mistralChunkToIR: empty line → null', () => { assert.equal(mistralChunkToIR(''), null); assert.equal(mistralChunkToIR(' '), null); }); it('mistralChunkToIR: malformed JSON → null (no throw)', () => { assert.doesNotThrow(() => { const result = mistralChunkToIR('{bad json'); assert.equal(result, null); }); }); // ── Test 8: mock spawn — JSON stream yields correct IR chunks ────────── it('spawn with mock: JSON lines → delta chunks then stop chunk', async () => { const fakeSpawn = makeMockMistralSpawn([ '{"text":"Hello"}', '{"text":" world"}', '{"type":"stop"}', ]); mistralSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'devstral-2', stream: true, messages: [{ role: 'user', content: 'Hi' }], }); const authCtx = { apiKey: '' }; const chunks = []; for await (const chunk of mistral.spawn(ir, authCtx)) { chunks.push(chunk); } const deltas = chunks.filter(c => c.type === 'delta'); const stops = chunks.filter(c => c.type === 'stop'); assert.ok(deltas.length >= 1, `Expected at least 1 delta, got ${deltas.length}`); assert.equal(stops.length, 1, `Expected 1 stop, got ${stops.length}`); const allContent = deltas.map(c => c.content).join(''); assert.equal(allContent, 'Hello world'); } finally { mistralResetSpawnImpl(); } }); it('spawn with mock: first delta chunk has role=assistant', async () => { const fakeSpawn = makeMockMistralSpawn(['{"text":"Test output"}']); mistralSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'devstral-2', stream: true, messages: [{ role: 'user', content: 'Hello' }], }); const authCtx = { apiKey: '' }; const chunks = []; for await (const chunk of mistral.spawn(ir, authCtx)) { chunks.push(chunk); } const firstDelta = chunks.find(c => c.type === 'delta'); assert.ok(firstDelta, 'No delta chunk found'); assert.equal(firstDelta.role, 'assistant'); } finally { mistralResetSpawnImpl(); } }); it('spawn with mock: stop event present → no extra synthetic stop appended', async () => { const fakeSpawn = makeMockMistralSpawn([ '{"text":"done"}', '{"type":"stop"}', ]); mistralSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'devstral-2', stream: false, messages: [{ role: 'user', content: 'Hello' }], }); const authCtx = { apiKey: '' }; const chunks = []; for await (const chunk of mistral.spawn(ir, authCtx)) { chunks.push(chunk); } const stops = chunks.filter(c => c.type === 'stop'); assert.equal(stops.length, 1, `Expected exactly 1 stop chunk, got ${stops.length}`); } finally { mistralResetSpawnImpl(); } }); it('spawn with mock: synthetic stop emitted when JSON stream has no stop event', async () => { const fakeSpawn = makeMockMistralSpawn([ '{"text":"only content, no stop"}', ]); mistralSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'devstral-2', stream: true, messages: [{ role: 'user', content: 'Hello' }], }); const authCtx = { apiKey: '' }; const chunks = []; for await (const chunk of mistral.spawn(ir, authCtx)) { chunks.push(chunk); } const stops = chunks.filter(c => c.type === 'stop'); assert.equal(stops.length, 1, 'Should have exactly 1 synthetic stop chunk'); assert.equal(stops[0].finish_reason, 'stop'); } finally { mistralResetSpawnImpl(); } }); it('spawn with mock: non-zero exit code throws ProviderError(SPAWN_FAILED)', async () => { const fakeSpawn = makeMockMistralSpawn([], 1); mistralSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'devstral-2', stream: true, messages: [{ role: 'user', content: 'Hi' }], }); const authCtx = { apiKey: '' }; let caught = null; try { for await (const _chunk of mistral.spawn(ir, authCtx)) { // eslint-disable-line no-unused-vars // drain } } catch (e) { caught = e; } assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`); assert.equal(caught.code, 'SPAWN_FAILED'); } finally { mistralResetSpawnImpl(); } }); it('spawn throws ProviderError(AUTH_MISSING) when no auth context and no env/file', async () => { const fakeSpawn = makeMockMistralSpawn(['{"text":"test"}']); mistralSetSpawnImpl(fakeSpawn); // Override auth path to nonexistent + clear MISTRAL_API_KEY env to guarantee missing auth const savedApiKey = process.env.MISTRAL_API_KEY; const savedAuthPath = process.env.MISTRAL_VIBE_AUTH_PATH; delete process.env.MISTRAL_API_KEY; process.env.MISTRAL_VIBE_AUTH_PATH = '/nonexistent/path/.env'; try { const ir = makeIR({ model: 'devstral-2', stream: false, messages: [{ role: 'user', content: 'Hi' }], }); let caught = null; try { for await (const _chunk of mistral.spawn(ir, null)) { // eslint-disable-line no-unused-vars // drain } } catch (e) { caught = e; } assert.ok(caught instanceof ProviderError, `Expected ProviderError, got ${caught?.constructor?.name}`); assert.equal(caught.code, 'AUTH_MISSING'); } finally { if (savedApiKey !== undefined) process.env.MISTRAL_API_KEY = savedApiKey; else delete process.env.MISTRAL_API_KEY; if (savedAuthPath !== undefined) process.env.MISTRAL_VIBE_AUTH_PATH = savedAuthPath; else delete process.env.MISTRAL_VIBE_AUTH_PATH; mistralResetSpawnImpl(); } }); it('spawn with mock: progress events silently ignored, only content emitted', async () => { const fakeSpawn = makeMockMistralSpawn([ '{"type":"progress","step":1}', '{"type":"progress","step":2}', '{"text":"actual response"}', '{"type":"stop"}', ]); mistralSetSpawnImpl(fakeSpawn); try { const ir = makeIR({ model: 'devstral-2', stream: true, messages: [{ role: 'user', content: 'Do something' }], }); const authCtx = { apiKey: '' }; const chunks = []; for await (const chunk of mistral.spawn(ir, authCtx)) { chunks.push(chunk); } const deltas = chunks.filter(c => c.type === 'delta'); assert.equal(deltas.length, 1); assert.equal(deltas[0].content, 'actual response'); } finally { mistralResetSpawnImpl(); } }); // ── Test 9: healthCheck ─────────────────────────────────────────────── it('healthCheck returns {ok: false, error: "vibe binary not found"} when binary absent', async () => { const result = await mistralHealthCheck({ _binaryExistsFn: () => false, _authReadFn: () => ({ apiKey: '' }), }); assert.equal(result.ok, false); assert.equal(result.error, 'vibe binary not found'); assert.ok(typeof result.latencyMs === 'number'); }); it('healthCheck returns {ok: false, error: "auth artifact missing"} when auth missing', async () => { const result = await mistralHealthCheck({ _binaryExistsFn: () => true, _authReadFn: () => null, }); assert.equal(result.ok, false); assert.equal(result.error, 'auth artifact missing'); assert.ok(typeof result.latencyMs === 'number'); }); it('healthCheck returns {ok: true} when binary and auth both present', async () => { const result = await mistralHealthCheck({ _binaryExistsFn: () => true, _authReadFn: () => ({ apiKey: '' }), }); assert.equal(result.ok, true); assert.ok(typeof result.latencyMs === 'number'); }); // ── Test 10: estimateCost ───────────────────────────────────────────── it('estimateCost returns shape with currency USD', () => { const request = makeIR({ model: 'devstral-2', messages: [ { role: 'system', content: 'You are a coding assistant.' }, { role: 'user', content: 'Write hello world in Python.' }, ], }); const result = mistralEstimateCost(request); assert.ok(result !== null, 'estimateCost returned null'); assert.ok('inputTokens' in result, 'missing inputTokens'); assert.ok('outputTokensEstimate' in result, 'missing outputTokensEstimate'); assert.ok('currency' in result, 'missing currency'); assert.ok('usd' in result, 'missing usd'); assert.equal(result.currency, 'USD'); assert.equal(result.usd, null); // not pinned at D8 assert.ok(result.inputTokens > 0, 'inputTokens should be > 0'); assert.ok(result.outputTokensEstimate >= 0); }); it('estimateCost returns null for null/missing request', () => { assert.equal(mistralEstimateCost(null), null); assert.equal(mistralEstimateCost({}), null); }); // ── Test 11: quotaStatus ────────────────────────────────────────────── it('quotaStatus returns null at D8 (Le Chat Pro budget not exposed via API)', async () => { const result = await mistralQuotaStatus({}); assert.equal(result, null); }); // ── Test 12: auth object shape ──────────────────────────────────────── it('mistral.auth has correct shape', () => { // Authority: DOCS-2 § auth type is api-key, path is ~/.vibe/.env assert.equal(mistral.auth.type, 'api-key'); assert.equal(mistral.auth.storage, 'file'); assert.equal(typeof mistral.auth.path, 'string'); assert.ok(mistral.auth.path.includes('.vibe'), 'auth.path should reference .vibe directory'); assert.ok(mistral.auth.path.includes('.env'), 'auth.path should reference .env file'); // Portability check: path must start with homedir() (no hardcoded literal) assert.ok( mistral.auth.path.startsWith(homedir()), `auth.path "${mistral.auth.path}" should start with homedir() "${homedir()}"`, ); }); // ── Test 13: hints shape ────────────────────────────────────────────── it('mistral.hints has correct shape', () => { assert.equal(typeof mistral.hints.requiresTTY, 'boolean'); assert.equal(typeof mistral.hints.concurrentSpawnSafe, 'boolean'); assert.ok(Number.isInteger(mistral.hints.maxConcurrent) && mistral.hints.maxConcurrent > 0); assert.equal(mistral.hints.requiresTTY, false); assert.equal(mistral.hints.concurrentSpawnSafe, true); }); // ── Test 14: STATIC_REGISTRY length after D8 ───────────────────────── it('STATIC_REGISTRY.length === 3 after D8 (anthropic + openai + mistral)', () => { assert.equal(listAllProviderNames().length, 3); assert.ok(listAllProviderNames().includes('anthropic')); assert.ok(listAllProviderNames().includes('openai')); assert.ok(listAllProviderNames().includes('mistral')); }); // ── Test 15: loadProviders with mistral enabled ─────────────────────── it('loadProviders with {enabled: {mistral: true}} returns Map of size 1 with mistral', () => { const loaded = loadProviders({ enabled: { mistral: true } }); assert.equal(loaded.size, 1); assert.ok(loaded.has('mistral')); }); it('loadProviders with all 3 providers enabled returns Map of size 3', () => { const loaded = loadProviders({ enabled: { anthropic: true, openai: true, mistral: true } }); assert.equal(loaded.size, 3); assert.ok(loaded.has('anthropic')); assert.ok(loaded.has('openai')); assert.ok(loaded.has('mistral')); }); it('mistral loaded via loadProviders passes contract validation', () => { const loaded = loadProviders({ enabled: { mistral: true } }); const p = loaded.get('mistral'); const { valid, errors } = validateProvider(p); assert.equal(valid, true, `Contract errors: ${errors.join('; ')}`); assert.ok(p.models.includes('devstral-2')); }); // ── Test 16: auth artifact reading helpers ──────────────────────────── it('readAuthArtifact: MISTRAL_API_KEY env var → returns {apiKey}', () => { const saved = process.env.MISTRAL_API_KEY; process.env.MISTRAL_API_KEY = ''; try { const result = mistralReadAuthArtifact(); assert.ok(result !== null, 'Expected auth result'); assert.equal(result.apiKey, ''); } finally { if (saved !== undefined) process.env.MISTRAL_API_KEY = saved; else delete process.env.MISTRAL_API_KEY; } }); it('readAuthArtifact: MISTRAL_VIBE_AUTH_PATH pointing to nonexistent file → null', () => { const savedApiKey = process.env.MISTRAL_API_KEY; const savedAuthPath = process.env.MISTRAL_VIBE_AUTH_PATH; delete process.env.MISTRAL_API_KEY; process.env.MISTRAL_VIBE_AUTH_PATH = '/definitely/not/a/real/path/.env'; try { const result = mistralReadAuthArtifact(); assert.equal(result, null); } finally { if (savedApiKey !== undefined) process.env.MISTRAL_API_KEY = savedApiKey; else delete process.env.MISTRAL_API_KEY; if (savedAuthPath !== undefined) process.env.MISTRAL_VIBE_AUTH_PATH = savedAuthPath; else delete process.env.MISTRAL_VIBE_AUTH_PATH; } }); // ── Test 17: Suite 5 registry test updated for D8 ───────────────────── // (This re-tests the registry length assertion which now expects 3.) it('listAllProviderNames() now returns 3-element array with mistral included', () => { const names = listAllProviderNames(); assert.equal(names.length, 3); assert.deepEqual(names, ['anthropic', 'openai', 'mistral']); }); });