// OCP multimodal helpers — OpenAI `image_url` content parts → Anthropic image // blocks fed to `claude -p --input-format stream-json`. (issue #110) // // Class B.1 (OpenAI-compatibility surface). Protocol authority is OpenAI's // chat/completions spec — the multimodal `content` parts shape // (https://platform.openai.com/docs/guides/vision and // https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages, // `image_url` part with `image_url.url` = data URI or http(s) URL). Authorized // by ADR 0006. This module introduces NO field beyond OpenAI's published shape: // the OpenAI-side vocabulary read here is `type:"image_url"` + // `image_url:{url, detail?}`; the Anthropic-side vocabulary written here // (`type:"image", source:{type:"base64"|"url", ...}`) is the CLI's native // stream-json input contract, not an OCP invention. // // Kept as a pure module (no I/O, no network, no process state) mirroring the // lib/*.mjs pattern so it is unit-testable without a live server. server.mjs is // the only consumer; it owns spawning, caps configuration, and HTTP status. // Anthropic vision-supported image media types. A data URI whose media type is // outside this set is rejected with a clear 4xx rather than forwarded (the API // would reject it anyway; failing early gives a better error). export const SUPPORTED_IMAGE_TYPES = new Set([ "image/jpeg", "image/png", "image/gif", "image/webp", ]); // Default caps. server.mjs overrides these from env; they live here so the // pure transform is self-contained and testable. export const DEFAULT_MULTIMODAL_OPTS = { allowRemoteUrl: false, // http(s) image URLs are OFF by default (v1: data URIs only) maxImageBytes: 5 * 1024 * 1024, // per-image decoded-byte cap maxImages: 20, // max image parts across the whole request maxTotalImageBytes: 20 * 1024 * 1024, // aggregate decoded-byte cap maxTextChars: Infinity, // text-char budget (server passes MAX_PROMPT_CHARS); Infinity = no truncation }; // Typed error so server.mjs can map to the right HTTP status + OpenAI-shaped // error body. `status` is the HTTP code; `type` is the OpenAI error `type`. export class MultimodalError extends Error { constructor(code, status, message) { super(message); this.name = "MultimodalError"; this.code = code; this.status = status; this.type = "invalid_request_error"; // OpenAI error `type` for 4xx client errors } } // True if any message carries an OpenAI `image_url` content part. Cheap guard so // the byte-for-byte text path is only left when an image is genuinely present. export function hasImageContent(messages) { if (!Array.isArray(messages)) return false; for (const m of messages) { if (m && Array.isArray(m.content)) { for (const part of m.content) { if (part && part.type === "image_url") return true; } } } return false; } // Extract the URL string from an OpenAI image_url part. Spec form is // `{type:"image_url", image_url:{url, detail?}}`; many OpenAI-compatible clients // also send `image_url` as a bare string. Accept both (input leniency — no new // output field). `detail` (auto|low|high) is OpenAI-only and has no Anthropic // analogue, so it is read-and-ignored. function imageUrlOf(part) { const iu = part.image_url; if (typeof iu === "string") return iu; if (iu && typeof iu.url === "string") return iu.url; return null; } // Parse a base64 data URI: `data:[][;base64],`. // Returns { mediaType, data (base64), bytes (decoded size) } or throws MultimodalError. function parseDataUri(uri) { const comma = uri.indexOf(","); if (comma === -1) { throw new MultimodalError("invalid_data_uri", 400, "Malformed image data URI (no comma)."); } const meta = uri.slice(5, comma); // strip leading "data:" const segs = meta.split(";"); const mediaType = (segs[0] || "").trim().toLowerCase(); const isBase64 = segs.slice(1).some((s) => s.trim().toLowerCase() === "base64"); if (!isBase64) { throw new MultimodalError("invalid_data_uri", 400, "Only base64-encoded image data URIs are supported."); } if (!SUPPORTED_IMAGE_TYPES.has(mediaType)) { throw new MultimodalError( "unsupported_image_type", 400, `Unsupported image media type '${mediaType || "(none)"}'. Supported: ${[...SUPPORTED_IMAGE_TYPES].join(", ")}.` ); } // Strip incidental whitespace/newlines some encoders insert into data URIs. const data = uri.slice(comma + 1).replace(/\s/g, ""); if (data.length === 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(data)) { throw new MultimodalError("invalid_data_uri", 400, "Image data URI payload is not valid base64."); } // Decoded size from base64 length (minus padding); avoids decoding the buffer // just to measure it. const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0; const bytes = Math.floor((data.length * 3) / 4) - padding; return { mediaType, data, bytes }; } // Convert a single OpenAI image_url part to an Anthropic image block, enforcing // caps via the mutable `acc` accumulator ({ images, bytes }). Throws MultimodalError. function imagePartToBlock(part, opts, acc) { const url = imageUrlOf(part); if (!url) { throw new MultimodalError("invalid_image_url", 400, "image_url part is missing a URL."); } acc.images += 1; if (acc.images > opts.maxImages) { throw new MultimodalError("too_many_images", 413, `Too many images in request (max ${opts.maxImages}).`); } if (url.startsWith("data:")) { const { mediaType, data, bytes } = parseDataUri(url); if (bytes > opts.maxImageBytes) { throw new MultimodalError("image_too_large", 413, `Image exceeds per-image size limit (${opts.maxImageBytes} bytes).`); } acc.bytes += bytes; if (acc.bytes > opts.maxTotalImageBytes) { throw new MultimodalError("images_too_large", 413, `Total image payload exceeds limit (${opts.maxTotalImageBytes} bytes).`); } return { type: "image", source: { type: "base64", media_type: mediaType, data } }; } if (/^https?:\/\//i.test(url)) { if (!opts.allowRemoteUrl) { throw new MultimodalError( "remote_url_disabled", 400, "Remote image URLs are disabled. Enable CLAUDE_IMAGE_ALLOW_URL=1 to allow http(s) image URLs, or pass the image as a base64 data URI." ); } // Passthrough as an Anthropic url-source block. OCP does NOT fetch the URL // itself (no OCP-side SSRF surface); the fetch is performed upstream by the // Anthropic API. Best-effort: unreachable/blocked URLs surface as an API error. return { type: "image", source: { type: "url", url } }; } throw new MultimodalError("unsupported_url_scheme", 400, "image_url must be a base64 data URI or an http(s) URL."); } // Role prefix mirrors messagesToPrompt()'s text-path labeling so a multi-turn // conversation reads the same whether or not it carries images. System messages // are handled by the caller via --system-prompt and never reach here. function rolePrefix(role) { if (role === "assistant") return "[Assistant] "; return ""; // user / tool / anything else: verbatim, as in the text path } // Build the Anthropic content-block array for a single stream-json user // envelope. Mirrors the text path's "collapse the whole conversation into one // turn passed via stdin" model (OCP runs stateless, full context per spawn), but // preserves image position relative to text and keeps images out of the text // char budget entirely. Returns { blocks, stats } or throws MultimodalError. export function buildImageBlocks(messages, opts = {}) { const o = { ...DEFAULT_MULTIMODAL_OPTS, ...opts }; const blocks = []; const acc = { images: 0, bytes: 0 }; let textChars = 0; let firstMessage = true; const pushText = (text) => { if (!text) return; blocks.push({ type: "text", text }); textChars += text.length; }; for (const m of messages) { const prefix = rolePrefix(m.role); // Separate messages with a blank line, matching messagesToPrompt's "\n\n" join. const sep = firstMessage ? "" : "\n\n"; firstMessage = false; let prefixEmitted = false; const emitPrefixWith = (t) => { if (prefixEmitted) return t; prefixEmitted = true; return sep + prefix + t; }; if (typeof m.content === "string") { pushText(emitPrefixWith(m.content)); continue; } if (!Array.isArray(m.content)) { // null / object content: mirror contentToText's fallback. const t = m.content == null ? "" : JSON.stringify(m.content); pushText(emitPrefixWith(t)); continue; } for (const part of m.content) { if (part && part.type === "text" && typeof part.text === "string") { pushText(emitPrefixWith(part.text)); } else if (part && part.type === "image_url") { // Ensure the role prefix isn't lost when a message leads with an image. if (!prefixEmitted && prefix) pushText(emitPrefixWith("")); blocks.push(imagePartToBlock(part, o, acc)); } else { // audio / file / unknown parts: preserve the existing placeholder // behavior (issue #110) — deferred to a future version. pushText(emitPrefixWith("[non-text content omitted]")); } } } // Defensive: a stream-json user turn must have at least one content block. if (blocks.length === 0) blocks.push({ type: "text", text: "" }); // Enforce the text-char budget (PR #154 review F2). Without this, attaching a // single tiny image would let unbounded text bypass the gateway's runaway-context // guard entirely — messagesToPrompt truncates the text path, so the multimodal // path must too. Image blocks are preserved and are NOT counted (they are bounded // by the byte/count caps above). const budgeted = enforceTextBudget(blocks, o.maxTextChars); return { blocks: budgeted.blocks, stats: { imageCount: acc.images, totalImageBytes: acc.bytes, textChars: budgeted.textChars, originalTextChars: budgeted.originalTextChars, truncated: budgeted.truncated, }, }; } // Enforce a text-char budget over already-built content blocks, mirroring // messagesToPrompt's "keep the tail, drop the oldest" truncation. Image blocks are // preserved in place; only text blocks count against the budget. A truncation note // is prepended when anything is dropped. Returns // { blocks, truncated, originalTextChars, textChars }. function enforceTextBudget(blocks, maxTextChars) { let originalTextChars = 0; for (const b of blocks) if (b.type === "text") originalTextChars += b.text.length; if (!(maxTextChars > 0) || originalTextChars <= maxTextChars) { return { blocks, truncated: false, originalTextChars, textChars: originalTextChars }; } // Keep the most recent text up to the budget; trim within the boundary block and // drop older text blocks. Non-text (image) blocks always survive, in order. let budget = maxTextChars; const out = []; for (let i = blocks.length - 1; i >= 0; i--) { const b = blocks[i]; if (b.type !== "text") { out.unshift(b); continue; } if (budget <= 0) continue; // older text fully dropped if (b.text.length <= budget) { out.unshift(b); budget -= b.text.length; } else { out.unshift({ type: "text", text: b.text.slice(b.text.length - budget) }); budget = 0; } } out.unshift({ type: "text", text: "[System] Note: older text content was truncated to fit the context limit." }); let textChars = 0; for (const b of out) if (b.type === "text") textChars += b.text.length; return { blocks: out, truncated: true, originalTextChars, textChars }; } // Serialize the non-system conversation to a single newline-terminated // stream-json user message for `claude -p --input-format stream-json` stdin. // Returns { payload, stats } or throws MultimodalError. export function buildStreamJsonInput(messages, opts = {}) { const { blocks, stats } = buildImageBlocks(messages, opts); const envelope = { type: "user", message: { role: "user", content: blocks } }; return { payload: JSON.stringify(envelope) + "\n", stats }; }