feat(models): commit models.schema.json and validate the SPOT against it in CI (#196)

models.json has declared `"$schema": "./models.schema.json"` since v3.11.0, but
that file was never committed — `git log --all -- models.schema.json` is empty.
So the SPOT that ADR 0003 makes canonical had NO structural validation: a
missing contextWindow, a typo'd openclawName, or a boolean written as a string
would pass every existing test and only surface downstream — in OpenClaw's
registry, or silently inside a truncation budget.

Validated with the repo's OWN validator (lib/structured-output.mjs, shipped for
#153) rather than adding ajv. AGENTS.md requires minimal dependencies and the
repo currently has ZERO; this keeps it that way and exercises that validator on
a second real input. Capability-probed first rather than assumed: it enforces
type / required / enum / const / additionalProperties (both the boolean and the
schema form) / items / minItems, which covers everything the SPOT needs.

Three tests:
  - the $schema reference resolves to a committed file (the #196 bug itself)
  - models.json validates against models.schema.json, strict
  - the schema actually REJECTS 8 corruption shapes — missing required field,
    wrong scalar type, non-integer window, unknown extra field, alias mapped to
    a non-string, empty models array, wrong document version, unknown
    top-level key

That third test is the guard-on-the-guard: without it a vacuous schema (`{}` or
a typo'd `properties`) would make the second test pass on anything.

Mutation-proven in three directions:
  schema replaced with {}              -> "schema failed to reject: missing required field"
  additionalProperties:false removed   -> guard fails (loosened constraint caught)
  models.json corrupted (drop a field) -> strict validation fails

What the schema deliberately does NOT encode: referential integrity — that
every aliases/legacyAliases target exists as a models[].id. That is not a JSON
Schema concept; it is already covered by two dedicated tests, and the schema
says so in its description so the next reader doesn't assume it is covered.

The field descriptions carry the non-obvious consequences that have bitten this
repo before: contextWindow is a GLOBAL lever (max across all entries x 3 sets
MAX_PROMPT_CHARS for every model, ADR 0009), maxTokens is advertising only and
is enforced by OpenClaw rather than OCP, and models[] order is load-bearing
because ocp-connect uses model_ids[0] as a fallback.

Documented per release_kit (new file/SPOT/schema -> contributor section):
AGENTS.md "Key files to know" and README's Available Models section.

Tests: 460 passed, 0 failed (457 + 3).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017gbqUZ8HfBZpjjbzQ85oH8
This commit is contained in:
2026-07-27 08:48:49 +10:00
co-authored by Claude Opus 5
parent c998370b8a
commit 544cc3dad9
4 changed files with 113 additions and 1 deletions
+46
View File
@@ -4305,6 +4305,52 @@ test("models.json: every legacyAliases value resolves to a real models[].id (ref
}
});
// ── models.json validates against models.schema.json (#196) ─────────────────
// models.json carried `"$schema": "./models.schema.json"` while that file had never been
// committed, so the SPOT that ADR 0003 makes canonical had no structural validation at all —
// a missing contextWindow or a typo'd openclawName would only surface downstream, in OpenClaw
// or in a truncation budget. Validated with the repo's OWN validator (lib/structured-output.mjs,
// shipped for #153) rather than a new dependency: zero deps added, and it exercises that
// validator on a second real input.
//
// The schema deliberately does NOT try to express referential integrity (alias -> models[].id);
// that is not a JSON Schema concept and is covered by the two tests directly above.
import { validateJsonSchema as _spotValidate } from "./lib/structured-output.mjs";
const _spotSchema = JSON.parse(spotReadFileSync(spotJoin(_spotDir, "models.schema.json"), "utf8"));
test("models.json: the $schema reference resolves to a committed file", () => {
assert.equal(_spotModels.$schema, "./models.schema.json", "models.json must point at the schema");
assert.ok(_ltExists(spotJoin(_spotDir, "models.schema.json")),
"models.schema.json must exist — a dangling $schema is what #196 was filed for");
});
test("models.json validates against models.schema.json (strict)", () => {
const errors = _spotValidate(_spotModels, _spotSchema, "$", true);
assert.deepEqual(errors, [], `models.json violates its own schema:\n ${errors.join("\n ")}`);
});
// Guards the guard: if the schema were vacuous (e.g. `{}` or a typo'd `properties`), the test
// above would pass on anything. Each corruption below must be caught.
test("models.schema.json actually rejects malformed entries (guard is not vacuous)", () => {
const clone = () => JSON.parse(JSON.stringify(_spotModels));
const cases = [
["missing required field", m => { delete m.models[0].contextWindow; }],
["wrong scalar type", m => { m.models[0].reasoning = "yes"; }],
["non-integer window", m => { m.models[0].contextWindow = 200000.5; }],
["unknown extra field", m => { m.models[0].tokensPerSecond = 42; }],
["alias mapped to non-string", m => { m.aliases.opus = { id: "x" }; }],
["empty models array", m => { m.models = []; }],
["wrong document version", m => { m.version = 2; }],
["unknown top-level key", m => { m.providers = {}; }],
];
for (const [label, corrupt] of cases) {
const bad = clone(); corrupt(bad);
const errs = _spotValidate(bad, _spotSchema, "$", true);
assert.ok(errs.length > 0, `schema failed to reject: ${label}`);
}
});
// ── escapeHtml + key-name validator (issue #114) ────────────────────────────
// Replicated verbatim from dashboard.html so tests run without a browser.
function escapeHtml(s) {