Files
dtzp555-maxGitHubtaodengClaude <claude-opus-4-8> <noreply@anthropic.com>
f14f4ec754 fix(upgrade): doctor fetches tags before deciding latest + post-flight asserts served version (#173) (#174)
Two fixes for the two halves of issue #173, both from live incidents during the
2026-07-17 fleet update:

1. scripts/doctor.mjs — `git show origin/main:package.json` reads the LOCALLY
   CACHED remote ref; without a fetch first, any machine that hadn't pulled since
   the last release saw latest == current and reported "Already at latest" (live
   repro: Oracle VM at 3.21.1 with v3.22.1 released). The doctor now runs
   `git fetch --tags --quiet` (15s timeout) before comparing, gated on
   !opts.skipNetwork; on failure (offline/auth) it falls through to the cached
   ref — the pre-existing behavior. All existing doctor tests pass mockLatest +
   skipNetwork, so no test touches the network.

2. scripts/upgrade.mjs — post-flight accepted any healthy /health (auth.ok only),
   so a stale process holding the port passed post-flight while still serving the
   OLD version (live repro: a Jul-7 nohup-fallback orphan held :3456; upgrade
   "succeeded", /health kept serving 3.21.1). New exported predicate
   postFlightOk(body, target): auth.ok AND /health.version === target (leading-v
   tolerant; empty target degrades to the auth-only check, never blocks). The
   failure message now reports the last-seen version and points at the
   stale-process diagnosis (`ss -ltnp` / `lsof -i`).

Tests: +4, mutation-proven — reverting the predicate to auth-only fails the
"orphan case" test (337/1). Full suite 338 passed / 0 failed.

No server.mjs change — scripts layer only; no cli.js operation involved, so no
citation applies.

Closes #173

Co-authored-by: dtzp555 <dtzp555@gmail.com>
Co-authored-by: Claude <claude-opus-4-8> <noreply@anthropic.com>
2026-07-17 18:52:45 +10:00

366 lines
14 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env node
/**
* scripts/upgrade.mjs — OCP unified upgrade dispatcher.
*
* Paths:
* noop current == latest, exit 0
* light same major.minor, patch bump only (existing fast path; delegated to bash)
* full cross-minor (snapshot + setup.mjs + post-flight)
* fresh_install from-version < v3.4.0 (--yes required for non-interactive)
* rollback restore from snapshot
*/
import { runDoctor } from "./doctor.mjs";
import { execSync } from "node:child_process";
import { homedir } from "node:os";
import { join } from "node:path";
import { existsSync, copyFileSync } from "node:fs";
import { writeSnapshot, listSnapshots, readSnapshot, gcSnapshots } from "./lib/snapshot.mjs";
import { DEFAULT_PORT } from "../lib/constants.mjs";
// Post-flight acceptance predicate (issue #173). A health probe passes ONLY when the server
// is authed AND actually serving the TARGET version. auth.ok alone is not enough: a stale
// process holding the port answers auth.ok=true while still running the OLD code — exactly
// what a nohup-fallback orphan did on 2026-07-17 (upgrade "succeeded", /health kept serving
// 3.21.1). Comparing /health.version to the checkout target catches orphan-holds-port,
// restart-didn't-take, and wrong-unit-restarted alike. `target` tolerates a leading "v"
// (doctor reports "v3.22.1"; /health reports "3.22.1"); an empty/unknown target degrades to
// the old auth-only check rather than blocking an otherwise-good upgrade.
export function postFlightOk(body, target) {
if (body?.auth?.ok !== true) return false;
const want = String(target || "").replace(/^v/, "");
return !want || body?.version === want;
}
export async function runUpgrade(opts = {}) {
const dryRun = !!opts.dryRun;
const yes = !!opts.yes;
// yes is reserved for Bundle 3 (fresh-install / rollback interactive gate); not used in upgrade-path here.
const plan = [];
// --- rollback path (no doctor needed; snapshot is authoritative) ---
if (opts.rollback) {
return await runRollback(opts);
}
// --- doctor pre-flight ---
const doctor = opts.mockDoctor || await runDoctor();
if (!doctor.ready_to_upgrade && doctor.next_action.kind !== "fresh_install") {
throw new Error(`doctor FAIL: ${doctor.next_action.kind} (run "ocp doctor" for details)`);
}
const kind = doctor.next_action.kind;
plan.push(`[doctor] from=${doctor.current_version} to=${doctor.latest_version} kind=${kind}`);
// --- noop ---
if (kind === "noop") {
plan.push(`[noop] already at latest (${doctor.latest_version})`);
return { path: "noop", executed: true, changed: false, plan };
}
// --- dry-run early exit ---
if (dryRun) {
plan.push(`[plan] would proceed with ${kind} path`);
if (kind === "upgrade") {
plan.push(`[plan] phase 1: snapshot to ~/.ocp/upgrade-snapshot-<ts>/`);
plan.push(`[plan] phase 2: git checkout ${doctor.latest_version} && npm install`);
plan.push(`[plan] phase 3: node setup.mjs`);
plan.push(`[plan] phase 4: launchctl bootout/bootstrap`);
plan.push(`[plan] phase 5: post-flight /health + /v1/models`);
} else if (kind === "update") {
plan.push(`[plan] light path: git pull + npm install + restart`);
} else if (kind === "fresh_install") {
plan.push(`[plan] fresh-install ai_executable[]:`);
for (const cmd of doctor.next_action.ai_executable) plan.push(` - ${cmd}`);
}
return { path: kind, executed: false, plan };
}
// --- non-dry-run paths ---
if (kind === "update") {
return { path: "update", executed: true, changed: true, plan: [...plan, "[light] delegated to bash cmd_update existing logic"] };
}
if (kind === "upgrade") {
return await runFullUpgrade({ doctor, opts });
}
if (kind === "fresh_install") {
return await runFreshInstall({ doctor, opts });
}
throw new Error(`path ${kind} not yet implemented`);
}
async function runFullUpgrade({ doctor, opts }) {
const phases = [];
let snapshotPath = null;
const exec = (cmd, label) => {
if (opts.mockExec) {
phases.push({ name: label, cmd, status: "skipped-mock" });
return "";
}
try {
const out = execSync(cmd, { stdio: ["pipe", "pipe", "pipe"] }).toString();
phases.push({ name: label, cmd, status: "ok" });
return out;
} catch (err) {
const detail = err.stderr?.toString().trim();
phases.push({ name: label, cmd, status: "fail", stderr: detail });
throw Object.assign(
new Error(`phase ${label} failed: ${detail || err.message}`),
{ phases, cmd }
);
}
};
const ocpDir = opts.ocpDir || join(homedir(), "ocp");
try {
// phase 1: pre-flight (doctor already passed; just record)
phases.push({ name: "pre-flight", status: "ok", note: `kind=upgrade from=${doctor.current_version} to=${doctor.latest_version}` });
// phase 2: snapshot
const fromCommit = opts.mockExec
? "mock-commit"
: execSync(`git -C ${ocpDir} rev-parse HEAD`).toString().trim();
snapshotPath = opts.mockExec
? "/tmp/mock-snapshot"
: writeSnapshot({ homeDir: homedir(), fromCommit, fromVersion: doctor.current_version, toVersion: doctor.latest_version });
phases.push({ name: "snapshot", path: snapshotPath, status: "ok" });
// phase 3: fetch + install
exec(`git -C ${ocpDir} fetch --tags --quiet`, "fetch+install");
exec(`git -C ${ocpDir} checkout ${doctor.latest_version}`, "fetch+install");
exec(`npm --prefix ${ocpDir} install --no-audit --no-fund`, "fetch+install");
// phase 4: reconfigure
exec(`node ${ocpDir}/setup.mjs`, "reconfigure");
// phase 5: restart (heads-up note printed before invoking)
if (!opts.mockExec) {
console.error(`[heads-up] restarting OCP service in 3s — expect ~510s blip on requests in flight.`);
await new Promise(r => setTimeout(r, 3000));
}
if (process.platform === "darwin") {
exec(`launchctl bootout gui/$(id -u)/dev.ocp.proxy 2>/dev/null || true`, "restart");
exec(`launchctl bootstrap gui/$(id -u) ${join(homedir(), "Library", "LaunchAgents", "dev.ocp.proxy.plist")}`, "restart");
} else {
exec(`systemctl --user restart ocp-proxy.service`, "restart");
}
// phase 6: post-flight (10s budget; skipped under mockExec)
if (!opts.mockExec) {
const port = process.env.CLAUDE_PROXY_PORT || String(DEFAULT_PORT);
let ok = false;
let lastSeen = null;
for (let i = 0; i < 10; i++) {
try {
const out = execSync(`curl -sf --max-time 2 http://127.0.0.1:${port}/health`).toString();
const body = JSON.parse(out);
lastSeen = body.version;
if (postFlightOk(body, doctor.latest_version)) { ok = true; break; }
} catch { /* retry */ }
await new Promise(r => setTimeout(r, 1000));
}
if (!ok) {
phases.push({
name: "post-flight", status: "fail",
message: `health did not return auth.ok=true AND version=${doctor.latest_version} within 10s`
+ (lastSeen ? ` (last saw version=${lastSeen} — a stale process may still hold the port; check \`ss -ltnp\` / \`lsof -i\`)` : ""),
});
throw new Error("post-flight failed");
}
execSync(`curl -sf --max-time 3 http://127.0.0.1:${port}/v1/models > /dev/null`);
phases.push({ name: "post-flight", status: "ok" });
} else {
phases.push({ name: "post-flight", status: "skipped-mock" });
}
// Auto-GC old snapshots after successful upgrade (best-effort, never throws).
try {
const gc = gcSnapshots(homedir(), { keepCount: 5, keepDays: 30 });
if (gc.removed.length > 0) {
console.error(`[gc] removed ${gc.removed.length} old snapshots; kept ${gc.kept.length}`);
}
} catch (e) {
console.error(`[gc] warn: snapshot GC failed: ${e.message}`);
}
return { path: "upgrade", executed: true, changed: true, snapshotPath, phases };
} catch (err) {
if (snapshotPath && !err.snapshotPath) {
Object.assign(err, {
snapshotPath,
phases,
hint: "Working tree may be at new version. Run `ocp update --rollback` to restore from snapshot."
});
}
throw err;
}
}
async function runFreshInstall({ doctor, opts }) {
if (!opts.yes) {
throw new Error("fresh_install requires --yes for non-interactive execution (or run interactively and answer y)");
}
const steps = [];
for (const cmd of doctor.next_action.ai_executable) {
if (opts.mockExec) {
steps.push({ cmd, status: "skipped-mock" });
} else {
try {
execSync(cmd, { stdio: "inherit" });
steps.push({ cmd, status: "ok" });
} catch (e) {
const detail = e.stderr?.toString().trim() || e.message;
steps.push({ cmd, status: "fail", error: String(detail) });
throw Object.assign(new Error(`fresh_install step failed: ${cmd}${detail}`), { steps });
}
}
}
return { path: "fresh_install", executed: true, changed: true, steps };
}
async function runRollback(opts) {
const homeDir = opts.homeDir || homedir();
const snapshots = opts.mockSnapshots ?? listSnapshots(homeDir);
if (opts.gc) {
const result = gcSnapshots(homeDir, { dryRun: opts.dryRun });
return { path: opts.dryRun ? "rollback-gc-dry-run" : "rollback-gc", ...result };
}
if (opts.list) {
return { path: "rollback-list", snapshots };
}
if (snapshots.length === 0) {
throw new Error("no upgrade snapshots found in ~/.ocp/upgrade-snapshot-*");
}
const target = opts.snapshotPath
? snapshots.find(s => s.path === opts.snapshotPath)
: snapshots[snapshots.length - 1];
if (!target) throw new Error(`snapshot not found: ${opts.snapshotPath} (must be inside ~/.ocp/upgrade-snapshot-*)`);
const meta = opts.mockSnapshotMeta ?? readSnapshot(target.path);
if (!meta.fromCommit) throw new Error(`snapshot ${target.path} has no from-commit.txt`);
const phases = [];
if (opts.dryRun) {
return {
path: "rollback-dry-run",
executed: false,
target: target.path,
plan: [
`git checkout ${meta.fromCommit}`,
`cp ${target.path}/plist ~/Library/LaunchAgents/dev.ocp.proxy.plist`,
`cp ${target.path}/db.bak ~/.ocp/ocp.db`,
`launchctl bootout/bootstrap`,
`ocp doctor`
]
};
}
if (!opts.yes) throw new Error("rollback requires --yes for non-interactive execution");
const exec = (cmd, label) => {
if (opts.mockExec) {
phases.push({ name: label, cmd, status: "skipped-mock" });
return "";
}
try {
execSync(cmd, { stdio: ["pipe", "pipe", "pipe"] });
phases.push({ name: label, cmd, status: "ok" });
} catch (err) {
const detail = err.stderr?.toString().trim();
phases.push({ name: label, cmd, status: "fail", stderr: detail });
throw Object.assign(
new Error(`rollback phase ${label} failed: ${detail || err.message}`),
{ phases, target: target.path }
);
}
};
const ocpDir = opts.ocpDir || join(homedir(), "ocp");
exec(`git -C ${ocpDir} checkout ${meta.fromCommit}`, "git-checkout");
if (!opts.mockExec) {
const tryCopy = (src, dst) => {
try {
if (existsSync(src)) copyFileSync(src, dst);
} catch (err) {
console.error(`[rollback] warn: could not restore ${src}${dst} (${err.code || err.message})`);
}
};
tryCopy(join(target.path, "plist"), join(homeDir, "Library", "LaunchAgents", "dev.ocp.proxy.plist"));
tryCopy(join(target.path, "service"), join(homeDir, ".config", "systemd", "user", "ocp-proxy.service"));
tryCopy(join(target.path, "db.bak"), join(homeDir, ".ocp", "ocp.db"));
tryCopy(join(target.path, "admin-key"), join(homeDir, ".ocp", "admin-key"));
phases.push({ name: "restore-files", status: "ok" });
} else {
phases.push({ name: "restore-files", status: "skipped-mock" });
}
exec(`npm --prefix ${ocpDir} install --no-audit --no-fund`, "npm-install");
if (!opts.mockExec) {
console.error(`[heads-up] restarting OCP service in 3s — expect ~510s blip on requests in flight.`);
await new Promise(r => setTimeout(r, 3000));
}
if (process.platform === "darwin") {
exec(`launchctl bootout gui/$(id -u)/dev.ocp.proxy 2>/dev/null || true`, "restart");
exec(`launchctl bootstrap gui/$(id -u) ${join(homedir(), "Library", "LaunchAgents", "dev.ocp.proxy.plist")}`, "restart");
} else {
exec(`systemctl --user restart ocp-proxy.service`, "restart");
}
return { path: "rollback", executed: true, changed: true, target: target.path, phases };
}
// CLI entrypoint — use fileURLToPath + realpath to handle symlinked install paths.
import { fileURLToPath } from "node:url";
import { realpathSync } from "node:fs";
function _isMain() {
if (!process.argv[1]) return false;
try {
return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1]);
} catch { return false; }
}
if (_isMain()) {
const args = process.argv.slice(2);
const dryRun = args.includes("--dry-run");
const yes = args.includes("--yes");
const rollback = args.includes("--rollback");
const list = args.includes("--list");
const gc = args.includes("--gc");
const targetIdx = args.indexOf("--target");
const target = targetIdx !== -1 ? args[targetIdx + 1] : undefined;
// First non-flag positional after --rollback is the snapshot path
let snapshotPath;
if (rollback) {
const rb = args.indexOf("--rollback");
const cand = args[rb + 1];
if (cand && !cand.startsWith("--")) snapshotPath = cand;
}
try {
const result = await runUpgrade({ dryRun, yes, rollback, list, gc, snapshotPath, target });
if (result.plan) for (const line of result.plan) console.log(line);
if (result.phases) for (const p of result.phases) console.log(`[${p.name}] ${p.status}${p.cmd ? `: ${p.cmd}` : ""}`);
if (result.steps) for (const s of result.steps) console.log(` ${s.status === "ok" ? "✓" : s.status === "skipped-mock" ? "·" : "✗"} ${s.cmd}`);
if (result.snapshots) {
console.log(`Found ${result.snapshots.length} snapshots:`);
for (const s of result.snapshots) console.log(` ${s.name}`);
}
if (result.removed && result.kept) {
console.log(`Snapshots: kept ${result.kept.length}, ${result.dryRun ? "would remove" : "removed"} ${result.removed.length}`);
for (const s of result.removed) console.log(` - ${s.name}`);
}
process.exit(0);
} catch (e) {
console.error(`✗ ${e.message}`);
if (e.snapshotPath) console.error(` snapshot: ${e.snapshotPath}`);
if (e.target) console.error(` target: ${e.target}`);
if (e.hint) console.error(` hint: ${e.hint}`);
process.exit(1);
}
}