diff --git a/.github/workflows/flake-hunt.yml b/.github/workflows/flake-hunt.yml new file mode 100644 index 0000000..5c18c60 --- /dev/null +++ b/.github/workflows/flake-hunt.yml @@ -0,0 +1,145 @@ +name: flake hunt (#203) + +# Manual only. This never runs on push or PR — it exists to reproduce a flake that has +# been seen FOUR times, always on Linux CI, and never once on macOS (0/1000+ across two +# independent experiments). The suspected variables are the platform and the Node version, +# so both are inputs. +# +# Background: an attempt to run this on a Linux VM was invalidated because Node 22 emits a +# `node:sqlite` ExperimentalWarning on stderr, which the boot gate read as "server did not +# start" — ~23 of 50 runs failed spuriously. That is exactly why `node` is an input here +# rather than pinned: 22-vs-24 is a comparison worth running deliberately, not an accident +# to stumble into. Interpret any 22 result against that known confound. +# +# Concurrency is the load-bearing knob, not round count. test-features.mjs makes test() +# fire-and-forget for async bodies, so one suite run already spawns 15 concurrent +# `server.mjs` children across 11 ltBoot tests; running several suites at once is what +# multiplies cross-process contention. See AGENTS.md § "Testing: reaching faults inside +# server.mjs". + +on: + workflow_dispatch: + inputs: + node: + description: 'Node major version (24 = what CI and released OCP run; 22 has the SQLite stderr confound)' + default: '24' + type: choice + options: ['24', '22', '26'] + rounds: + description: 'Rounds; each round runs suites at once and waits' + default: '25' + concurrency: + description: 'Concurrent suite processes per round (this is the knob that reproduces)' + default: '4' + +jobs: + hunt: + name: 'hunt: node ${{ inputs.node }} x ${{ inputs.rounds }} rounds x ${{ inputs.concurrency }}' + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node }} + + - name: Record the environment + run: | + set -euo pipefail + { + echo "node $(node --version)" + echo "kernel $(uname -srm)" + echo "cpus $(nproc)" + echo "ephemeral $(cat /proc/sys/net/ipv4/ip_local_port_range)" + } | tee env.txt + # The ephemeral range matters: the fixed ports this suite used before #204 + # (39321-39364) sit INSIDE Linux's default 32768-60999 but OUTSIDE macOS's + # 49152-65535, which is independent support for "CI is more exposed than a Mac". + + - name: Hunt + run: | + set -uo pipefail # NOT -e: a failing suite run is the DATA, not an error + mkdir -p logs + R=${{ inputs.rounds }} + C=${{ inputs.concurrency }} + for r in $(seq 1 "$R"); do + for c in $(seq 1 "$C"); do + npm test > "logs/r${r}c${c}.log" 2>&1 & + done + wait + printf '.' + done + echo + + - name: Classify + if: always() + run: | + set -uo pipefail + + # EVERY pattern is anchored on the failure marker AND on the test's name, because + # the categories cross-contaminate otherwise. ltDiag samples the first 900B of the + # child's stdout (PR #204), which is the boot banner — so a log where only the GATE + # test failed still contains the string "Local tools: ON", and an unanchored + # `grep -l 'Local tools'` scores it as an announcement failure too. Verified against + # real logs: 3 clean runs + 1 gate-mutation run gave + # unanchored 'Local tools' -> 1 hit, attributed to the WRONG category + # anchored -> gate=1, announce=0 (correct) + # A diagnostic that quotes the program's own output makes substring classification + # unsound; anchor on what the runner prints, not on what the child printed. + n() { grep -l "✗.*$1" logs/*.log 2>/dev/null | wc -l | tr -d ' '; } + + total=$(ls logs/*.log | wc -l | tr -d ' ') + clean=$(grep -l ', 0 failed ===' logs/*.log 2>/dev/null | wc -l | tr -d ' ') + + # A category scoring 0 has NOT been proven absent — it may simply not fire in this + # configuration. Only a non-zero count is evidence. + gate=$(n 'boot gate REFUSES') + tui=$(n 'announced INERT') + announce=$(n 'BOOTS past the gate') + # These two are runtime errors rather than named tests, so they are matched on the + # error string anywhere in the log, not on a ✗ line. + eaddr=$(grep -l 'EADDRINUSE' logs/*.log 2>/dev/null | wc -l | tr -d ' ') + enotempty=$(grep -l 'ENOTEMPTY' logs/*.log 2>/dev/null | wc -l | tr -d ' ') + + { + echo "## flake hunt — node ${{ inputs.node }}" + echo + echo '```' + cat env.txt + echo '```' + echo + echo "| category | runs affected |" + echo "|---|---|" + echo "| **clean (0 failed)** | **$clean / $total** |" + echo "| #203 boot gate | $gate |" + echo "| #199 TUI inert warning | $tui |" + echo "| \`Local tools\` announce | $announce |" + echo "| EADDRINUSE | $eaddr |" + echo "| ENOTEMPTY teardown | $enotempty |" + echo + echo "A zero is not proof of absence — only a non-zero count is evidence." + } >> "$GITHUB_STEP_SUMMARY" + + # Surface the diagnostics inline so a repro is readable without downloading. + # ltDiag prints exit/signal/closed/closeMs/node plus both streams (PR #204). + if [ "$clean" -lt "$total" ]; then + echo "### first failing run" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + grep -h '✗' $(grep -L ', 0 failed ===' logs/*.log | head -1) | head -20 >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: flake-hunt-node${{ inputs.node }} + path: | + logs/ + env.txt + retention-days: 14 + + # This job does NOT fail on a reproduction. Reproducing is the goal, and a red X + # would read as "the hunt is broken" rather than "the flake was caught".