wave 1 (platform stream): background-observability accuracy + owner-driven follow-ups

- observability: liveness-gated stale-job recovery, extraction run registry on globalThis with
  explicit 'embedding' phase, /api/jobs force-dynamic, client refresh nudges at every seam
- headless: providerSafePng canonicalizer (multi-IDAT Chrome PNGs → single-IDAT) fixing qwen 400s
- composer: Enter routes through pure enterAction — accepts a pending plan verdict (owner decision)
- markdown chrome: session-scoped mermaid overlay + md-blocks component set + pan-zoom math,
  corner-copy (clipboard grant in electron main), math/table/code block fixes, heading weight 900

543/543 tests; typecheck/lint/build clean
This commit is contained in:
amosblomqvist
2026-09-19 13:16:34 +03:00
parent cbecb8df03
commit 487f5d4cea
27 changed files with 1754 additions and 111 deletions
+16 -8
View File
@@ -42,7 +42,7 @@ import {
type SteerMessage,
} from "../core/contracts.ts";
import { findPendingToolCall } from "../core/pending.ts";
import { fireJobAbort } from "../agent/subagents/aborts.ts";
import { fireJobAbort, isJobLive } from "../agent/subagents/aborts.ts";
import { stores } from "./stores.ts";
import { isTurnRunning } from "./turn-liveness.ts";
@@ -445,16 +445,24 @@ async function waitForIdle(chatId: string): Promise<void> {
// ─── crash recovery ──────────────────────────────────────────────────────────
/** Jobs this old with status "running" are orphans (crash/restart). */
const STALE_JOB_MS = 15 * 60_000;
const SWEEP_THROTTLE_MS = 30_000;
/**
* Lazy sweep (called off the /api/jobs poll, throttled): every stale
* "running" job either has a result (the wake never fired — re-deliver,
* idempotently) or doesn't (the loop itself died — re-spawn it).
* Lazy sweep (called off the /api/jobs poll, throttled), LIVENESS-GATED
* (accuracy wave, 2026-09-19): a "running" record without a live loop
* registration (subagents/aborts.ts) is an orphan of a process death —
* re-deliver its stored result (idempotently) or re-spawn the loop, at
* once (the old 15-minute age heuristic waited on ghosts AND re-ran
* hung-but-alive loops beside themselves — the figure-maker incident,
* 2026-09-18). A record WITH a live registration is provably executing
* in THIS process — never touched, however old: a hung first model call
* (zero usage ever billed) stays honestly "running" in the dock until
* END kills it. The abort registration spans loop start to delivery
* completion, so a live registration can never belong to a dead run.
*/
function recoverStaleJobs(): void {
/** Re-run a job's sub-agent loop — exposed for the sweep's tests (the
* /api/jobs poll is its only production caller, disabled under NODE_ENV=test). */
export function recoverStaleJobs(): void {
const s = state();
const now = Date.now();
if (now - s.lastSweepAt < SWEEP_THROTTLE_MS) return;
@@ -463,7 +471,7 @@ function recoverStaleJobs(): void {
for (const chat of stores.chatStore.listChats()) {
for (const job of stores.jobStore.listJobs(chat.id)) {
if (job.status !== "running") continue;
if (now - Date.parse(job.updatedAt) < STALE_JOB_MS) continue;
if (isJobLive(job.id)) continue; // provably executing here — never re-run beside it
if (!stores.chatStore.getChat(job.chatId)) continue;
if (isTurnRunning(job.chatId)) continue; // a wake turn may be delivering it right now
if (job.kind === "graph-context") {