Merge durable background job delivery and recovery

This commit is contained in:
amosblomqvist
2026-10-26 02:39:59 +02:00
19 changed files with 730 additions and 329 deletions
+5 -7
View File
@@ -41,18 +41,16 @@ export function unregisterJobAbort(jobId: string): void {
* the liveness gate for the stale-job sweep: a "running" record with a
* live registration belongs to a loop this process owns (hung or slow, it
* is the one truth — never re-spawn beside it); without one the record is
* an orphan of a process death and recovers immediately. The span is
* loop-start to delivery-completion: the runner's unregistering finally
* runs only after deliver() resolves, by which point finalizeJob has
* flipped the record — so absence of registration is unambiguous. */
* an orphan of a process death and recovers immediately. The registration
* spans computation and cleanup only. Finished outcomes have independent,
* persisted delivery obligations; no worker waits for their consumption. */
export function isJobLive(jobId: string): boolean {
return registry().has(jobId);
}
/** Abort a running job's loop. Returns false when no controller is live —
* the job's loop is not executing (crashed, or in the delivering-result
* phase where the runner has already returned); the caller then owns the
* whole abort on the record side. */
* the job's loop is not executing. The endpoint separately refuses abort
* for a finished worker, even while its session delivery remains pending. */
export function fireJobAbort(jobId: string): boolean {
const controller = registry().get(jobId);
if (!controller) return false;
+35 -27
View File
@@ -3,6 +3,7 @@ import type { SearchProvider } from "../../core/search.ts";
import {
accumulateJobUsage,
buildSubagentSteer,
isSubagentFailure,
piUsageToStep,
type DemoResult,
type DemoToolInput,
@@ -40,13 +41,13 @@ import { runVisualMaker } from "./visual-maker.ts";
* (the job tools' execute fires this via the spawn seam, fire-and-forget),
* updating the job's activity line on every iteration. On completion the
* runner builds the STEER MESSAGE (buildSubagentSteer) and hands it to
* `deliver`; the wake pipeline fires the server-side teacher turn and
* flips the job to done/error when the lesson has actually resumed.
* `deliver` without waiting. The worker's final outcome and pending
* delivery are saved together; the session owns delivery from then on.
*
* On failure the result is the failure shape (empty html/svg + "FAILED: …"
* notes; research: failure brief) — and it STILL steers, because the
* teacher must learn the outcome. If delivery itself fails, the job is
* marked error (the learner's activity card shows it).
* teacher must learn the outcome. Failed delivery starts stay quietly
* pending and never rewrite the worker's computation outcome.
*/
/**
@@ -67,9 +68,9 @@ export interface JobRunnerDeps {
chatStore: ChatStore;
jobStore: JobStore;
/**
* Hand a finished job's steer to the wake pipeline — resolves when the
* steer is delivered (the wake turn completed and the job status
* flipped); rejects when delivery fails unrecoverably.
* Hand a finished job's steer to the session delivery registry. The
* promise tracks durable marker acceptance, but the worker never waits
* on it. Its persisted pending obligation survives process restart.
*/
deliver: (steer: SteerMessage) => Promise<void>;
/** Settings-resolved engine per role (model slots wave, 2026-09-20):
@@ -96,8 +97,8 @@ export interface JobRunnerDeps {
export interface JobRunner {
/** Run one job's sub-agent loop to completion (background; never throws).
* The tool input rides along from dispatch (the spawn fires mid-stream,
* before the tool part is persisted — only crash-recovery re-spawns
* resolve the input from the transcript). */
* before the tool part is persisted). Recovery reads saved job input,
* with transcript fallback for legacy jobs. Delivery is not awaited. */
runJob(jobId: string, input?: JobToolInput): Promise<void>;
}
@@ -270,14 +271,15 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
}
}
/** Resolve the job's tool input: dispatch passes it directly; only a
* crash-recovery re-spawn (no input) falls back to scanning the
* transcript (by then onFinish has long persisted the part). */
/** Dispatch input wins, then the job's saved arguments. Legacy jobs
* without saved input fall back to the durable transcript, which may
* lack a call if the dispatching turn died before settlement. */
function resolveInput(
job: JobRecord,
input: JobToolInput | undefined,
): JobToolInput | undefined {
if (input) return input;
if (job.input !== undefined) return job.input as JobToolInput;
const chat = chatStore.getChat(job.chatId);
if (!chat) return undefined;
// Wave D: the record is native history — the scan reads the native
@@ -295,27 +297,28 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
}
/**
* The tail of every job: store the result, build the steer, hand it to
* the wake pipeline. The pipeline flips the status when the lesson has
* actually resumed — until then the job reads "running" so the client's
* activity card keeps polling and re-syncs once, at the end.
* Save the computation outcome and session delivery obligation together.
* Worker liveness ends after cleanup, not after a lesson consumes the result.
*/
async function finishJob(jobId: string, result: ResearchResult | DemoResult | VisualResult | PdfResult): Promise<void> {
const job = jobStore.getJob(jobId);
if (!job) return;
// Aborted mid-run or mid-delivery (abortJob flips the record before
// Aborted mid-run (abortJob flips the record before
// the loop unwinds): no result write, no steer — the abort path owns
// the record and queues the abort steer itself.
if (job.status !== "running") return;
jobStore.updateJob(jobId, { result, activity: "delivering result" });
const steer = buildSubagentSteer(job, result);
try {
await deliver(steer);
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
console.error(`[job ${jobId}] steer delivery failed:`, err);
jobStore.updateJob(jobId, { status: "error", error: reason, activity: "delivery failed" });
}
const failed = isSubagentFailure(job.kind, result);
jobStore.updateJob(jobId, {
result,
status: failed ? "error" : "done",
activity: failed ? "failed" : "finished",
delivery: { kind: "result", status: "pending" },
});
// The saved result now belongs to the session. Do not keep the worker,
// abort registration, or temporary resources alive behind a question.
void deliver(buildSubagentSteer(job, result)).catch((err: unknown) => {
console.error(`[job ${jobId}] pending delivery:`, err);
});
}
/** Chat-record + input resolution shared by all three runners. */
@@ -343,7 +346,12 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
const { job } = ctx;
const input = (passedInput ?? resolveInput(job, passedInput)) as ResearchToolInput | undefined;
const question = typeof input?.question === "string" && input.question.trim() ? input.question : "(no question)";
if (!(typeof input?.question === "string" && input.question.trim())) {
jobStore.updateJob(jobId, { error: "research question unavailable" });
await finishJob(jobId, { brief: "Research failed: the research question is unavailable.", sources: [] });
return;
}
const question = input.question;
const context = typeof input?.context === "string" && input.context.trim() ? input.context : undefined;
const prompt = [
+1 -1
View File
@@ -78,7 +78,7 @@ function dispatch(
input: ResearchInput | DemoInput | VisualInput | PdfInput,
): string {
const existing = ctx.jobStore.findJobByToolCallId(ctx.chatId, toolCallId);
const job = existing ?? ctx.jobStore.createJob({ kind, chatId: ctx.chatId, toolCallId });
const job = existing ?? ctx.jobStore.createJob({ kind, chatId: ctx.chatId, toolCallId, input });
if (!existing) ctx.spawn(job, input);
return `Dispatched (job ${job.id}). ${ack}`;
}
+28 -32
View File
@@ -21,6 +21,12 @@ export interface JobRecord {
chatId: string;
/** The tool part this job fulfills. */
toolCallId: string;
/** Dispatch-time arguments, saved before spawn. Older jobs resolve from history. */
input?: unknown;
/** Session-owned delivery, independent of the worker's terminal outcome.
* Absent on legacy records: old terminal jobs must never be replayed. */
delivery?: { kind: "result" | "abort"; status: "pending" | "delivered" };
/** Worker computation only; a pending delivery does not keep it running. */
status: JobStatus;
/** Latest human-readable activity line, e.g. `web_search: "sigma.js v3"`. */
activity: string;
@@ -148,42 +154,32 @@ export function accumulateJobUsage(
* sourced briefs) — tracking a live activity line in the job record.
* ONE central client poll of GET /api/jobs?chatId=… (in Chat) drives every
* job surface: the transcript dispatch cards (live activity by
* toolCallId, quiet one-liner after delivery), the sub-agents sidebar
* toolCallId, quiet one-liner after worker completion), the sub-agents sidebar
* panel, and the single re-sync when a job settles.
*
* 3. On completion the runner builds a STEER MESSAGE (buildSubagentSteer)
* and hands it to the wake pipeline (src/server/jobs.ts): a user-role
* result-carrying native marker — the content is what the MODEL needs
* (the research brief, the builder's notes), plus a data-subagent-artifact
* part with what the UI renders (the full result: brief + sources /
* SVG / demo HTML). Provider conversion drops data parts, so artifact
* payloads never enter the teacher's context. metadata.subagentResult
* identifies the job; the UI renders these as sub-agent result cards,
* NEVER as learner interjections.
* 3. Dispatch saves input on the job before spawn. On completion the
* runner saves result + terminal worker status + pending delivery in
* one job update, then queues buildSubagentSteer WITHOUT waiting for
* delivery. The native custom marker's content is model-facing text;
* details carry the full artifact payload for the UI. Worker resources
* are released even if a question keeps the result pending indefinitely.
* Only computing workers appear in Background agents or accept abort.
*
* 4. The wake pipeline delivers the steer by firing a SERVER-SIDE teacher
* turn through the same turn composition as a client request
* (handleChatRequest) — write-ahead persist, turn liveness, extraction
* trigger: one code path. A per-chat serialized queue: a steer waits
* while a turn is in flight OR a question tool part is pending (an
* unanswered client-side tool call breaks provider conversion — the
* steer fires right after the learner's continuation), then fires; a 409
* (a client turn raced in) retries. Delivery is idempotent per jobId, so
* a crashed wake can be re-entered by the stale-job recovery sweep. The
* job's status flips to done/error only AFTER the wake turn completes —
* a polling client that sees the flip re-syncs once and gets the
* finished record.
* 4. Session delivery owns one registry entry per deterministic steer ID,
* including while a wake is in flight or a 409/403 returns its cargo.
* A live agent takes it via pi steering; registration restores saved
* pending jobs before the answer run's first poll. Otherwise the idle
* wake uses the normal turn composition and respects pending questions,
* dismissal, and background consent. A saved transcript marker completes
* delivery; HTTP 200 or streamed visibility alone does not.
*
* CONTINUATION COALESCING: the idle window above can starve — a
* quiz-chaining teacher ends every turn with a new pending check, so
* the gates never stay open long enough for a wake to fire. The fix:
* when a client turn wins the turn registry, the handler CLAIMS every
* queued, undelivered steer (jobs.ts claimSteers) and appends it after
* the learner's answer in that very turn — conversion-safe, because the
* answered quiz carries its output. Every answer flushes pending
* steers immediately; the wake pump remains for turns that end with no
* pending question. The pump re-checks steerDelivered right before
* firing, so a claim racing the pump converges to exactly one delivery.
* Failed starts remain quietly pending until a learner run or restart
* recovery, without new timers, visible delivery errors, or worker-status
* changes. Existing 409 backoff and consent deferral remain. Accepted
* turns are never automatically replayed. Legacy running records with a
* result become finished/pending; legacy terminal records with no
* delivery metadata remain untouched. Session deletion discards pending
* deliveries rather than treating them as running work.
*
* 5. No pending job parts ever exist, so the composer never gates on jobs
* (the learner can interject mid-job; the steer queues behind their
+2
View File
@@ -136,6 +136,8 @@ export interface CreateJobInit {
toolCallId: string;
/** First activity line (defaults to "starting"). */
activity?: string;
/** Saved on the job before its worker is spawned. */
input?: unknown;
}
/** Mergeable fields — identity (id, kind, chatId, toolCallId, createdAt) is immutable. */
+15 -10
View File
@@ -17,7 +17,7 @@ import { createExtractionPipeline } from "../agent/subagents/extraction.ts";
import type { ModelRole } from "../../config.ts";
import {
deliverJobResult,
noteSteerVisible,
noteSteersDurable,
reconcileUserSteers,
registerLiveAgent,
restoreUncommittedAutomatic,
@@ -58,7 +58,7 @@ import {
type TurnInput,
} from "../core/input-events.ts";
import { steerDelivered } from "../core/contracts.ts";
import type {CallUsage, SteerMessage} from "../core/contracts.ts";
import type {CallUsage} from "../core/contracts.ts";
/**
* One chat turn over the server-owned native record. Framework-free (plain
@@ -259,7 +259,8 @@ export async function handleChatInput(
chatId,
);
const compactedRecord = stores.chatStore.getChat(chatId);
if (compactedRecord) record = compactedRecord;
if (!compactedRecord) return Response.json({ error: "Unknown chat" }, { status: 404 });
record = compactedRecord;
// The input's native appends (pure; the honest 400 for an answer with no
// pending question call). Computed BEFORE beginTurn so a refused input
@@ -329,9 +330,7 @@ export async function handleChatInput(
// Visibility bookkeeping is not the canonical save. Its failure must
// neither misreport a saved input as lost nor prevent the turn running.
try {
for (const appended of stagedAutomatic ? [] : appends) {
if ((appended as { role?: string }).role === "custom") noteSteerVisible(appended as SteerMessage);
}
if (!stagedAutomatic) noteSteersDurable(chatId, merged);
} catch (err) {
console.error(`[chat ${chatId}] input visibility bookkeeping failed:`, err);
}
@@ -383,7 +382,11 @@ export async function handleChatInput(
foregroundOwed: !callPurpose.background,
purpose: callPurpose,
scopeCurrent: (scope) => connectionScopeCurrent(conversation.provider, scope),
commitStaged: () => { stores.chatStore.saveMessages(chatId, merged); persistBaseCount = merged.length; },
commitStaged: () => {
stores.chatStore.saveMessages(chatId, merged);
persistBaseCount = merged.length;
noteSteersDurable(chatId, merged);
},
restoreAutomatic: (automaticIds, retainedIds) => restoreUncommittedAutomatic(chatId, constructed, automaticIds, retainedIds),
});
const agentOpts: AgentBuildOptions = {
@@ -481,7 +484,8 @@ export async function handleChatInput(
// run's first LLM call (the native replacement for continuation
// coalescing). Settlement still cannot beat the flow below: it needs
// a streamed response, which never happens inside the constructor.
registerLiveAgent(chatId, constructed);
registerLiveAgent(chatId, constructed, input.type !== "steers" && input.type !== "continue",
input.type === "steers" ? new Set(appends.map((message) => message.id)) : undefined);
// Debug recorder run marker (owner, 2026-10-04): correlates the
// JSONL event stream with the record's turns; no-op unless
// LH_PROVIDER_EVENTS=1.
@@ -522,12 +526,13 @@ export async function handleChatInput(
// merely because usage/extraction/other settlement work failed.
const persisted = stores.chatStore.getChat(chatId)?.messages ?? [];
reconcileUserSteers(chatId, persisted);
settleSteers(chatId, persisted);
settleSteers(chatId, persisted, true, constructed);
},
onFinish: (turnMessages, isAborted) => {
// Wave B: the record write is the agent's own final state —
// the new messages are the slice past the merged base.
const stamped = persistTurn(stores.chatStore, chatId, persistBaseCount, turnMessages, isAborted);
noteSteersDurable(chatId, stores.chatStore.getChat(chatId)?.messages ?? stamped);
// usage log append (idempotent per turn id) — the stats bar
// refolds this on the client after the turn settles. The
// provenance message is the turn's FIRST new assistant message.
@@ -573,7 +578,7 @@ export async function handleChatInput(
// resolve; stranded markers return to pending), then let
// the pump take any stragglers down the idle wake.
unregisterLiveAgent(chatId);
settleSteers(chatId, stamped);
settleSteers(chatId, stamped, false, constructed);
// Composer steers (2026-10-03): delivered ids dropped, stranded
// ones stay parked for the chat's next run (the registry is the
// stranding safety net — the agent's undrained copy died above).
+26 -6
View File
@@ -1,4 +1,5 @@
import { abortExtraction, extractionStatuses } from "../agent/subagents/extraction.ts";
import { isJobLive } from "../agent/subagents/aborts.ts";
import { coverageOf } from "../core/coverage.ts";
import { deleteChatTraces, type ChatDeleteMode, type ChatDeletionStats } from "../core/graph-deletion.ts";
import { titleFromPrompt } from "../core/contracts.ts";
@@ -6,6 +7,7 @@ import { graphRevision } from "./graph.ts";
import { stores } from "./stores.ts";
import { isTurnRunning } from "./turn-liveness.ts";
import { clearTurnFailure } from "./turn-runs.ts";
import { discardChatDeliveries } from "./jobs.ts";
/**
* Server-side chat entries for the transport layer (app/api/chats*,
@@ -66,10 +68,7 @@ export type ChatDeleteOutcome =
* the wind-down finds the chat gone at its merge and aborts with
* nothing merged.
*/
export async function deleteChatSession(chatId: string, mode: ChatDeleteMode): Promise<ChatDeleteOutcome> {
if (!stores.chatStore.getChat(chatId)) {
return { ok: false, status: 404, code: "not_found", error: "No such session" };
}
function runningWorkBlocker(chatId: string): Extract<ChatDeleteOutcome, { status: 409 }> | null {
if (isTurnRunning(chatId)) {
return {
ok: false,
@@ -79,7 +78,11 @@ export async function deleteChatSession(chatId: string, mode: ChatDeleteMode): P
jobIds: [],
};
}
const runningJobs = stores.jobStore.listJobs(chatId).filter((job) => job.status === "running");
// Legacy records kept completed workers running until delivery. A saved
// result without a live loop is pending session data, not computing work;
// deletion must not depend on a prior recovery poll to recognize it.
const runningJobs = stores.jobStore.listJobs(chatId).filter((job) =>
job.status === "running" && (!job.result || isJobLive(job.id)));
if (runningJobs.length > 0) {
return {
ok: false,
@@ -89,6 +92,15 @@ export async function deleteChatSession(chatId: string, mode: ChatDeleteMode): P
jobIds: runningJobs.map((job) => job.id),
};
}
return null;
}
export async function deleteChatSession(chatId: string, mode: ChatDeleteMode): Promise<ChatDeleteOutcome> {
if (!stores.chatStore.getChat(chatId)) {
return { ok: false, status: 404, code: "not_found", error: "No such session" };
}
const blocker = runningWorkBlocker(chatId);
if (blocker) return blocker;
if (extractionStatuses().some((run) => run.chatId === chatId)) {
abortExtraction(chatId); // true cancellation — spend stops, nothing merges
const deadline = Date.now() + WIND_DOWN_TIMEOUT_MS;
@@ -105,11 +117,19 @@ export async function deleteChatSession(chatId: string, mode: ChatDeleteMode): P
}
}
// A result wake or a worker may have started during extraction wind-down.
// Recheck immediately before the synchronous purge, with no await gap.
const lateBlocker = runningWorkBlocker(chatId);
if (lateBlocker) return lateBlocker;
// THE SYNC BLOCK — no awaits below (see the doc comment). Order:
// ledgers → jobs → the chat record.
const stats = deleteChatTraces(stores.graphStore, chatId, mode);
const jobsRemoved = stores.jobStore.deleteJobsForChat(chatId);
if (stores.chatStore.deleteChat(chatId)) clearTurnFailure(chatId);
if (stores.chatStore.deleteChat(chatId)) {
discardChatDeliveries(chatId);
clearTurnFailure(chatId);
}
return { ok: true, mode, stats, jobsRemoved, graphRevision: graphRevision() };
}
+185 -163
View File
@@ -1,7 +1,7 @@
/**
* THE WAKE PIPELINE (contracts.ts "THE ASYNC PROTOCOL" §4) — server-side
* steer delivery for finished sub-agent jobs. Wave C (2026-09-26):
* delivery rides pi's OWN follow-up queue when a run is live, and the
* delivery rides pi's OWN steering queue when a run is live, and the
* idle wake otherwise. One registry, two vehicles.
*
* deliverJobResult(steer) — the runner calls this on job completion.
@@ -40,15 +40,17 @@
*
* settleSteers(chatId, persisted) — the handler calls this at turn
* settlement (after unregisterLiveAgent): entries the run DELIVERED
* resolve + flip their jobs; entries that were handed to the dying
* resolve + acknowledge delivery; entries handed to the dying
* run but never injected (stopper exit, abort, a missed final poll)
* return to pending and the pump restarts — no stranding, no loss.
*
* Crash recovery (recoverStaleJobs): a process restart can orphan jobs —
* "running" with a result (the wake never fired) or without one (the
* sub-agent loop itself died). The sweep re-delivers finished results
* (idempotently) and re-spawns unfinished loops, both via injected
* seams. It runs lazily off the /api/jobs poll, throttled.
* Worker status ends at computation completion; JobRecord.delivery owns
* the pending result or abort notice independently. Registry entries
* survive in-flight wakes, so duplicate submissions share one promise.
* Failed starts stay pending without timers or worker errors. Recovery
* restores known pending deliveries and re-spawns unfinished workers;
* registration also restores pending delivery before the first poll.
* Legacy terminal jobs without delivery metadata are never replayed.
*
* All mutable state lives on globalThis (the turn-liveness lesson: Next dev
* gives each route entry its own module instances — a module-level Map would
@@ -83,6 +85,14 @@ export type JobRespawn = (jobId: string) => void;
interface QueuedSteer {
steer: SteerMessage;
promise: Promise<void>;
/** Wake cargo remains owned here, but must not steer into its own run. */
waking?: boolean;
/** The constructed vehicle, once startup claims a turn. Until then only
* the pump owns waking: an unrelated foreground settlement cannot free it. */
wakeAgent?: Agent;
/** A failed start waits for a learner run or process restart, not a timer. */
deferred?: boolean;
resolve: () => void;
reject: (reason: Error) => void;
/** The live agent this marker was steered into (agent.steer) — set by
@@ -166,18 +176,37 @@ function jobIdOf(steer: SteerMessage): string | null {
return typeof details?.jobId === "string" ? details.jobId : null;
}
/** Dock bookkeeping: the marker reached the learner's view — flip its job
* NOW instead of waiting for the delivering turn to settle. Called from
* BOTH visibility seams: the marker's message_end wire event (live steer
* injection — the card is on screen while the teacher keeps teaching) and
* the write-ahead persist in handleChatInput (the wake's input appends —
* the card is durable in the record). The registry's delivery promise
* still settles via settleSteers / the runner-200 path; finalizeJob is
* idempotent, so the later flip is a no-op. Owner ruling (2026-09-26): a
* delivered subagent leaves the background dock the moment its result is
* visible, not when the teacher's turn ends. */
export function noteSteerVisible(steer: SteerMessage): void {
finalizeSteer(steer);
/** Call only after canonical conversation persistence. Streaming visibility
* is not durable acceptance. The saved marker wins over delivery-bookkeeping
* failures; acknowledging delivery never changes the worker's outcome. */
export function noteSteersDurable(chatId: string, persisted: readonly LhMessage[]): void {
for (const message of persisted) {
if (message.role === "custom") finalizeSteer(message);
}
for (const entry of [...(state().queues.get(chatId) ?? [])]) {
if (persisted.some((message) => message.id === entry.steer.id)) completeEntry(chatId, entry);
}
}
function removeEntry(chatId: string, entry: QueuedSteer): void {
const remaining = (state().queues.get(chatId) ?? []).filter((candidate) => candidate !== entry);
if (remaining.length > 0) state().queues.set(chatId, remaining);
else state().queues.delete(chatId);
}
function completeEntry(chatId: string, entry: QueuedSteer): void {
finalizeSteer(entry.steer);
removeEntry(chatId, entry);
entry.resolve();
}
/** Deletion discards the session's obligations, never wakes a deleted chat. */
export function discardChatDeliveries(chatId: string): void {
const s = state();
const queue = s.queues.get(chatId) ?? [];
s.queues.delete(chatId);
s.userSteers.delete(chatId);
for (const entry of queue) entry.resolve();
}
/** Finalize a delivered steer's job bookkeeping — job steers only. */
@@ -186,24 +215,30 @@ function finalizeSteer(steer: SteerMessage): void {
if (jobId) finalizeJob(jobId);
}
/** Fail a steer's job on unrecoverable delivery failure — job steers only. */
function failSteer(steer: SteerMessage, err: Error): void {
const jobId = jobIdOf(steer);
if (jobId) failJob(jobId, err);
/** Queue one logical delivery per ID, including in-flight wake cargo.
* Job promises resolve on durable marker acceptance (or session deletion);
* failed starts stay pending. Non-job probe handoffs keep their existing
* wake-acceptance/error semantics. Workers never await this promise. */
export function deliverJobResult(steer: SteerMessage): Promise<void> {
const chatId = chatIdOf(steer);
const entry = queueSteer(chatId, steer);
void pump(chatId);
return entry.promise;
}
/** Queue ANY steer (result, abort, probe handoff) through the pipeline.
* Resolves when the wake turn has completed; rejects when delivery fails
* unrecoverably. Never rejects for queueing itself. */
export function deliverJobResult(steer: SteerMessage): Promise<void> {
return new Promise<void>((resolve, reject) => {
const s = state();
const chatId = chatIdOf(steer);
const queue = s.queues.get(chatId) ?? [];
queue.push({ steer, resolve, reject });
s.queues.set(chatId, queue);
void pump(chatId);
});
function queueSteer(chatId: string, steer: SteerMessage): QueuedSteer {
const queue = state().queues.get(chatId) ?? [];
const existing = queue.find((entry) => entry.steer.id === steer.id);
if (existing) return existing;
let resolve!: () => void;
let reject!: (error: Error) => void;
const promise = new Promise<void>((res, rej) => { resolve = res; reject = rej; });
// Recovery/registration may own an entry with no waiting caller.
void promise.catch(() => undefined);
const entry: QueuedSteer = { steer, promise, resolve, reject };
queue.push(entry);
state().queues.set(chatId, queue);
return entry;
}
/** Hand every eligible pending marker to a live agent's steering queue
@@ -227,7 +262,7 @@ function handOffEligible(chatId: string, agent: Agent): void {
// remain deliverable, including the first poll of an answer run.
if (automaticFollowUpAllowed(chatId, agent) && queue && queue.length > 0) {
for (const entry of queue) {
if (entry.handedTo !== agent && entry.steer.customType !== "probe-handoff") {
if (!entry.waking && !entry.deferred && entry.handedTo !== agent && entry.steer.customType !== "probe-handoff") {
agent.steer(entry.steer);
entry.handedTo = agent;
}
@@ -254,8 +289,13 @@ function handOffEligible(chatId: string, agent: Agent): void {
* markers are steered into the run right here. The settle seam cannot
* beat the registration: settlement needs a streamed response, which
* never happens inside the constructor call. */
export function registerLiveAgent(chatId: string, agent: Agent): void {
export function registerLiveAgent(chatId: string, agent: Agent, allowDeferred = true, wakeSteerIds?: ReadonlySet<string>): void {
state().liveAgents.set(chatId, agent);
restorePendingDeliveries(chatId);
for (const entry of state().queues.get(chatId) ?? []) {
if (allowDeferred) entry.deferred = false;
if (entry.waking && wakeSteerIds?.has(entry.steer.id)) entry.wakeAgent = agent;
}
handOffEligible(chatId, agent);
}
@@ -326,7 +366,7 @@ export function restoreUncommittedAutomatic(chatId: string, agent: Agent, automa
* unregisterLiveAgent, with the run's persisted messages. Two jobs:
*
* - entries whose marker the run DELIVERED (in-run follow-up injection —
* their ids are in the persisted slice) resolve and flip their jobs;
* their ids are in the persisted slice) resolve and acknowledge delivery;
* - entries handed to the dying run but never injected (the question
* stopper exited before the follow-up poll; an abort; a delivery that
* raced the final poll) return to PENDING (handedTo cleared) and the
@@ -335,7 +375,7 @@ export function restoreUncommittedAutomatic(chatId: string, agent: Agent, automa
* This is what makes the live hand-off safe: pi owns no retry for a marker
* stranded in a settled agent's queue, so the registry stays the truth and
* the dead agent's copy is simply abandoned (GC'd with the agent). */
export function settleSteers(chatId: string, persisted: { id?: string }[]): void {
export function settleSteers(chatId: string, persisted: { id?: string }[], failed = false, settledAgent?: Agent): void {
const queue = state().queues.get(chatId);
if (!queue || queue.length === 0) return;
const deliveredIds = new Set(persisted.map((m) => m.id).filter(Boolean));
@@ -345,7 +385,11 @@ export function settleSteers(chatId: string, persisted: { id?: string }[]): void
finalizeSteer(entry.steer);
entry.resolve();
} else {
entry.handedTo = undefined;
const handedHere = entry.handedTo !== undefined && (!settledAgent || entry.handedTo === settledAgent);
const wakeHere = settledAgent !== undefined && entry.wakeAgent === settledAgent;
if (handedHere) entry.handedTo = undefined;
if (wakeHere) { entry.waking = false; entry.wakeAgent = undefined; }
if (failed && (handedHere || wakeHere) && jobIdOf(entry.steer)) entry.deferred = true;
remaining.push(entry);
}
}
@@ -373,16 +417,10 @@ export interface AbortOutcome {
* runner registered it at loop start) — the in-flight model call
* aborts, not just the next loop check. The runner unwinds silently
* (its catch checks abortSignal.aborted; finishJob checks status).
* 2. DROP any queued result steer for this job (the delivering-result
* phase): resolve its entry without firing the wake. If the wake
* turn is ALREADY in flight (the pump shifted the entry), it's too
* late — the result steer lands, and the cancellation steer dedupes
* against it via steerDelivered (same jobId metadata).
* 3. FLIP the record to error "Aborted by learner" immediately — the
* poll reflects the user's action at once (the user chose
* error-with-reason over a distinct cancelled status). finalizeJob
* never overwrites a non-running status, so the flip is terminal no
* matter what the wake does.
* 2. REFUSE if computation already finished. Queued results belong to
* the session, not to an abortable background worker.
* 3. SAVE error "Aborted by learner" and a pending abort delivery together.
* Delivery acknowledgement never overwrites the worker's outcome.
* 4. STEER the teacher: a one-line abort note (buildAbortSteer) through
* the same wake pipeline, so the teacher's promise of a result
* doesn't hang silently in the lesson.
@@ -394,28 +432,18 @@ export function abortJob(jobId: string): AbortOutcome {
const job = stores.jobStore.getJob(jobId);
if (!job) return { ok: false, reason: "not_found" };
if (job.status !== "running") return { ok: false, reason: "not_running" };
fireJobAbort(jobId); // no-op when the loop already returned (delivering)
// Drop any queued steers for this job (their deliver() resolves unfired).
// HANDED entries (already in a live run's follow-up queue) are past
// recall — same ruling as a fired wake: the result lands, and the abort
// note below dedupes against it via steerDelivered (same jobId).
const queue = state().queues.get(job.chatId);
if (queue) {
const remaining = queue.filter(
(e) => jobIdOf(e.steer) !== jobId || e.handedTo !== undefined,
);
if (remaining.length === 0) state().queues.delete(job.chatId);
else state().queues.set(job.chatId, remaining);
for (const entry of queue) {
if (jobIdOf(entry.steer) === jobId && entry.handedTo === undefined) entry.resolve();
}
if (job.result) {
normalizeLegacyResult(job);
return { ok: false, reason: "not_running" };
}
stores.jobStore.updateJob(jobId, { status: "error", error: "Aborted by learner", activity: "aborted" });
fireJobAbort(jobId);
stores.jobStore.updateJob(jobId, {
status: "error", error: "Aborted by learner", activity: "aborted",
delivery: { kind: "abort", status: "pending" },
});
void deliverJobResult(buildAbortSteer(job)).catch(() => {
/* deliverJobResult marks the job error on unrecoverable failure */
/* The saved pending abort notice remains owned by the session. */
});
return { ok: true };
}
@@ -470,18 +498,9 @@ async function pump(chatId: string): Promise<void> {
// Idempotent delivery: the transcript already carries these steers
// (an in-run injection persisted, a wake completed, or a duplicate
// deliver raced in). Resolve and flip without re-firing anything.
const stillPending = queue.filter((e) => !steerDelivered(record.messages, e.steer.id));
for (const entry of queue) {
if (!stillPending.includes(entry)) {
finalizeSteer(entry.steer);
entry.resolve();
}
}
if (stillPending.length === 0) {
s.queues.delete(chatId);
return;
}
s.queues.set(chatId, stillPending);
noteSteersDurable(chatId, record.messages);
const stillPending = (s.queues.get(chatId) ?? []).filter((entry) => !entry.waking && !entry.deferred);
if (stillPending.length === 0) { parked = true; return; }
if (!automaticFollowUpAllowed(chatId, s.liveAgents.get(chatId))) { parked = true; return; }
// VEHICLE 1 — THE LIVE RUN: pi's steering queue. The agent of the
@@ -526,28 +545,17 @@ async function pump(chatId: string): Promise<void> {
// The runner is required — chat-handler sets it at init. If some
// module instance missed the wiring, fail loudly rather than hang.
if (!s.turnRunner) {
s.queues.delete(chatId);
const err = new Error("wake pipeline: no turn runner wired (setTurnRunner)");
for (const entry of stillPending) {
failSteer(entry.steer, err);
entry.reject(err);
}
for (const entry of stillPending) deferOrReject(chatId, entry, err);
parked = true;
return;
}
// A live run may have delivered some markers while the gates were
// closed — re-check: only still-missing markers ride the wake.
const fire = stillPending.filter((e) => !steerDelivered(fresh.messages, e.steer.id));
for (const entry of stillPending) {
if (!fire.includes(entry)) {
finalizeSteer(entry.steer);
entry.resolve();
}
}
if (fire.length === 0) {
s.queues.delete(chatId);
return;
}
noteSteersDurable(chatId, fresh.messages);
const fire = (s.queues.get(chatId) ?? []).filter((entry) => !entry.waking && !entry.deferred && !entry.handedTo);
if (fire.length === 0) return;
// THE DISMISSAL PAUSE (owner design 2026-10-03, "A2"): the record
// ends in a user dismissal — the session waits for the USER. Hold
@@ -561,50 +569,37 @@ async function pump(chatId: string): Promise<void> {
return;
}
// DEQUEUE-AT-FIRE (the Wave C lesson from the Wave B regression):
// the carried markers leave the registry BEFORE the turn runs —
// the wake run's OWN follow-up poll must never see its own cargo,
// and a 409 (client raced in) must be able to push them back.
s.queues.delete(chatId);
// Keep ownership through the wake. The flag excludes cargo from
// its own live-agent handoff while duplicate callers share it.
for (const entry of fire) entry.waking = true;
const steers = fire.map((e) => e.steer);
// Fire the wake (every marker as the turn's input appends). A 409 =
// a client turn raced in between idle and firing — push the cargo
// back and retry after backoff (the racing turn's own settlement
// re-pumps if it beats us to the re-queue).
// Fire the wake with cargo still registered. A 409 releases its
// vehicle flag and retries after the existing backoff, without
// appending another registry entry.
const status = await s.turnRunner(chatId, { type: "steers", steers }).catch((err: unknown) => {
console.error(`[steer ${steers.map((x) => x.id).join(",")}] wake turn crashed:`, err);
return -1;
});
if (status === 403) {
// Consent may have changed after the last gate. Preserve completed
// job/result ownership and wait for an explicit connection change.
const back = s.queues.get(chatId) ?? [];
back.push(...fire);
s.queues.set(chatId, back);
parked = true;
return;
}
if (status === 409) {
const back = s.queues.get(chatId) ?? [];
back.push(...fire);
s.queues.set(chatId, back);
const saved = stores.chatStore.getChat(chatId);
if (saved) noteSteersDurable(chatId, saved.messages);
const owned = fire.filter((entry) => s.queues.get(chatId)?.includes(entry));
if (status === 403 || status === 409) {
for (const entry of owned) { entry.waking = false; entry.wakeAgent = undefined; }
if (status === 403) { parked = true; return; }
await delay(RETRY_BACKOFF_MS[Math.min(attempt++, RETRY_BACKOFF_MS.length - 1)]!);
continue;
}
attempt = 0;
if (status === 200) {
for (const entry of fire) {
finalizeSteer(entry.steer);
entry.resolve();
}
} else {
const err = new Error(`wake turn failed with status ${status}`);
for (const entry of fire) {
failSteer(entry.steer, err);
entry.reject(err);
for (const entry of owned) {
if (status === 200 && !jobIdOf(entry.steer)) {
// Preserve the non-job probe handoff's acceptance semantics.
completeEntry(chatId, entry);
} else if (status !== 200 || !isTurnRunning(chatId)) {
deferOrReject(chatId, entry, new Error(`wake turn did not persist steer (status ${status})`));
}
// An accepted live turn still owns its cargo until canonical save
// or settlement. HTTP 200 alone is not a durable marker receipt.
}
}
} finally {
@@ -614,7 +609,7 @@ async function pump(chatId: string): Promise<void> {
// a live run are NOT stragglers (settleSteers owns their resurrection)
// — restarting for them would just park in waitForIdle forever.
const q = state().queues.get(chatId) ?? [];
if (!parked && automaticFollowUpAllowed(chatId, state().liveAgents.get(chatId)) && q.some((e) => e.handedTo === undefined)) void pump(chatId);
if (!parked && automaticFollowUpAllowed(chatId, state().liveAgents.get(chatId)) && q.some((e) => e.handedTo === undefined && !e.waking && !e.deferred)) void pump(chatId);
}
}
@@ -638,25 +633,56 @@ state().stopConnectionWatch = onSubscriptionChange(() => {
}
});
/** Flip the job to its final status now that the lesson has resumed. */
function finalizeJob(jobId: string): void {
const job = stores.jobStore.getJob(jobId);
if (!job || job.status !== "running") return;
const result = job.result;
const failed = result ? isSubagentFailure(job.kind, result as never) : true;
stores.jobStore.updateJob(jobId, {
status: failed ? "error" : "done",
activity: failed ? "failed" : "delivered",
});
function deferOrReject(chatId: string, entry: QueuedSteer, error: Error): void {
entry.waking = false;
entry.wakeAgent = undefined;
if (jobIdOf(entry.steer)) {
entry.deferred = true;
console.error(`[steer ${entry.steer.id}] delivery remains pending:`, error.message);
} else {
removeEntry(chatId, entry);
entry.reject(error);
}
}
function failJob(jobId: string, err: Error): void {
// An aborted job stays aborted even when its (abort) steer
// delivery fails — the user's action is the terminal fact, not the
// delivery error. Non-running records are never overwritten here.
const job = stores.jobStore.getJob(jobId);
if (!job || job.status !== "running") return;
stores.jobStore.updateJob(jobId, { status: "error", error: err.message, activity: "delivery failed" });
/** Acknowledge session delivery without changing the worker outcome. */
function finalizeJob(jobId: string): void {
try {
let job = stores.jobStore.getJob(jobId);
if (!job) return;
if (!job.delivery) job = normalizeLegacyResult(job);
if (job.delivery?.status !== "pending") return;
stores.jobStore.updateJob(jobId, { delivery: { ...job.delivery, status: "delivered" } });
} catch (error) {
// The conversation is already durable. A bookkeeping failure must
// never replay that marker or turn a successful worker into a failure.
console.error(`[job ${jobId}] delivery acknowledgement failed:`, error);
}
}
function normalizeLegacyResult(job: JobRecord): JobRecord {
if (job.delivery || job.status !== "running" || !job.result) return job;
const failed = isSubagentFailure(job.kind, job.result as never);
return stores.jobStore.updateJob(job.id, {
status: failed ? "error" : "done", activity: failed ? "failed" : "finished",
delivery: { kind: "result", status: "pending" },
}) ?? job;
}
/** Restore known obligations only; old terminal records are never replayed. */
function restorePendingDeliveries(chatId: string): void {
const record = stores.chatStore.getChat(chatId);
if (!record) return;
noteSteersDurable(chatId, record.messages);
for (const stored of stores.jobStore.listJobs(chatId)) {
const job = isJobLive(stored.id) ? stored : normalizeLegacyResult(stored);
if (job.delivery?.status !== "pending") continue;
const steer = job.delivery.kind === "abort" ? buildAbortSteer(job)
: job.result ? buildSubagentSteer(job, job.result as never) : undefined;
if (!steer) continue;
if (steerDelivered(record.messages, steer.id)) finalizeSteer(steer);
else queueSteer(chatId, steer);
}
}
/**
@@ -669,7 +695,7 @@ function failJob(jobId: string, err: Error): void {
async function waitForIdle(chatId: string): Promise<void> {
for (;;) {
const record = stores.chatStore.getChat(chatId);
if (!record) return;
if (!record || !(state().queues.get(chatId) ?? []).some((entry) => !entry.waking && !entry.deferred && !entry.handedTo)) return;
const blocked = isTurnRunning(chatId) || danglingQuestionOf(record.messages) !== null;
if (!blocked) {
await delay(GRACE_MS);
@@ -699,9 +725,9 @@ const SWEEP_THROTTLE_MS = 30_000;
* hung-but-alive loops beside themselves — the figure-maker incident,
* 2026-09-18). A record WITH a live registration is provably executing
* in THIS process — never touched, however old: a hung first model call
* (zero usage ever billed) stays honestly "running" in the dock until
* END kills it. The abort registration spans loop start to delivery
* completion, so a live registration can never belong to a dead run.
* stays honestly running until END kills it. Terminal workers with known
* pending delivery are restored independently; old terminal records without
* delivery metadata are never inferred to owe a delivery.
*/
/** Re-run a job's sub-agent loop — exposed for the sweep's tests (the
* /api/jobs poll is its only production caller, disabled under NODE_ENV=test). */
@@ -713,18 +739,14 @@ export function recoverStaleJobs(): void {
try {
for (const chat of stores.chatStore.listChats()) {
for (const job of stores.jobStore.listJobs(chat.id)) {
if (job.status !== "running") continue;
if (isJobLive(job.id)) continue; // provably executing here — never re-run beside it
if (isJobLive(job.id)) continue; // never re-run beside live computation
if (!stores.chatStore.getChat(job.chatId)) continue;
if (isTurnRunning(job.chatId)) continue; // a wake turn may be delivering it right now
if (job.result) {
// The wake never fired (or crashed mid-flight): re-deliver.
const steer = buildSubagentSteer(job, job.result as never);
stores.jobStore.updateJob(job.id, { activity: "recovering: re-delivering" });
void deliverJobResult(steer).catch(() => {
/* deliverJobResult marks the job error on failure */
});
} else {
if (isTurnRunning(job.chatId)) continue;
const recovered = normalizeLegacyResult(job);
if (recovered.delivery?.status === "pending") {
restorePendingDeliveries(job.chatId);
void pump(job.chatId);
} else if (job.status === "running" && !job.result) {
// The sub-agent loop itself died with the process: re-spawn.
if (!s.respawn) continue;
stores.jobStore.updateJob(job.id, { activity: "recovering: re-running" });
-7
View File
@@ -1,6 +1,4 @@
import type { Agent, AgentEvent, AgentMessage } from "@earendil-works/pi-agent-core";
import type { SteerMessage } from "../../core/contracts.ts";
import { noteSteerVisible } from "../jobs.ts";
import { ToolCallIdUniquifier } from "./call-ids.ts";
import type { ChatGPTRequestCycle } from "../../agent/chatgpt-request-cycle.ts";
@@ -255,11 +253,6 @@ export function runPiTurn(spec: PiTurnSpec): PiTurnHandle {
if (event.type === "message_end") {
const m = event.message as { role?: string; stopReason?: string } | undefined;
if (m?.role === "assistant" && m.stopReason === "aborted") sawAbort = true;
// Dock fix (owner ruling 2026-09-26): an injected steer marker's
// message_end = the card is visible to the learner — flip the
// job's bookkeeping now (the turn may keep teaching for minutes;
// settleSteers re-runs the flip as a no-op idempotently).
if (m?.role === "custom") noteSteerVisible(event.message as SteerMessage);
}
if (event.type === "agent_end") {
// Persist/report before the terminal closes every view to new frames.
+1
View File
@@ -35,6 +35,7 @@ function createJob(init: CreateJobInit): JobRecord {
kind: init.kind,
chatId: init.chatId,
toolCallId: init.toolCallId,
...(init.input !== undefined ? { input: init.input } : {}),
status: "running",
activity: init.activity ?? "starting",
createdAt: now,