sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Merge durable background job delivery and recovery

Browse Source
This commit is contained in:
amosblomqvist
2026-10-26 02:39:59 +02:00
parent 85329290f2 90828cef49
commit 6370c9d618
19 changed files with 730 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent/subagents/aborts.ts
+5 -7
View File
@@ -41,18 +41,16 @@ export function unregisterJobAbort(jobId: string): void {
* the liveness gate for the stale-job sweep: a "running" record with a
* live registration belongs to a loop this process owns (hung or slow, it
* is the one truth — never re-spawn beside it); without one the record is
* an orphan of a process death and recovers immediately. The span is
* loop-start to delivery-completion: the runner's unregistering finally
* runs only after deliver() resolves, by which point finalizeJob has
* flipped the record — so absence of registration is unambiguous. */
* an orphan of a process death and recovers immediately. The registration
* spans computation and cleanup only. Finished outcomes have independent,
* persisted delivery obligations; no worker waits for their consumption. */
export function isJobLive(jobId: string): boolean {
return registry().has(jobId);
}
/** Abort a running job's loop. Returns false when no controller is live —
* the job's loop is not executing (crashed, or in the delivering-result
* phase where the runner has already returned); the caller then owns the
* whole abort on the record side. */
* the job's loop is not executing. The endpoint separately refuses abort
* for a finished worker, even while its session delivery remains pending. */
export function fireJobAbort(jobId: string): boolean {
const controller = registry().get(jobId);
if (!controller) return false;
src/agent/subagents/runner.ts
+35 -27
View File
@@ -3,6 +3,7 @@ import type { SearchProvider } from "../../core/search.ts";
import {
accumulateJobUsage,
buildSubagentSteer,
isSubagentFailure,
piUsageToStep,
type DemoResult,
type DemoToolInput,
@@ -40,13 +41,13 @@ import { runVisualMaker } from "./visual-maker.ts";
* (the job tools' execute fires this via the spawn seam, fire-and-forget),
* updating the job's activity line on every iteration. On completion the
* runner builds the STEER MESSAGE (buildSubagentSteer) and hands it to
* `deliver`; the wake pipeline fires the server-side teacher turn and
* flips the job to done/error when the lesson has actually resumed.
* `deliver` without waiting. The worker's final outcome and pending
* delivery are saved together; the session owns delivery from then on.
*
* On failure the result is the failure shape (empty html/svg + "FAILED: …"
* notes; research: failure brief) — and it STILL steers, because the
* teacher must learn the outcome. If delivery itself fails, the job is
* marked error (the learner's activity card shows it).
* teacher must learn the outcome. Failed delivery starts stay quietly
* pending and never rewrite the worker's computation outcome.
*/
/**
@@ -67,9 +68,9 @@ export interface JobRunnerDeps {
chatStore: ChatStore;
jobStore: JobStore;
/**
* Hand a finished job's steer to the wake pipeline — resolves when the
* steer is delivered (the wake turn completed and the job status
* flipped); rejects when delivery fails unrecoverably.
* Hand a finished job's steer to the session delivery registry. The
* promise tracks durable marker acceptance, but the worker never waits
* on it. Its persisted pending obligation survives process restart.
*/
deliver: (steer: SteerMessage) => Promise<void>;
/** Settings-resolved engine per role (model slots wave, 2026-09-20):
@@ -96,8 +97,8 @@ export interface JobRunnerDeps {
export interface JobRunner {
/** Run one job's sub-agent loop to completion (background; never throws).
* The tool input rides along from dispatch (the spawn fires mid-stream,
* before the tool part is persisted — only crash-recovery re-spawns
* resolve the input from the transcript). */
* before the tool part is persisted). Recovery reads saved job input,
* with transcript fallback for legacy jobs. Delivery is not awaited. */
runJob(jobId: string, input?: JobToolInput): Promise<void>;
}
@@ -270,14 +271,15 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
}
}
/** Resolve the job's tool input: dispatch passes it directly; only a
* crash-recovery re-spawn (no input) falls back to scanning the
* transcript (by then onFinish has long persisted the part). */
/** Dispatch input wins, then the job's saved arguments. Legacy jobs
* without saved input fall back to the durable transcript, which may
* lack a call if the dispatching turn died before settlement. */
function resolveInput(
job: JobRecord,
input: JobToolInput | undefined,
): JobToolInput | undefined {
if (input) return input;
if (job.input !== undefined) return job.input as JobToolInput;
const chat = chatStore.getChat(job.chatId);
if (!chat) return undefined;
// Wave D: the record is native history — the scan reads the native
@@ -295,27 +297,28 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
}
/**
* The tail of every job: store the result, build the steer, hand it to
* the wake pipeline. The pipeline flips the status when the lesson has
* actually resumed — until then the job reads "running" so the client's
* activity card keeps polling and re-syncs once, at the end.
* Save the computation outcome and session delivery obligation together.
* Worker liveness ends after cleanup, not after a lesson consumes the result.
*/
async function finishJob(jobId: string, result: ResearchResult | DemoResult | VisualResult | PdfResult): Promise<void> {
const job = jobStore.getJob(jobId);
if (!job) return;
// Aborted mid-run or mid-delivery (abortJob flips the record before
// Aborted mid-run (abortJob flips the record before
// the loop unwinds): no result write, no steer — the abort path owns
// the record and queues the abort steer itself.
if (job.status !== "running") return;
jobStore.updateJob(jobId, { result, activity: "delivering result" });
const steer = buildSubagentSteer(job, result);
try {
await deliver(steer);
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
console.error(`[job ${jobId}] steer delivery failed:`, err);
jobStore.updateJob(jobId, { status: "error", error: reason, activity: "delivery failed" });
}
const failed = isSubagentFailure(job.kind, result);
jobStore.updateJob(jobId, {
result,
status: failed ? "error" : "done",
activity: failed ? "failed" : "finished",
delivery: { kind: "result", status: "pending" },
});
// The saved result now belongs to the session. Do not keep the worker,
// abort registration, or temporary resources alive behind a question.
void deliver(buildSubagentSteer(job, result)).catch((err: unknown) => {
console.error(`[job ${jobId}] pending delivery:`, err);
});
}
/** Chat-record + input resolution shared by all three runners. */
@@ -343,7 +346,12 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
const { job } = ctx;
const input = (passedInput ?? resolveInput(job, passedInput)) as ResearchToolInput | undefined;
const question = typeof input?.question === "string" && input.question.trim() ? input.question : "(no question)";
if (!(typeof input?.question === "string" && input.question.trim())) {
jobStore.updateJob(jobId, { error: "research question unavailable" });
await finishJob(jobId, { brief: "Research failed: the research question is unavailable.", sources: [] });
return;
}
const question = input.question;
const context = typeof input?.context === "string" && input.context.trim() ? input.context : undefined;
const prompt = [
src/agent/tools/jobs.ts
+1 -1
View File
@@ -78,7 +78,7 @@ function dispatch(
input: ResearchInput | DemoInput | VisualInput | PdfInput,
): string {
const existing = ctx.jobStore.findJobByToolCallId(ctx.chatId, toolCallId);
const job = existing ?? ctx.jobStore.createJob({ kind, chatId: ctx.chatId, toolCallId });
const job = existing ?? ctx.jobStore.createJob({ kind, chatId: ctx.chatId, toolCallId, input });
if (!existing) ctx.spawn(job, input);
return `Dispatched (job ${job.id}). ${ack}`;
}