sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Merge durable background job delivery and recovery

Browse Source
This commit is contained in:
amosblomqvist
2026-10-26 02:39:59 +02:00
parent 85329290f2 90828cef49
commit 6370c9d618
19 changed files with 730 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/jobs.test.ts
+68 -50
View File
@@ -52,18 +52,20 @@ import { runningTurn } from "../src/server/turn-runs.ts";
import {
abortJob,
deliverJobResult,
noteSteerVisible,
noteSteersDurable,
discardChatDeliveries,
reconcileUserSteers,
recoverStaleJobs,
registerLiveAgent,
setJobRespawn,
setTurnRunner,
setTurnRunner as setRawTurnRunner,
type TurnRunner,
settleSteers,
steerLiveChat,
unregisterLiveAgent,
} from "../src/server/jobs.ts";
import { registerJobAbort, unregisterJobAbort } from "../src/agent/subagents/aborts.ts";
import { beginTurn, endTurn } from "../src/server/turn-liveness.ts";
import { isJobLive, registerJobAbort, unregisterJobAbort } from "../src/agent/subagents/aborts.ts";
import { beginTurn, endTurn, isTurnRunning } from "../src/server/turn-liveness.ts";
import { persistTurn } from "../src/server/turn-service.ts";
import { chatStore } from "../src/store/chats.ts";
import { jobStore } from "../src/store/jobs.ts";
@@ -141,11 +143,13 @@ afterEach(() => {
if (job && job.status === "running") {
jobStore.updateJob(id, { status: "done", activity: "settled (fence)" });
}
if (job?.delivery?.status === "pending") jobStore.updateJob(id, { delivery: { ...job.delivery, status: "delivered" } });
unregisterJobAbort(id);
} catch {
/* already gone */
}
}
for (const chatId of createdChats) discardChatDeliveries(chatId);
// Late wakes from a prior test must be inert: a no-op respawn can't
// increment anyone's counter, a no-op runner can't spawn real work.
setJobRespawn(() => {});
@@ -455,16 +459,25 @@ function fakeDeliver() {
steers,
deliver: async (steer: SteerMessage) => {
steers.push(steer);
const jobId = (steer.details as { jobId: string }).jobId;
const job = jobStore.getJob(jobId);
if (job && job.status === "running") {
const failed = job.result ? isSubagentFailure(job.kind, job.result as never) : true;
jobStore.updateJob(jobId, { status: failed ? "error" : "done", activity: failed ? "failed" : "delivered" });
}
const job = jobStore.getJob((steer.details as { jobId: string }).jobId)!;
chatStore.saveMessages(job.chatId, [...chatStore.getChat(job.chatId)!.messages, steer]);
noteSteersDurable(job.chatId, [steer]);
},
};
}
/** Fake wakes model canonical input acceptance, not just HTTP acknowledgement. */
function setTurnRunner(run: TurnRunner): void {
setRawTurnRunner(async (chatId, input) => {
const status = await run(chatId, input);
if (status === 200 && input.type === "steers") {
const record = chatStore.getChat(chatId);
if (record) chatStore.saveMessages(chatId, [...record.messages, ...input.steers.filter((s) => !steerDelivered(record.messages, s.id))]);
}
return status;
});
}
// ─── 3. dispatch: the tool call acks mid-turn and spawns ─────────────────────
const RESEARCH_INPUT = {
@@ -527,7 +540,7 @@ describe("dispatch (async protocol §2)", () => {
const done = jobStore.getJob(job!.id)!;
assert.equal(done.status, "done");
assert.equal(done.activity, "delivered");
assert.equal(done.activity, "finished");
// The steer carries the brief on its model-facing content; the
// artifact payload rides details for the (UI) renderer.
@@ -604,7 +617,7 @@ describe("research runner (async protocol §3)", () => {
// The job is done, with the activity line from the tool call.
const done = jobStore.getJob(job.id)!;
assert.equal(done.status, "done");
assert.equal(done.activity, "delivered", "the fake deliver flips the status like the pipeline");
assert.equal(done.activity, "finished", "delivery does not own the worker status");
const result = done.result as ResearchResult;
assert.equal(result.brief, BRIEF);
assert.ok(result.sources.length > 0, "sources parsed from the brief");
@@ -768,7 +781,7 @@ describe("pdf runner (sources v0)", () => {
const done = jobStore.getJob(job.id)!;
assert.equal(done.status, "done");
assert.equal(done.activity, "delivered");
assert.equal(done.activity, "finished");
const result = done.result as PdfResult;
assert.equal(result.path, PDF_FIXTURE);
assert.match(result.brief, /integral rule/);
@@ -880,7 +893,7 @@ describe("wake pipeline (src/server/jobs.ts)", () => {
const job = jobStore.createJob({ kind, chatId: chat.id, toolCallId: `call-${jobStore.listJobs(chat.id).length}` });
trackJob(job.id);
const result: ResearchResult = { brief: "Done: it works.", sources: [{ title: "s", url: "https://x.test" }] };
jobStore.updateJob(job.id, { result, activity: "delivering result" });
jobStore.updateJob(job.id, { result, status: "done", activity: "finished", delivery: { kind: "result", status: "pending" } });
const steer = buildSubagentSteer(job, result);
return { chatId: chat.id, jobId: job.id, steer };
}
@@ -935,7 +948,7 @@ describe("wake pipeline (src/server/jobs.ts)", () => {
await waitFor(() => jobStore.getJob(jobId)!.status !== "running", 8000, "job flip after delivery");
assert.equal(jobStore.getJob(jobId)!.status, "done");
assert.equal(jobStore.getJob(jobId)!.activity, "delivered");
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "delivered");
assert.ok(seen, "the turn runner fired");
assert.equal(seen.chatId, chatId);
assert.equal(seen.steers.length, 1);
@@ -973,7 +986,8 @@ describe("wake pipeline (src/server/jobs.ts)", () => {
await delay(1200);
assert.equal(fired, 0, "no wake while the quiz is pending");
assert.equal(jobStore.getJob(jobId)!.status, "running", "job stays running while blocked");
assert.equal(jobStore.getJob(jobId)!.status, "done", "worker is finished while delivery waits");
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending");
// The learner answers: the real tool result clears the dangling call.
const record = chatStore.getChat(chatId)!;
@@ -1037,7 +1051,7 @@ describe("wake pipeline (src/server/jobs.ts)", () => {
await delay(1200);
assert.equal(fired, 0, "no wake while the session sits dismissal-paused");
assert.equal(jobStore.getJob(jobId)!.status, "running", "the delivery holds (queued, not lost)");
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending", "the delivery holds (queued, not lost)");
// The user's next message is the wake — its turn settles, and the
// settlement re-pump lands the held delivery right after it.
@@ -1109,7 +1123,8 @@ describe("Wave C: live steering handoff + settlement reconciliation", () => {
await waitFor(() => handed.length === 1, 8000, "marker handed to the live agent");
await delay(1200);
assert.equal(fired, 0, "no wake while the run is live");
assert.equal(jobStore.getJob(jobId)!.status, "running", "job flips only at delivery");
assert.equal(jobStore.getJob(jobId)!.status, "done", "worker finished before delivery");
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending");
// Settlement: the run persisted the injected marker.
unregisterLiveAgent(chatId);
@@ -1222,20 +1237,15 @@ describe("Wave C: live steering handoff + settlement reconciliation", () => {
assert.equal(jobStore.getJob(jobId)!.status, "done");
});
it("noteSteerVisible flips the job at the visibility seam — before any settle (owner ruling 2026-09-26)", async () => {
// The dock fix: the marker reaching the learner's view (wire
// message_end / write-ahead persist) flips the job NOW — the
// delivering turn may keep teaching for minutes. Idempotent with
// the settle-path finalize.
it("only a durable marker acknowledges delivery, independently of worker completion", () => {
const { chatId, jobId, steer } = setup();
assert.equal(jobStore.getJob(jobId)!.status, "running");
noteSteerVisible(steer);
assert.equal(jobStore.getJob(jobId)!.status, "done", "flipped at the seam, no turn settled");
noteSteerVisible(steer); // the settle-path flip re-runs — no-op
assert.equal(jobStore.getJob(jobId)!.status, "done");
// A probe-handoff (no jobId) is a harmless no-op.
const handoff = buildProbeHandoffSteer(chatId);
noteSteerVisible(handoff);
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending");
chatStore.saveMessages(chatId, [...chatStore.getChat(chatId)!.messages, steer]);
noteSteersDurable(chatId, [steer]);
noteSteersDurable(chatId, [steer]);
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "delivered");
assert.equal(jobStore.getJob(jobId)!.activity, "finished");
});
it("the idle wake carries EVERY pending marker in ONE turn", async () => {
@@ -1261,17 +1271,25 @@ describe("Wave C: live steering handoff + settlement reconciliation", () => {
});
it("fails the job (error status) when the wake turn errors unrecoverably", async () => {
const { jobId, steer } = setup();
setTurnRunner(async () => 500);
await deliverJobResult(steer).then(
() => assert.fail("expected delivery to reject"),
() => undefined,
);
const errored = jobStore.getJob(jobId)!;
assert.equal(errored.status, "error");
assert.match(errored.error ?? "", /status 500/);
it("a failed start stays pending without changing the worker outcome", async () => {
const { chatId, jobId, steer } = setup();
let attempts = 0;
setTurnRunner(async () => { attempts++; return 500; });
const delivery = deliverJobResult(steer);
await waitFor(() => attempts === 1, 8000, "failed start attempted");
await delay(1000);
assert.equal(attempts, 1, "no automatic retry loop");
assert.equal(jobStore.getJob(jobId)!.status, "done");
assert.equal(jobStore.getJob(jobId)!.error, undefined);
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending");
const handed: SteerMessage[] = [];
assert.ok(beginTurn(chatId));
registerLiveAgent(chatId, { steer: (m: SteerMessage) => handed.push(m) } as never);
assert.deepEqual(handed.map((m) => m.id), [steer.id]);
unregisterLiveAgent(chatId);
settleSteers(chatId, [steer]);
endTurn(chatId);
await delivery;
});
});
@@ -1607,14 +1625,14 @@ describe("abort (src/server/jobs.ts abortJob)", () => {
}
});
it("aborting a delivering job drops the queued result steer and wakes with the abort note instead", async () => {
it("a finished worker with a queued result is no longer abortable", async () => {
const chat = chatStore.createChat("Test: abort delivering job");
trackChat(chat.id);
chatStore.saveMessages(chat.id, [{ id: "u1", role: "user", content: "hello", timestamp: 1 } as never]);
const job = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "call-abort3" });
trackJob(job.id);
const result: ResearchResult = { brief: "Done: it works.", sources: [{ title: "s", url: "https://x.test" }] };
jobStore.updateJob(job.id, { result, activity: "delivering result" });
jobStore.updateJob(job.id, { result, status: "done", activity: "finished", delivery: { kind: "result", status: "pending" } });
// Block the pump's idle gates so the result steer sits QUEUED, not in flight.
assert.ok(beginTurn(chat.id));
@@ -1626,18 +1644,18 @@ describe("abort (src/server/jobs.ts abortJob)", () => {
void deliverJobResult(buildSubagentSteer(job, result)).catch(() => {
/* dropped by the abort below */
});
await waitFor(() => jobStore.getJob(job.id)!.activity === "delivering result", 8000, "queued");
assert.equal(jobStore.getJob(job.id)!.delivery?.status, "pending");
const outcome = abortJob(job.id);
assert.ok(outcome.ok);
assert.equal(jobStore.getJob(job.id)!.status, "error");
assert.equal(jobStore.getJob(job.id)!.error, "Aborted by learner");
assert.deepEqual(outcome, { ok: false, reason: "not_running" });
assert.equal(jobStore.getJob(job.id)!.status, "done");
assert.equal(jobStore.getJob(job.id)!.error, undefined);
endTurn(chat.id); // gates open — whatever is left in the queue fires
await waitFor(() => fired.length > 0, 8000, "wake fired");
assert.equal(fired.length, 1, "exactly one delivery — the result steer was dropped");
assert.equal((fired[0]!.details as { aborted: boolean }).aborted, true, "the abort steer, not the result");
assert.equal(jobStore.getJob(job.id)!.status, "error", "terminal after the wake");
assert.equal(fired.length, 1, "exactly one result delivery");
assert.equal(fired[0]!.customType, "subagent-steer");
assert.equal(jobStore.getJob(job.id)!.status, "done");
});
it("outcome reasons: not_found for unknown, not_running for settled jobs", () => {