fix: separate worker completion from durable session delivery
This commit is contained in:
amosblomqvist
@@ -522,7 +522,7 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
const job = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "call-mid" });
|
||||
jobIdMid = job.id;
|
||||
// The runner stores the result BEFORE delivery (finalizeJob reads it).
|
||||
jobStore.updateJob(job.id, { result: { brief: "Mid-run brief.", sources: [] } });
|
||||
jobStore.updateJob(job.id, { status: "done", result: { brief: "Mid-run brief.", sources: [] }, delivery: { kind: "result", status: "pending" } });
|
||||
const steer = buildSubagentSteer(job, { brief: "Mid-run brief.", sources: [] });
|
||||
const delivery = deliverJobResult(steer);
|
||||
await new Promise((r) => setTimeout(r, 300)); // vehicle 1 hands off
|
||||
@@ -534,7 +534,7 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
release2();
|
||||
await drain(await res);
|
||||
await delivery; // resolves at settleSteers (the run persisted the marker)
|
||||
assert.equal(jobFlippedMidTurn, "done", "the job flipped at the marker's message_end — WHILE the turn still streamed");
|
||||
assert.equal(jobFlippedMidTurn, "done", "worker completion is independent of the streaming teacher turn");
|
||||
|
||||
// Record shape: the marker sits BETWEEN completed turn units — after
|
||||
// the toolResult it must never split, adjacent to it.
|
||||
|
||||
@@ -43,7 +43,7 @@ for (const revokedAfterQueue of [false, true]) {
|
||||
stores.chatStore.saveSettings(chat.id, { model: { provider: "chatgpt", id: "gpt-5" } });
|
||||
const job = stores.jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "scratch-job-call" });
|
||||
const result = { brief: "Scratch completed job result", sources: [] };
|
||||
stores.jobStore.updateJob(job.id, { status: "running", result });
|
||||
stores.jobStore.updateJob(job.id, { status: "done", result, delivery: { kind: "result", status: "pending" } });
|
||||
const marker = buildSubagentSteer(job, result);
|
||||
const contexts: string[] = [];
|
||||
const native = piModels().getModel("openai", "gpt-5")!;
|
||||
@@ -83,7 +83,8 @@ for (const revokedAfterQueue of [false, true]) {
|
||||
unregisterLiveAgent(chat.id); endTurn(chat.id); settleSteers(chat.id, persisted);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(wakes, 0, "no background spin or spend while consent is absent");
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.status, "running", "delivery ownership must not be finalized or failed by deferral");
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "pending", "delivery ownership must not be finalized or failed by deferral");
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.status, "done", "worker completion is independent of consent deferral");
|
||||
assert.deepEqual(stores.jobStore.getJob(job.id)!.result, result);
|
||||
connection(true);
|
||||
await delivered;
|
||||
|
||||
@@ -274,14 +274,14 @@ describe("dispatch card rendering (the tool-call ack box)", () => {
|
||||
assert.match(html, /Dispatched…/);
|
||||
});
|
||||
|
||||
it("settles to a quiet delivered one-liner when done (artifact lives in the steer)", () => {
|
||||
it("settles to a quiet finished one-liner when the worker is done", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<JobsContext.Provider value={[mkJob("done", "delivered")]}>
|
||||
<JobCard part={ackPart as never} />
|
||||
</JobsContext.Provider>,
|
||||
);
|
||||
assert.match(html, /Research agent/);
|
||||
assert.match(html, /delivered/);
|
||||
assert.match(html, /finished/);
|
||||
assert.ok(!html.includes("web_search"), "no stale activity line once settled");
|
||||
});
|
||||
|
||||
@@ -325,7 +325,7 @@ describe("dispatch card rendering (the tool-call ack box)", () => {
|
||||
assert.match(html, /pdf_render/);
|
||||
});
|
||||
|
||||
it("pdf dispatch settles to the delivered one-liner", () => {
|
||||
it("pdf dispatch settles to the finished one-liner", () => {
|
||||
const pdfJob: JobRecord = ({
|
||||
id: "jp1",
|
||||
kind: "pdf",
|
||||
@@ -342,7 +342,7 @@ describe("dispatch card rendering (the tool-call ack box)", () => {
|
||||
</JobsContext.Provider>,
|
||||
);
|
||||
assert.match(html, /PDF reader/);
|
||||
assert.match(html, /delivered/);
|
||||
assert.match(html, /finished/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -848,7 +848,7 @@ describe("demo/visual job runners", () => {
|
||||
|
||||
const done = jobStore.getJob(job.id)!;
|
||||
assert.equal(done.status, "done");
|
||||
assert.equal(done.activity, "delivered", "the fake deliver flips status like the wake pipeline");
|
||||
assert.equal(done.activity, "finished", "the worker finishes independently of delivery");
|
||||
const result = done.result as { title: string; html: string; tested: boolean; notes?: string };
|
||||
assert.equal(result.title, "Compound interest");
|
||||
assert.equal(result.html, GOOD_HTML);
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { after, afterEach, test } from "node:test";
|
||||
import { createFauxCore, fauxAssistantMessage } from "@earendil-works/pi-ai";
|
||||
import type { LhMessage } from "../src/core/native-messages.ts";
|
||||
import { buildSubagentSteer, type JobKind, type SteerMessage } from "../src/core/contracts.ts";
|
||||
|
||||
// Install isolation before importing any storage/configuration composition.
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "lh-job-delivery-"));
|
||||
process.env.LH_DATA_DIR = path.join(directory, "data");
|
||||
process.env.LH_AUTH_FILE = path.join(directory, "auth.json");
|
||||
fs.writeFileSync(process.env.LH_AUTH_FILE, "{}");
|
||||
const { stores } = await import("../src/server/stores.ts");
|
||||
const { jobTools } = await import("../src/agent/tools/jobs.ts");
|
||||
const { createJobRunner, researchModelOverride } = await import("../src/agent/subagents/runner.ts");
|
||||
const { isJobLive } = await import("../src/agent/subagents/aborts.ts");
|
||||
const { beginTurn, endTurn } = await import("../src/server/turn-liveness.ts");
|
||||
const {
|
||||
deliverJobResult, discardChatDeliveries, noteSteersDurable, recoverStaleJobs,
|
||||
registerLiveAgent, unregisterLiveAgent, settleSteers, setTurnRunner,
|
||||
} = await import("../src/server/jobs.ts");
|
||||
const { activityFeed } = await import("../src/server/observability.ts");
|
||||
const { deleteChatSession } = await import("../src/server/chats.ts");
|
||||
const chats: string[] = [];
|
||||
const result = { brief: "Fixture research result", sources: [] };
|
||||
const question = {
|
||||
id: "question-message", role: "assistant", content: [{ type: "toolCall", id: "question-call", name: "quiz", arguments: {} }],
|
||||
api: "openai-completions", provider: "faux", model: "faux", timestamp: 1,
|
||||
} as unknown as LhMessage;
|
||||
const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
||||
async function until(predicate: () => boolean): Promise<void> {
|
||||
const end = Date.now() + 8000;
|
||||
while (!predicate()) { assert(Date.now() < end, "fixture condition timed out"); await delay(10); }
|
||||
}
|
||||
function sweep(): void {
|
||||
const global = globalThis as { __lhWakePipeline?: { lastSweepAt: number } };
|
||||
if (global.__lhWakePipeline) global.__lhWakePipeline.lastSweepAt = 0;
|
||||
recoverStaleJobs();
|
||||
}
|
||||
function chat(blocked = false) {
|
||||
const value = stores.chatStore.createChat("Isolated delivery fixture");
|
||||
chats.push(value.id);
|
||||
if (blocked) stores.chatStore.saveMessages(value.id, [question]);
|
||||
return value;
|
||||
}
|
||||
function pending(chatId: string) {
|
||||
const value = stores.jobStore.createJob({ kind: "research", chatId, toolCallId: "fixture-call" });
|
||||
const job = stores.jobStore.updateJob(value.id, {
|
||||
status: "done", result, activity: "finished", delivery: { kind: "result", status: "pending" },
|
||||
})!;
|
||||
return { job, steer: buildSubagentSteer(job, result) };
|
||||
}
|
||||
function agent() {
|
||||
const handed: SteerMessage[] = [];
|
||||
return { handed, value: { steer: (marker: SteerMessage) => handed.push(marker) } as unknown as Parameters<typeof registerLiveAgent>[1] };
|
||||
}
|
||||
function persist(chatId: string, steers: readonly LhMessage[]): void {
|
||||
stores.chatStore.saveMessages(chatId, [...stores.chatStore.getChat(chatId)!.messages, ...steers]);
|
||||
noteSteersDurable(chatId, steers);
|
||||
}
|
||||
afterEach(() => {
|
||||
researchModelOverride.current = undefined;
|
||||
for (const id of chats.splice(0)) {
|
||||
unregisterLiveAgent(id); endTurn(id); discardChatDeliveries(id);
|
||||
stores.jobStore.deleteJobsForChat(id); stores.chatStore.deleteChat(id);
|
||||
}
|
||||
setTurnRunner(async () => 500);
|
||||
});
|
||||
after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
|
||||
for (const [kind, input] of Object.entries({
|
||||
research: { question: "Saved research question", context: "Saved context" },
|
||||
demo: { title: "Fixture", brief: "Saved demo brief" },
|
||||
visual: { brief: "Saved figure brief" },
|
||||
pdf: { path: "fixture.pdf", question: "Saved PDF question" },
|
||||
})) test(`${kind}: dispatch saves input before spawn and does not overwrite it on repeated dispatch`, async () => {
|
||||
const owner = chat();
|
||||
let spawns = 0;
|
||||
const tools = jobTools({ chatId: owner.id, jobStore: stores.jobStore, spawn: (job) => {
|
||||
spawns++;
|
||||
assert.deepEqual(stores.jobStore.getJob(job.id)!.input, input);
|
||||
} });
|
||||
await tools[kind]!.execute!("fixture-dispatch", input);
|
||||
await tools[kind]!.execute!("fixture-dispatch", input);
|
||||
assert.equal(spawns, 1);
|
||||
assert.equal(stores.jobStore.listJobs(owner.id)[0]!.kind, kind as JobKind);
|
||||
});
|
||||
|
||||
test("saved question survives absent transcript; runner ends while delivery is still pending", async () => {
|
||||
const owner = chat(true);
|
||||
const input = { question: "The dispatch turn was never saved", context: "Fixture context" };
|
||||
const job = stores.jobStore.createJob({ kind: "research", chatId: owner.id, toolCallId: "lost-turn", input });
|
||||
const faux = createFauxCore({});
|
||||
let seen = "";
|
||||
faux.setResponses([(context) => { seen = JSON.stringify(context.messages); return fauxAssistantMessage("Fixture brief"); }]);
|
||||
researchModelOverride.current = { model: faux.getModel(), streamFn: faux.streamSimple };
|
||||
let delivery: Promise<void> | undefined;
|
||||
const runner = createJobRunner({ ...stores, deliver: (steer) => { delivery = deliverJobResult(steer); return delivery; } });
|
||||
await runner.runJob(job.id);
|
||||
assert(seen.includes(input.question)); assert(seen.includes(input.context));
|
||||
assert.equal(isJobLive(job.id), false, "worker registration no longer spans delivery");
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.status, "done");
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "pending");
|
||||
assert(!activityFeed().jobs.some((item) => item.id === job.id), "no running background-agent row");
|
||||
const next = agent(); assert(beginTurn(owner.id)); registerLiveAgent(owner.id, next.value);
|
||||
assert.equal(next.handed.length, 1);
|
||||
persist(owner.id, next.handed); unregisterLiveAgent(owner.id); settleSteers(owner.id, next.handed); endTurn(owner.id);
|
||||
await delivery;
|
||||
});
|
||||
|
||||
test("missing or blank research question records an honest failure without invoking a model", async () => {
|
||||
for (const input of [undefined, { question: " " }]) {
|
||||
const owner = chat();
|
||||
const job = stores.jobStore.createJob({ kind: "research", chatId: owner.id, toolCallId: "missing-question", input });
|
||||
const faux = createFauxCore({}); let calls = 0;
|
||||
faux.setResponses([() => { calls++; return fauxAssistantMessage("must not run"); }]);
|
||||
researchModelOverride.current = { model: faux.getModel(), streamFn: faux.streamSimple };
|
||||
const steers: SteerMessage[] = [];
|
||||
await createJobRunner({ ...stores, deliver: async (steer) => { steers.push(steer); } }).runJob(job.id);
|
||||
assert.equal(calls, 0);
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.error, "research question unavailable");
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.status, "error");
|
||||
assert.match(steers[0]!.content, /^Research failed: the research question is unavailable\./);
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "pending");
|
||||
}
|
||||
});
|
||||
|
||||
test("repeated recovery behind a question yields one handed and one durable marker", async () => {
|
||||
const owner = chat(true); const { job, steer } = pending(owner.id);
|
||||
let wakes = 0; setTurnRunner(async () => { wakes++; return 500; });
|
||||
sweep(); sweep(); sweep();
|
||||
const first = deliverJobResult(steer);
|
||||
assert.strictEqual(deliverJobResult(steer), first, "duplicate callers share one logical delivery");
|
||||
const next = agent(); assert(beginTurn(owner.id)); registerLiveAgent(owner.id, next.value);
|
||||
assert.deepEqual(next.handed.map((message) => message.id), [steer.id]);
|
||||
persist(owner.id, next.handed); unregisterLiveAgent(owner.id); settleSteers(owner.id, next.handed); endTurn(owner.id);
|
||||
await first;
|
||||
assert.equal(wakes, 0);
|
||||
assert.equal(stores.chatStore.getChat(owner.id)!.messages.filter((message) => message.id === steer.id).length, 1);
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "delivered");
|
||||
});
|
||||
|
||||
for (const refused of [409, 403]) test(`duplicate during a ${refused} wake retains one logical owner`, async () => {
|
||||
const owner = chat(); const { steer } = pending(owner.id);
|
||||
let release!: () => void; const gate = new Promise<void>((resolve) => { release = resolve; });
|
||||
let attempts = 0;
|
||||
setTurnRunner(async (id, input) => {
|
||||
assert.equal(input.type, "steers"); if (input.type !== "steers") return 500;
|
||||
assert.equal(input.steers.length, 1);
|
||||
attempts++;
|
||||
if (attempts === 1) { await gate; return refused; }
|
||||
persist(id, input.steers); return 200;
|
||||
});
|
||||
const first = deliverJobResult(steer);
|
||||
await until(() => attempts === 1);
|
||||
assert.strictEqual(deliverJobResult(steer), first, "wake cargo is still in the registry");
|
||||
release();
|
||||
if (refused === 403) {
|
||||
await delay(30);
|
||||
const next = agent(); assert(beginTurn(owner.id)); registerLiveAgent(owner.id, next.value);
|
||||
assert.equal(next.handed.length, 1);
|
||||
persist(owner.id, next.handed); unregisterLiveAgent(owner.id); settleSteers(owner.id, next.handed); endTurn(owner.id);
|
||||
}
|
||||
await first;
|
||||
assert.equal(stores.chatStore.getChat(owner.id)!.messages.filter((message) => message.id === steer.id).length, 1);
|
||||
});
|
||||
|
||||
test("failed start stays quiet across sweeps, then joins a learner run", async () => {
|
||||
const owner = chat(); const { job, steer } = pending(owner.id);
|
||||
let attempts = 0; setTurnRunner(async () => { attempts++; return 503; });
|
||||
const delivery = deliverJobResult(steer); await until(() => attempts === 1); await delay(30);
|
||||
sweep(); sweep(); await delay(900);
|
||||
assert.equal(attempts, 1);
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.status, "done");
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.error, undefined);
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "pending");
|
||||
const next = agent(); assert(beginTurn(owner.id)); registerLiveAgent(owner.id, next.value);
|
||||
assert.equal(next.handed.length, 1);
|
||||
persist(owner.id, next.handed); unregisterLiveAgent(owner.id); settleSteers(owner.id, next.handed); endTurn(owner.id);
|
||||
await delivery;
|
||||
});
|
||||
|
||||
test("HTTP acceptance without a saved marker is not delivery completion or a retry trigger", async () => {
|
||||
const owner = chat(); const { job, steer } = pending(owner.id);
|
||||
let attempts = 0;
|
||||
setTurnRunner(async () => { attempts++; assert(beginTurn(owner.id)); return 200; });
|
||||
const delivery = deliverJobResult(steer); await until(() => attempts === 1); await delay(30);
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "pending");
|
||||
unregisterLiveAgent(owner.id); endTurn(owner.id); settleSteers(owner.id, [], true);
|
||||
sweep(); await delay(900); assert.equal(attempts, 1);
|
||||
const next = agent(); assert(beginTurn(owner.id)); registerLiveAgent(owner.id, next.value);
|
||||
assert.equal(next.handed.length, 1);
|
||||
persist(owner.id, next.handed); unregisterLiveAgent(owner.id); endTurn(owner.id); await delivery;
|
||||
});
|
||||
|
||||
test("canonical save wins even if delivery bookkeeping fails; no marker replay", async () => {
|
||||
const owner = chat(true); const { job, steer } = pending(owner.id);
|
||||
const delivery = deliverJobResult(steer);
|
||||
const update = stores.jobStore.updateJob;
|
||||
stores.jobStore.updateJob = (id, patch) => {
|
||||
if (id === job.id && patch.delivery?.status === "delivered") throw new Error("fixture acknowledgement write failure");
|
||||
return update(id, patch);
|
||||
};
|
||||
try { persist(owner.id, [steer]); await delivery; }
|
||||
finally { stores.jobStore.updateJob = update; }
|
||||
let wakes = 0; setTurnRunner(async () => { wakes++; return 500; });
|
||||
sweep();
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "delivered");
|
||||
assert.equal(wakes, 0);
|
||||
const next = agent(); assert(beginTurn(owner.id)); registerLiveAgent(owner.id, next.value);
|
||||
assert.equal(next.handed.length, 0); unregisterLiveAgent(owner.id); endTurn(owner.id);
|
||||
});
|
||||
|
||||
test("old running results become finished pending work; old terminal records are untouched", () => {
|
||||
const owner = chat(true);
|
||||
const old = stores.jobStore.createJob({ kind: "research", chatId: owner.id, toolCallId: "legacy-pending" });
|
||||
stores.jobStore.updateJob(old.id, { result });
|
||||
const terminal = stores.jobStore.createJob({ kind: "research", chatId: owner.id, toolCallId: "legacy-terminal" });
|
||||
stores.jobStore.updateJob(terminal.id, { status: "done", result });
|
||||
sweep();
|
||||
assert.equal(stores.jobStore.getJob(old.id)!.status, "done");
|
||||
assert.equal(stores.jobStore.getJob(old.id)!.delivery?.status, "pending");
|
||||
assert.equal(stores.jobStore.getJob(terminal.id)!.delivery, undefined);
|
||||
const next = agent(); assert(beginTurn(owner.id)); registerLiveAgent(owner.id, next.value);
|
||||
assert.deepEqual(next.handed.map((message) => message.id), [`steer_${old.id}`]);
|
||||
unregisterLiveAgent(owner.id); endTurn(owner.id);
|
||||
});
|
||||
|
||||
test("fresh process restores a pending delivery on registration without a jobs poll", () => {
|
||||
const owner = chat(true); const { job, steer } = pending(owner.id);
|
||||
const script = `
|
||||
const { stores } = await import('./src/server/stores.ts');
|
||||
const { registerLiveAgent, noteSteersDurable, unregisterLiveAgent } = await import('./src/server/jobs.ts');
|
||||
const { beginTurn, endTurn } = await import('./src/server/turn-liveness.ts');
|
||||
const handed = []; beginTurn(${JSON.stringify(owner.id)});
|
||||
registerLiveAgent(${JSON.stringify(owner.id)}, { steer: marker => handed.push(marker) });
|
||||
const record = stores.chatStore.getChat(${JSON.stringify(owner.id)});
|
||||
stores.chatStore.saveMessages(record.id, [...record.messages, ...handed]);
|
||||
noteSteersDurable(record.id, handed); unregisterLiveAgent(record.id); endTurn(record.id);
|
||||
console.log(JSON.stringify(handed.map(marker => marker.id)));
|
||||
`;
|
||||
const child = spawnSync(process.execPath, ["--import", "tsx", "--input-type=module", "-e", script], {
|
||||
cwd: process.cwd(), encoding: "utf8", timeout: 15000,
|
||||
env: { PATH: process.env.PATH, HOME: path.join(directory, "home"), NODE_ENV: "test", LH_DATA_DIR: process.env.LH_DATA_DIR, LH_AUTH_FILE: process.env.LH_AUTH_FILE },
|
||||
});
|
||||
assert.equal(child.status, 0, child.stderr);
|
||||
assert.deepEqual(JSON.parse(child.stdout.trim()), [steer.id]);
|
||||
assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "delivered");
|
||||
assert.equal(stores.chatStore.getChat(owner.id)!.messages.filter((message) => message.id === steer.id).length, 1);
|
||||
});
|
||||
|
||||
test("deleting a session discards its pending deliveries without starting a turn", async () => {
|
||||
const owner = chat(true); const { job, steer } = pending(owner.id);
|
||||
let wakes = 0; setTurnRunner(async () => { wakes++; return 500; });
|
||||
const delivery = deliverJobResult(steer);
|
||||
const outcome = await deleteChatSession(owner.id, "preserve-graph");
|
||||
assert(outcome.ok);
|
||||
await delivery; await delay(450);
|
||||
assert.equal(stores.jobStore.getJob(job.id), null);
|
||||
assert.equal(stores.chatStore.getChat(owner.id), null);
|
||||
assert.equal(wakes, 0);
|
||||
});
|
||||
+68
-50
@@ -51,18 +51,20 @@ import { runningTurn } from "../src/server/turn-runs.ts";
|
||||
import {
|
||||
abortJob,
|
||||
deliverJobResult,
|
||||
noteSteerVisible,
|
||||
noteSteersDurable,
|
||||
discardChatDeliveries,
|
||||
reconcileUserSteers,
|
||||
recoverStaleJobs,
|
||||
registerLiveAgent,
|
||||
setJobRespawn,
|
||||
setTurnRunner,
|
||||
setTurnRunner as setRawTurnRunner,
|
||||
type TurnRunner,
|
||||
settleSteers,
|
||||
steerLiveChat,
|
||||
unregisterLiveAgent,
|
||||
} from "../src/server/jobs.ts";
|
||||
import { registerJobAbort, unregisterJobAbort } from "../src/agent/subagents/aborts.ts";
|
||||
import { beginTurn, endTurn } from "../src/server/turn-liveness.ts";
|
||||
import { isJobLive, registerJobAbort, unregisterJobAbort } from "../src/agent/subagents/aborts.ts";
|
||||
import { beginTurn, endTurn, isTurnRunning } from "../src/server/turn-liveness.ts";
|
||||
import { persistTurn } from "../src/server/turn-service.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { jobStore } from "../src/store/jobs.ts";
|
||||
@@ -140,11 +142,13 @@ afterEach(() => {
|
||||
if (job && job.status === "running") {
|
||||
jobStore.updateJob(id, { status: "done", activity: "settled (fence)" });
|
||||
}
|
||||
if (job?.delivery?.status === "pending") jobStore.updateJob(id, { delivery: { ...job.delivery, status: "delivered" } });
|
||||
unregisterJobAbort(id);
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
}
|
||||
for (const chatId of createdChats) discardChatDeliveries(chatId);
|
||||
// Late wakes from a prior test must be inert: a no-op respawn can't
|
||||
// increment anyone's counter, a no-op runner can't spawn real work.
|
||||
setJobRespawn(() => {});
|
||||
@@ -454,16 +458,25 @@ function fakeDeliver() {
|
||||
steers,
|
||||
deliver: async (steer: SteerMessage) => {
|
||||
steers.push(steer);
|
||||
const jobId = (steer.details as { jobId: string }).jobId;
|
||||
const job = jobStore.getJob(jobId);
|
||||
if (job && job.status === "running") {
|
||||
const failed = job.result ? isSubagentFailure(job.kind, job.result as never) : true;
|
||||
jobStore.updateJob(jobId, { status: failed ? "error" : "done", activity: failed ? "failed" : "delivered" });
|
||||
}
|
||||
const job = jobStore.getJob((steer.details as { jobId: string }).jobId)!;
|
||||
chatStore.saveMessages(job.chatId, [...chatStore.getChat(job.chatId)!.messages, steer]);
|
||||
noteSteersDurable(job.chatId, [steer]);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Fake wakes model canonical input acceptance, not just HTTP acknowledgement. */
|
||||
function setTurnRunner(run: TurnRunner): void {
|
||||
setRawTurnRunner(async (chatId, input) => {
|
||||
const status = await run(chatId, input);
|
||||
if (status === 200 && input.type === "steers") {
|
||||
const record = chatStore.getChat(chatId);
|
||||
if (record) chatStore.saveMessages(chatId, [...record.messages, ...input.steers.filter((s) => !steerDelivered(record.messages, s.id))]);
|
||||
}
|
||||
return status;
|
||||
});
|
||||
}
|
||||
|
||||
// ─── 3. dispatch: the tool call acks mid-turn and spawns ─────────────────────
|
||||
|
||||
const RESEARCH_INPUT = {
|
||||
@@ -526,7 +539,7 @@ describe("dispatch (async protocol §2)", () => {
|
||||
|
||||
const done = jobStore.getJob(job!.id)!;
|
||||
assert.equal(done.status, "done");
|
||||
assert.equal(done.activity, "delivered");
|
||||
assert.equal(done.activity, "finished");
|
||||
|
||||
// The steer carries the brief on its model-facing content; the
|
||||
// artifact payload rides details for the (UI) renderer.
|
||||
@@ -603,7 +616,7 @@ describe("research runner (async protocol §3)", () => {
|
||||
// The job is done, with the activity line from the tool call.
|
||||
const done = jobStore.getJob(job.id)!;
|
||||
assert.equal(done.status, "done");
|
||||
assert.equal(done.activity, "delivered", "the fake deliver flips the status like the pipeline");
|
||||
assert.equal(done.activity, "finished", "delivery does not own the worker status");
|
||||
const result = done.result as ResearchResult;
|
||||
assert.equal(result.brief, BRIEF);
|
||||
assert.ok(result.sources.length > 0, "sources parsed from the brief");
|
||||
@@ -746,7 +759,7 @@ describe("pdf runner (sources v0)", () => {
|
||||
|
||||
const done = jobStore.getJob(job.id)!;
|
||||
assert.equal(done.status, "done");
|
||||
assert.equal(done.activity, "delivered");
|
||||
assert.equal(done.activity, "finished");
|
||||
const result = done.result as PdfResult;
|
||||
assert.equal(result.path, PDF_FIXTURE);
|
||||
assert.match(result.brief, /integral rule/);
|
||||
@@ -827,7 +840,7 @@ describe("wake pipeline (src/server/jobs.ts)", () => {
|
||||
const job = jobStore.createJob({ kind, chatId: chat.id, toolCallId: `call-${jobStore.listJobs(chat.id).length}` });
|
||||
trackJob(job.id);
|
||||
const result: ResearchResult = { brief: "Done: it works.", sources: [{ title: "s", url: "https://x.test" }] };
|
||||
jobStore.updateJob(job.id, { result, activity: "delivering result" });
|
||||
jobStore.updateJob(job.id, { result, status: "done", activity: "finished", delivery: { kind: "result", status: "pending" } });
|
||||
const steer = buildSubagentSteer(job, result);
|
||||
return { chatId: chat.id, jobId: job.id, steer };
|
||||
}
|
||||
@@ -882,7 +895,7 @@ describe("wake pipeline (src/server/jobs.ts)", () => {
|
||||
await waitFor(() => jobStore.getJob(jobId)!.status !== "running", 8000, "job flip after delivery");
|
||||
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "done");
|
||||
assert.equal(jobStore.getJob(jobId)!.activity, "delivered");
|
||||
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "delivered");
|
||||
assert.ok(seen, "the turn runner fired");
|
||||
assert.equal(seen.chatId, chatId);
|
||||
assert.equal(seen.steers.length, 1);
|
||||
@@ -920,7 +933,8 @@ describe("wake pipeline (src/server/jobs.ts)", () => {
|
||||
|
||||
await delay(1200);
|
||||
assert.equal(fired, 0, "no wake while the quiz is pending");
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "running", "job stays running while blocked");
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "done", "worker is finished while delivery waits");
|
||||
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending");
|
||||
|
||||
// The learner answers: the real tool result clears the dangling call.
|
||||
const record = chatStore.getChat(chatId)!;
|
||||
@@ -984,7 +998,7 @@ describe("wake pipeline (src/server/jobs.ts)", () => {
|
||||
|
||||
await delay(1200);
|
||||
assert.equal(fired, 0, "no wake while the session sits dismissal-paused");
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "running", "the delivery holds (queued, not lost)");
|
||||
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending", "the delivery holds (queued, not lost)");
|
||||
|
||||
// The user's next message is the wake — its turn settles, and the
|
||||
// settlement re-pump lands the held delivery right after it.
|
||||
@@ -1056,7 +1070,8 @@ describe("Wave C: live steering handoff + settlement reconciliation", () => {
|
||||
await waitFor(() => handed.length === 1, 8000, "marker handed to the live agent");
|
||||
await delay(1200);
|
||||
assert.equal(fired, 0, "no wake while the run is live");
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "running", "job flips only at delivery");
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "done", "worker finished before delivery");
|
||||
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending");
|
||||
|
||||
// Settlement: the run persisted the injected marker.
|
||||
unregisterLiveAgent(chatId);
|
||||
@@ -1169,20 +1184,15 @@ describe("Wave C: live steering handoff + settlement reconciliation", () => {
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "done");
|
||||
});
|
||||
|
||||
it("noteSteerVisible flips the job at the visibility seam — before any settle (owner ruling 2026-09-26)", async () => {
|
||||
// The dock fix: the marker reaching the learner's view (wire
|
||||
// message_end / write-ahead persist) flips the job NOW — the
|
||||
// delivering turn may keep teaching for minutes. Idempotent with
|
||||
// the settle-path finalize.
|
||||
it("only a durable marker acknowledges delivery, independently of worker completion", () => {
|
||||
const { chatId, jobId, steer } = setup();
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "running");
|
||||
noteSteerVisible(steer);
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "done", "flipped at the seam, no turn settled");
|
||||
noteSteerVisible(steer); // the settle-path flip re-runs — no-op
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "done");
|
||||
// A probe-handoff (no jobId) is a harmless no-op.
|
||||
const handoff = buildProbeHandoffSteer(chatId);
|
||||
noteSteerVisible(handoff);
|
||||
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending");
|
||||
chatStore.saveMessages(chatId, [...chatStore.getChat(chatId)!.messages, steer]);
|
||||
noteSteersDurable(chatId, [steer]);
|
||||
noteSteersDurable(chatId, [steer]);
|
||||
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "delivered");
|
||||
assert.equal(jobStore.getJob(jobId)!.activity, "finished");
|
||||
});
|
||||
|
||||
it("the idle wake carries EVERY pending marker in ONE turn", async () => {
|
||||
@@ -1208,17 +1218,25 @@ describe("Wave C: live steering handoff + settlement reconciliation", () => {
|
||||
});
|
||||
|
||||
|
||||
it("fails the job (error status) when the wake turn errors unrecoverably", async () => {
|
||||
const { jobId, steer } = setup();
|
||||
setTurnRunner(async () => 500);
|
||||
|
||||
await deliverJobResult(steer).then(
|
||||
() => assert.fail("expected delivery to reject"),
|
||||
() => undefined,
|
||||
);
|
||||
const errored = jobStore.getJob(jobId)!;
|
||||
assert.equal(errored.status, "error");
|
||||
assert.match(errored.error ?? "", /status 500/);
|
||||
it("a failed start stays pending without changing the worker outcome", async () => {
|
||||
const { chatId, jobId, steer } = setup();
|
||||
let attempts = 0;
|
||||
setTurnRunner(async () => { attempts++; return 500; });
|
||||
const delivery = deliverJobResult(steer);
|
||||
await waitFor(() => attempts === 1, 8000, "failed start attempted");
|
||||
await delay(1000);
|
||||
assert.equal(attempts, 1, "no automatic retry loop");
|
||||
assert.equal(jobStore.getJob(jobId)!.status, "done");
|
||||
assert.equal(jobStore.getJob(jobId)!.error, undefined);
|
||||
assert.equal(jobStore.getJob(jobId)!.delivery?.status, "pending");
|
||||
const handed: SteerMessage[] = [];
|
||||
assert.ok(beginTurn(chatId));
|
||||
registerLiveAgent(chatId, { steer: (m: SteerMessage) => handed.push(m) } as never);
|
||||
assert.deepEqual(handed.map((m) => m.id), [steer.id]);
|
||||
unregisterLiveAgent(chatId);
|
||||
settleSteers(chatId, [steer]);
|
||||
endTurn(chatId);
|
||||
await delivery;
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1554,14 +1572,14 @@ describe("abort (src/server/jobs.ts abortJob)", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("aborting a delivering job drops the queued result steer and wakes with the abort note instead", async () => {
|
||||
it("a finished worker with a queued result is no longer abortable", async () => {
|
||||
const chat = chatStore.createChat("Test: abort delivering job");
|
||||
trackChat(chat.id);
|
||||
chatStore.saveMessages(chat.id, [{ id: "u1", role: "user", content: "hello", timestamp: 1 } as never]);
|
||||
const job = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "call-abort3" });
|
||||
trackJob(job.id);
|
||||
const result: ResearchResult = { brief: "Done: it works.", sources: [{ title: "s", url: "https://x.test" }] };
|
||||
jobStore.updateJob(job.id, { result, activity: "delivering result" });
|
||||
jobStore.updateJob(job.id, { result, status: "done", activity: "finished", delivery: { kind: "result", status: "pending" } });
|
||||
|
||||
// Block the pump's idle gates so the result steer sits QUEUED, not in flight.
|
||||
assert.ok(beginTurn(chat.id));
|
||||
@@ -1573,18 +1591,18 @@ describe("abort (src/server/jobs.ts abortJob)", () => {
|
||||
void deliverJobResult(buildSubagentSteer(job, result)).catch(() => {
|
||||
/* dropped by the abort below */
|
||||
});
|
||||
await waitFor(() => jobStore.getJob(job.id)!.activity === "delivering result", 8000, "queued");
|
||||
assert.equal(jobStore.getJob(job.id)!.delivery?.status, "pending");
|
||||
|
||||
const outcome = abortJob(job.id);
|
||||
assert.ok(outcome.ok);
|
||||
assert.equal(jobStore.getJob(job.id)!.status, "error");
|
||||
assert.equal(jobStore.getJob(job.id)!.error, "Aborted by learner");
|
||||
assert.deepEqual(outcome, { ok: false, reason: "not_running" });
|
||||
assert.equal(jobStore.getJob(job.id)!.status, "done");
|
||||
assert.equal(jobStore.getJob(job.id)!.error, undefined);
|
||||
|
||||
endTurn(chat.id); // gates open — whatever is left in the queue fires
|
||||
await waitFor(() => fired.length > 0, 8000, "wake fired");
|
||||
assert.equal(fired.length, 1, "exactly one delivery — the result steer was dropped");
|
||||
assert.equal((fired[0]!.details as { aborted: boolean }).aborted, true, "the abort steer, not the result");
|
||||
assert.equal(jobStore.getJob(job.id)!.status, "error", "terminal after the wake");
|
||||
assert.equal(fired.length, 1, "exactly one result delivery");
|
||||
assert.equal(fired[0]!.customType, "subagent-steer");
|
||||
assert.equal(jobStore.getJob(job.id)!.status, "done");
|
||||
});
|
||||
|
||||
it("outcome reasons: not_found for unknown, not_running for settled jobs", () => {
|
||||
|
||||
@@ -156,9 +156,9 @@ describe("DispatchCard — the BYOK refusal tone (owner, 2026-09-22)", () => {
|
||||
assert.ok(!html.includes("dispatch-refused"));
|
||||
});
|
||||
|
||||
it("a delivered job reads delivered, untouched by the wave", () => {
|
||||
it("a finished worker reads finished, independently of delivery", () => {
|
||||
const html = renderToStaticMarkup(inJobs([job({ status: "done", error: undefined })]));
|
||||
assert.ok(html.includes("delivered"));
|
||||
assert.ok(html.includes("finished"));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ for (const p of NATIVE_SUBSCRIPTIONS) for (const revokeAfterQueue of [false, tru
|
||||
const chat = stores.chatStore.createChat("Fixture mixed queue"); const model = { ...piModels().getModels(p.providerId)[0]!, provider: p.logicalId };
|
||||
stores.chatStore.saveSettings(chat.id, { model: { provider: p.logicalId, id: model.id } });
|
||||
const job = stores.jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "fixture-job-call" }); const result = { brief: "Fixture helper result", sources: [] };
|
||||
stores.jobStore.updateJob(job.id, { status: "running", result }); const steer = buildSubagentSteer(job, result); const contexts: string[] = [];
|
||||
stores.jobStore.updateJob(job.id, { status: "done", result, delivery: { kind: "result", status: "pending" } }); const steer = buildSubagentSteer(job, result); const contexts: string[] = [];
|
||||
const agent = new Agent({ initialState: { model, messages: [{ id: "fixture-initial", role: "user", content: "Initial learner input", timestamp: 1 } as AgentMessage] }, convertToLlm: toLlmMessages, streamFn: () => {
|
||||
contexts.push(JSON.stringify(agent.state.messages)); const stream = new AssistantMessageEventStream();
|
||||
const output = { role: "assistant" as const, provider: p.logicalId, api: model.api, model: model.id, content: [{ type: "text" as const, text: "Foreground response" }], stopReason: "stop" as const, timestamp: 2, usage: { input: 2, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 3, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } };
|
||||
@@ -57,6 +57,6 @@ for (const p of NATIVE_SUBSCRIPTIONS) for (const revokeAfterQueue of [false, tru
|
||||
assert.equal(contexts.length, 1); assert(contexts[0]!.includes("Continue my learning")); assert(!contexts[0]!.includes(steer.id));
|
||||
const persisted = persistTurn(stores.chatStore, chat.id, 0, agent.state.messages, false);
|
||||
unregisterLiveAgent(chat.id); endTurn(chat.id); settleSteers(chat.id, persisted); await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(wakes, 0); assert.equal(stores.jobStore.getJob(job.id)!.status, "running");
|
||||
assert.equal(wakes, 0); assert.equal(stores.jobStore.getJob(job.id)!.status, "done"); assert.equal(stores.jobStore.getJob(job.id)!.delivery?.status, "pending");
|
||||
connection(p.providerId, true); await delivered; assert.equal(wakes, 1); assert.equal(stores.jobStore.getJob(job.id)!.status, "done");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user