Swap the server engine to pi-ai/pi-agent-core (Band B+C of docs/pi-engine-swap.md)
The Vercel AI SDK survives only as Band D (useChat client + UIMessage
record types); @openrouter/ai-sdk-provider is uninstalled.
- src/agent/models.ts: PiEngine {model, streamFn} via createModels +
openrouterProvider subpath; thinkingLevelFor maps config effort none->off
- src/agent/lh-tool.ts: neutral LhTool (zod authoring, optional execute,
clientQuestion flag); toPiTool converts via z.toJSONSchema and re-runs
zod inside execute so refinements ride the error-result retry path
- src/server/pi/record-adapter.ts: UIMessage records -> pi Messages
(dangling tool calls get synthesized interrupted results)
- src/server/pi/ui-stream.ts: SSE bridge emitting AI-SDK-shaped chunks so
the useChat client parses unchanged (continuation id rule, question
tools never emit output, tool-input-error for bad authoring)
- teacher-agent.ts: stateless per-turn pi Agents; shouldStopAfterTurn
stops only when every question-tool call resolved cleanly (errors retry)
- chat-handler.ts: buildAgent(outgoing) factory seam (stateless doctrine);
assertValidUiMessages owns the retired SDK validation gate
- sub-agents via shared runSubagent (pi-run.ts); extraction stages stop
after a clean submit; visual-maker keeps its hand-rolled loop over
direct streamSimple calls; aborted model calls bill no usage
- tests: all model seams ported to pi-ai faux provider (scripted
FauxResponseSteps, agent factories); handleChatRequest second param
is now an agent factory
Gates: tsc, lint, build, 305/305. Live GLM smoke (disposable driver,
deleted): prober streaming + persistence, question seam, turn-2
continuation same message id, per-turn real token usage - ALL PASS.
Data wiped at the pivot (chats/graph/jobs, .gitkeeps kept).
This commit is contained in:
amosblomqvist
+98
-124
@@ -1,28 +1,38 @@
|
||||
/**
|
||||
* Scripted integration test — the full teacher loop with zero tokens:
|
||||
*
|
||||
* learner opens chat → mock model calls `quiz` → server streams + persists
|
||||
* learner opens chat → faux model calls `quiz` → server streams + persists
|
||||
* → client grades and sends the answer back as the tool output
|
||||
* → mock model reacts → transcript persisted.
|
||||
* → faux model reacts → transcript persisted.
|
||||
*
|
||||
* Drives handleChatRequest directly (plain Request/Response — no Next server
|
||||
* needed) with a MockLanguageModelV4. This is the AI SDK equivalent of the
|
||||
* old pi dispatch contract test: the quiz bridge is gone, the tool part IS
|
||||
* the pending state, and the persisted transcript is the resume record.
|
||||
* needed) with a scripted pi engine (pi-ai's faux provider, docs/
|
||||
* pi-engine-swap.md §7): the quiz bridge is gone, the tool part IS the
|
||||
* pending state, and the persisted transcript is the resume record.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { MockLanguageModelV4, simulateReadableStream } from "ai/test";
|
||||
import type { LanguageModelV4StreamPart } from "@ai-sdk/provider";
|
||||
import {
|
||||
createFauxCore,
|
||||
fauxAssistantMessage,
|
||||
fauxText,
|
||||
fauxThinking,
|
||||
fauxToolCall,
|
||||
type FauxResponseStep,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
|
||||
import { handleChatRequest } from "../src/server/chat-handler.ts";
|
||||
import { isTurnRunning } from "../src/server/turn-liveness.ts";
|
||||
import { recordToPiMessages } from "../src/server/pi/record-adapter.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { buildAnswer, coerceCorrectAnswer, normalizeOptions, resolveCorrect, type QuestionData } from "../src/core/question-core.ts";
|
||||
import { findPendingToolCall } from "../src/core/pending.ts";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
import type { TeachingAgent } from "../src/agent/teacher-agent.ts";
|
||||
import type { UIMessage } from "ai";
|
||||
import { PATHS } from "../config.ts";
|
||||
|
||||
const QUIZ_INPUT = {
|
||||
@@ -36,52 +46,41 @@ const QUIZ_INPUT = {
|
||||
explanation: "Mercury is the innermost planet.",
|
||||
};
|
||||
|
||||
function usage() {
|
||||
return {
|
||||
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
|
||||
outputTokens: { total: 5, text: 5, reasoning: 0 },
|
||||
};
|
||||
/** Turn 1: a bit of reasoning, then the quiz tool call. Turn 2: plain text.
|
||||
* Each response factory records the prompt it received (the transcript the
|
||||
* model saw) — the old doStream capture, engine-shaped. */
|
||||
function scriptedTeacher(steps: (calls: string[][]) => FauxResponseStep[]): {
|
||||
engine: PiEngine;
|
||||
calls: string[][];
|
||||
} {
|
||||
const calls: string[][] = [];
|
||||
const faux = createFauxCore({});
|
||||
faux.setResponses(steps(calls));
|
||||
return { engine: { model: faux.getModel(), streamFn: faux.streamSimple }, calls };
|
||||
}
|
||||
|
||||
/** Turn 1: a bit of reasoning, then the quiz tool call. Turn 2: plain text. */
|
||||
function scriptedModel(calls: string[][]) {
|
||||
let turn = 0;
|
||||
return new MockLanguageModelV4({
|
||||
doStream: async (options) => {
|
||||
const promptTexts = options.prompt.map((m) => JSON.stringify(m.content));
|
||||
calls.push(promptTexts);
|
||||
turn++;
|
||||
const chunks: LanguageModelV4StreamPart[] =
|
||||
turn === 1
|
||||
? [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "reasoning-start", id: "r1" },
|
||||
{ type: "reasoning-delta", id: "r1", delta: "Probe the basics first." },
|
||||
{ type: "reasoning-end", id: "r1" },
|
||||
// Same shape real providers use: input streams as deltas,
|
||||
// then the completed call. (This is what makes the client's
|
||||
// 'Writing a question…' indicator possible.)
|
||||
{ type: "tool-input-start", id: "call-1", toolName: "quiz" },
|
||||
{ type: "tool-input-delta", id: "call-1", delta: JSON.stringify(QUIZ_INPUT) },
|
||||
{ type: "tool-input-end", id: "call-1" },
|
||||
{
|
||||
type: "tool-call",
|
||||
toolCallId: "call-1",
|
||||
toolName: "quiz",
|
||||
input: JSON.stringify(QUIZ_INPUT),
|
||||
},
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
]
|
||||
: [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "text-start", id: "t1" },
|
||||
{ type: "text-delta", id: "t1", delta: "Right — Mercury it is." },
|
||||
{ type: "text-end", id: "t1" },
|
||||
{ type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: usage() },
|
||||
];
|
||||
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
|
||||
/** The default two-turn lesson: probe quiz, then react to the answer. */
|
||||
function defaultLesson(calls: string[][]): FauxResponseStep[] {
|
||||
return [
|
||||
(context) => {
|
||||
calls.push(context.messages.map((m) => JSON.stringify((m as { content: unknown }).content)));
|
||||
return fauxAssistantMessage(
|
||||
[fauxThinking("Probe the basics first."), fauxToolCall("quiz", QUIZ_INPUT, { id: "call-1" })],
|
||||
{ stopReason: "toolUse" },
|
||||
);
|
||||
},
|
||||
});
|
||||
(context) => {
|
||||
calls.push(context.messages.map((m) => JSON.stringify((m as { content: unknown }).content)));
|
||||
return fauxAssistantMessage([fauxText("Right — Mercury it is.")]);
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/** The handler seam (stateless per turn): each request builds a fresh agent
|
||||
* off the transcript it is about to run — the factory tests inject. */
|
||||
function agentFactory(engine: PiEngine) {
|
||||
return (outgoing: UIMessage[]): TeachingAgent =>
|
||||
createTeacherAgent(undefined, { engine, messages: recordToPiMessages(outgoing) });
|
||||
}
|
||||
|
||||
async function drain(res: Response): Promise<string> {
|
||||
@@ -112,14 +111,13 @@ after(() => {
|
||||
|
||||
describe("teacher chat flow (scripted model)", () => {
|
||||
it("quiz tool call persists as a pending part; the answer continues the lesson", async () => {
|
||||
const calls: string[][] = [];
|
||||
const agent = createTeacherAgent(undefined, scriptedModel(calls));
|
||||
const { engine, calls } = scriptedTeacher(defaultLesson);
|
||||
const chat = chatStore.createChat("Test: solar system");
|
||||
createdChats.push(chat.id);
|
||||
|
||||
// ── Turn 1: learner opens with a topic; teacher probes with a quiz ──
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] };
|
||||
await drain(await handleChatRequest(post(chat.id, [firstUser]), agent));
|
||||
await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)));
|
||||
|
||||
const saved1 = chatStore.getChat(chat.id)!;
|
||||
assert.equal(saved1.messages.length, 2, "user message + assistant response persisted");
|
||||
@@ -128,12 +126,12 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
|
||||
const reasoningPart = assistantMsg.parts.find((p) => p.type === "reasoning");
|
||||
assert.ok(reasoningPart && reasoningPart.type === "reasoning", "reasoning part streamed + persisted");
|
||||
assert.match(reasoningPart.text, /Probe the basics/);
|
||||
assert.match((reasoningPart as { text?: string }).text!, /Probe the basics/);
|
||||
|
||||
const quizPart = assistantMsg.parts.find((p) => p.type === "tool-quiz");
|
||||
assert.ok(quizPart && quizPart.type === "tool-quiz", "quiz tool part persisted");
|
||||
assert.equal(quizPart.state, "input-available", "unanswered quiz = pending state (resume record)");
|
||||
assert.deepEqual(quizPart.input, QUIZ_INPUT);
|
||||
assert.equal((quizPart as { state?: string }).state, "input-available", "unanswered quiz = pending state (resume record)");
|
||||
assert.deepEqual((quizPart as { input?: unknown }).input, QUIZ_INPUT);
|
||||
|
||||
// The pending-question gate sees it.
|
||||
const pending = findPendingToolCall(saved1.messages);
|
||||
@@ -166,7 +164,7 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
}
|
||||
: m,
|
||||
);
|
||||
await drain(await handleChatRequest(post(chat.id, messages2), agent));
|
||||
await drain(await handleChatRequest(post(chat.id, messages2), agentFactory(engine)));
|
||||
|
||||
// ── The teacher reacts — as a CONTINUATION of the same assistant message
|
||||
// (the last client message is the assistant message carrying the tool
|
||||
@@ -176,14 +174,14 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
const finalParts = saved2.messages[1]!.parts;
|
||||
const textPart = finalParts.find((p) => p.type === "text");
|
||||
assert.ok(textPart && textPart.type === "text", "reaction text appended after the tool part");
|
||||
assert.match(textPart.text, /Mercury it is/);
|
||||
assert.match((textPart as { text?: string }).text!, /Mercury it is/);
|
||||
// The answered quiz part is still there, now with its output.
|
||||
const answeredQuiz = finalParts.find((p) => p.type === "tool-quiz");
|
||||
assert.ok(answeredQuiz && answeredQuiz.type === "tool-quiz" && answeredQuiz.state === "output-available");
|
||||
assert.ok(answeredQuiz && answeredQuiz.type === "tool-quiz" && (answeredQuiz as { state?: string }).state === "output-available");
|
||||
|
||||
// The model actually received the graded answer as a tool result.
|
||||
assert.equal(calls.length, 2, "model saw exactly two prompts");
|
||||
const toolResultSent = calls[1]!.some((c) => c.includes("tool-result") || c.includes("Mercury"));
|
||||
const toolResultSent = calls[1]!.some((c) => c.includes("toolResult") || c.includes("Mercury"));
|
||||
assert.ok(toolResultSent, "continuation prompt carries the tool result to the model");
|
||||
|
||||
// No pending question remains.
|
||||
@@ -200,11 +198,11 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
// 'input-streaming', which the stream processor sets from the
|
||||
// tool-input-start chunk. If a transport change ever stopped emitting
|
||||
// it, the indicator would silently die — pin the protocol.
|
||||
const agent = createTeacherAgent(undefined, scriptedModel([]));
|
||||
const { engine } = scriptedTeacher(defaultLesson);
|
||||
const chat = chatStore.createChat("Test: protocol pin");
|
||||
createdChats.push(chat.id);
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] };
|
||||
const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agent));
|
||||
const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)));
|
||||
const startIdx = body.indexOf('"type":"tool-input-start"');
|
||||
const availIdx = body.indexOf('"type":"tool-input-available"');
|
||||
assert.ok(startIdx !== -1, "tool-input-start chunk present in the stream");
|
||||
@@ -213,20 +211,19 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
});
|
||||
|
||||
it("persists even when the stream is aborted (resume keeps partial state)", async () => {
|
||||
const calls: string[][] = [];
|
||||
const agent = createTeacherAgent(undefined, scriptedModel(calls));
|
||||
const { engine } = scriptedTeacher(defaultLesson);
|
||||
const chat = chatStore.createChat("Test: abort");
|
||||
createdChats.push(chat.id);
|
||||
|
||||
const req = post(chat.id, [{ id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] }]);
|
||||
const res = await handleChatRequest(req, agent);
|
||||
const res = await handleChatRequest(req, agentFactory(engine));
|
||||
assert.equal(res.status, 200);
|
||||
// Read one chunk then cancel — simulates the browser closing mid-stream.
|
||||
const reader = res.body!.getReader();
|
||||
await reader.read();
|
||||
await reader.cancel();
|
||||
// Give onFinish a tick to fire.
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
// Give the aborted run a tick to settle (agent_end → onFinish).
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
const saved = chatStore.getChat(chat.id)!;
|
||||
assert.ok(saved.messages.length >= 1, "partial transcript persisted");
|
||||
});
|
||||
@@ -234,8 +231,8 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
it("an ungradeable quiz is rejected at the call boundary, the error reaches the model, and the model retries", async () => {
|
||||
// The authoring contract (question-core superRefine): a quiz whose
|
||||
// correctAnswer matches no option value must NEVER reach the dock —
|
||||
// the SDK validates client-side tool inputs against the schema, marks
|
||||
// the call invalid, and returns the validation error to the teacher as
|
||||
// toPiTool re-runs the zod schema at the call boundary, marks the
|
||||
// call invalid, and returns the validation error to the teacher as
|
||||
// a tool result. The loop re-prompts the model, which retries in the
|
||||
// same turn. This pins that whole mechanism (the live Nietzsche bug).
|
||||
const BAD_QUIZ_INPUT = {
|
||||
@@ -245,49 +242,28 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
explanation: "Genealogy diagnoses, it does not convert.",
|
||||
};
|
||||
|
||||
const calls: string[][] = [];
|
||||
let step = 0;
|
||||
const retryModel = new MockLanguageModelV4({
|
||||
doStream: async (options) => {
|
||||
calls.push(options.prompt.map((m) => JSON.stringify(m.content)));
|
||||
step++;
|
||||
const chunks: LanguageModelV4StreamPart[] =
|
||||
step === 1
|
||||
? [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "tool-input-start", id: "call-bad", toolName: "quiz" },
|
||||
{ type: "tool-input-delta", id: "call-bad", delta: JSON.stringify(BAD_QUIZ_INPUT) },
|
||||
{ type: "tool-input-end", id: "call-bad" },
|
||||
{
|
||||
type: "tool-call",
|
||||
toolCallId: "call-bad",
|
||||
toolName: "quiz",
|
||||
input: JSON.stringify(BAD_QUIZ_INPUT),
|
||||
},
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
]
|
||||
: [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "tool-input-start", id: "call-good", toolName: "quiz" },
|
||||
{ type: "tool-input-delta", id: "call-good", delta: JSON.stringify(QUIZ_INPUT) },
|
||||
{ type: "tool-input-end", id: "call-good" },
|
||||
{
|
||||
type: "tool-call",
|
||||
toolCallId: "call-good",
|
||||
toolName: "quiz",
|
||||
input: JSON.stringify(QUIZ_INPUT),
|
||||
},
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
];
|
||||
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
|
||||
const steps = (calls: string[][]): FauxResponseStep[] => [
|
||||
(context) => {
|
||||
calls.push(context.messages.map((m) => JSON.stringify((m as { content: unknown }).content)));
|
||||
return fauxAssistantMessage(
|
||||
[fauxToolCall("quiz", BAD_QUIZ_INPUT, { id: "call-bad" })],
|
||||
{ stopReason: "toolUse" },
|
||||
);
|
||||
},
|
||||
});
|
||||
(context) => {
|
||||
calls.push(context.messages.map((m) => JSON.stringify((m as { content: unknown }).content)));
|
||||
return fauxAssistantMessage(
|
||||
[fauxToolCall("quiz", QUIZ_INPUT, { id: "call-good" })],
|
||||
{ stopReason: "toolUse" },
|
||||
);
|
||||
},
|
||||
];
|
||||
const { engine, calls } = scriptedTeacher(steps);
|
||||
|
||||
const agent = createTeacherAgent(undefined, retryModel);
|
||||
const chat = chatStore.createChat("Test: bad quiz retry");
|
||||
createdChats.push(chat.id);
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] };
|
||||
await drain(await handleChatRequest(post(chat.id, [firstUser]), agent));
|
||||
await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)));
|
||||
|
||||
// The model was re-prompted (the error did not end the turn)…
|
||||
assert.equal(calls.length, 2, "the invalid call triggered a second model prompt");
|
||||
@@ -304,8 +280,8 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
const assistantMsg = saved.messages[1]!;
|
||||
const quizParts = assistantMsg.parts.filter((p) => p.type === "tool-quiz");
|
||||
const failed = quizParts.find(
|
||||
(p) => p.type === "tool-quiz" && (p as { state?: string }).state === "output-error",
|
||||
);
|
||||
(p) => p.type === "tool-quiz" && (p as { state?: string }).state === "output-error",
|
||||
);
|
||||
assert.ok(failed, "the invalid call persisted as an error-state part");
|
||||
assert.match(
|
||||
String((failed as { errorText?: string }).errorText),
|
||||
@@ -330,36 +306,34 @@ describe("chat store", () => {
|
||||
|
||||
describe("mid-turn navigation (turn liveness + write-ahead)", () => {
|
||||
it("one turn per chat: a concurrent request is 409'd; the next one after finish is accepted", async () => {
|
||||
const calls: string[][] = [];
|
||||
const agent = createTeacherAgent(undefined, scriptedModel(calls));
|
||||
const { engine } = scriptedTeacher(defaultLesson);
|
||||
const chat = chatStore.createChat("Test: turn guard");
|
||||
createdChats.push(chat.id);
|
||||
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] };
|
||||
const res1 = await handleChatRequest(post(chat.id, [firstUser]), agent); // NOT drained → in flight
|
||||
const res1 = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)); // NOT drained → in flight
|
||||
assert.equal(isTurnRunning(chat.id), true, "turn registered as in flight");
|
||||
|
||||
// A stale client (remount racing a live turn) tries to start a second turn.
|
||||
const res2 = await handleChatRequest(post(chat.id, [firstUser]), agent);
|
||||
const res2 = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine));
|
||||
assert.equal(res2.status, 409, "concurrent turn rejected — last-writer-wins persist would clobber it");
|
||||
|
||||
await drain(res1); // finish → onFinish → endTurn
|
||||
assert.equal(isTurnRunning(chat.id), false, "registry cleared at finish");
|
||||
|
||||
const res3 = await handleChatRequest(post(chat.id, [firstUser]), agent);
|
||||
const res3 = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine));
|
||||
assert.equal(res3.status, 200, "the next turn is accepted after the previous finished");
|
||||
await drain(res3);
|
||||
});
|
||||
|
||||
it("write-ahead: the learner's answer is durable BEFORE the turn finishes", async () => {
|
||||
const calls: string[][] = [];
|
||||
const agent = createTeacherAgent(undefined, scriptedModel(calls));
|
||||
const { engine } = scriptedTeacher(defaultLesson);
|
||||
const chat = chatStore.createChat("Test: write-ahead");
|
||||
createdChats.push(chat.id);
|
||||
|
||||
// Turn 1: the teacher asks the quiz (pending part in the record).
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] };
|
||||
await drain(await handleChatRequest(post(chat.id, [firstUser]), agent));
|
||||
await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)));
|
||||
const saved1 = chatStore.getChat(chat.id)!;
|
||||
const assistantMsg = saved1.messages[1]!;
|
||||
|
||||
@@ -384,17 +358,17 @@ describe("mid-turn navigation (turn liveness + write-ahead)", () => {
|
||||
p.type === "tool-quiz" ? { ...p, state: "output-available" as const, output: answer } : p,
|
||||
),
|
||||
}
|
||||
: m,
|
||||
: m,
|
||||
);
|
||||
|
||||
// NOT drained: the teacher turn is streaming, but the answer is already
|
||||
// on disk — a remount (or crash) mid-turn sees the answered quiz, never
|
||||
// the stale pre-answer question.
|
||||
const res = await handleChatRequest(post(chat.id, messages2), agent);
|
||||
const res = await handleChatRequest(post(chat.id, messages2), agentFactory(engine));
|
||||
const midTurn = chatStore.getChat(chat.id)!;
|
||||
const midQuiz = midTurn.messages[1]!.parts.find((p) => p.type === "tool-quiz");
|
||||
assert.ok(midQuiz && midQuiz.type === "tool-quiz");
|
||||
assert.equal(midQuiz.state, "output-available", "write-ahead persisted the answer before finish");
|
||||
assert.equal((midQuiz as { state?: string }).state, "output-available", "write-ahead persisted the answer before finish");
|
||||
|
||||
await drain(res); // finish → persist the full turn → endTurn
|
||||
const saved2 = chatStore.getChat(chat.id)!;
|
||||
@@ -405,19 +379,19 @@ describe("mid-turn navigation (turn liveness + write-ahead)", () => {
|
||||
it("a turn that fails to start (malformed messages) clears the registry — no lockout", async () => {
|
||||
const chat = chatStore.createChat("Test: turn failed to start");
|
||||
createdChats.push(chat.id);
|
||||
// A text part with no text: fails UIMessage validation before the stream
|
||||
// A text part with no text: fails message validation before the stream
|
||||
// exists — the path where no onFinish will ever run.
|
||||
const bad = [{ id: "m1", role: "user", parts: [{ type: "text" }] }];
|
||||
const res = await handleChatRequest(post(chat.id, bad), createTeacherAgent(undefined, scriptedModel([])));
|
||||
const res = await handleChatRequest(post(chat.id, bad), agentFactory(scriptedTeacher(defaultLesson).engine));
|
||||
assert.equal(res.status, 500, "turn-start failure is a 500, not a hang");
|
||||
assert.equal(isTurnRunning(chat.id), false, "registry cleared when the stream never started");
|
||||
|
||||
// And the chat is NOT locked: the next (valid) turn is accepted.
|
||||
const ok = await handleChatRequest(
|
||||
post(chat.id, [{ id: "m1", role: "user", parts: [{ type: "text", text: "teach me" }] }]),
|
||||
createTeacherAgent(undefined, scriptedModel([])),
|
||||
agentFactory(scriptedTeacher(defaultLesson).engine),
|
||||
);
|
||||
assert.equal(ok.status, 200);
|
||||
await drain(ok);
|
||||
});
|
||||
});
|
||||
});
|
||||
+46
-79
@@ -27,8 +27,7 @@ import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { MockLanguageModelV4, simulateReadableStream } from "ai/test";
|
||||
import type { LanguageModelV4StreamPart } from "@ai-sdk/provider";
|
||||
import { createFauxCore, fauxAssistantMessage, fauxToolCall, type FauxResponseStep } from "@earendil-works/pi-ai";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { PATHS } from "../config.ts";
|
||||
import {
|
||||
@@ -54,7 +53,10 @@ import { createTeacherTools } from "../src/agent/tools.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { jobStore } from "../src/store/jobs.ts";
|
||||
import { handleChatRequest } from "../src/server/chat-handler.ts";
|
||||
import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
|
||||
import { createTeacherAgent, type TeachingAgent } from "../src/agent/teacher-agent.ts";
|
||||
import { recordToPiMessages } from "../src/server/pi/record-adapter.ts";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
import type { UIMessage } from "ai";
|
||||
|
||||
/**
|
||||
* A fake deliver seam + a runner wired like the chat-handler's, per test:
|
||||
@@ -104,37 +106,22 @@ after(() => {
|
||||
visualModelOverride.current = undefined;
|
||||
});
|
||||
|
||||
function usage() {
|
||||
return {
|
||||
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
|
||||
outputTokens: { total: 5, text: 5, reasoning: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
/** A model that answers a scripted sequence of doGenerate turns (tool calls or final text). */
|
||||
function scriptedModel(turns: Array<{ toolCalls?: Array<{ toolCallId: string; toolName: string; input: unknown }>; text?: string }>): MockLanguageModelV4 {
|
||||
let turn = -1;
|
||||
return new MockLanguageModelV4({
|
||||
doGenerate: async () => {
|
||||
turn++;
|
||||
const step = turns[turn];
|
||||
if (!step) throw new Error(`scripted model exhausted after ${turns.length} turns`);
|
||||
if (step.toolCalls) {
|
||||
return {
|
||||
content: step.toolCalls.map((c) => ({ type: "tool-call" as const, toolCallId: c.toolCallId, toolName: c.toolName, input: JSON.stringify(c.input) })),
|
||||
finishReason: { unified: "tool-calls" as const, raw: "tool-calls" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
return {
|
||||
content: [{ type: "text" as const, text: step.text ?? "" }],
|
||||
finishReason: { unified: "stop" as const, raw: "stop" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
};
|
||||
},
|
||||
/** A scripted pi engine (faux provider): the response queue replaces the old
|
||||
* doGenerate turn counter — one queued response per model call (tool-call
|
||||
* turn or final text), consumed in order. */
|
||||
function scriptedModel(turns: Array<{ toolCalls?: Array<{ toolCallId: string; toolName: string; input: unknown }>; text?: string }>): PiEngine {
|
||||
const steps: FauxResponseStep[] = turns.map((step) => {
|
||||
if (step.toolCalls) {
|
||||
return fauxAssistantMessage(
|
||||
step.toolCalls.map((c) => fauxToolCall(c.toolName, c.input as Record<string, unknown>, { id: c.toolCallId })),
|
||||
{ stopReason: "toolUse" },
|
||||
);
|
||||
}
|
||||
return fauxAssistantMessage(step.text ?? "");
|
||||
});
|
||||
const faux = createFauxCore({ tokenSize: { min: 64, max: 64 } });
|
||||
faux.setResponses(steps);
|
||||
return { model: faux.getModel(), streamFn: faux.streamSimple };
|
||||
}
|
||||
|
||||
// ─── fake headless ───────────────────────────────────────────────────────────
|
||||
@@ -424,7 +411,7 @@ describe("demo builder (fake model + fake headless)", () => {
|
||||
|
||||
const result = await runDemoBuilder(
|
||||
{ title: "Damped oscillator", brief: "A mass on a spring with a damping slider." },
|
||||
{ model, headless, onActivity: (line) => activities.push(line) },
|
||||
{ engine: model, headless, onActivity: (line) => activities.push(line) },
|
||||
);
|
||||
|
||||
assert.equal(result.title, "Damped oscillator");
|
||||
@@ -442,7 +429,7 @@ describe("demo builder (fake model + fake headless)", () => {
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||
{ text: "Done." },
|
||||
]);
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless });
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { engine: model, headless });
|
||||
assert.equal(result.html, GOOD_HTML);
|
||||
assert.equal(result.tested, true, "tested by the final verification gate");
|
||||
assert.equal(headless.demoRuns.length, 1, "the gate ran the demo once itself");
|
||||
@@ -454,7 +441,7 @@ describe("demo builder (fake model + fake headless)", () => {
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: BROKEN_HTML } }] },
|
||||
{ text: "It works, trust me." },
|
||||
]);
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless });
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { engine: model, headless });
|
||||
assert.equal(result.html, "");
|
||||
assert.equal(result.tested, true, "the loop ran");
|
||||
assert.match(result.notes ?? "", /^FAILED:/);
|
||||
@@ -463,14 +450,14 @@ describe("demo builder (fake model + fake headless)", () => {
|
||||
|
||||
it("fails fast with a clear message when no Chrome is available (never skips testing)", async () => {
|
||||
const model = scriptedModel([{ text: "should never be reached" }]);
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless: noChromeHeadless() });
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { engine: model, headless: noChromeHeadless() });
|
||||
assert.equal(result.html, "");
|
||||
assert.match(result.notes ?? "", /^FAILED: .*Chrome/);
|
||||
});
|
||||
|
||||
it("fails when the model never writes a demo", async () => {
|
||||
const model = scriptedModel([{ text: "I cannot build this." }]);
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless: fakeHeadless() });
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { engine: model, headless: fakeHeadless() });
|
||||
assert.equal(result.html, "");
|
||||
assert.match(result.notes ?? "", /^FAILED: the builder never wrote a demo/);
|
||||
});
|
||||
@@ -489,7 +476,7 @@ describe("visual maker (fake model + fake headless)", () => {
|
||||
{ text: "A number line from -3 to 3 with the root marked at 0." },
|
||||
]);
|
||||
const activities: string[] = [];
|
||||
const result = await runVisualMaker({ brief: "Number line, root at 0." }, { model, headless, onActivity: (l) => activities.push(l) });
|
||||
const result = await runVisualMaker({ brief: "Number line, root at 0." }, { engine: model, headless, onActivity: (l) => activities.push(l) });
|
||||
|
||||
assert.equal(result.svg, SVG_1);
|
||||
assert.equal(result.notes, "A number line from -3 to 3 with the root marked at 0.");
|
||||
@@ -504,7 +491,7 @@ describe("visual maker (fake model + fake headless)", () => {
|
||||
{ text: "Done." }, // premature finish — no render yet
|
||||
{ text: "Looked at it — correct." }, // after the forced look
|
||||
]);
|
||||
const result = await runVisualMaker({ brief: "b" }, { model, headless });
|
||||
const result = await runVisualMaker({ brief: "b" }, { engine: model, headless });
|
||||
assert.equal(result.svg, SVG_1, "delivered after the forced look");
|
||||
assert.equal(result.notes, "Looked at it — correct.");
|
||||
assert.equal(headless.svgRenders.length, 1, "the gate rendered the svg once itself");
|
||||
@@ -522,7 +509,7 @@ describe("visual maker (fake model + fake headless)", () => {
|
||||
{ toolCalls: [{ toolCallId: "r2", toolName: "render_svg", input: {} }] },
|
||||
{ text: "Moved the start of the line." },
|
||||
]);
|
||||
const result = await runVisualMaker({ brief: "b" }, { model, headless });
|
||||
const result = await runVisualMaker({ brief: "b" }, { engine: model, headless });
|
||||
|
||||
assert.equal(result.svg, SVG_1.replace('<line x1="10"', '<line x1="20"'), "the edit was applied to the delivered svg");
|
||||
assert.equal(result.notes, "Moved the start of the line.");
|
||||
@@ -536,7 +523,7 @@ describe("visual maker (fake model + fake headless)", () => {
|
||||
{ toolCalls: [{ toolCallId: "e0", toolName: "edit_svg", input: { old_text: "svg", new_text: "x" } }] },
|
||||
{ text: "Cannot proceed." },
|
||||
]);
|
||||
const result = await runVisualMaker({ brief: "b" }, { model, headless });
|
||||
const result = await runVisualMaker({ brief: "b" }, { engine: model, headless });
|
||||
assert.equal(result.svg, "", "the failed edit produced no deliverable");
|
||||
assert.match(result.notes ?? "", /never wrote an SVG/);
|
||||
});
|
||||
@@ -550,20 +537,20 @@ describe("visual maker (fake model + fake headless)", () => {
|
||||
{ toolCalls: [{ toolCallId: "r2", toolName: "render_svg", input: {} }] },
|
||||
{ text: "Fixed and verified." },
|
||||
]);
|
||||
const result = await runVisualMaker({ brief: "b" }, { model, headless });
|
||||
const result = await runVisualMaker({ brief: "b" }, { engine: model, headless });
|
||||
assert.equal(result.svg, SVG_1);
|
||||
assert.equal(result.notes, "Fixed and verified.");
|
||||
});
|
||||
|
||||
it("fails when the model never writes an svg", async () => {
|
||||
const model = scriptedModel([{ text: "This idea is purely relational — belongs in mermaid." }]);
|
||||
const result = await runVisualMaker({ brief: "b" }, { model, headless: fakeHeadless() });
|
||||
const result = await runVisualMaker({ brief: "b" }, { engine: model, headless: fakeHeadless() });
|
||||
assert.equal(result.svg, "");
|
||||
assert.match(result.notes ?? "", /^FAILED: the maker never wrote an SVG/);
|
||||
});
|
||||
|
||||
it("fails with a clear message when no Chrome is available", async () => {
|
||||
const result = await runVisualMaker({ brief: "b" }, { model: scriptedModel([{ text: "x" }]), headless: noChromeHeadless() });
|
||||
const result = await runVisualMaker({ brief: "b" }, { engine: scriptedModel([{ text: "x" }]), headless: noChromeHeadless() });
|
||||
assert.equal(result.svg, "");
|
||||
assert.match(result.notes ?? "", /^FAILED: .*Chrome/);
|
||||
});
|
||||
@@ -685,30 +672,10 @@ describe("demo/visual job runners", () => {
|
||||
createdChats.push(chat.id);
|
||||
|
||||
// A teacher that calls demo on turn 1, then continues in the SAME turn.
|
||||
let turn = 0;
|
||||
const teacher = new MockLanguageModelV4({
|
||||
doStream: async () => {
|
||||
turn++;
|
||||
const chunks: LanguageModelV4StreamPart[] =
|
||||
turn === 1
|
||||
? [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "tool-input-start", id: "call-d2", toolName: "demo" },
|
||||
{ type: "tool-input-delta", id: "call-d2", delta: JSON.stringify(DEMO_INPUT) },
|
||||
{ type: "tool-input-end", id: "call-d2" },
|
||||
{ type: "tool-call", toolCallId: "call-d2", toolName: "demo", input: JSON.stringify(DEMO_INPUT) },
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
]
|
||||
: [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "text-start", id: "t2" },
|
||||
{ type: "text-delta", id: "t2", delta: "Here it is." },
|
||||
{ type: "text-end", id: "t2" },
|
||||
{ type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: usage() },
|
||||
];
|
||||
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
|
||||
},
|
||||
});
|
||||
const teacher = scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "call-d2", toolName: "demo", input: DEMO_INPUT }] },
|
||||
{ text: "Here it is." },
|
||||
]);
|
||||
|
||||
// The overrides must be live BEFORE the request — dispatch fires the
|
||||
// runner mid-stream, inside the tool execute.
|
||||
@@ -731,17 +698,17 @@ describe("demo/visual job runners", () => {
|
||||
messages: [{ id: "u1", role: "user", parts: [{ type: "text", text: "teach me compounding" }] }],
|
||||
}),
|
||||
}),
|
||||
createTeacherAgent(
|
||||
undefined,
|
||||
teacher,
|
||||
undefined,
|
||||
createTeacherTools({
|
||||
chatId: chat.id,
|
||||
chatStore,
|
||||
jobStore,
|
||||
spawn: (job, input) => void runner.runJob(job.id, input),
|
||||
(outgoing: UIMessage[]): TeachingAgent =>
|
||||
createTeacherAgent(undefined, {
|
||||
engine: teacher,
|
||||
messages: recordToPiMessages(outgoing),
|
||||
tools: createTeacherTools({
|
||||
chatId: chat.id,
|
||||
chatStore,
|
||||
jobStore,
|
||||
spawn: (job, input) => void runner.runJob(job.id, input),
|
||||
}),
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
|
||||
+63
-86
@@ -5,7 +5,7 @@
|
||||
*
|
||||
* transcriptView — the compact view both LLM stages see (pure):
|
||||
* message-id-tagged verbatim text + quiz lines, NO tree
|
||||
* pipeline — MockLanguageModelV4 drives both agents: findings out,
|
||||
* pipeline — scripted faux engines drive both agents: findings out,
|
||||
* audit verdict (with linkTo identity) in, deterministic
|
||||
* merge applied; failure paths checkpoint nothing;
|
||||
* single-flight per chat
|
||||
@@ -22,8 +22,9 @@
|
||||
import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { MockLanguageModelV4 } from "ai/test";
|
||||
import { createFauxCore, fauxAssistantMessage, fauxToolCall, type FauxResponseStep } from "@earendil-works/pi-ai";
|
||||
import type { UIMessage } from "ai";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
import type {
|
||||
AuditResult,
|
||||
GraphEdge,
|
||||
@@ -42,11 +43,14 @@ import { PATHS } from "../config.ts";
|
||||
|
||||
const createdChats: string[] = [];
|
||||
|
||||
function usage() {
|
||||
return {
|
||||
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
|
||||
outputTokens: { total: 5, text: 5, reasoning: 0 },
|
||||
};
|
||||
/** A scripted pi engine (faux provider, docs/pi-engine-swap.md §7): the
|
||||
* response queue replaces the old doGenerate turn counter. Usage is
|
||||
* ESTIMATED by faux from the prompt/response text (deterministic, > 0) —
|
||||
* the tests assert structure and positivity, not flat mock numbers. */
|
||||
function fauxEngine(steps: FauxResponseStep[]): PiEngine {
|
||||
const faux = createFauxCore({});
|
||||
faux.setResponses(steps);
|
||||
return { model: faux.getModel(), streamFn: faux.streamSimple };
|
||||
}
|
||||
|
||||
/** An in-memory GraphStore (core/ports.ts) — this suite never touches data/graph. */
|
||||
@@ -163,68 +167,46 @@ const FINDINGS: SessionFindings = {
|
||||
attribution: { quiz: { "probe-1": "p1", "check-1": "p2" } },
|
||||
};
|
||||
|
||||
/** A model whose ONE tool call is `name` with the given input; then text.
|
||||
* `onCall` (optional) fires at the moment of the first generate — the
|
||||
* observation point for mid-run assertions. */
|
||||
function submitModel(name: string, input: unknown, onCall?: () => void): MockLanguageModelV4 {
|
||||
let turn = 0;
|
||||
return new MockLanguageModelV4({
|
||||
doGenerate: async () => {
|
||||
turn++;
|
||||
if (turn === 1) {
|
||||
onCall?.();
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "tool-call" as const,
|
||||
toolCallId: `${name}-call`,
|
||||
toolName: name,
|
||||
input: JSON.stringify(input),
|
||||
},
|
||||
],
|
||||
finishReason: { unified: "tool-calls" as const, raw: "tool-calls" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
return {
|
||||
content: [{ type: "text" as const, text: "submitted" }],
|
||||
finishReason: { unified: "stop" as const, raw: "stop" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
};
|
||||
/** An engine whose ONE response is the `name` tool call with the given
|
||||
* input — the whole deliverable; the pipeline stops after a clean submit
|
||||
* (shouldStop), so no trailing text step exists anymore. `onCall` fires at
|
||||
* the moment of the model call — the observation point for mid-run
|
||||
* assertions. */
|
||||
function submitModel(name: string, input: unknown, onCall?: () => void): PiEngine {
|
||||
return fauxEngine([
|
||||
(context) => {
|
||||
onCall?.();
|
||||
void context;
|
||||
return fauxAssistantMessage(
|
||||
[fauxToolCall(name, input as Record<string, unknown>, { id: `${name}-call` })],
|
||||
{ stopReason: "toolUse" },
|
||||
);
|
||||
},
|
||||
});
|
||||
]);
|
||||
}
|
||||
|
||||
/** A model that hangs until its abort signal fires, then rejects —
|
||||
* mimicking a cancelled provider request (the abort path). Like a real
|
||||
* provider, an ALREADY-aborted signal rejects immediately (the abort
|
||||
* can win the race before doGenerate attaches its listener). */
|
||||
function hangingModel(): MockLanguageModelV4 {
|
||||
const abortReject = (reject: (reason: Error) => void) =>
|
||||
reject(Object.assign(new Error("The operation was aborted"), { name: "AbortError" }));
|
||||
return new MockLanguageModelV4({
|
||||
doGenerate: async (options): Promise<never> => {
|
||||
await new Promise<never>((_, reject) => {
|
||||
if (options.abortSignal?.aborted) abortReject(reject);
|
||||
else options.abortSignal?.addEventListener("abort", () => abortReject(reject));
|
||||
});
|
||||
throw new Error("unreachable");
|
||||
},
|
||||
});
|
||||
/** An engine whose model call never resolves until its abort signal fires
|
||||
* — mimicking a cancelled provider request (the abort path). Like a real
|
||||
* provider, an ALREADY-aborted signal resolves immediately (the abort can
|
||||
* win the race before the call attaches its listener); the faux stream
|
||||
* then sees the aborted signal and ends the message as aborted. */
|
||||
function hangingModel(): PiEngine {
|
||||
return fauxEngine([
|
||||
(_context, options) =>
|
||||
new Promise((resolve) => {
|
||||
const sig = options?.signal;
|
||||
if (sig?.aborted) {
|
||||
resolve(fauxAssistantMessage("late"));
|
||||
return;
|
||||
}
|
||||
sig?.addEventListener("abort", () => resolve(fauxAssistantMessage("late")), { once: true });
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
/** A model that never calls a tool (the failure path). */
|
||||
function silentModel(): MockLanguageModelV4 {
|
||||
return new MockLanguageModelV4({
|
||||
doGenerate: async () => ({
|
||||
content: [{ type: "text" as const, text: "I refuse to use tools" }],
|
||||
finishReason: { unified: "stop" as const, raw: "stop" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
}),
|
||||
});
|
||||
/** An engine that never calls a tool (the failure path). */
|
||||
function silentModel(): PiEngine {
|
||||
return fauxEngine([fauxAssistantMessage("I refuse to use tools"), fauxAssistantMessage("…")]);
|
||||
}
|
||||
|
||||
// ─── transcriptView (pure) ───────────────────────────────────────────────────
|
||||
@@ -427,9 +409,11 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
});
|
||||
await pipeline.runExtraction(chatId);
|
||||
|
||||
// Live: mid-audit the extractor's usage was fully accumulated
|
||||
// (the mock bills 10 input / 5 output per step, flat per the SDK).
|
||||
assert.deepEqual(usageSeen, [10]);
|
||||
// Live: mid-audit the extractor's usage was fully accumulated (faux
|
||||
// estimates tokens from the prompt/response text — deterministic,
|
||||
// positive, no cache).
|
||||
assert.equal(usageSeen.length, 1);
|
||||
assert.ok(typeof usageSeen[0] === "number" && usageSeen[0]! > 0, "the extractor's step billed input tokens");
|
||||
|
||||
// Persisted: the run appended ONE usage entry to the chat's
|
||||
// extractions log — per-stage model + steps, with merge stats.
|
||||
@@ -438,16 +422,14 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const run = record.extractions![0]!;
|
||||
assert.equal(run.stats?.nodes, 0); // linked, not created
|
||||
assert.equal(run.stats?.events, 3);
|
||||
assert.equal(run.stages.extractor.model, "mock-model-id"); // the mock's model id
|
||||
assert.equal(run.stages.extractor.steps.length, 2); // tool-call step + text step
|
||||
assert.deepEqual(run.stages.extractor.steps[0], {
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
cacheReadTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
});
|
||||
assert.equal(run.stages.auditor.steps.length, 2);
|
||||
assert.equal(run.stages.extractor.model, "faux-1"); // the scripted engine's model id
|
||||
assert.equal(run.stages.extractor.steps.length, 1); // the submit call — the run STOPS after a clean submit
|
||||
const step = run.stages.extractor.steps[0]!;
|
||||
assert.ok(step.inputTokens! > 0 && step.outputTokens! > 0, "estimated usage captured");
|
||||
assert.equal(step.cacheReadTokens, 0);
|
||||
assert.equal(step.cacheWriteTokens, 0);
|
||||
assert.equal(step.reasoningTokens, 0);
|
||||
assert.equal(run.stages.auditor.steps.length, 1);
|
||||
assert.ok(!Number.isNaN(Date.parse(run.at)));
|
||||
});
|
||||
|
||||
@@ -497,18 +479,13 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
release = resolve;
|
||||
});
|
||||
let calls = 0;
|
||||
const gatedExtractor = new MockLanguageModelV4({
|
||||
doGenerate: async () => {
|
||||
const gatedExtractor = fauxEngine([
|
||||
async () => {
|
||||
calls++;
|
||||
if (calls === 1) await gate;
|
||||
return {
|
||||
content: [{ type: "text" as const, text: "…" }],
|
||||
finishReason: { unified: "stop" as const, raw: "stop" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
};
|
||||
return fauxAssistantMessage("…");
|
||||
},
|
||||
});
|
||||
]);
|
||||
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
|
||||
+80
-139
@@ -28,8 +28,7 @@ import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import type { UIMessage } from "ai";
|
||||
import { MockLanguageModelV4, simulateReadableStream } from "ai/test";
|
||||
import type { LanguageModelV4StreamPart } from "@ai-sdk/provider";
|
||||
import { createFauxCore, fauxAssistantMessage, fauxText, fauxToolCall, type AssistantMessage, type FauxResponseStep } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
accumulateJobUsage,
|
||||
buildAbortSteer,
|
||||
@@ -45,16 +44,18 @@ import {
|
||||
buildProbeHandoffSteer,
|
||||
type ProbeHandoffSteerMessage,
|
||||
} from "../src/core/contracts.ts";
|
||||
import { composeInstructions, createTeacherAgent } from "../src/agent/teacher-agent.ts";
|
||||
import { composeInstructions, createTeacherAgent, type TeachingAgent } from "../src/agent/teacher-agent.ts";
|
||||
import { createTeacherTools } from "../src/agent/tools.ts";
|
||||
import { webTools } from "../src/agent/tools/web.ts";
|
||||
import { handleChatRequest } from "../src/server/chat-handler.ts";
|
||||
import { recordToPiMessages } from "../src/server/pi/record-adapter.ts";
|
||||
import { abortJob, claimSteers, deliverJobResult, setTurnRunner } from "../src/server/jobs.ts";
|
||||
import { beginTurn, endTurn } from "../src/server/turn-liveness.ts";
|
||||
import { persistTurn, withMessageIds } from "../src/server/turn-service.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { jobStore } from "../src/store/jobs.ts";
|
||||
import { createJobRunner, researchModelOverride } from "../src/agent/subagents/runner.ts";
|
||||
import { createResearcherAgent } from "../src/agent/subagents/researcher.ts";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
|
||||
const JOBS_DIR = path.join(PATHS.data, "jobs");
|
||||
@@ -88,11 +89,30 @@ after(() => {
|
||||
researchModelOverride.current = undefined;
|
||||
});
|
||||
|
||||
function usage() {
|
||||
return {
|
||||
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
|
||||
outputTokens: { total: 5, text: 5, reasoning: 0 },
|
||||
};
|
||||
/** A scripted pi engine (faux provider): the response queue replaces the
|
||||
* old doStream/doGenerate turn counters. tokenSize is set so every test
|
||||
* string streams as ONE delta — the SSE-body assertions ("Back with the
|
||||
* brief.") read contiguous text. */
|
||||
function fauxEngine(steps: FauxResponseStep[]): PiEngine {
|
||||
const faux = createFauxCore({ tokenSize: { min: 64, max: 64 } });
|
||||
faux.setResponses(steps);
|
||||
return { model: faux.getModel(), streamFn: faux.streamSimple };
|
||||
}
|
||||
|
||||
/** A scripted response factory whose model call never resolves until its
|
||||
* abort signal fires (a cancelled provider request); an already-aborted
|
||||
* signal resolves at once — the faux stream then ends the message aborted. */
|
||||
function abortableFactory(): FauxResponseStep {
|
||||
const factory = (_context: unknown, options: { signal?: AbortSignal } | undefined) =>
|
||||
new Promise<AssistantMessage>((resolve) => {
|
||||
const sig = options?.signal;
|
||||
if (sig?.aborted) {
|
||||
resolve(fauxAssistantMessage("late"));
|
||||
return;
|
||||
}
|
||||
sig?.addEventListener("abort", () => resolve(fauxAssistantMessage("late")), { once: true });
|
||||
});
|
||||
return factory as unknown as FauxResponseStep;
|
||||
}
|
||||
|
||||
async function drain(res: Response): Promise<string> {
|
||||
@@ -295,34 +315,14 @@ const RESEARCH_INPUT = {
|
||||
};
|
||||
|
||||
/** A teacher that calls research (text → call), then plain text thereafter. */
|
||||
function researchCallingTeacher(): MockLanguageModelV4 {
|
||||
let turn = 0;
|
||||
return new MockLanguageModelV4({
|
||||
doStream: async () => {
|
||||
turn++;
|
||||
const chunks: LanguageModelV4StreamPart[] =
|
||||
turn === 1
|
||||
? [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "text-start", id: "t1" },
|
||||
{ type: "text-delta", id: "t1", delta: "Let me check the current state of the library." },
|
||||
{ type: "text-end", id: "t1" },
|
||||
{ type: "tool-input-start", id: "call-r1", toolName: "research" },
|
||||
{ type: "tool-input-delta", id: "call-r1", delta: JSON.stringify(RESEARCH_INPUT) },
|
||||
{ type: "tool-input-end", id: "call-r1" },
|
||||
{ type: "tool-call", toolCallId: "call-r1", toolName: "research", input: JSON.stringify(RESEARCH_INPUT) },
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
]
|
||||
: [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "text-start", id: "t2" },
|
||||
{ type: "text-delta", id: "t2", delta: "Back with the brief." },
|
||||
{ type: "text-end", id: "t2" },
|
||||
{ type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: usage() },
|
||||
];
|
||||
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
|
||||
},
|
||||
});
|
||||
function researchCallingTeacher(): PiEngine {
|
||||
return fauxEngine([
|
||||
fauxAssistantMessage(
|
||||
[fauxText("Let me check the current state of the library."), fauxToolCall("research", RESEARCH_INPUT, { id: "call-r1" })],
|
||||
{ stopReason: "toolUse" },
|
||||
),
|
||||
fauxAssistantMessage([fauxText("Back with the brief.")]),
|
||||
]);
|
||||
}
|
||||
|
||||
describe("dispatch (async protocol §2)", () => {
|
||||
@@ -332,17 +332,15 @@ describe("dispatch (async protocol §2)", () => {
|
||||
|
||||
const delivery = fakeDeliver();
|
||||
const testRunner = createJobRunner({ chatStore, jobStore, deliver: delivery.deliver });
|
||||
const agent = createTeacherAgent(
|
||||
undefined,
|
||||
researchCallingTeacher(),
|
||||
undefined,
|
||||
createTeacherTools({
|
||||
chatId: chat.id,
|
||||
chatStore,
|
||||
jobStore,
|
||||
spawn: (job, input) => void testRunner.runJob(job.id, input),
|
||||
}),
|
||||
);
|
||||
const engine = researchCallingTeacher();
|
||||
const tools = createTeacherTools({
|
||||
chatId: chat.id,
|
||||
chatStore,
|
||||
jobStore,
|
||||
spawn: (job, input) => void testRunner.runJob(job.id, input),
|
||||
});
|
||||
const agent = (outgoing: UIMessage[]): TeachingAgent =>
|
||||
createTeacherAgent(undefined, { engine, tools, messages: recordToPiMessages(outgoing) });
|
||||
|
||||
const restoreFetch = stubFetch();
|
||||
researchModelOverride.current = researcherModel();
|
||||
@@ -394,38 +392,15 @@ sigma.js v3 is current.
|
||||
## Sources
|
||||
- sigma.js docs (https://www.sigmajs.org/docs)`;
|
||||
|
||||
/**
|
||||
* The researcher loop runs via agent.generate (non-streaming) — the mock
|
||||
* therefore scripts doGenerate. Turn 1: a web_search tool call; turn 2: the
|
||||
* final text brief.
|
||||
*/
|
||||
function researcherModel(): MockLanguageModelV4 {
|
||||
let turn = 0;
|
||||
return new MockLanguageModelV4({
|
||||
doGenerate: async () => {
|
||||
turn++;
|
||||
return turn === 1
|
||||
? {
|
||||
content: [
|
||||
{
|
||||
type: "tool-call" as const,
|
||||
toolCallId: "ws1",
|
||||
toolName: "web_search",
|
||||
input: JSON.stringify({ query: "sigma.js latest version" }),
|
||||
},
|
||||
],
|
||||
finishReason: { unified: "tool-calls" as const, raw: "tool-calls" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
}
|
||||
: {
|
||||
content: [{ type: "text" as const, text: BRIEF }],
|
||||
finishReason: { unified: "stop" as const, raw: "stop" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
};
|
||||
},
|
||||
});
|
||||
/** The researcher loop's scripted engine. Turn 1: a web_search tool call;
|
||||
* turn 2: the final text brief. */
|
||||
function researcherModel(): PiEngine {
|
||||
return fauxEngine([
|
||||
fauxAssistantMessage([fauxToolCall("web_search", { query: "sigma.js latest version" }, { id: "ws1" })], {
|
||||
stopReason: "toolUse",
|
||||
}),
|
||||
fauxAssistantMessage([fauxText(BRIEF)]),
|
||||
]);
|
||||
}
|
||||
|
||||
function stubFetch() {
|
||||
@@ -516,11 +491,11 @@ describe("research runner (async protocol §3)", () => {
|
||||
|
||||
const delivery = fakeDeliver();
|
||||
const runner = createJobRunner({ chatStore, jobStore, deliver: delivery.deliver });
|
||||
researchModelOverride.current = new MockLanguageModelV4({
|
||||
doGenerate: async () => {
|
||||
researchModelOverride.current = fauxEngine([
|
||||
() => {
|
||||
throw new Error("model exploded");
|
||||
},
|
||||
});
|
||||
]);
|
||||
try {
|
||||
await runner.runJob(job.id);
|
||||
} finally {
|
||||
@@ -578,11 +553,12 @@ describe("research runner (async protocol §3)", () => {
|
||||
// ─── the researcher agent construction (model resolution seam) ──────────────
|
||||
|
||||
describe("researcher agent", () => {
|
||||
it("builds with the web tool set (no network touched)", () => {
|
||||
const agent = createResearcherAgent(researcherModel());
|
||||
assert.ok(agent.tools.web_search, "web_search available");
|
||||
assert.ok(agent.tools.web_fetch, "web_fetch available");
|
||||
assert.equal(Object.keys(agent.tools).length, 2, "ONLY the web tools — no question/plan tools");
|
||||
it("runs with the web tool set (no network touched)", () => {
|
||||
// The researcher is a runSubagent call over the webTools registry —
|
||||
// the tool set is what the loop sees.
|
||||
assert.ok(webTools.web_search, "web_search available");
|
||||
assert.ok(webTools.web_fetch, "web_fetch available");
|
||||
assert.equal(Object.keys(webTools).length, 2, "ONLY the web tools — no question/plan tools");
|
||||
// reset any accidental global override
|
||||
researchModelOverride.current = undefined;
|
||||
});
|
||||
@@ -950,57 +926,20 @@ describe("prompt composition", () => {
|
||||
// ─── abort (2026-09 semantics: real kill, cancelled status, teacher told) ───
|
||||
|
||||
describe("abort (src/server/jobs.ts abortJob)", () => {
|
||||
/** A hanging model that only settles when its abort signal fires — the
|
||||
* real-kill path: the in-flight call itself is killed. */
|
||||
function hangingModel(): MockLanguageModelV4 {
|
||||
return new MockLanguageModelV4({
|
||||
doGenerate: ({ abortSignal }) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
// an ALREADY-aborted signal never fires the event — real
|
||||
// providers reject an aborted fetch at once, the mock must too
|
||||
if (abortSignal?.aborted) {
|
||||
reject(new DOMException("The operation was aborted", "AbortError"));
|
||||
return;
|
||||
}
|
||||
abortSignal?.addEventListener("abort", () => {
|
||||
reject(new DOMException("The operation was aborted", "AbortError"));
|
||||
});
|
||||
}),
|
||||
});
|
||||
/** A hanging model whose call never resolves until its abort signal
|
||||
* fires — the real-kill path: the in-flight call itself is killed. */
|
||||
function hangingModel(): PiEngine {
|
||||
return fauxEngine([abortableFactory()]);
|
||||
}
|
||||
|
||||
/** One step (with usage) then a hang — proves usage-so-far survives the kill. */
|
||||
function oneStepThenHangModel(): MockLanguageModelV4 {
|
||||
let turn = 0;
|
||||
return new MockLanguageModelV4({
|
||||
doGenerate: ({ abortSignal }) => {
|
||||
turn++;
|
||||
if (turn === 1) {
|
||||
return Promise.resolve({
|
||||
content: [
|
||||
{
|
||||
type: "tool-call" as const,
|
||||
toolCallId: "ws1",
|
||||
toolName: "web_search",
|
||||
input: JSON.stringify({ query: "anything" }),
|
||||
},
|
||||
],
|
||||
finishReason: { unified: "tool-calls" as const, raw: "tool-calls" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
});
|
||||
}
|
||||
return new Promise((_resolve, reject) => {
|
||||
if (abortSignal?.aborted) {
|
||||
reject(new DOMException("The operation was aborted", "AbortError"));
|
||||
return;
|
||||
}
|
||||
abortSignal?.addEventListener("abort", () => {
|
||||
reject(new DOMException("The operation was aborted", "AbortError"));
|
||||
});
|
||||
});
|
||||
},
|
||||
});
|
||||
/** One step (billed) then a hang — proves usage-so-far survives the kill. */
|
||||
function oneStepThenHangModel(): PiEngine {
|
||||
return fauxEngine([
|
||||
fauxAssistantMessage([fauxToolCall("web_search", { query: "anything" }, { id: "ws1" })], {
|
||||
stopReason: "toolUse",
|
||||
}),
|
||||
abortableFactory(),
|
||||
]);
|
||||
}
|
||||
|
||||
function newChatWithResearchJob(label: string, toolCallId: string): { chatId: string; jobId: string } {
|
||||
@@ -1082,8 +1021,10 @@ describe("abort (src/server/jobs.ts abortJob)", () => {
|
||||
assert.equal(job.error, "Aborted by learner");
|
||||
assert.equal(job.usage!.steps, 1, "usage-so-far persisted by the runner's finally");
|
||||
assert.equal(job.usage!.model, "z-ai/glm-5.3-flash", "the config role's OpenRouter id");
|
||||
assert.equal(job.usage!.inputTokens, 10);
|
||||
assert.equal(job.usage!.outputTokens, 5);
|
||||
// faux estimates tokens from the prompt/response text — the
|
||||
// pinned fact is that the ONE completed step's spend survived.
|
||||
assert.ok(job.usage!.inputTokens! > 0, "input tokens billed");
|
||||
assert.ok(job.usage!.outputTokens! > 0, "output tokens billed");
|
||||
} finally {
|
||||
researchModelOverride.current = undefined;
|
||||
}
|
||||
|
||||
@@ -50,10 +50,10 @@ const validQuiz = {
|
||||
};
|
||||
|
||||
interface QuizToolSpec {
|
||||
inputSchema: { safeParse: (v: unknown) => { success: boolean; error?: { issues: Array<{ message: string }> } } };
|
||||
schema: { safeParse: (v: unknown) => { success: boolean; error?: { issues: Array<{ message: string }> } } };
|
||||
}
|
||||
/** teachQuizToolFor returns the single quiz tool; createProberTools a registry. */
|
||||
const schemaOf = (toolLike: unknown) => (toolLike as QuizToolSpec).inputSchema;
|
||||
const schemaOf = (toolLike: unknown) => (toolLike as QuizToolSpec).schema;
|
||||
const quizSchemaOf = (tools: Record<string, unknown>) => schemaOf(tools.quiz);
|
||||
|
||||
describe("teachQuizToolFor: the dynamic plan-nodeId validation (position signal)", () => {
|
||||
|
||||
Reference in New Issue
Block a user