fix: let the learner retry a turn whose model call failed

2026-10-26 14:31:51 +02:00
parent 3dc88472ee
commit a572675900
4 changed files with 330 additions and 2 deletions
@@ -0,0 +1,240 @@
+/**
+ * The failed-turn retry (issue #110): a model call that fails persists as an
+ * assistant message with stopReason "error". The learner's input survives
+ * (write-ahead), but pre-fix nothing could re-run the turn — re-answering
+ * 400'd (the question is no longer dangling) and `continue` died in pi
+ * ("Cannot continue from message role: assistant"). The `retry` input drops
+ * the failure and re-runs the turn from the input that preceded it.
+ *
+ * Drives handleChatRequest with a scripted pi engine (the chat-flow pattern).
+ */
+
+import assert from "node:assert/strict";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { after, describe, it } from "node:test";
+import { createFauxCore, fauxAssistantMessage, fauxText, fauxToolCall, type FauxResponseStep } from "@earendil-works/pi-ai";
+import { SessionModel } from "../components/session-core.ts";
+import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
+import { handleChatRequest } from "../src/server/chat-handler.ts";
+import { runningTurn } from "../src/server/turn-runs.ts";
+import { chatStore } from "../src/store/chats.ts";
+import { buildAnswer, normalizeOptions, resolveCorrect, type QuestionData } from "../src/core/question-core.ts";
+import { ChatInputError, endsInFailedTurn, parseChatInput, retryDrops } from "../src/core/input-events.ts";
+import { newPlanBlockMessage, newUserMessage, type LhMessage } from "../src/core/native-messages.ts";
+import { newCompactionMarker } from "../src/core/compaction.ts";
+import type { PiEngine } from "../src/agent/models.ts";
+import { PATHS } from "../config.ts";
+
+process.env.OPENROUTER_API_KEY ??= "test-key-presence-gate";
+
+const QUIZ_INPUT = {
+	question: "Which planet is closest to the sun?",
+	options: [{ label: "Venus" }, { label: "Mercury" }],
+	correctAnswer: "Mercury",
+	explanation: "Mercury is the innermost planet.",
+	type: "check",
+	nodeId: "planet-order",
+};
+
+/** A response step that fails the way a dropped connection does. */
+const networkFailure = (() => {
+	throw new Error("fetch failed");
+}) as unknown as FauxResponseStep;
+
+/** Scripted engine; each step records the conversation roles it saw
+ *  (the composed system message aside). */
+function scripted(steps: FauxResponseStep[]): { engine: PiEngine; contexts: string[][] } {
+	const contexts: string[][] = [];
+	const faux = createFauxCore({});
+	faux.setResponses(
+		steps.map((step) => ((context: { messages: Array<{ role: string }> }, ...rest: unknown[]) => {
+			contexts.push(context.messages.map((m) => m.role).filter((r) => r !== "system"));
+			return (step as (...a: unknown[]) => unknown)(context, ...rest);
+		}) as unknown as FauxResponseStep),
+	);
+	return { engine: { model: faux.getModel(), streamFn: faux.streamSimple }, contexts };
+}
+
+function post(chatId: string, input: unknown): Request {
+	return new Request("http://test/api/chat", {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify({ chatId, input }),
+	});
+}
+
+/** POST one input event; on a 200 ack, drain the turn to completion. */
+async function turn(engine: PiEngine, chatId: string, input: unknown): Promise<Response> {
+	const ack = await handleChatRequest(post(chatId, input), (opts) =>
+		createTeacherAgent(undefined, { engine, messages: opts.messages }),
+	);
+	if (ack.status === 200) {
+		const handle = runningTurn(chatId);
+		if (handle) await new Response(handle.subscribe()).text();
+	}
+	return ack;
+}
+
+const createdChats: string[] = [];
+after(() => {
+	for (const id of createdChats) {
+		try {
+			fs.unlinkSync(path.join(PATHS.chats, `${id}.json`));
+		} catch {
+			/* already gone */
+		}
+	}
+});
+
+function createConfiguredChat(title: string) {
+	const chat = chatStore.createChat(title);
+	createdChats.push(chat.id);
+	chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } });
+	chatStore.saveTree(chat.id, { nodes: [{ id: "planet-order", title: "Planet order" }], edges: [], version: 1 });
+	return chat;
+}
+
+function gradedAnswer() {
+	const options = normalizeOptions(QUIZ_INPUT.options);
+	const qd: QuestionData = {
+		id: "call-1",
+		kind: "quiz",
+		question: QUIZ_INPUT.question,
+		mode: "single-select",
+		options,
+		correctIndices: resolveCorrect(QUIZ_INPUT.correctAnswer, options).indices,
+		explanation: QUIZ_INPUT.explanation,
+		createdAt: new Date().toISOString(),
+	};
+	return buildAnswer(qd, { values: ["Mercury"] });
+}
+
+const roles = (chatId: string) => chatStore.getChat(chatId)!.messages.map((m) => m.role);
+
+describe("failed-turn retry (issue #110)", () => {
+	it("a failed answer turn re-runs from the saved answer — no re-answering needed", async () => {
+		const { engine, contexts } = scripted([
+			() => fauxAssistantMessage([fauxToolCall("quiz", QUIZ_INPUT, { id: "call-1" })], { stopReason: "toolUse" }),
+			networkFailure,
+			() => fauxAssistantMessage([fauxText("Right — Mercury it is.")]),
+		]);
+		const chat = createConfiguredChat("Test: retry after failed answer");
+
+		await turn(engine, chat.id, { type: "text", text: "teach me the planets" });
+		await turn(engine, chat.id, { type: "answer", toolCallId: "call-1", answer: gradedAnswer() });
+
+		// The failure as #110 met it: answer durable, turn failed, and the
+		// pre-fix paths refused.
+		assert.deepEqual(roles(chat.id), ["user", "assistant", "toolResult", "assistant"]);
+		assert.ok(endsInFailedTurn(chatStore.getChat(chat.id)!.messages));
+		const reanswer = await turn(engine, chat.id, { type: "answer", toolCallId: "call-1", answer: gradedAnswer() });
+		assert.equal(reanswer.status, 400, "the answered question cannot be answered again");
+
+		const retry = await turn(engine, chat.id, { type: "retry" });
+		assert.equal(retry.status, 200);
+
+		const saved = chatStore.getChat(chat.id)!.messages;
+		assert.deepEqual(saved.map((m) => m.role), ["user", "assistant", "toolResult", "assistant"]);
+		assert.equal(endsInFailedTurn(saved), false, "the failure left the record");
+		assert.match(JSON.stringify(saved.at(-1)), /Mercury it is/);
+		assert.deepEqual(contexts.at(-1), ["user", "assistant", "toolResult"], "the retried call saw the answer, not the failure");
+	});
+
+	it("a failed text turn re-runs from the learner's message", async () => {
+		const { engine, contexts } = scripted([networkFailure, () => fauxAssistantMessage([fauxText("Hello again.")])]);
+		const chat = createConfiguredChat("Test: retry after failed text");
+
+		await turn(engine, chat.id, { type: "text", text: "hello" });
+		assert.deepEqual(roles(chat.id), ["user", "assistant"]);
+		assert.ok(endsInFailedTurn(chatStore.getChat(chat.id)!.messages));
+
+		assert.equal((await turn(engine, chat.id, { type: "retry" })).status, 200);
+		assert.deepEqual(roles(chat.id), ["user", "assistant"]);
+		assert.match(JSON.stringify(chatStore.getChat(chat.id)!.messages.at(-1)), /Hello again/);
+		assert.deepEqual(contexts.at(-1), ["user"]);
+	});
+
+	it("refuses a retry with nothing failed — 400, record untouched", async () => {
+		const { engine } = scripted([() => fauxAssistantMessage([fauxText("Fine.")])]);
+		const chat = createConfiguredChat("Test: retry refused");
+
+		await turn(engine, chat.id, { type: "text", text: "hello" });
+		const before = chatStore.getChat(chat.id)!.messages;
+		const res = await turn(engine, chat.id, { type: "retry" });
+		assert.equal(res.status, 400);
+		assert.match(await res.text(), /requires a failed turn/);
+		assert.deepEqual(chatStore.getChat(chat.id)!.messages, before);
+	});
+
+	it("a plan block persisted after the failure stays, and the turn still retries", async () => {
+		const { engine, contexts } = scripted([networkFailure, () => fauxAssistantMessage([fauxText("Planned reply.")])]);
+		const chat = createConfiguredChat("Test: retry past a plan block");
+
+		await turn(engine, chat.id, { type: "text", text: "hello" });
+		// persistTurn's shape when plan_init succeeded and the next call
+		// failed: the plan block lands AFTER the failure.
+		const failedRecord = chatStore.getChat(chat.id)!.messages;
+		chatStore.saveMessages(chat.id, [...failedRecord, newPlanBlockMessage("plan-1", {})]);
+		assert.ok(endsInFailedTurn(chatStore.getChat(chat.id)!.messages), "the failure is still found past the marker");
+
+		assert.equal((await turn(engine, chat.id, { type: "retry" })).status, 200);
+		const saved = chatStore.getChat(chat.id)!.messages;
+		assert.deepEqual(saved.map((m) => m.role), ["user", "custom", "assistant"], "failure dropped, plan block kept");
+		assert.match(JSON.stringify(saved.at(-1)), /Planned reply/);
+		assert.deepEqual(contexts.at(-1), ["user"], "the plan block is not model input");
+	});
+});
+
+describe("the Retry button's client path", () => {
+	it("after a dropped request, Retry posts straight from the error status", async () => {
+		const failedTurn = [
+			{ ...newUserMessage("hello"), id: "u1" },
+			{ ...fauxAssistantMessage("", { stopReason: "error", errorMessage: "fetch failed" }), id: "a1" },
+		] as unknown as LhMessage[];
+		const posted: string[] = [];
+		let online = false;
+		const fetch = (async (_url: string, init?: RequestInit) => {
+			if (!online) throw new TypeError("Failed to fetch");
+			if (init?.method === "POST") posted.push(String(init.body));
+			return new Response("{}", { status: 500 }); // the reattach is not under test
+		}) as typeof globalThis.fetch;
+		const m = new SessionModel({ chatId: "chat_x", initialMessages: failedTurn, fetch });
+
+		await m.send({ type: "retry" });
+		assert.equal(m.getState().status, "error", "the dropped request leaves the model in error");
+		online = true;
+		await m.send({ type: "retry" });
+		assert.equal(posted.length, 1, "send() recovers from error without stop()");
+		assert.deepEqual(JSON.parse(posted[0]!).input, { type: "retry" });
+	});
+});
+
+describe("retryDrops (pure)", () => {
+	const failed = (id: string): LhMessage =>
+		({ ...fauxAssistantMessage("", { stopReason: "error", errorMessage: "boom" }), id }) as unknown as LhMessage;
+
+	it("parses the retry input event", () => {
+		assert.deepEqual(parseChatInput({ type: "retry" }), { type: "retry" });
+	});
+
+	it("drops every trailing failure back to the input", () => {
+		const user = { ...newUserMessage("hi"), id: "u1" } as LhMessage;
+		assert.deepEqual([...retryDrops([user, failed("a1"), failed("a2")])].sort(), ["a1", "a2"]);
+	});
+
+	it("looks past trailing UI-only markers, and keeps them", () => {
+		const user = { ...newUserMessage("hi"), id: "u1" } as LhMessage;
+		const block = newPlanBlockMessage("plan-1", {});
+		assert.deepEqual([...retryDrops([user, failed("a1"), block])], ["a1"]);
+		assert.equal(endsInFailedTurn([user, block]), false, "a marker alone is no failure");
+		const compaction = newCompactionMarker("summary", { firstKeptMessageId: "u1", tokensBefore: 100, model: "fixture", at: "2026-01-01T00:00:00Z" });
+		assert.deepEqual([...retryDrops([user, failed("a1"), compaction])], ["a1"], "the settle kick's marker too");
+	});
+
+	it("throws when the record does not end in a failure, or nothing precedes it", () => {
+		const user = { ...newUserMessage("hi"), id: "u1" } as LhMessage;
+		assert.throws(() => retryDrops([user]), ChatInputError);
+		assert.throws(() => retryDrops([failed("a1")]), ChatInputError);
+	});
+});