sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

fix: preserve failed responses when retrying model calls

Browse Source 
Keep failed attempts in the saved transcript and omit only trailing failures from Retry's temporary agent history. Use the matching temporary prefix length when persisting genuinely new output.

Preserve the error notice across plan/compaction markers, refuse Retry after later compaction, and avoid synthesizing results for tool calls inside failed generations while retaining Pi's native filtering.

Verified: 173 focused Retry/session/compaction-projection/practice-record tests, typecheck, changed-file lint, and diff --check. No app launch or real provider calls.
This commit is contained in:
amosblomqvist
2026-10-27 16:49:29 +02:00
parent a572675900
commit 82c6999a63
7 changed files with 559 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/chat.tsx
+6 -7
View File
@@ -460,13 +460,6 @@ export function Chat({
const busy = localStream || turnInFlight; const busy = localStream || turnInFlight;
const busyRef = useRef(busy); const busyRef = useRef(busy);
busyRef.current = busy; busyRef.current = busy;
// A failed model call (issue #110): the banner offers Retry, which
// re-runs the turn from the saved input — the only path back, since an
// answered question cannot be answered again. Offered from "error" too:
// a dropped stream leaves the model there, and send() recovers from it.
const retryable = (status === "ready" || status === "error") && !busy && endsInFailedTurn(nativeMessages);
const retryFailedTurn = () => void send({ type: "retry" });
// The attach handoff (stream-reattach wave): the moment a resumed // The attach handoff (stream-reattach wave): the moment a resumed
// stream takes over the rendering — the SDK left "ready", so // stream takes over the rendering — the SDK left "ready", so
// submitted/streaming means chunks are flowing — localStream covers // submitted/streaming means chunks are flowing — localStream covers
@@ -689,6 +682,12 @@ export function Chat({
// server's registry mutex is the structural gate; this is the // server's registry mutex is the structural gate; this is the
// affordance). // affordance).
const [compacting, setCompacting] = useState(initiallyCompacting); const [compacting, setCompacting] = useState(initiallyCompacting);
// A failed model response (issue #110): Retry keeps the saved input
// and completed work, without another learner message. Offered from
// "error" too: a dropped stream leaves the model there, and send()
// recovers from it. Later compaction requires new input instead.
const retryable = (status === "ready" || status === "error") && !busy && !compacting && endsInFailedTurn(nativeMessages);
const retryFailedTurn = () => void send({ type: "retry" });
const canSend = const canSend =
(text.trim().length > 0 || dictation.active) && !pending && !activeProblem && !compacting; (text.trim().length > 0 || dictation.active) && !pending && !activeProblem && !compacting;
components/session-core.ts
+6 -5
View File
@@ -19,6 +19,7 @@ import {
needsContinuation, needsContinuation,
danglingQuestionOf, danglingQuestionOf,
answeredCallIds, answeredCallIds,
failedResponseAtEnd,
measuredContextTokens, measuredContextTokens,
newAnswerResult, newAnswerResult,
newNextMessage, newNextMessage,
@@ -286,12 +287,12 @@ export class SessionModel {
? { executions: this.executions, inFlight: true, streamingIds: this.streamingIds } ? { executions: this.executions, inFlight: true, streamingIds: this.streamingIds }
: undefined; : undefined;
const messages = [...this.base, ...this.tail]; const messages = [...this.base, ...this.tail];
const latest = messages.at(-1); const latest = failedResponseAtEnd(messages);
// One rule for live and restored failures: only the final completed // Presentation/compaction markers do not resolve a failure. While
// assistant can own the banner. While trying again, hide the old // trying again, hide the old banner from the first Thinking frame,
// failure from the first Thinking frame, but show a new failure at end. // but show a new completed failure from this consume immediately.
const providerError = const providerError =
latest?.role === "assistant" && latest.stopReason === "error" && latest !== undefined &&
!this.incompleteUsageIds.has(latest.id) && (!live || latest.id === this.liveFailureId) !this.incompleteUsageIds.has(latest.id) && (!live || latest.id === this.liveFailureId)
? { message: latest.errorMessage ?? "An error occurred." } ? { message: latest.errorMessage ?? "An error occurred." }
: null; : null;
src/core/input-events.ts
+42 -25
View File
@@ -16,9 +16,9 @@
* continue — run the next turn on the current record (mount * continue — run the next turn on the current record (mount
* continuations, job-settle re-syncs: the record ends at an * continuations, job-settle re-syncs: the record ends at an
* answered question with no teacher reaction yet). * answered question with no teacher reaction yet).
* retry — re-run a turn whose model call failed (issue #110): the * retry — retry a failed model response (issue #110), without new
* trailing failed assistant message(s) leave the record and * input. Failed attempts stay in the record; only the retry's
* the turn re-runs from the input that preceded them. * temporary agent history omits the trailing failures.
* *
* (The plan verdict input died 2026-09-26 with the verdict itself — the * (The plan verdict input died 2026-09-26 with the verdict itself — the
* learner's post-plan reply is a natural text message.) * learner's post-plan reply is a natural text message.)
@@ -218,8 +218,8 @@ export function inputAppends(
case "continue": case "continue":
return dismissalConclusion(record); return dismissalConclusion(record);
case "retry": case "retry":
// The failed turn's own input already landed write-ahead; the // The input already landed write-ahead. retryDrops affects only
// handler trims the failure (retryDrops) — nothing new appends. // the temporary agent history — nothing new appends here.
return []; return [];
case "thinking": { case "thinking": {
const active = activeProblemOf(record.messages); const active = activeProblemOf(record.messages);
@@ -244,40 +244,57 @@ export function inputAppends(
} }
} }
/** Does the record end in a failed model call (a retryable turn)? */ /** The failure owning the ephemeral banner. Plan presentation and
export function endsInFailedTurn(messages: readonly LhMessage[]): boolean { * compaction do not resolve an error; substantive later input/output does.
return failedTail(messages).ids.size > 0; * This is deliberately broader than Retry eligibility below. */
export function failedResponseAtEnd(messages: readonly LhMessage[]): Extract<LhMessage, { role: "assistant" }> | undefined {
for (let i = messages.length - 1; i >= 0; i--) {
const m = messages[i]!;
if (m.role === "custom" && (m.customType === "plan-block" || m.customType === "compaction")) continue;
return m.role === "assistant" && m.stopReason === "error" ? m : undefined;
}
return undefined;
} }
/** The ids a retry drops (issue #110, pure): the failed assistant /** Does the record end in a failure with input that can be retried?
* message(s) ending the record. A provider failure persists as an * Aborts are deliberate stops; compaction after a failure requires new
* assistant message with stopReason "error", and pi refuses to continue * learner input rather than a retry using a changed context. */
* from an assistant — so neither re-answering (the question is no longer export function endsInFailedTurn(messages: readonly LhMessage[]): boolean {
* dangling) nor `continue` can resume the turn. Ids, not a cut index: const { ids, before } = failedTail(messages);
* UI-only markers may follow the failure, and they stay. Honest 400s return ids.size > 0 && before !== undefined && before.role !== "assistant";
* when there is no failure to retry, or when dropping it leaves nothing }
* to continue from. */
/** The ids omitted ONLY from the retry's temporary agent history.
* Failed attempts (including partial text and their ids) stay on disk.
* Pi's provider conversion already skips error/aborted responses, but
* continue() rejects an assistant tail BEFORE conversion. Omitting the
* trailing failures lets that continuation start from the saved input or
* completed tool results without replaying tools or resubmitting input.
* Plan blocks may follow the failure and stay; later compaction blocks
* Retry without changing compaction itself. */
export function retryDrops(messages: readonly LhMessage[]): Set<string> { export function retryDrops(messages: readonly LhMessage[]): Set<string> {
const { ids, before } = failedTail(messages); const { ids, before } = failedTail(messages);
if (ids.size === 0) throw new ChatInputError("retry input requires a failed turn to retry"); if (ids.size === 0) {
const failure = failedResponseAtEnd(messages);
if (failure) {
throw new ChatInputError(`${failure.errorMessage ?? "An error occurred."}\n\nThis response cannot be retried after compaction. Send a new message to continue.`);
}
throw new ChatInputError("retry input requires a failed turn to retry");
}
if (!before || before.role === "assistant") throw new ChatInputError("retry input leaves no input to continue from"); if (!before || before.role === "assistant") throw new ChatInputError("retry input leaves no input to continue from");
return ids; return ids;
} }
/** Markers that can land AFTER a failed call and are never model input: /** Walk back over failed assistants and UI-only plan blocks. A compaction
* persistTurn's plan block (plan_init succeeded, the next call failed) * is a boundary, not a marker Retry may cross. A failure AFTER a compaction
* and the settle kick's compaction marker. */ * can still retry: that compacted context was already its starting state. */
const TRAILING_MARKERS: readonly string[] = ["plan-block", "compaction"];
/** Walk back from the end over failed assistants and trailing markers:
* the failures' ids, and the message the run stops at. */
function failedTail(messages: readonly LhMessage[]): { ids: Set<string>; before: LhMessage | undefined } { function failedTail(messages: readonly LhMessage[]): { ids: Set<string>; before: LhMessage | undefined } {
const ids = new Set<string>(); const ids = new Set<string>();
let i = messages.length - 1; let i = messages.length - 1;
for (; i >= 0; i--) { for (; i >= 0; i--) {
const m = messages[i]!; const m = messages[i]!;
if (m.role === "assistant" && m.stopReason === "error") ids.add(m.id); if (m.role === "assistant" && m.stopReason === "error") ids.add(m.id);
else if (!(m.role === "custom" && TRAILING_MARKERS.includes(m.customType))) break; else if (!(m.role === "custom" && m.customType === "plan-block")) break;
} }
return { ids, before: messages[i] }; return { ids, before: messages[i] };
} }
src/core/native-messages.ts
+8 -6
View File
@@ -551,9 +551,10 @@ function resultIdsFrom(messages: readonly AgentMessage[]): Set<string> {
* - markers flatten to a user message carrying their content (the * - markers flatten to a user message carrying their content (the
* canonical text single-sourced in core/contracts); plan blocks are * canonical text single-sourced in core/contracts); plan blocks are
* UI-only and filtered. * UI-only and filtered.
* - dangling tool calls (an aborted turn's unanswered questions) get the * - dangling calls from successfully generated assistant messages (for
* synthesized interrupted result — the provider protocol's hard * example, an unanswered question) get a synthesized interrupted result.
* invariant: a tool call must be followed by its result. * Error/aborted generations do not: Pi drops those assistant messages,
* so manufacturing their results would leave orphan results on the wire.
* *
* Contract (pi-agent-core): never throws — an unknown shape passes * Contract (pi-agent-core): never throws — an unknown shape passes
* through untouched rather than killing the loop. * through untouched rather than killing the loop.
@@ -619,11 +620,12 @@ export function toLlmMessages(messages: readonly AgentMessage[]): Message[] {
// Strip the record id: the wire carries the native shape only. // Strip the record id: the wire carries the native shape only.
const { id: _id, ...native } = m as LhMessage; const { id: _id, ...native } = m as LhMessage;
out.push(native as Message); out.push(native as Message);
// Dangling-call synthesis: an assistant message whose toolCalls have // Repair unanswered calls from completed generations, not calls in
// no results in the remaining tail (abort mid-question) — the // failed generations. Pi filters error/aborted assistants itself;
// provider rejects unanswered calls outright. // adding results for their unexecuted calls would leave orphans.
if (role === "assistant") { if (role === "assistant") {
const assistant = m as AssistantMessage; const assistant = m as AssistantMessage;
if (assistant.stopReason === "error" || assistant.stopReason === "aborted") continue;
const calls = assistant.content.filter((b) => b.type === "toolCall"); const calls = assistant.content.filter((b) => b.type === "toolCall");
if (calls.length === 0) continue; if (calls.length === 0) continue;
const answered = resultIdsFrom(messages.slice(i + 1)); const answered = resultIdsFrom(messages.slice(i + 1));
src/server/chat-handler.ts
+28 -25
View File
@@ -187,6 +187,17 @@ export async function handleChatInput(
return Response.json({ error: "Unknown chat" }, { status: 404 }); return Response.json({ error: "Unknown chat" }, { status: 404 });
} }
// Refuse an ineligible Retry before catalog refresh, compaction, or any
// turn/write effects. Revalidate after the awaited gates below too.
if (input.type === "retry") {
try {
retryDrops(record.messages);
} catch (err) {
if (err instanceof ChatInputError) return Response.json({ error: err.message }, { status: 400 });
throw err;
}
}
// THE UNIVERSAL TURN GATE (BYOK, owner, 2026-09-22): a turn needs the // THE UNIVERSAL TURN GATE (BYOK, owner, 2026-09-22): a turn needs the
// CONVERSATION chain — the chat's own model override over the strong // CONVERSATION chain — the chat's own model override over the strong
// slot — and there is no factory floor; when both are empty the turn // slot — and there is no factory floor; when both are empty the turn
@@ -224,18 +235,6 @@ export async function handleChatInput(
return Response.json(envelope, { status: 503 }); return Response.json(envelope, { status: 503 });
} }
// THE RETRY GATE (issue #110): a retry with no failure to retry
// refuses here, BEFORE compaction and beginTurn — no effects. The
// trim itself re-validates on the record compaction re-reads below.
if (input.type === "retry") {
try {
retryDrops(record.messages);
} catch (err) {
if (err instanceof ChatInputError) return Response.json({ error: err.message }, { status: 400 });
throw err;
}
}
// THE COMPACTION GATE (owner round 14, 2026-10-05 — the structural // THE COMPACTION GATE (owner round 14, 2026-10-05 — the structural
// ruling: NOTHING model-facing for this chat interleaves with a // ruling: NOTHING model-facing for this chat interleaves with a
// running compaction): await any in-flight compaction (the settle // running compaction): await any in-flight compaction (the settle
@@ -274,18 +273,17 @@ export async function handleChatInput(
const compactedRecord = stores.chatStore.getChat(chatId); const compactedRecord = stores.chatStore.getChat(chatId);
if (!compactedRecord) return Response.json({ error: "Unknown chat" }, { status: 404 }); if (!compactedRecord) return Response.json({ error: "Unknown chat" }, { status: 404 });
record = compactedRecord; record = compactedRecord;
// THE RETRY TRIM: recomputed on the current record (the awaits above // Recompute Retry eligibility on current truth, still before beginTurn.
// let other writers land), still before beginTurn. The trimmed record // A compaction marker or another writer may have landed during awaits.
// reaches disk through the write-ahead. // These omissions affect ONLY the agent context, never the saved record.
let retryOmissions = new Set<string>();
if (input.type === "retry") { if (input.type === "retry") {
let drop: Set<string>;
try { try {
drop = retryDrops(record.messages); retryOmissions = retryDrops(record.messages);
} catch (err) { } catch (err) {
if (err instanceof ChatInputError) return Response.json({ error: err.message }, { status: 400 }); if (err instanceof ChatInputError) return Response.json({ error: err.message }, { status: 400 });
throw err; throw err;
} }
record = { ...record, messages: record.messages.filter((m) => !drop.has(m.id)) };
} }
// The input's native appends (pure; the honest 400 for an answer with no // The input's native appends (pure; the honest 400 for an answer with no
@@ -344,8 +342,13 @@ export async function handleChatInput(
// ride this same write-ahead - the pump de-queued them at fire, so this // ride this same write-ahead - the pump de-queued them at fire, so this
// is the ONLY copy ever appended.) // is the ONLY copy ever appended.)
const merged = [...record.messages, ...appends]; const merged = [...record.messages, ...appends];
// Retry preserves every persisted attempt. Pi needs a non-assistant
// tail to start continue(), so its temporary history omits the trailing
// failures only. Count that SAME temporary prefix at settlement, or the
// retained failures would make persistTurn skip genuinely new output.
const agentMessages = input.type === "retry" ? merged.filter((m) => !retryOmissions.has(m.id)) : merged;
const stagedAutomatic = isSubscriptionProvider(conversation.provider) && input.type === "steers"; const stagedAutomatic = isSubscriptionProvider(conversation.provider) && input.type === "steers";
let persistBaseCount = stagedAutomatic ? record.messages.length : merged.length; let persistBaseCount = stagedAutomatic ? record.messages.length : agentMessages.length;
try { try {
if (!stagedAutomatic) stores.chatStore.saveMessages(chatId, merged); if (!stagedAutomatic) stores.chatStore.saveMessages(chatId, merged);
} catch (err) { } catch (err) {
@@ -384,7 +387,7 @@ export async function handleChatInput(
// the session is still probing: build the PROBER (quiz/ask + probe_done). // the session is still probing: build the PROBER (quiz/ask + probe_done).
// Both share this whole composition (write-ahead persist, liveness, // Both share this whole composition (write-ahead persist, liveness,
// steers, usage, extraction) — only the agent differs. // steers, usage, extraction) — only the agent differs.
const teaching = hasProbeDone(merged); const teaching = hasProbeDone(agentMessages);
// BYOK (owner, 2026-09-22): the conversation chain was resolved by the // BYOK (owner, 2026-09-22): the conversation chain was resolved by the
// turn gate above (chat.model override → strong slot → refused) — the // turn gate above (chat.model override → strong slot → refused) — the
// engine build is pure from here; thinking levels clamp natively. // engine build is pure from here; thinking levels clamp natively.
@@ -403,7 +406,7 @@ export async function handleChatInput(
// sub-agent JOB tools (research/demo/visual/pdf; adjudicated with the // sub-agent JOB tools (research/demo/visual/pdf; adjudicated with the
// user: quiz/ask/fs/plan/probe_done are always-on). // user: quiz/ask/fs/plan/probe_done are always-on).
if (isSubscriptionProvider(conversation.provider)) requestCycle = new SubscriptionRequestCycle({ if (isSubscriptionProvider(conversation.provider)) requestCycle = new SubscriptionRequestCycle({
initialMessages: asAgentMessages(merged), initialMessages: asAgentMessages(agentMessages),
staged: stagedAutomatic ? asAgentMessages(appends) : undefined, staged: stagedAutomatic ? asAgentMessages(appends) : undefined,
foregroundOwed: !callPurpose.background, foregroundOwed: !callPurpose.background,
purpose: callPurpose, purpose: callPurpose,
@@ -419,7 +422,7 @@ export async function handleChatInput(
engine, engine,
callPurpose, callPurpose,
requestCycle, requestCycle,
messages: asAgentMessages(merged), messages: asAgentMessages(agentMessages),
onUsage: collectUsage, onUsage: collectUsage,
toolToggles: effective.agent.tools, toolToggles: effective.agent.tools,
// Debug (owner, 2026-10-04): teacher/prober raw provider events under // Debug (owner, 2026-10-04): teacher/prober raw provider events under
@@ -515,7 +518,7 @@ export async function handleChatInput(
// Debug recorder run marker (owner, 2026-10-04): correlates the // Debug recorder run marker (owner, 2026-10-04): correlates the
// JSONL event stream with the record's turns; no-op unless // JSONL event stream with the record's turns; no-op unless
// LH_PROVIDER_EVENTS=1. // LH_PROVIDER_EVENTS=1.
providerEventNote(chatId, "turn", `run start (base: ${merged.length} messages)`); providerEventNote(chatId, "turn", `run start (base: ${agentMessages.length} messages)`);
const handle = runPiTurn({ const handle = runPiTurn({
agent: constructed, agent: constructed,
requestCycle, requestCycle,
@@ -555,8 +558,8 @@ export async function handleChatInput(
settleSteers(chatId, persisted, true, constructed); settleSteers(chatId, persisted, true, constructed);
}, },
onFinish: (turnMessages, isAborted) => { onFinish: (turnMessages, isAborted) => {
// Wave B: the record write is the agent's own final state — // Wave B: append only messages past the temporary agent prefix.
// the new messages are the slice past the merged base. // Retry's saved record still includes its earlier failed attempts.
const stamped = persistTurn(stores.chatStore, chatId, persistBaseCount, turnMessages, isAborted); const stamped = persistTurn(stores.chatStore, chatId, persistBaseCount, turnMessages, isAborted);
noteSteersDurable(chatId, stores.chatStore.getChat(chatId)?.messages ?? stamped); noteSteersDurable(chatId, stores.chatStore.getChat(chatId)?.messages ?? stamped);
// usage log append (idempotent per turn id) — the stats bar // usage log append (idempotent per turn id) — the stats bar
tests/retry-failed-turn.test.ts
+440 -126
View File
@@ -1,32 +1,47 @@
/** /**
* The failed-turn retry (issue #110): a model call that fails persists as an * Failed-response Retry (issue #110). Input is already durable. Retry
* assistant message with stopReason "error". The learner's input survives * omits the trailing failures from the temporary agent context, never
* (write-ahead), but pre-fix nothing could re-run the turn — re-answering * deletes them from the transcript, and appends only genuinely new output.
* 400'd (the question is no longer dangling) and `continue` died in pi * Scripted faux engines: no real provider calls or learning data.
* ("Cannot continue from message role: assistant"). The `retry` input drops
* the failure and re-runs the turn from the input that preceded it.
*
* Drives handleChatRequest with a scripted pi engine (the chat-flow pattern).
*/ */
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import * as fs from "node:fs"; import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path"; import * as path from "node:path";
import { after, describe, it } from "node:test"; import { after, describe, it, mock } from "node:test";
import { createFauxCore, fauxAssistantMessage, fauxText, fauxToolCall, type FauxResponseStep } from "@earendil-works/pi-ai"; import { createFauxCore, fauxAssistantMessage, fauxText, fauxToolCall, type FauxResponseStep } from "@earendil-works/pi-ai";
import { SessionModel } from "../components/session-core.ts"; import { transformMessages } from "@earendil-works/pi-ai/api/transform-messages";
import { createTeacherAgent } from "../src/agent/teacher-agent.ts"; import { Type } from "typebox";
import { handleChatRequest } from "../src/server/chat-handler.ts"; import type { AgentBuildOptions, TeachingAgent } from "../src/agent/teacher-agent.ts";
import { runningTurn } from "../src/server/turn-runs.ts";
import { chatStore } from "../src/store/chats.ts";
import { buildAnswer, normalizeOptions, resolveCorrect, type QuestionData } from "../src/core/question-core.ts";
import { ChatInputError, endsInFailedTurn, parseChatInput, retryDrops } from "../src/core/input-events.ts";
import { newPlanBlockMessage, newUserMessage, type LhMessage } from "../src/core/native-messages.ts";
import { newCompactionMarker } from "../src/core/compaction.ts";
import type { PiEngine } from "../src/agent/models.ts"; import type { PiEngine } from "../src/agent/models.ts";
import { PATHS } from "../config.ts"; import type { LhMessage } from "../src/core/native-messages.ts";
import type { QuestionData } from "../src/core/question-core.ts";
process.env.OPENROUTER_API_KEY ??= "test-key-presence-gate"; // Set the storage/auth seams BEFORE any app/config-loading imports.
assert.equal(process.env.NODE_ENV, "test", "Run with NODE_ENV=test; these tests must not invoke real providers.");
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-retry-failed-turn-"));
process.env.LH_DATA_DIR = path.join(scratch, "data");
process.env.LH_AUTH_FILE = path.join(scratch, "auth.json");
process.env.OPENROUTER_API_KEY = "test-key-presence-gate";
fs.writeFileSync(process.env.LH_AUTH_FILE, "{}");
const { SessionModel } = await import("../components/session-core.ts");
const { createPracticeAgent, createProberAgent, createTeacherAgent } = await import("../src/agent/teacher-agent.ts");
const { handleChatRequest } = await import("../src/server/chat-handler.ts");
const { clearTurnFailure, rememberTurnFailure, runningTurn, turnFailure } = await import("../src/server/turn-runs.ts");
const { isTurnRunning } = await import("../src/server/turn-liveness.ts");
const { chatStore } = await import("../src/store/chats.ts");
const { jobStore } = await import("../src/store/jobs.ts");
const { subscribe } = await import("../src/core/events.ts");
const { buildAnswer, normalizeOptions, resolveCorrect } = await import("../src/core/question-core.ts");
const { ChatInputError, endsInFailedTurn, failedResponseAtEnd, inputAppends, parseChatInput, retryDrops } = await import("../src/core/input-events.ts");
const { INTERRUPTED_TEXT, asAgentMessages, newHandoffMessage, newPlanBlockMessage, newUserMessage, reconstructRequestCause, toLlmMessages } = await import("../src/core/native-messages.ts");
const { newCompactionMarker } = await import("../src/core/compaction.ts");
after(() => {
fs.rmSync(scratch, { recursive: true, force: true });
});
const QUIZ_INPUT = { const QUIZ_INPUT = {
question: "Which planet is closest to the sun?", question: "Which planet is closest to the sun?",
@@ -37,23 +52,20 @@ const QUIZ_INPUT = {
nodeId: "planet-order", nodeId: "planet-order",
}; };
/** A response step that fails the way a dropped connection does. */ const networkFailure: FauxResponseStep = () => { throw new Error("fetch failed"); };
const networkFailure = (() => {
throw new Error("fetch failed");
}) as unknown as FauxResponseStep;
/** Scripted engine; each step records the conversation roles it saw /** Record the actual model context after the app's native conversion. */
* (the composed system message aside). */ function scripted(steps: FauxResponseStep[]) {
function scripted(steps: FauxResponseStep[]): { engine: PiEngine; contexts: string[][] } {
const contexts: string[][] = []; const contexts: string[][] = [];
const histories: string[] = [];
const faux = createFauxCore({}); const faux = createFauxCore({});
faux.setResponses( faux.setResponses(steps.map((step) => (context, options, state, model) => {
steps.map((step) => ((context: { messages: Array<{ role: string }> }, ...rest: unknown[]) => { contexts.push(context.messages.map((m) => m.role).filter((role) => role !== "system"));
contexts.push(context.messages.map((m) => m.role).filter((r) => r !== "system")); histories.push(JSON.stringify(context.messages));
return (step as (...a: unknown[]) => unknown)(context, ...rest); return typeof step === "function" ? step(context, options, state, model) : step;
}) as unknown as FauxResponseStep), }));
); const engine: PiEngine = { model: faux.getModel(), streamFn: faux.streamSimple };
return { engine: { model: faux.getModel(), streamFn: faux.streamSimple }, contexts }; return { engine, contexts, histories };
} }
function post(chatId: string, input: unknown): Request { function post(chatId: string, input: unknown): Request {
@@ -64,10 +76,13 @@ function post(chatId: string, input: unknown): Request {
}); });
} }
/** POST one input event; on a 200 ack, drain the turn to completion. */ type AgentFactory = (opts: AgentBuildOptions & { teaching: boolean }) => TeachingAgent;
async function turn(engine: PiEngine, chatId: string, input: unknown): Promise<Response> {
const ack = await handleChatRequest(post(chatId, input), (opts) => /** POST one event; after acceptance drain the server-owned turn. */
createTeacherAgent(undefined, { engine, messages: opts.messages }), async function turn(engine: PiEngine, chatId: string, input: unknown, buildAgent?: AgentFactory): Promise<Response> {
const ack = await handleChatRequest(
post(chatId, input),
buildAgent ?? ((opts) => createTeacherAgent(undefined, { ...opts, engine })),
); );
if (ack.status === 200) { if (ack.status === 200) {
const handle = runningTurn(chatId); const handle = runningTurn(chatId);
@@ -76,20 +91,8 @@ async function turn(engine: PiEngine, chatId: string, input: unknown): Promise<R
return ack; return ack;
} }
const createdChats: string[] = []; function createConfiguredChat(title: string, kind?: "practice") {
after(() => { const chat = chatStore.createChat(title, kind);
for (const id of createdChats) {
try {
fs.unlinkSync(path.join(PATHS.chats, `${id}.json`));
} catch {
/* already gone */
}
}
});
function createConfiguredChat(title: string) {
const chat = chatStore.createChat(title);
createdChats.push(chat.id);
chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } }); chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } });
chatStore.saveTree(chat.id, { nodes: [{ id: "planet-order", title: "Planet order" }], edges: [], version: 1 }); chatStore.saveTree(chat.id, { nodes: [{ id: "planet-order", title: "Planet order" }], edges: [], version: 1 });
return chat; return chat;
@@ -107,134 +110,445 @@ function gradedAnswer() {
explanation: QUIZ_INPUT.explanation, explanation: QUIZ_INPUT.explanation,
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}; };
return buildAnswer(qd, { values: ["Mercury"] }); return buildAnswer(qd, { values: ["Mercury"], thinking: "Mercury is the innermost", readingThinking: "I followed the planet ordering", confidence: 80 });
} }
const roles = (chatId: string) => chatStore.getChat(chatId)!.messages.map((m) => m.role); const roles = (chatId: string) => chatStore.getChat(chatId)!.messages.map((m) => m.role);
const failed = (id: string, text = ""): LhMessage => ({
...fauxAssistantMessage(text, { stopReason: "error", errorMessage: "fetch failed" }), id,
});
const compaction = (firstKeptMessageId: string) => newCompactionMarker("summary", {
firstKeptMessageId, tokensBefore: 100, model: "fixture", at: "2026-01-01T00:00:00Z",
});
describe("failed-turn retry (issue #110)", () => { function assertPreserved(before: LhMessage[], saved: LhMessage[]): void {
it("a failed answer turn re-runs from the saved answer — no re-answering needed", async () => { assert.deepEqual(saved.slice(0, before.length), before, "all previous messages, ids and partial content stay intact");
const { engine, contexts } = scripted([ }
describe("failed-response Retry (issue #110)", () => {
it("retries from the saved answer, preserving the full input and failed attempt", async () => {
const { engine, contexts, histories } = scripted([
() => fauxAssistantMessage([fauxToolCall("quiz", QUIZ_INPUT, { id: "call-1" })], { stopReason: "toolUse" }), () => fauxAssistantMessage([fauxToolCall("quiz", QUIZ_INPUT, { id: "call-1" })], { stopReason: "toolUse" }),
networkFailure, networkFailure,
() => fauxAssistantMessage([fauxText("Right — Mercury it is.")]), () => fauxAssistantMessage([fauxText("Right — Mercury it is.")]),
]); ]);
const chat = createConfiguredChat("Test: retry after failed answer"); const chat = createConfiguredChat("Test: retry after failed answer");
const answer = gradedAnswer();
await turn(engine, chat.id, { type: "text", text: "teach me the planets" }); await turn(engine, chat.id, { type: "text", text: "teach me the planets" });
await turn(engine, chat.id, { type: "answer", toolCallId: "call-1", answer: gradedAnswer() }); await turn(engine, chat.id, { type: "answer", toolCallId: "call-1", answer });
const before = chatStore.getChat(chat.id)!.messages;
// The failure as #110 met it: answer durable, turn failed, and the
// pre-fix paths refused.
assert.deepEqual(roles(chat.id), ["user", "assistant", "toolResult", "assistant"]); assert.deepEqual(roles(chat.id), ["user", "assistant", "toolResult", "assistant"]);
assert.ok(endsInFailedTurn(chatStore.getChat(chat.id)!.messages)); assert.ok(endsInFailedTurn(before));
const reanswer = await turn(engine, chat.id, { type: "answer", toolCallId: "call-1", answer: gradedAnswer() }); assert.equal((await turn(engine, chat.id, { type: "answer", toolCallId: "call-1", answer })).status, 400);
assert.equal(reanswer.status, 400, "the answered question cannot be answered again"); assert.equal((await turn(engine, chat.id, { type: "retry" })).status, 200);
const retry = await turn(engine, chat.id, { type: "retry" });
assert.equal(retry.status, 200);
const saved = chatStore.getChat(chat.id)!.messages; const saved = chatStore.getChat(chat.id)!.messages;
assert.deepEqual(saved.map((m) => m.role), ["user", "assistant", "toolResult", "assistant"]); assertPreserved(before, saved);
assert.equal(endsInFailedTurn(saved), false, "the failure left the record"); assert.deepEqual(roles(chat.id), ["user", "assistant", "toolResult", "assistant", "assistant"]);
assert.equal(endsInFailedTurn(saved), false, "a new successful response supersedes the retained failure");
const results = saved.filter((m) => m.role === "toolResult" && m.toolCallId === "call-1");
assert.equal(results.length, 1, "the learner's answer was not submitted twice");
const result = results[0]!;
assert.ok(result.role === "toolResult");
assert.deepEqual(result.details, JSON.parse(JSON.stringify(answer)));
assert.match(JSON.stringify(saved.at(-1)), /Mercury it is/); assert.match(JSON.stringify(saved.at(-1)), /Mercury it is/);
assert.deepEqual(contexts.at(-1), ["user", "assistant", "toolResult"], "the retried call saw the answer, not the failure"); assert.deepEqual(contexts.at(-1), ["user", "assistant", "toolResult"]);
assert.match(histories.at(-1)!, /Mercury is the innermost/);
}); });
it("a failed text turn re-runs from the learner's message", async () => { it("a failed text turn retries without adding another learner message", async () => {
const { engine, contexts } = scripted([networkFailure, () => fauxAssistantMessage([fauxText("Hello again.")])]); const { engine, contexts } = scripted([networkFailure, () => fauxAssistantMessage("Hello again.")]);
const chat = createConfiguredChat("Test: retry after failed text"); const chat = createConfiguredChat("Test: retry after failed text");
await turn(engine, chat.id, { type: "text", text: "hello" }); await turn(engine, chat.id, { type: "text", text: "hello" });
assert.deepEqual(roles(chat.id), ["user", "assistant"]); const before = chatStore.getChat(chat.id)!.messages;
assert.ok(endsInFailedTurn(chatStore.getChat(chat.id)!.messages)); assert.ok(endsInFailedTurn(before));
assert.equal((await turn(engine, chat.id, { type: "retry" })).status, 200); assert.equal((await turn(engine, chat.id, { type: "retry" })).status, 200);
assert.deepEqual(roles(chat.id), ["user", "assistant"]); const saved = chatStore.getChat(chat.id)!.messages;
assert.match(JSON.stringify(chatStore.getChat(chat.id)!.messages.at(-1)), /Hello again/); assertPreserved(before, saved);
assert.deepEqual(roles(chat.id), ["user", "assistant", "assistant"]);
assert.deepEqual(contexts.at(-1), ["user"]);
assert.match(JSON.stringify(saved.at(-1)), /Hello again/);
});
it("preserves partial text and its usage reference before the replacement succeeds", { timeout: 5_000 }, async () => {
const chat = createConfiguredChat("Test: retained partial reply");
const before = [newUserMessage("hello"), failed("partial", "A partial explanation already shown.")];
chatStore.saveMessages(chat.id, before);
const usage = { id: "original-turn", messageId: "partial", at: "2026-01-01T00:00:00Z", model: "faux:fixture", steps: [] };
chatStore.appendTurnUsage(chat.id, usage);
let release!: () => void;
let started!: () => void;
const waiting = new Promise<void>((resolve) => { release = resolve; });
const entered = new Promise<void>((resolve) => { started = resolve; });
const { engine, histories } = scripted([async () => {
started();
await waiting;
return fauxAssistantMessage("Replacement response.");
}]);
const retry = turn(engine, chat.id, { type: "retry" });
await entered;
try {
assert.deepEqual(chatStore.getChat(chat.id)!.messages, before, "write-ahead must not delete the failed attempt");
assert.deepEqual(chatStore.getChat(chat.id)!.turns, [usage]);
assert.doesNotMatch(histories.at(-1)!, /partial explanation/);
} finally {
release();
await retry;
}
const saved = chatStore.getChat(chat.id)!;
assertPreserved(before, saved.messages);
assert.ok(saved.turns?.some((entry) => entry.id === usage.id && entry.messageId === "partial"));
assert.equal(saved.messages.length, before.length + 1, "the persistence offset counts the temporary prefix, not the full saved record");
});
it("keeps every failed attempt across repeated retries, then appends success once", async () => {
const { engine, contexts } = scripted([networkFailure, networkFailure, () => fauxAssistantMessage("Recovered.")]);
const chat = createConfiguredChat("Test: repeated model failure");
await turn(engine, chat.id, { type: "text", text: "hello" });
const first = chatStore.getChat(chat.id)!.messages;
await turn(engine, chat.id, { type: "retry" });
const second = chatStore.getChat(chat.id)!.messages;
assertPreserved(first, second);
assert.equal(second.length, first.length + 1);
assert.ok(endsInFailedTurn(second));
await turn(engine, chat.id, { type: "retry" });
const saved = chatStore.getChat(chat.id)!.messages;
assertPreserved(second, saved);
assert.equal(saved.length, second.length + 1);
assert.deepEqual(contexts, [["user"], ["user"], ["user"]]);
assert.equal(new SessionModel({ chatId: chat.id, initialMessages: saved }).getState().error, null);
});
it("preserves a trailing plan block while retrying from the preceding input", async () => {
const { engine, contexts } = scripted([networkFailure, () => fauxAssistantMessage("Planned reply.")]);
const chat = createConfiguredChat("Test: retry past a plan block");
await turn(engine, chat.id, { type: "text", text: "hello" });
const before = [...chatStore.getChat(chat.id)!.messages, newPlanBlockMessage("plan-1", {})];
chatStore.saveMessages(chat.id, before);
assert.ok(endsInFailedTurn(before));
assert.equal((await turn(engine, chat.id, { type: "retry" })).status, 200);
const saved = chatStore.getChat(chat.id)!.messages;
assertPreserved(before, saved);
assert.deepEqual(roles(chat.id), ["user", "assistant", "custom", "assistant"]);
assert.deepEqual(contexts.at(-1), ["user"], "the plan block remains UI-only");
});
it("does not reexecute completed tools before a later model failure", async () => {
let executions = 0;
const { engine, contexts } = scripted([
() => fauxAssistantMessage([fauxToolCall("save_note", {}, { id: "note-1" })], { stopReason: "toolUse" }),
networkFailure,
() => fauxAssistantMessage("The saved note is ready."),
]);
const tools: AgentBuildOptions["tools"] = {
save_note: {
name: "save_note", label: "save_note", description: "Record one fixture side effect.", parameters: Type.Object({}),
execute: async () => {
executions++;
return { content: [{ type: "text", text: "Note saved" }], details: { saved: true } };
},
},
};
const factory: AgentFactory = (opts) => createTeacherAgent(undefined, { ...opts, engine, tools });
const chat = createConfiguredChat("Test: no completed tool replay");
await turn(engine, chat.id, { type: "text", text: "save a note" }, factory);
assert.equal(executions, 1);
const before = chatStore.getChat(chat.id)!.messages;
assert.deepEqual(roles(chat.id), ["user", "assistant", "toolResult", "assistant"]);
assert.equal((await turn(engine, chat.id, { type: "retry" }, factory)).status, 200);
assert.equal(executions, 1, "Retry requests another response, not execution of stored calls");
assertPreserved(before, chatStore.getChat(chat.id)!.messages);
assert.deepEqual(contexts.at(-1), ["user", "assistant", "toolResult"]);
});
it("never executes tool calls carried by a failed partial response", async () => {
let executions = 0;
const { engine, contexts } = scripted([
() => fauxAssistantMessage([fauxToolCall("save_note", {}, { id: "unexecuted" })], { stopReason: "error", errorMessage: "cut off" }),
() => fauxAssistantMessage("Recovered without running the partial call."),
]);
const factory: AgentFactory = (opts) => createTeacherAgent(undefined, {
...opts, engine,
tools: {
save_note: {
name: "save_note", label: "save_note", description: "Must not execute.", parameters: Type.Object({}),
execute: async () => { executions++; return { content: [], details: {} }; },
},
},
});
const chat = createConfiguredChat("Test: unexecuted partial tool call");
await turn(engine, chat.id, { type: "text", text: "hello" }, factory);
assert.equal(executions, 0);
const before = chatStore.getChat(chat.id)!.messages;
assert.equal((await turn(engine, chat.id, { type: "retry" }, factory)).status, 200);
assert.equal(executions, 0);
assertPreserved(before, chatStore.getChat(chat.id)!.messages);
assert.deepEqual(contexts.at(-1), ["user"]); assert.deepEqual(contexts.at(-1), ["user"]);
}); });
it("refuses a retry with nothing failed — 400, record untouched", async () => { for (const kind of ["prober", "teacher", "practice"] as const) {
const { engine } = scripted([() => fauxAssistantMessage([fauxText("Fine.")])]); it(`uses the same retained-history retry boundary for the ${kind}`, async () => {
const chat = createConfiguredChat("Test: retry refused"); const { engine, contexts } = scripted([() => fauxAssistantMessage("Recovered response.")]);
const chat = createConfiguredChat(`Test: ${kind} retry`, kind === "practice" ? "practice" : undefined);
const before: LhMessage[] = [newUserMessage("hello")];
if (kind === "teacher") {
before.push({ ...fauxAssistantMessage([fauxToolCall("probe_done", {}, { id: "probe-1" })], { stopReason: "toolUse" }), id: "probe-response" });
before.push({ id: "probe-result", role: "toolResult", toolCallId: "probe-1", toolName: "probe_done", content: [], details: { ok: true }, isError: false, timestamp: 1 });
}
before.push(failed("failed-response", "Partial reply."));
chatStore.saveMessages(chat.id, before);
const factory: AgentFactory = (opts) => {
assert.equal(opts.teaching, kind === "teacher");
assert.ok(opts.messages?.every((m) => m.role !== "assistant" || m.stopReason !== "error"));
if (kind === "practice") return createPracticeAgent({ chatId: chat.id, chatStore }, { ...opts, engine });
if (kind === "prober") return createProberAgent({ chatId: chat.id, chatStore, jobStore, deliver: () => {}, spawn: () => {} }, { ...opts, engine });
return createTeacherAgent(undefined, { ...opts, engine });
};
assert.equal((await turn(engine, chat.id, { type: "retry" }, factory)).status, 200);
assertPreserved(before, chatStore.getChat(chat.id)!.messages);
assert.equal(chatStore.getChat(chat.id)!.messages.length, before.length + 1);
assert.deepEqual(contexts.at(-1), kind === "teacher" ? ["user", "assistant", "toolResult"] : ["user"]);
});
}
await turn(engine, chat.id, { type: "text", text: "hello" }); it("treats an explicit Retry of an automatic main-agent response as foreground", async () => {
const before = chatStore.getChat(chat.id)!.messages; const { engine } = scripted([() => fauxAssistantMessage("Explicitly requested response.")]);
const res = await turn(engine, chat.id, { type: "retry" }); const chat = createConfiguredChat("Test: foreground manual retry");
assert.equal(res.status, 400); const before = [newUserMessage("hello"), newHandoffMessage(chat.id), failed("automatic-failure")];
assert.match(await res.text(), /requires a failed turn/); chatStore.saveMessages(chat.id, before);
assert.deepEqual(chatStore.getChat(chat.id)!.messages, before); assert.equal(reconstructRequestCause(before), "background");
let built = false;
const factory: AgentFactory = (opts) => {
built = true;
assert.equal(opts.callPurpose?.background, false);
return createTeacherAgent(undefined, { ...opts, engine });
};
assert.equal((await turn(engine, chat.id, { type: "retry" }, factory)).status, 200);
assert.ok(built);
assertPreserved(before, chatStore.getChat(chat.id)!.messages);
});
});
describe("retained generations and Pi's native provider filtering", () => {
for (const stopReason of ["error", "aborted"] as const) {
it(`does not fabricate results for ${stopReason} partial tool calls on later requests`, () => {
const faux = createFauxCore({});
const partial: LhMessage = {
...fauxAssistantMessage([fauxToolCall("save_note", {}, { id: "partial-call" })], { stopReason }), id: "partial",
};
const recovered: LhMessage = { ...fauxAssistantMessage("Recovered."), id: "recovered" };
const messages = [newUserMessage("hello"), partial, recovered, newUserMessage("next question")];
const before = JSON.parse(JSON.stringify(messages));
const converted = toLlmMessages(asAgentMessages(messages));
assert.ok(converted.some((m) => m.role === "assistant" && m.stopReason === stopReason), "the app still leaves failed-generation filtering to Pi");
assert.equal(converted.filter((m) => m.role === "toolResult").length, 0, "no orphan synthetic result for an unexecuted partial call");
const providerHistory = transformMessages(converted, faux.getModel());
assert.deepEqual(providerHistory.map((m) => m.role), ["user", "assistant", "user"]);
assert.doesNotMatch(JSON.stringify(providerHistory), /partial-call/);
assert.deepEqual(messages, before, "model conversion never changes the saved attempts");
});
}
it("keeps an actual tool-cancellation result and its completed calling message", () => {
const faux = createFauxCore({});
const call: LhMessage = {
...fauxAssistantMessage([fauxToolCall("save_note", {}, { id: "completed-call" })], { stopReason: "toolUse" }), id: "caller",
};
const cancellation: LhMessage = {
id: "cancelled-result", role: "toolResult", toolCallId: "completed-call", toolName: "save_note",
content: [{ type: "text", text: "Tool execution was aborted." }], isError: true, timestamp: 1,
};
const messages = [newUserMessage("hello"), call, cancellation, newUserMessage("continue")];
const providerHistory = transformMessages(toLlmMessages(asAgentMessages(messages)), faux.getModel());
assert.deepEqual(providerHistory.map((m) => m.role), ["user", "assistant", "toolResult", "user"]);
const result = providerHistory.find((m) => m.role === "toolResult");
assert.ok(result?.role === "toolResult");
assert.equal(result.toolCallId, "completed-call");
assert.equal(result.isError, true);
assert.deepEqual(result.content, cancellation.content);
}); });
it("a plan block persisted after the failure stays, and the turn still retries", async () => { it("still repairs an unanswered call from a completed generation", () => {
const { engine, contexts } = scripted([networkFailure, () => fauxAssistantMessage([fauxText("Planned reply.")])]); const faux = createFauxCore({});
const chat = createConfiguredChat("Test: retry past a plan block"); const question: LhMessage = {
...fauxAssistantMessage([fauxToolCall("quiz", QUIZ_INPUT, { id: "unanswered-question" })], { stopReason: "toolUse" }), id: "question",
};
const messages = [newUserMessage("hello"), question, newUserMessage("move on")];
const before = JSON.parse(JSON.stringify(messages));
const providerHistory = transformMessages(toLlmMessages(asAgentMessages(messages)), faux.getModel());
assert.deepEqual(providerHistory.map((m) => m.role), ["user", "assistant", "toolResult", "user"]);
const result = providerHistory.find((m) => m.role === "toolResult");
assert.ok(result?.role === "toolResult");
assert.equal(result.toolCallId, "unanswered-question");
assert.deepEqual(result.content, [{ type: "text", text: INTERRUPTED_TEXT }]);
assert.deepEqual(messages, before, "the repair is temporary, never persisted");
});
});
describe("Retry refusal and revalidation", () => {
it("rejects an invalid Retry before provider gates, turn claims or writes", async () => {
const chat = chatStore.createChat("Test: retry refused before subscription refresh");
chatStore.saveSettings(chat.id, { model: { provider: "chatgpt", id: "not-a-fixture-model" } });
const before = chatStore.getChat(chat.id)!;
rememberTurnFailure(chat.id, "Existing settlement notice");
const events: string[] = [];
const unsubscribe = subscribe((event) => { if ("chatId" in event && event.chatId === chat.id) events.push(event.type); });
try {
const res = await handleChatRequest(post(chat.id, { type: "retry" }), () => { throw new Error("must not construct an agent"); });
assert.equal(res.status, 400, "invalid retry wins even with an unavailable subscription model");
assert.match(await res.text(), /requires a failed turn/);
assert.deepEqual(chatStore.getChat(chat.id), before);
assert.equal(turnFailure(chat.id), "Existing settlement notice", "beginTurn did not clear recovery state");
assert.equal(isTurnRunning(chat.id), false);
assert.deepEqual(events, []);
} finally {
unsubscribe();
clearTurnFailure(chat.id);
}
});
it("rejects Retry after later compaction without removing the error or its boundary", async () => {
const { engine, contexts } = scripted([]);
const chat = createConfiguredChat("Test: compacted failure");
const failure = failed("partial", "Partial explanation.");
const marker = compaction(failure.id);
const before = [newUserMessage("hello"), failure, marker];
chatStore.saveMessages(chat.id, before);
const snapshot = chatStore.getChat(chat.id)!;
const res = await turn(engine, chat.id, { type: "retry" });
assert.equal(res.status, 400);
const body = await res.json() as { error: string };
assert.equal(body.error, "fetch failed\n\nThis response cannot be retried after compaction. Send a new message to continue.");
assert.deepEqual(chatStore.getChat(chat.id), snapshot);
assert.equal(contexts.length, 0);
assert.equal(new SessionModel({ chatId: chat.id, initialMessages: before }).getState().error?.message, "fetch failed");
assert.equal(endsInFailedTurn(before), false);
});
for (const later of ["compaction", "learner-input"] as const) {
it(`revalidates against ${later} landing on the post-compaction reread`, async () => {
const { engine, contexts } = scripted([]);
const chat = createConfiguredChat(`Test: retry race with ${later}`);
const before = [newUserMessage("hello"), failed("failed-response")];
chatStore.saveMessages(chat.id, before);
const appended = later === "compaction" ? compaction("failed-response") : newUserMessage("A newer input");
const read = chatStore.getChat;
let reads = 0;
// Simulate another writer landing during the awaited gate. Do not
// change or invoke the compaction implementation to exercise Retry.
const reader = mock.method(chatStore, "getChat", (id: string) => {
if (id === chat.id && ++reads === 2) chatStore.saveMessages(id, [...before, appended]);
return read(id);
});
const events: string[] = [];
const unsubscribe = subscribe((event) => { if ("chatId" in event && event.chatId === chat.id) events.push(event.type); });
try {
const res = await turn(engine, chat.id, { type: "retry" });
assert.equal(res.status, 400);
if (later === "compaction") assert.match(await res.text(), /fetch failed.*cannot be retried after compaction/);
assert.deepEqual(read(chat.id)!.messages, [...before, appended]);
assert.equal(contexts.length, 0);
assert.equal(isTurnRunning(chat.id), false);
assert.deepEqual(events, [], "no turn was claimed after final validation refused");
} finally {
reader.mock.restore();
unsubscribe();
}
});
}
it("a sibling's successful retry makes a later stale Retry refuse without changing history", async () => {
const { engine, contexts } = scripted([networkFailure, () => fauxAssistantMessage("Recovered.")]);
const chat = createConfiguredChat("Test: stale sibling Retry");
await turn(engine, chat.id, { type: "text", text: "hello" }); await turn(engine, chat.id, { type: "text", text: "hello" });
// persistTurn's shape when plan_init succeeded and the next call await turn(engine, chat.id, { type: "retry" });
// failed: the plan block lands AFTER the failure. const before = chatStore.getChat(chat.id)!;
const failedRecord = chatStore.getChat(chat.id)!.messages; assert.equal((await turn(engine, chat.id, { type: "retry" })).status, 400);
chatStore.saveMessages(chat.id, [...failedRecord, newPlanBlockMessage("plan-1", {})]); assert.deepEqual(chatStore.getChat(chat.id), before);
assert.ok(endsInFailedTurn(chatStore.getChat(chat.id)!.messages), "the failure is still found past the marker"); assert.equal(contexts.length, 2);
assert.equal((await turn(engine, chat.id, { type: "retry" })).status, 200);
const saved = chatStore.getChat(chat.id)!.messages;
assert.deepEqual(saved.map((m) => m.role), ["user", "custom", "assistant"], "failure dropped, plan block kept");
assert.match(JSON.stringify(saved.at(-1)), /Planned reply/);
assert.deepEqual(contexts.at(-1), ["user"], "the plan block is not model input");
}); });
}); });
describe("the Retry button's client path", () => { describe("the Retry button's client path", () => {
it("keeps the original model error plus the explanation when compaction wins the click race", async () => {
const messages = [newUserMessage("hello"), failed("failed-response")];
const compacted = [...messages, compaction("failed-response")];
const explanation = "This response cannot be retried after compaction. Send a new message to continue.";
const refused = `fetch failed\n\n${explanation}`;
const m = new SessionModel({
chatId: "chat_x", initialMessages: messages,
fetch: async (_url, init) => {
assert.equal(init?.method, "POST");
assert.deepEqual(JSON.parse(String(init?.body)).input, { type: "retry" });
m.adoptRecord(compacted);
return Response.json({ error: refused }, { status: 400 });
},
});
await m.send({ type: "retry" });
assert.equal(m.getState().status, "error");
assert.equal(JSON.parse(m.getState().error!.message).error, refused, "the existing refusal-envelope path carries both messages");
assert.deepEqual(m.getState().nativeMessages, compacted);
assert.equal(endsInFailedTurn(m.getState().nativeMessages), false);
});
it("after a dropped request, Retry posts straight from the error status", async () => { it("after a dropped request, Retry posts straight from the error status", async () => {
const failedTurn = [ const failedTurn = [newUserMessage("hello"), failed("a1")];
{ ...newUserMessage("hello"), id: "u1" },
{ ...fauxAssistantMessage("", { stopReason: "error", errorMessage: "fetch failed" }), id: "a1" },
] as unknown as LhMessage[];
const posted: string[] = []; const posted: string[] = [];
let online = false; let online = false;
const fetch = (async (_url: string, init?: RequestInit) => { const fetch = (async (_url: string, init?: RequestInit) => {
if (!online) throw new TypeError("Failed to fetch"); if (!online) throw new TypeError("Failed to fetch");
if (init?.method === "POST") posted.push(String(init.body)); if (init?.method === "POST") posted.push(String(init.body));
return new Response("{}", { status: 500 }); // the reattach is not under test return new Response("{}", { status: 500 });
}) as typeof globalThis.fetch; }) as typeof globalThis.fetch;
const m = new SessionModel({ chatId: "chat_x", initialMessages: failedTurn, fetch }); const m = new SessionModel({ chatId: "chat_x", initialMessages: failedTurn, fetch });
await m.send({ type: "retry" }); await m.send({ type: "retry" });
assert.equal(m.getState().status, "error", "the dropped request leaves the model in error"); assert.equal(m.getState().status, "error");
online = true; online = true;
await m.send({ type: "retry" }); await m.send({ type: "retry" });
assert.equal(posted.length, 1, "send() recovers from error without stop()"); assert.equal(posted.length, 1, "send() recovers from error without stop()");
assert.deepEqual(JSON.parse(posted[0]!).input, { type: "retry" }); assert.deepEqual(JSON.parse(posted[0]!).input, { type: "retry" });
assert.deepEqual(m.getState().nativeMessages, failedTurn, "there is no optimistic learner message or deletion");
}); });
}); });
describe("retryDrops (pure)", () => { describe("Retry eligibility and temporary omissions (pure)", () => {
const failed = (id: string): LhMessage => it("parses Retry without appending input", () => {
({ ...fauxAssistantMessage("", { stopReason: "error", errorMessage: "boom" }), id }) as unknown as LhMessage; const input = parseChatInput({ type: "retry" });
assert.deepEqual(input, { type: "retry" });
it("parses the retry input event", () => { assert.deepEqual(inputAppends({ messages: [newUserMessage("hi"), failed("a1")] }, input), []);
assert.deepEqual(parseChatInput({ type: "retry" }), { type: "retry" });
}); });
it("drops every trailing failure back to the input", () => { it("omits consecutive failures back to their input without mutating the record", () => {
const user = { ...newUserMessage("hi"), id: "u1" } as LhMessage; const messages = [newUserMessage("hi"), failed("a1"), failed("a2")];
assert.deepEqual([...retryDrops([user, failed("a1"), failed("a2")])].sort(), ["a1", "a2"]); const before = JSON.parse(JSON.stringify(messages));
assert.deepEqual([...retryDrops(messages)].sort(), ["a1", "a2"]);
assert.deepEqual(messages, before);
}); });
it("looks past trailing UI-only markers, and keeps them", () => { it("looks past plan blocks, but compaction after the latest failure blocks Retry", () => {
const user = { ...newUserMessage("hi"), id: "u1" } as LhMessage; const user = newUserMessage("hi");
const failure = failed("a1");
const block = newPlanBlockMessage("plan-1", {}); const block = newPlanBlockMessage("plan-1", {});
assert.deepEqual([...retryDrops([user, failed("a1"), block])], ["a1"]); const marker = compaction(user.id);
assert.equal(endsInFailedTurn([user, block]), false, "a marker alone is no failure"); assert.deepEqual([...retryDrops([user, failure, block])], ["a1"]);
const compaction = newCompactionMarker("summary", { firstKeptMessageId: "u1", tokensBefore: 100, model: "fixture", at: "2026-01-01T00:00:00Z" }); assert.equal(endsInFailedTurn([user, block]), false);
assert.deepEqual([...retryDrops([user, failed("a1"), compaction])], ["a1"], "the settle kick's marker too"); for (const messages of [[user, failure, marker], [user, failure, marker, block], [user, failure, block, marker]]) {
assert.equal(endsInFailedTurn(messages), false);
assert.equal(failedResponseAtEnd(messages)?.id, failure.id, "only the Retry button disappears, not the banner");
assert.throws(() => retryDrops(messages), /fetch failed[\s\S]*cannot be retried after compaction/);
}
assert.deepEqual([...retryDrops([user, failed("old"), marker, failed("new")])], ["new"], "a failure after compaction can retry its already-compacted starting state");
}); });
it("throws when the record does not end in a failure, or nothing precedes it", () => { it("excludes aborts, healthy responses, missing input and assistant-only predecessors", () => {
const user = { ...newUserMessage("hi"), id: "u1" } as LhMessage; const user = newUserMessage("hi");
assert.throws(() => retryDrops([user]), ChatInputError); const aborted = { ...failed("aborted"), stopReason: "aborted" } as LhMessage;
assert.throws(() => retryDrops([failed("a1")]), ChatInputError); const healthy = { ...fauxAssistantMessage("Done"), id: "healthy" } as LhMessage;
for (const messages of [[user], [failed("a1")], [user, aborted], [user, healthy], [healthy, failed("a1")]]) {
assert.equal(endsInFailedTurn(messages), false);
assert.throws(() => retryDrops(messages), ChatInputError);
}
assert.equal(failedResponseAtEnd([user, failed("old"), aborted]), undefined);
}); });
}); });
tests/session-core.test.ts
+29 -6
View File
@@ -25,7 +25,7 @@ import { absorbed, pendingCallTool, SessionModel } from "../components/session-c
import type { ChatInput } from "../src/core/input-events.ts"; import type { ChatInput } from "../src/core/input-events.ts";
import type { LhMessage } from "../src/core/native-messages.ts"; import type { LhMessage } from "../src/core/native-messages.ts";
import { newUserMessage, newAnswerResult, newThinkingMessage, needsContinuation, endsInDismissal } from "../src/core/native-messages.ts"; import { newUserMessage, newAnswerResult, newThinkingMessage, needsContinuation, endsInDismissal } from "../src/core/native-messages.ts";
import { newCompactionMarker, newPlanBlockMessage, projectView } from "../src/core/index.ts"; import { endsInFailedTurn, newCompactionMarker, newPlanBlockMessage, projectView } from "../src/core/index.ts";
import { dismissedAnswer } from "../src/core/question-core.ts"; import { dismissedAnswer } from "../src/core/question-core.ts";
import { buildAnswer, type QuestionData } from "../src/core/question-core.ts"; import { buildAnswer, type QuestionData } from "../src/core/question-core.ts";
@@ -483,7 +483,7 @@ it("Wave C: markers injected mid-run (custom role, follow-up delivery) append to
}); });
}); });
// ─── provider failures: one final-entry rule for live and saved records ────── // ─── provider failures: one marker-aware rule for live and saved records ────
describe("provider-error banner (live and saved)", () => { describe("provider-error banner (live and saved)", () => {
function failed(id = "failed", errorMessage: string | undefined = "Google 500: fixture failure"): LhMessage { function failed(id = "failed", errorMessage: string | undefined = "Google 500: fixture failure"): LhMessage {
@@ -512,7 +512,7 @@ describe("provider-error banner (live and saved)", () => {
assert.equal(model().getState().error, null); assert.equal(model().getState().error, null);
}); });
it("any later entry supersedes the failure, including plan/compaction markers and aborts", () => { it("substantive later input/output supersedes the failure, including aborts", () => {
const failure = failed(); const failure = failed();
const later: LhMessage[] = [ const later: LhMessage[] = [
newUserMessage("retry"), newUserMessage("retry"),
@@ -520,8 +520,6 @@ describe("provider-error banner (live and saved)", () => {
{ ...assistantMsg("aborted", ""), stopReason: "aborted", errorMessage: "Aborted" } as LhMessage, { ...assistantMsg("aborted", ""), stopReason: "aborted", errorMessage: "Aborted" } as LhMessage,
newThinkingMessage("problem", "my attempt"), newThinkingMessage("problem", "my attempt"),
resultMsg("call"), resultMsg("call"),
newPlanBlockMessage("plan", { kind: "full", nodes: [], edges: [], current: null }),
newCompactionMarker("summary", { firstKeptMessageId: failure.id, tokensBefore: 100, model: "fixture", at: "2026-01-01T00:00:00Z" }),
]; ];
for (const entry of later) { for (const entry of later) {
const m = model({ initialMessages: [failure] }); const m = model({ initialMessages: [failure] });
@@ -533,6 +531,31 @@ describe("provider-error banner (live and saved)", () => {
} }
}); });
it("plan and compaction markers preserve the banner on sync, adoption and reopen", async () => {
const failure = failed();
const record = [newUserMessage("topic"), failure];
const markers = [
newPlanBlockMessage("plan", { kind: "full", nodes: [], edges: [], current: null }),
newCompactionMarker("summary", { firstKeptMessageId: failure.id, tokensBefore: 100, model: "fixture", at: "2026-01-01T00:00:00Z" }),
];
for (const marker of markers) {
const saved = [...record, marker];
let posts = 0;
const m = model({ initialMessages: record, fetch: async () => { posts++; throw new Error("unexpected request"); } });
m.setMessages(saved);
assert.equal(m.getState().error?.message, failure.role === "assistant" ? failure.errorMessage : undefined);
assert.equal(model({ initialMessages: saved }).getState().error?.message, m.getState().error?.message);
const adopted = model({ initialMessages: record });
adopted.adoptRecord(saved);
assert.equal(adopted.getState().error?.message, m.getState().error?.message);
assert.equal(endsInFailedTurn(saved), marker.customType === "plan-block", "later compaction removes Retry eligibility, not the error notice");
await m.continueIfOwed(saved);
assert.equal(posts, 0, "restoring the banner is never an automatic retry");
m.setMessages([...saved, newUserMessage("continue from here")]);
assert.equal(m.getState().error, null, "new learner input supersedes the notice");
}
});
it("a newer live message supersedes the failure before its response finishes", () => { it("a newer live message supersedes the failure before its response finishes", () => {
const m = model(); const m = model();
m.applyWireEvent({ type: "message_end", message: failed() }); m.applyWireEvent({ type: "message_end", message: failed() });
@@ -554,7 +577,7 @@ describe("provider-error banner (live and saved)", () => {
assert.equal(m.getState().error?.message, "Google 500: fixture failure"); assert.equal(m.getState().error?.message, "Google 500: fixture failure");
}); });
for (const input of [{ type: "text", text: "retry" }, { type: "continue" }] as const) { for (const input of [{ type: "text", text: "retry" }, { type: "continue" }, { type: "retry" }] as const) {
it(`hides the old failure during the optimistic ${input.type} wait, then shows a new live failure`, async () => { it(`hides the old failure during the optimistic ${input.type} wait, then shows a new live failure`, async () => {
let acknowledge!: (response: Response) => void; let acknowledge!: (response: Response) => void;
const ack = new Promise<Response>((resolve) => { acknowledge = resolve; }); const ack = new Promise<Response>((resolve) => { acknowledge = resolve; });