Files
learnharness/tests/dismissal-park.test.ts
amosblomqvist 85022ae143 practice hardening: weak-model argument heal + parked dismissals mid-turn
The GLM incident (2026-10-04): glm-5.3 sent mode as a JSON-string-encoded
enum ('"recall"') on 19 consecutive problem calls; TypeBox's error never
named the expected values, so the model re-emitted the identical payload
until the dev server was killed. And mid-turn the learner was trapped:
Esc's question/problem rungs only dismissed — the dismiss route 409'd
against the turn-owned record and nothing could stop the running turn.

- healEncodedArguments (practice.ts, the prepareArguments seam): a STRING
  where the schema expects a literal union / boolean / null / number /
  array is JSON.parse-d and kept when it satisfies the schema. Valid args
  pass through untouched (same reference); unhealable values stay for
  validation to reject honestly. Wired on BOTH problem and conclude.
- dismissal-park.ts: the dismiss route parks a mid-turn dismissal (202);
  the turn's settle path (onFinish, after persistTurn, before endTurn —
  the same ordering law as the record write itself) applies it against
  the persisted record, only if the dismissed thing really is pending.
- confirmAbort: question/problem discard while busy now ALSO aborts the
  in-flight turn; the dialog line says so (busy prop).

No error-result cap (owner ruling). 16 new pins (heal incl. pi-ai
validateToolArguments end-to-end; park registry, apply paths, the 202
route contract).
2026-10-04 23:57:00 +03:00

203 lines
7.4 KiB
TypeScript

/**
* PARKED DISMISSALS (owner, 2026-10-04): ESC-discard while the turn is
* still working. Mid-turn the stored record holds no pending question or
* problem (the turn's messages land at settle), so the dismiss route
* cannot validate — it PARKS the intent (202) and the turn's settle path
* applies it against the freshly persisted record, only if the dismissed
* thing really is pending there.
*
* These pins hold both halves: the pure park registry + apply against
* fixture records, and the route's park contract under an isolated
* LH_DATA_DIR (the birth-stamp discipline — the seam is set before any
* config-loading import, every app-module import dynamic).
*/
import assert from "node:assert/strict";
import fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { after, describe, it } from "node:test";
// ── the seam, BEFORE any config-loading import ──────────────────────────────
const TMP_DATA = fs.mkdtempSync(path.join(os.tmpdir(), "lh-dismissal-park-"));
process.env.LH_DATA_DIR = TMP_DATA;
const { chatStore } = await import("../src/store/chats.ts");
const { applyParkedDismissal, parkDismissal, takeParkedDismissal } = await import(
"../src/server/dismissal-park.ts"
);
const { beginTurn, endTurn, isTurnRunning } = await import("../src/server/turn-liveness.ts");
const { subscribe } = await import("../src/core/events.ts");
const {
activeProblemOf,
answeredCallIds,
danglingQuestionOf,
dismissedAnswer,
} = await import("../src/core/index.ts");
const newMessageId = (await import("../src/core/native-messages.ts")).newMessageId;
after(() => {
fs.rmSync(TMP_DATA, { recursive: true, force: true });
});
/** A record whose newest assistant message carries a DANGLING quiz call —
* the settled state a mid-turn dismissal parks against. */
function chatWithDanglingQuiz(id: string): string {
const chat = chatStore.createChat("park: dangling quiz");
chatStore.saveMessages(chat.id, [{
id: newMessageId(),
role: "assistant",
content: [
{ type: "text", text: "Quick check:" },
{
type: "toolCall",
id: `${id}-q1`,
name: "quiz",
arguments: { question: "What is a cyclic group?" },
},
],
} as never]);
return chat.id;
}
/** A record with an ACTIVE problem — the settled practice state a
* dismiss-problem park applies against. */
function chatWithActiveProblem(): string {
const chat = chatStore.createChat("park: active problem");
const assistant = {
id: newMessageId(),
role: "assistant",
content: [
{
type: "toolCall",
id: "call-prob-1",
name: "problem",
arguments: { statement: "Prove |G|=p ⇒ cyclic", mode: "recall" },
},
],
} as never;
chatStore.saveMessages(chat.id, [assistant]);
return chat.id;
}
describe("the park registry", () => {
it("parks and takes (take drains — the second read is null)", () => {
parkDismissal("chat_x", { type: "dismiss-problem", problemCallId: "p" });
assert.deepEqual(takeParkedDismissal("chat_x"), { type: "dismiss-problem", problemCallId: "p" });
assert.equal(takeParkedDismissal("chat_x"), null);
});
it("apply with nothing parked is a no-op (no write, no emit)", () => {
const chat = chatWithDanglingQuiz("noop");
const before = chatStore.getChat(chat)!.messages.length;
let emitted = 0;
const off = subscribe((e) => {
if (e.type === "record-changed" && e.chatId === chat) emitted++;
});
applyParkedDismissal(chatStore, chat);
off();
assert.equal(emitted, 0);
assert.equal(chatStore.getChat(chat)!.messages.length, before);
});
});
describe("applyParkedDismissal — against real records", () => {
it("applies a parked question dismissal to the dangling quiz call", () => {
const chat = chatWithDanglingQuiz("apply-q");
const pending = danglingQuestionOf(chatStore.getChat(chat)!.messages);
assert.ok(pending);
parkDismissal(chat, {
type: "dismiss",
toolCallId: pending!.toolCallId,
answer: dismissedAnswer("quiz", "What is a cyclic group?"),
});
let emitted = 0;
const off = subscribe((e) => {
if (e.type === "record-changed" && e.chatId === chat) emitted++;
});
applyParkedDismissal(chatStore, chat);
off();
assert.equal(emitted, 1);
// The call is answered now — nothing dangles.
assert.equal(danglingQuestionOf(chatStore.getChat(chat)!.messages), null);
assert.ok(answeredCallIds(chatStore.getChat(chat)!.messages).has(pending!.toolCallId));
assert.equal(takeParkedDismissal(chat), null);
});
it("applies a parked problem dismissal to the active problem", () => {
const chat = chatWithActiveProblem();
parkDismissal(chat, { type: "dismiss-problem", problemCallId: "call-prob-1" });
applyParkedDismissal(chatStore, chat);
// The marker freed the slot — no active problem remains.
assert.equal(activeProblemOf(chatStore.getChat(chat)!.messages), null);
});
it("drops a park that targets nothing (never materialized — silence)", () => {
const chat = chatWithDanglingQuiz("drop");
const before = chatStore.getChat(chat)!.messages.length;
parkDismissal(chat, { type: "dismiss-problem", problemCallId: "call-ghost" });
let emitted = 0;
const off = subscribe((e) => {
if (e.type === "record-changed" && e.chatId === chat) emitted++;
});
applyParkedDismissal(chatStore, chat);
off();
assert.equal(emitted, 0);
assert.equal(chatStore.getChat(chat)!.messages.length, before);
assert.equal(takeParkedDismissal(chat), null);
});
});
describe("the dismiss route — the park contract (202)", () => {
it("parks (202) while a turn runs; the settle apply lands the dismissal", async () => {
const chat = chatWithDanglingQuiz("route");
const pending = danglingQuestionOf(chatStore.getChat(chat)!.messages);
assert.ok(pending);
const dismissRoute = await import("../app/api/chats/[id]/dismiss/route.ts");
// Mid-turn: the route cannot validate against stale truth — park.
assert.ok(!isTurnRunning(chat));
beginTurn(chat);
const res = await dismissRoute.POST(
new Request(`http://test/api/chats/${chat}/dismiss`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
input: {
type: "dismiss",
toolCallId: pending!.toolCallId,
answer: dismissedAnswer("quiz", "What is a cyclic group?"),
},
}),
}),
{ params: Promise.resolve({ id: chat }) },
);
assert.equal(res.status, 202);
assert.deepEqual(await res.json(), { ok: true, parked: true });
// Nothing landed yet — the record still dangles (the turn owns it).
assert.ok(danglingQuestionOf(chatStore.getChat(chat)!.messages));
// The settle: persistTurn (done) → applyParkedDismissal → endTurn.
applyParkedDismissal(chatStore, chat);
endTurn(chat);
assert.equal(danglingQuestionOf(chatStore.getChat(chat)!.messages), null);
assert.equal(takeParkedDismissal(chat), null);
});
it("still refuses honestly (400) with no turn and nothing dangling", async () => {
const chat = chatStore.createChat("park: nothing dangling").id;
const dismissRoute = await import("../app/api/chats/[id]/dismiss/route.ts");
const res = await dismissRoute.POST(
new Request(`http://test/api/chats/${chat}/dismiss`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
input: { type: "dismiss", toolCallId: "call-nowhere", answer: dismissedAnswer("quiz", "?") },
}),
}),
{ params: Promise.resolve({ id: chat }) },
);
assert.equal(res.status, 400);
});
});