practice hardening: weak-model argument heal + parked dismissals mid-turn
The GLM incident (2026-10-04): glm-5.3 sent mode as a JSON-string-encoded
enum ('"recall"') on 19 consecutive problem calls; TypeBox's error never
named the expected values, so the model re-emitted the identical payload
until the dev server was killed. And mid-turn the learner was trapped:
Esc's question/problem rungs only dismissed — the dismiss route 409'd
against the turn-owned record and nothing could stop the running turn.
- healEncodedArguments (practice.ts, the prepareArguments seam): a STRING
where the schema expects a literal union / boolean / null / number /
array is JSON.parse-d and kept when it satisfies the schema. Valid args
pass through untouched (same reference); unhealable values stay for
validation to reject honestly. Wired on BOTH problem and conclude.
- dismissal-park.ts: the dismiss route parks a mid-turn dismissal (202);
the turn's settle path (onFinish, after persistTurn, before endTurn —
the same ordering law as the record write itself) applies it against
the persisted record, only if the dismissed thing really is pending.
- confirmAbort: question/problem discard while busy now ALSO aborts the
in-flight turn; the dialog line says so (busy prop).
No error-result cap (owner ruling). 16 new pins (heal incl. pi-ai
validateToolArguments end-to-end; park registry, apply paths, the 202
route contract).
This commit is contained in:
amosblomqvist
@@ -0,0 +1,202 @@
|
||||
/**
|
||||
* PARKED DISMISSALS (owner, 2026-10-04): ESC-discard while the turn is
|
||||
* still working. Mid-turn the stored record holds no pending question or
|
||||
* problem (the turn's messages land at settle), so the dismiss route
|
||||
* cannot validate — it PARKS the intent (202) and the turn's settle path
|
||||
* applies it against the freshly persisted record, only if the dismissed
|
||||
* thing really is pending there.
|
||||
*
|
||||
* These pins hold both halves: the pure park registry + apply against
|
||||
* fixture records, and the route's park contract under an isolated
|
||||
* LH_DATA_DIR (the birth-stamp discipline — the seam is set before any
|
||||
* config-loading import, every app-module import dynamic).
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
|
||||
// ── the seam, BEFORE any config-loading import ──────────────────────────────
|
||||
const TMP_DATA = fs.mkdtempSync(path.join(os.tmpdir(), "lh-dismissal-park-"));
|
||||
process.env.LH_DATA_DIR = TMP_DATA;
|
||||
|
||||
const { chatStore } = await import("../src/store/chats.ts");
|
||||
const { applyParkedDismissal, parkDismissal, takeParkedDismissal } = await import(
|
||||
"../src/server/dismissal-park.ts"
|
||||
);
|
||||
const { beginTurn, endTurn, isTurnRunning } = await import("../src/server/turn-liveness.ts");
|
||||
const { subscribe } = await import("../src/core/events.ts");
|
||||
const {
|
||||
activeProblemOf,
|
||||
answeredCallIds,
|
||||
danglingQuestionOf,
|
||||
dismissedAnswer,
|
||||
} = await import("../src/core/index.ts");
|
||||
const newMessageId = (await import("../src/core/native-messages.ts")).newMessageId;
|
||||
|
||||
after(() => {
|
||||
fs.rmSync(TMP_DATA, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** A record whose newest assistant message carries a DANGLING quiz call —
|
||||
* the settled state a mid-turn dismissal parks against. */
|
||||
function chatWithDanglingQuiz(id: string): string {
|
||||
const chat = chatStore.createChat("park: dangling quiz");
|
||||
chatStore.saveMessages(chat.id, [{
|
||||
id: newMessageId(),
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: "Quick check:" },
|
||||
{
|
||||
type: "toolCall",
|
||||
id: `${id}-q1`,
|
||||
name: "quiz",
|
||||
arguments: { question: "What is a cyclic group?" },
|
||||
},
|
||||
],
|
||||
} as never]);
|
||||
return chat.id;
|
||||
}
|
||||
|
||||
/** A record with an ACTIVE problem — the settled practice state a
|
||||
* dismiss-problem park applies against. */
|
||||
function chatWithActiveProblem(): string {
|
||||
const chat = chatStore.createChat("park: active problem");
|
||||
const assistant = {
|
||||
id: newMessageId(),
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call-prob-1",
|
||||
name: "problem",
|
||||
arguments: { statement: "Prove |G|=p ⇒ cyclic", mode: "recall" },
|
||||
},
|
||||
],
|
||||
} as never;
|
||||
chatStore.saveMessages(chat.id, [assistant]);
|
||||
return chat.id;
|
||||
}
|
||||
|
||||
describe("the park registry", () => {
|
||||
it("parks and takes (take drains — the second read is null)", () => {
|
||||
parkDismissal("chat_x", { type: "dismiss-problem", problemCallId: "p" });
|
||||
assert.deepEqual(takeParkedDismissal("chat_x"), { type: "dismiss-problem", problemCallId: "p" });
|
||||
assert.equal(takeParkedDismissal("chat_x"), null);
|
||||
});
|
||||
|
||||
it("apply with nothing parked is a no-op (no write, no emit)", () => {
|
||||
const chat = chatWithDanglingQuiz("noop");
|
||||
const before = chatStore.getChat(chat)!.messages.length;
|
||||
let emitted = 0;
|
||||
const off = subscribe((e) => {
|
||||
if (e.type === "record-changed" && e.chatId === chat) emitted++;
|
||||
});
|
||||
applyParkedDismissal(chatStore, chat);
|
||||
off();
|
||||
assert.equal(emitted, 0);
|
||||
assert.equal(chatStore.getChat(chat)!.messages.length, before);
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyParkedDismissal — against real records", () => {
|
||||
it("applies a parked question dismissal to the dangling quiz call", () => {
|
||||
const chat = chatWithDanglingQuiz("apply-q");
|
||||
const pending = danglingQuestionOf(chatStore.getChat(chat)!.messages);
|
||||
assert.ok(pending);
|
||||
parkDismissal(chat, {
|
||||
type: "dismiss",
|
||||
toolCallId: pending!.toolCallId,
|
||||
answer: dismissedAnswer("quiz", "What is a cyclic group?"),
|
||||
});
|
||||
let emitted = 0;
|
||||
const off = subscribe((e) => {
|
||||
if (e.type === "record-changed" && e.chatId === chat) emitted++;
|
||||
});
|
||||
applyParkedDismissal(chatStore, chat);
|
||||
off();
|
||||
assert.equal(emitted, 1);
|
||||
// The call is answered now — nothing dangles.
|
||||
assert.equal(danglingQuestionOf(chatStore.getChat(chat)!.messages), null);
|
||||
assert.ok(answeredCallIds(chatStore.getChat(chat)!.messages).has(pending!.toolCallId));
|
||||
assert.equal(takeParkedDismissal(chat), null);
|
||||
});
|
||||
|
||||
it("applies a parked problem dismissal to the active problem", () => {
|
||||
const chat = chatWithActiveProblem();
|
||||
parkDismissal(chat, { type: "dismiss-problem", problemCallId: "call-prob-1" });
|
||||
applyParkedDismissal(chatStore, chat);
|
||||
// The marker freed the slot — no active problem remains.
|
||||
assert.equal(activeProblemOf(chatStore.getChat(chat)!.messages), null);
|
||||
});
|
||||
|
||||
it("drops a park that targets nothing (never materialized — silence)", () => {
|
||||
const chat = chatWithDanglingQuiz("drop");
|
||||
const before = chatStore.getChat(chat)!.messages.length;
|
||||
parkDismissal(chat, { type: "dismiss-problem", problemCallId: "call-ghost" });
|
||||
let emitted = 0;
|
||||
const off = subscribe((e) => {
|
||||
if (e.type === "record-changed" && e.chatId === chat) emitted++;
|
||||
});
|
||||
applyParkedDismissal(chatStore, chat);
|
||||
off();
|
||||
assert.equal(emitted, 0);
|
||||
assert.equal(chatStore.getChat(chat)!.messages.length, before);
|
||||
assert.equal(takeParkedDismissal(chat), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the dismiss route — the park contract (202)", () => {
|
||||
it("parks (202) while a turn runs; the settle apply lands the dismissal", async () => {
|
||||
const chat = chatWithDanglingQuiz("route");
|
||||
const pending = danglingQuestionOf(chatStore.getChat(chat)!.messages);
|
||||
assert.ok(pending);
|
||||
const dismissRoute = await import("../app/api/chats/[id]/dismiss/route.ts");
|
||||
|
||||
// Mid-turn: the route cannot validate against stale truth — park.
|
||||
assert.ok(!isTurnRunning(chat));
|
||||
beginTurn(chat);
|
||||
const res = await dismissRoute.POST(
|
||||
new Request(`http://test/api/chats/${chat}/dismiss`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
input: {
|
||||
type: "dismiss",
|
||||
toolCallId: pending!.toolCallId,
|
||||
answer: dismissedAnswer("quiz", "What is a cyclic group?"),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: chat }) },
|
||||
);
|
||||
assert.equal(res.status, 202);
|
||||
assert.deepEqual(await res.json(), { ok: true, parked: true });
|
||||
// Nothing landed yet — the record still dangles (the turn owns it).
|
||||
assert.ok(danglingQuestionOf(chatStore.getChat(chat)!.messages));
|
||||
|
||||
// The settle: persistTurn (done) → applyParkedDismissal → endTurn.
|
||||
applyParkedDismissal(chatStore, chat);
|
||||
endTurn(chat);
|
||||
assert.equal(danglingQuestionOf(chatStore.getChat(chat)!.messages), null);
|
||||
assert.equal(takeParkedDismissal(chat), null);
|
||||
});
|
||||
|
||||
it("still refuses honestly (400) with no turn and nothing dangling", async () => {
|
||||
const chat = chatStore.createChat("park: nothing dangling").id;
|
||||
const dismissRoute = await import("../app/api/chats/[id]/dismiss/route.ts");
|
||||
const res = await dismissRoute.POST(
|
||||
new Request(`http://test/api/chats/${chat}/dismiss`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
input: { type: "dismiss", toolCallId: "call-nowhere", answer: dismissedAnswer("quiz", "?") },
|
||||
}),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: chat }) },
|
||||
);
|
||||
assert.equal(res.status, 400);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* THE WEAK-MODEL ARGUMENT HEAL (owner, 2026-10-04 — the GLM practice-loop
|
||||
* incident): glm-5.3 sent `"mode": "\"recall\""` — a JSON-string-encoded
|
||||
* enum — on 19 consecutive `problem` calls; TypeBox's "must be equal to
|
||||
* constant" error never named the expected values, so the model re-emitted
|
||||
* the identical payload until the dev server was killed.
|
||||
*
|
||||
* These pins hold the heal that runs at pi's prepareArguments seam (BEFORE
|
||||
* validation): string values where the schema expects a literal union /
|
||||
* boolean / null / number / array are JSON.parse-d and kept when the
|
||||
* parsed value satisfies the schema. Well-formed args pass through
|
||||
* untouched (same reference); unhealable values stay for validation to
|
||||
* reject honestly — the END-TO-END pins drive pi-ai's own
|
||||
* validateToolArguments: the healed args pass where the raw ones threw.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
concludeInputObject,
|
||||
healEncodedArguments,
|
||||
problemInputObject,
|
||||
} from "../src/agent/tools/practice.ts";
|
||||
import { validateToolArguments } from "@earendil-works/pi-ai";
|
||||
|
||||
/** The incident's exact mode spelling: the string "recall" INCLUDING
|
||||
* its quote characters (what JSON.parse turns back into recall). */
|
||||
const QUOTED = (s: string) => `"${s}"`;
|
||||
|
||||
describe("healEncodedArguments — the GLM incident", () => {
|
||||
it("heals the double-encoded mode enum (the exact 19-call loop)", () => {
|
||||
const healed = healEncodedArguments({ mode: QUOTED("recall") }, problemInputObject) as {
|
||||
mode: unknown;
|
||||
};
|
||||
assert.equal(healed.mode, "recall");
|
||||
});
|
||||
|
||||
it("END-TO-END: the healed args pass pi-ai validation where the raw threw", () => {
|
||||
const raw = {
|
||||
statement: "Prove that every finite group of prime order is cyclic.",
|
||||
mode: QUOTED("recall"), // the incident: every field valid BUT mode
|
||||
target: ["group-order"],
|
||||
modelSolution: "Let |G| = p. Any x ≠ e generates a subgroup whose order divides p…",
|
||||
decisionPoints: ["pick an element and consider ⟨x⟩"],
|
||||
expectedWrongMoves: [
|
||||
{ move: "reaching for Sylow", resolution: "Sylow needs p² at least; order p is direct" },
|
||||
],
|
||||
strategy: "single-cue recall, minimal parse cost",
|
||||
rationale: "first problem of the session — baseline the cue",
|
||||
};
|
||||
// The raw shape is the incident: validation throws.
|
||||
assert.throws(() => {
|
||||
validateToolArguments({ name: "problem", parameters: problemInputObject } as never, {
|
||||
id: "call_x",
|
||||
type: "toolCall",
|
||||
name: "problem",
|
||||
arguments: raw,
|
||||
} as never);
|
||||
});
|
||||
const healed = healEncodedArguments(raw, problemInputObject);
|
||||
// The healed shape is valid — the loop never starts.
|
||||
const validated = validateToolArguments(
|
||||
{ name: "problem", parameters: problemInputObject } as never,
|
||||
{ id: "call_x", type: "toolCall", name: "problem", arguments: healed } as never,
|
||||
);
|
||||
assert.equal((validated as { mode: unknown }).mode, "recall");
|
||||
});
|
||||
|
||||
it("well-formed args pass through untouched — the SAME reference", () => {
|
||||
const args = { mode: "recall", target: ["a"], retest: false };
|
||||
assert.equal(healEncodedArguments(args, problemInputObject), args);
|
||||
assert.equal(healEncodedArguments(args, concludeInputObject), args);
|
||||
});
|
||||
|
||||
it("unhealable values stay — validation rejects them honestly", () => {
|
||||
const healed = healEncodedArguments({ mode: QUOTED("bogus") }, problemInputObject) as {
|
||||
mode: unknown;
|
||||
};
|
||||
assert.equal(healed.mode, QUOTED("bogus"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("healEncodedArguments — the sibling encodings", () => {
|
||||
it("heals the conclude enums (outcome) and the null rung", () => {
|
||||
const healed = healEncodedArguments(
|
||||
{ outcome: QUOTED("solved"), rung: "null" },
|
||||
concludeInputObject,
|
||||
) as { outcome: unknown; rung: unknown };
|
||||
assert.equal(healed.outcome, "solved");
|
||||
assert.equal(healed.rung, null);
|
||||
});
|
||||
|
||||
it("heals the rung enum and the stringified boolean retest", () => {
|
||||
const healed = healEncodedArguments(
|
||||
{ rung: QUOTED("R2"), retest: "true" },
|
||||
concludeInputObject,
|
||||
) as { rung: unknown; retest: unknown };
|
||||
assert.equal(healed.rung, "R2");
|
||||
assert.equal(healed.retest, true);
|
||||
});
|
||||
|
||||
it("heals a stringified array (the extraction-bug sibling)", () => {
|
||||
const healed = healEncodedArguments({ target: '["a","b"]' }, problemInputObject) as {
|
||||
target: unknown;
|
||||
};
|
||||
assert.deepEqual(healed.target, ["a", "b"]);
|
||||
});
|
||||
|
||||
it("leaves plain string properties alone (statement is a string schema)", () => {
|
||||
const args = { statement: '"not JSON-healed"' };
|
||||
const healed = healEncodedArguments(args, problemInputObject) as { statement: unknown };
|
||||
// A Type.String property has no literals and no scalar type match —
|
||||
// the string passes through as-is (same reference, nothing cloned).
|
||||
assert.equal(healed, args);
|
||||
});
|
||||
|
||||
it("passes non-object shapes through (null, arrays, strings)", () => {
|
||||
assert.equal(healEncodedArguments(null, problemInputObject), null);
|
||||
const arr = [1, 2];
|
||||
assert.equal(healEncodedArguments(arr, problemInputObject), arr);
|
||||
assert.equal(healEncodedArguments("mode", problemInputObject), "mode");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user