Files
learnharness/tests/tool-parts.test.ts
T
amosblomqvist 4ba594bab5 fix(arch): parse, don't cast — the UI parse seam for model-authored input
The architectural answer to the validation-crash question: validation IS
first at the model-facing boundary (pi validates arguments before
execute; failures return to the model same-turn — all three live
incidents retried successfully). But the same bytes also exist UNVALIDATED
by design in two other places: the live wire streams raw tool-input
deltas DURING generation, and the transcript keeps the raw input of
REJECTED calls (honesty + the replay invariant). The consumers were
supposed to trust part STATES — but that trust was enforced by
TypeScript casts ('part.input as X'), which are compile-time lies with
no runtime check behind them. That is the hole both crashes came
through, and it should not have been possible.

The structural rule now: model-authored INPUT is consumed only through
parsers running the SAME zod schema the model faced — boundary-valid ⇔
consumer-parseable, mechanically. Server/learner-authored OUTPUT stays
trusted (our code authors it).

- src/core/tool-parts.ts (new): parseQuizInput (display parse over the
  shared quizInputBase — strips the teacher-only nodeId, accepts both
  prober and teacher boundary shapes), parseAskInput, parseProbeDoneInput.
- contracts/probe.ts: probeDoneInputSchema is now core-owned zod — the
  probe_done tool declares it (single source of truth), the seam runs
  the literal same schema.
- question-dock.tsx: the dock parses its input at entry; unparsable
  renders the quiet malformed-question fallback, never a crash.
- probe-done-card.tsx: the input preview goes through the parse; the
  old state reasoning and shape guards collapse into it.

Pinned in tests/tool-parts.test.ts: both exact live crash shapes
(stringified quiz options, stringified assessment), boundary-parity
cases for all three schemas, nodeId stripping, mid-stream partials;
render-cards pins the dock fallback markup.

Gates: tsc, lint, build, 328/328.
2026-09-12 04:51:39 +03:00

122 lines
4.9 KiB
TypeScript

/**
* Unit tests — the UI parse seam (src/core/tool-parts.ts).
*
* The rule born of the 2026-09-12 live crashes: model-authored tool input
* is consumed only through these parsers, each running the SAME zod schema
* the model faced at the boundary. These pins hold the two exact live
* crash shapes (the model's double-encoding habit) and the boundary-parity
* property: anything a boundary schema accepts, the display parse accepts.
*/
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
parseAskInput,
parseProbeDoneInput,
parseQuizInput,
} from "../src/core/tool-parts.ts";
import { quizInputSchema, probeQuizInputSchema } from "../src/core/question-core.ts";
import { probeDoneInputSchema } from "../src/core/contracts/probe.ts";
const TEACHER_QUIZ = {
question: "What does Tarski's convention-T define?",
options: [
{ label: "Truth for a language", value: "truth" },
{ label: "Provability", value: "provability" },
],
correctAnswer: "truth",
explanation: "Convention-T schematizes truth for the object language.",
nodeId: "convention-t", // teacher-only field — the display parse must accept and strip it
};
const PROBER_QUIZ = {
question: "Which planet is closest to the sun?",
options: [{ label: "Venus" }, { label: "Mercury" }],
correctAnswer: "Mercury",
explanation: "Mercury is the innermost planet.",
};
const VALID_ASSESSMENT = {
assessment: {
strands: [
{ strand: "modal logic basics", assessment: "Floor: solid intuition; ceiling: notation." },
{ strand: "indexical 'actual'", assessment: "Genuine floor — prior exposure likely." },
],
summary: "Bracketed: teach the symbols on the intuition he has.",
},
};
describe("parseQuizInput", () => {
it("parses a boundary-valid TEACHER quiz and strips the teacher-only nodeId", () => {
const parsed = parseQuizInput(TEACHER_QUIZ);
assert.ok(parsed);
assert.equal(parsed.question, TEACHER_QUIZ.question);
assert.equal((parsed as Record<string, unknown>).nodeId, undefined, "unknown keys are stripped, not trusted");
});
it("parses a boundary-valid PROBER quiz (no nodeId at all)", () => {
assert.ok(parseQuizInput(PROBER_QUIZ));
});
it("boundary parity: anything the TEACH schema accepts, the display parse accepts", () => {
const boundary = quizInputSchema.safeParse(TEACHER_QUIZ);
assert.ok(boundary.success, "fixture must be boundary-valid for this pin to mean anything");
assert.ok(parseQuizInput(boundary.data));
});
it("boundary parity: anything the PROBE schema accepts, the display parse accepts", () => {
const boundary = probeQuizInputSchema.safeParse(PROBER_QUIZ);
assert.ok(boundary.success);
assert.ok(parseQuizInput(boundary.data));
});
it("the live crash shape — options as a JSON string — does not parse", () => {
const stringified = { ...PROBER_QUIZ, options: JSON.stringify(PROBER_QUIZ.options) };
assert.equal(parseQuizInput(stringified), undefined);
});
it("garbage never parses: non-object, missing question, missing correctAnswer", () => {
assert.equal(parseQuizInput("a string"), undefined);
assert.equal(parseQuizInput({ options: [] }), undefined);
assert.equal(parseQuizInput({ ...PROBER_QUIZ, correctAnswer: undefined }), undefined);
});
});
describe("parseAskInput", () => {
it("parses free-text mode (no options) and option mode", () => {
assert.ok(parseAskInput({ question: "What do you want to learn?" }));
assert.ok(parseAskInput({ question: "Which topics?", options: [{ label: "Physics" }, { label: "Logic" }], multiSelect: true }));
});
it("the live crash shape — stringified options — does not parse", () => {
assert.equal(parseAskInput({ question: "q", options: "[]" }), undefined);
});
});
describe("parseProbeDoneInput", () => {
it("parses the valid assessment", () => {
const parsed = parseProbeDoneInput(VALID_ASSESSMENT);
assert.ok(parsed);
assert.equal(parsed.assessment.strands.length, 2);
assert.equal(parsed.assessment.summary, "Bracketed: teach the symbols on the intuition he has.");
});
it("boundary parity: anything the tool schema accepts, the display parse accepts", () => {
const boundary = probeDoneInputSchema.safeParse(VALID_ASSESSMENT);
assert.ok(boundary.success);
assert.ok(parseProbeDoneInput(boundary.data));
});
it("the live crash shape — assessment STRINGIFIED — does not parse", () => {
// The exact double-encoding from the modal-realism run: strands died
// on `.map` because the card cast the shape. The parse kills the class.
const stringified = { assessment: JSON.stringify(VALID_ASSESSMENT.assessment) };
assert.equal(parseProbeDoneInput(stringified), undefined);
const garbage = JSON.stringify(VALID_ASSESSMENT.assessment).slice(0, 30);
assert.equal(parseProbeDoneInput(garbage), undefined);
});
it("a mid-stream partial (no strands yet) does not parse", () => {
assert.equal(parseProbeDoneInput({ assessment: { summary: "…" } }), undefined);
});
});