fix(arch): parse, don't cast — the UI parse seam for model-authored input

The architectural answer to the validation-crash question: validation IS
first at the model-facing boundary (pi validates arguments before
execute; failures return to the model same-turn — all three live
incidents retried successfully). But the same bytes also exist UNVALIDATED
by design in two other places: the live wire streams raw tool-input
deltas DURING generation, and the transcript keeps the raw input of
REJECTED calls (honesty + the replay invariant). The consumers were
supposed to trust part STATES — but that trust was enforced by
TypeScript casts ('part.input as X'), which are compile-time lies with
no runtime check behind them. That is the hole both crashes came
through, and it should not have been possible.

The structural rule now: model-authored INPUT is consumed only through
parsers running the SAME zod schema the model faced — boundary-valid ⇔
consumer-parseable, mechanically. Server/learner-authored OUTPUT stays
trusted (our code authors it).

- src/core/tool-parts.ts (new): parseQuizInput (display parse over the
  shared quizInputBase — strips the teacher-only nodeId, accepts both
  prober and teacher boundary shapes), parseAskInput, parseProbeDoneInput.
- contracts/probe.ts: probeDoneInputSchema is now core-owned zod — the
  probe_done tool declares it (single source of truth), the seam runs
  the literal same schema.
- question-dock.tsx: the dock parses its input at entry; unparsable
  renders the quiet malformed-question fallback, never a crash.
- probe-done-card.tsx: the input preview goes through the parse; the
  old state reasoning and shape guards collapse into it.

Pinned in tests/tool-parts.test.ts: both exact live crash shapes
(stringified quiz options, stringified assessment), boundary-parity
cases for all three schemas, nodeId stripping, mid-stream partials;
render-cards pins the dock fallback markup.

Gates: tsc, lint, build, 328/328.
This commit is contained in:
amosblomqvist
2026-09-12 04:51:39 +03:00
parent e821c3277d
commit 4ba594bab5
8 changed files with 274 additions and 36 deletions
+10 -8
View File
@@ -1,7 +1,7 @@
"use client"; "use client";
import { ClipboardCheckIcon } from "lucide-react"; import { ClipboardCheckIcon } from "lucide-react";
import type { ProbeDoneResult, ProbeDoneInput } from "@/src/core"; import { parseProbeDoneInput, type ProbeDoneResult } from "@/src/core";
/** /**
* The probe conclusion card (docs/probe-teach-split.md §2): renders the * The probe conclusion card (docs/probe-teach-split.md §2): renders the
@@ -22,16 +22,18 @@ import type { ProbeDoneResult, ProbeDoneInput } from "@/src/core";
* the UI wave; this card just guarantees the data is never invisible. * the UI wave; this card just guarantees the data is never invisible.
*/ */
export function ProbeDoneCard({ part }: { part: { input?: unknown; output?: unknown; state?: unknown } }) { export function ProbeDoneCard({ part }: { part: { input?: unknown; output?: unknown; state?: unknown } }) {
const input = part.input as ProbeDoneInput | undefined;
const output = part.output as ProbeDoneResult | undefined; const output = part.output as ProbeDoneResult | undefined;
const state = typeof part.state === "string" ? part.state : ""; const state = typeof part.state === "string" ? part.state : "";
const assessment = // Parse, don't cast (the seam rule, src/core/tool-parts.ts): the model
output?.ok // authored this input — it is consumer-trustworthy only if the SAME
? output.assessment // schema the tool declared says so. The parse subsumes the old state
: state === "output-error" // reasoning and shape guards: an errored call's stringified assessment,
? null // the rejected call — its input is unvalidated garbage // a mid-stream partial — anything that doesn't parse falls to the
: (input?.assessment ?? null); // fallback branch. Output stays a cast: OUR execute authored it.
const input = parseProbeDoneInput(part.input);
const assessment = output?.ok ? output.assessment : input?.assessment;
const strands = assessment && Array.isArray(assessment.strands) ? assessment.strands : null; const strands = assessment && Array.isArray(assessment.strands) ? assessment.strands : null;
return ( return (
+18 -2
View File
@@ -13,6 +13,8 @@ import {
resolveCorrect, resolveCorrect,
seededRandom, seededRandom,
shuffleOptions, shuffleOptions,
parseAskInput,
parseQuizInput,
toggleOption, toggleOption,
toggleOther, toggleOther,
type AnswerRequest, type AnswerRequest,
@@ -76,12 +78,26 @@ export function QuestionDock({
onSubmit, onSubmit,
phase = "probe", phase = "probe",
}: QuestionDockProps) { }: QuestionDockProps) {
// Parse, don't cast (src/core/tool-parts.ts): the model authored this
// input, and only the schema the tool declared vouches for its shape.
// The dock mounts on input-available parts (already validated at the
// bridge), so this is pure defense in depth — but it is STRUCTURAL now:
// anything that slips through renders the quiet fallback, never a
// crash on `.map` over model-authored bytes.
const parsed = toolName === "quiz" ? parseQuizInput(input) : parseAskInput(input);
if (!parsed) {
return (
<div data-testid="malformed-question" className="rounded-xl border border-dashed bg-card/60 p-3 text-sm text-muted-foreground">
This question arrived malformed — the transcript keeps the retry.
</div>
);
}
return ( return (
<> <>
{!expanded && ( {!expanded && (
<QuestionDockBar <QuestionDockBar
toolName={toolName} toolName={toolName}
question={input.question} question={parsed.question}
onExpand={onExpand} onExpand={onExpand}
variant={reveal ? "reveal" : "collapsed"} variant={reveal ? "reveal" : "collapsed"}
/> />
@@ -90,7 +106,7 @@ export function QuestionDock({
<QuestionDockForm <QuestionDockForm
toolName={toolName} toolName={toolName}
toolCallId={toolCallId} toolCallId={toolCallId}
input={input} input={parsed}
phase={phase} phase={phase}
onSubmit={onSubmit} onSubmit={onSubmit}
/> />
+4 -26
View File
@@ -1,6 +1,5 @@
import { z } from "zod";
import { lhTool, type LhTool } from "../lh-tool.ts"; import { lhTool, type LhTool } from "../lh-tool.ts";
import type { ProbeDoneInput, ProbeDoneResult } from "../../core/contracts.ts"; import { probeDoneInputSchema, type ProbeDoneInput, type ProbeDoneResult } from "../../core/contracts.ts";
import type { UIMessage } from "ai"; import type { UIMessage } from "ai";
import { findPendingToolCall } from "../../core/pending.ts"; import { findPendingToolCall } from "../../core/pending.ts";
import type { ChatStore } from "../../core/ports.ts"; import type { ChatStore } from "../../core/ports.ts";
@@ -56,30 +55,9 @@ export function probeDoneTool(ctx: ProbeDoneContext): Record<string, LhTool<any,
"(floor, ceiling, misconceptions, what to build from). It streams to the learner live and becomes " + "(floor, ceiling, misconceptions, what to build from). It streams to the learner live and becomes " +
"the handoff artifact the teacher plans from. Refused while a question is still open, and after " + "the handoff artifact the teacher plans from. Refused while a question is still open, and after " +
"the probe is already concluded. After it succeeds, say your closing sentence and end your turn.", "the probe is already concluded. After it succeeds, say your closing sentence and end your turn.",
schema: z.object({ // The core-owned schema (contracts/probe.ts) — the SAME schema the UI
assessment: z // parse seam (tool-parts.ts) runs, so boundary-valid ⇔ consumer-parseable.
.object({ schema: probeDoneInputSchema,
strands: z
.array(
z.object({
strand: z.string().describe("The dependency strand (e.g. 'the derivative as a limit') — a name the plan can hang a node on."),
assessment: z
.string()
.describe(
"Where his edge sits on this strand: the floor (what he reliably gets right), the ceiling (where it runs out), " +
"misconceptions found, and what the lesson should build from.",
),
}),
)
.min(1)
.describe("Every strand the planned lesson will depend on, each with your read of his edge on it."),
summary: z
.string()
.optional()
.describe("Optional overall synthesis: his apparent goal and where the teaching should enter."),
})
.describe("The learner assessment — the probe's distilled output."),
}) satisfies z.ZodType<ProbeDoneInput>,
execute: async (input): Promise<ProbeDoneResult> => { execute: async (input): Promise<ProbeDoneResult> => {
const record = ctx.chatStore.getChat(ctx.chatId); const record = ctx.chatStore.getChat(ctx.chatId);
if (!record) return { ok: false, error: `Chat not found: ${ctx.chatId}` }; if (!record) return { ok: false, error: `Chat not found: ${ctx.chatId}` };
+38
View File
@@ -15,6 +15,8 @@
* the learner's edge sits. * the learner's edge sits.
*/ */
import { z } from "zod";
/** One dependency strand the planned lesson will rest on. */ /** One dependency strand the planned lesson will rest on. */
export interface ProbeStrand { export interface ProbeStrand {
/** Name/description of the strand (e.g. "the derivative as a limit"). */ /** Name/description of the strand (e.g. "the derivative as a limit"). */
@@ -36,6 +38,42 @@ export interface ProbeDoneInput {
assessment: ProbeAssessment; assessment: ProbeAssessment;
} }
// The zod schema of that input — SINGLE SOURCE OF TRUTH, lived by the tool
// (src/agent/tools/probe.ts) and by the UI parse seam (src/core/tool-parts.ts):
// the boundary validation and the consumer parse run literally the same
// schema, so "schema-valid at the boundary" ⇔ "parseable by a consumer" is
// structural, not a convention. (Live crash 2026-09-12: the model sent
// `assessment` as a STRING; the card cast it and died on `.strands.map`.)
export const probeDoneInputSchema = z
.object({
assessment: z
.object({
strands: z
.array(
z.object({
strand: z
.string()
.describe(
"The dependency strand (e.g. 'the derivative as a limit') — a name the plan can hang a node on.",
),
assessment: z
.string()
.describe(
"Where his edge sits on this strand: the floor (what he reliably gets right), the ceiling (where it runs out), " +
"misconceptions found, and what the lesson should build from.",
),
}),
)
.min(1)
.describe("Every strand the planned lesson will depend on, each with your read of his edge on it."),
summary: z
.string()
.optional()
.describe("Optional overall synthesis: his apparent goal and where the teaching should enter."),
})
.describe("The learner assessment — the probe's distilled output."),
}) satisfies z.ZodType<ProbeDoneInput>;
/** probe_done tool output. */ /** probe_done tool output. */
export interface ProbeDoneResult { export interface ProbeDoneResult {
ok: boolean; ok: boolean;
+1
View File
@@ -24,4 +24,5 @@ export * from "./phases.ts";
export * from "./pending.ts"; export * from "./pending.ts";
export * from "./math-delims.ts"; export * from "./math-delims.ts";
export * from "./ui-message.ts"; export * from "./ui-message.ts";
export * from "./tool-parts.ts";
export * from "./prices.ts"; export * from "./prices.ts";
+68
View File
@@ -0,0 +1,68 @@
/**
* The UI parse seam — parse, don't cast (the rule born of the 2026-09-12
* live crashes).
*
* WHY THIS EXISTS. Validation IS first at the model-facing boundary — pi
* validates arguments before execute, and our zod re-run adds the
* refinements; a failure returns to the model as an error tool result,
* same turn. But the same bytes have two other consumers, and there they
* exist UNVALIDATED BY DESIGN:
*
* - the live wire streams raw tool-input deltas while the model is still
* typing the call (validation cannot run before the call is complete);
* - the transcript keeps the raw input of REJECTED calls too (transcript
* honesty + the replay invariant: every toolCall followed by a result).
*
* So invalid input is always observable, and part STATES carry the verdict
* (input-streaming = partial, input-available = validated, output-error =
* garbage). The flaw that made the crashes possible: consumers trusted
* those shapes via TypeScript CASTS — `part.input as X` is a compile-time
* lie the checker believes, with no runtime check behind it.
*
* THE RULE (mechanical, not vigilance): model-authored INPUT is consumed
* only through these parsers, each running the SAME zod schema the model
* faced — boundary-valid ⇔ consumer-parseable, structurally. Server- and
* learner-authored OUTPUT stays trusted (our own code authors it: the
* execute acks, the learner's buildAnswer). A parse failure renders the
* quiet fallback, never a crash.
*/
import { z } from "zod";
import { quizInputBase, askInputSchema, type AskToolInput, type QuizToolInput } from "./question-core.ts";
import { probeDoneInputSchema, type ProbeDoneInput } from "./contracts/probe.ts";
/**
* Quiz parse for DISPLAY: the shared authoring base, pre-refinement.
* Both boundary schemas (the prober's probeQuizInputSchema over the bare
* base, the teacher's quizInputSchema over base + optional nodeId) refine
* THE SAME quizInputBase, so anything that passed either boundary parses
* here: refinements don't weaken on re-run, and the teacher-only `nodeId`
* is an unknown key the base strips. This is deliberately the LENIENT
* common core — the strict per-agent refinements are the boundary's job.
*/
export function parseQuizInput(input: unknown): QuizToolInput | undefined {
const parsed = quizInputBase.safeParse(input);
return parsed.success ? (parsed.data as QuizToolInput) : undefined;
}
/**
* Ask parse for DISPLAY: the full ask schema (options optional — free-text
* mode is the options-less shape). The only refinement is option sanity
* (distinct values), which the boundary already enforced.
*/
export function parseAskInput(input: unknown): AskToolInput | undefined {
const parsed = askInputSchema.safeParse(input);
return parsed.success ? parsed.data : undefined;
}
/** probe_done parse — the core-owned schema the tool itself declares. */
export function parseProbeDoneInput(input: unknown): ProbeDoneInput | undefined {
const parsed = probeDoneInputSchema.safeParse(input);
return parsed.success ? parsed.data : undefined;
}
/** Parse helper for schemas this seam doesn't own (kept for future cards). */
export function parseWith<T>(schema: z.ZodType<T>, input: unknown): T | undefined {
const parsed = schema.safeParse(input);
return parsed.success ? parsed.data : undefined;
}
+13
View File
@@ -120,6 +120,19 @@ describe("render smoke tests", () => {
assert.ok(html.includes('data-testid="other-option"'), "Other is still an option row in multi-select"); assert.ok(html.includes('data-testid="other-option"'), "Other is still an option row in multi-select");
}); });
it("a malformed question input renders the quiet fallback, never a crash", () => {
// The seam rule (tool-parts.ts): the dock PARSES the model-authored
// input instead of casting it — the live crash shape (options as a
// JSON string) must render the fallback, not die on .map.
const html = renderDock({
toolName: "quiz",
toolCallId: "bad",
input: { question: "q?", options: JSON.stringify([{ label: "a" }, { label: "b" }]) } as never,
});
assert.ok(html.includes('data-testid="malformed-question"'));
assert.match(html, /malformed/);
});
it("phase scales the thinking field: teaching invites depth, probing stays quick", () => { it("phase scales the thinking field: teaching invites depth, probing stays quick", () => {
const teach = renderToStaticMarkup( const teach = renderToStaticMarkup(
<QuestionDock <QuestionDock
+122
View File
@@ -0,0 +1,122 @@
/**
* Unit tests — the UI parse seam (src/core/tool-parts.ts).
*
* The rule born of the 2026-09-12 live crashes: model-authored tool input
* is consumed only through these parsers, each running the SAME zod schema
* the model faced at the boundary. These pins hold the two exact live
* crash shapes (the model's double-encoding habit) and the boundary-parity
* property: anything a boundary schema accepts, the display parse accepts.
*/
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
parseAskInput,
parseProbeDoneInput,
parseQuizInput,
} from "../src/core/tool-parts.ts";
import { quizInputSchema, probeQuizInputSchema } from "../src/core/question-core.ts";
import { probeDoneInputSchema } from "../src/core/contracts/probe.ts";
const TEACHER_QUIZ = {
question: "What does Tarski's convention-T define?",
options: [
{ label: "Truth for a language", value: "truth" },
{ label: "Provability", value: "provability" },
],
correctAnswer: "truth",
explanation: "Convention-T schematizes truth for the object language.",
nodeId: "convention-t", // teacher-only field — the display parse must accept and strip it
};
const PROBER_QUIZ = {
question: "Which planet is closest to the sun?",
options: [{ label: "Venus" }, { label: "Mercury" }],
correctAnswer: "Mercury",
explanation: "Mercury is the innermost planet.",
};
const VALID_ASSESSMENT = {
assessment: {
strands: [
{ strand: "modal logic basics", assessment: "Floor: solid intuition; ceiling: notation." },
{ strand: "indexical 'actual'", assessment: "Genuine floor — prior exposure likely." },
],
summary: "Bracketed: teach the symbols on the intuition he has.",
},
};
describe("parseQuizInput", () => {
it("parses a boundary-valid TEACHER quiz and strips the teacher-only nodeId", () => {
const parsed = parseQuizInput(TEACHER_QUIZ);
assert.ok(parsed);
assert.equal(parsed.question, TEACHER_QUIZ.question);
assert.equal((parsed as Record<string, unknown>).nodeId, undefined, "unknown keys are stripped, not trusted");
});
it("parses a boundary-valid PROBER quiz (no nodeId at all)", () => {
assert.ok(parseQuizInput(PROBER_QUIZ));
});
it("boundary parity: anything the TEACH schema accepts, the display parse accepts", () => {
const boundary = quizInputSchema.safeParse(TEACHER_QUIZ);
assert.ok(boundary.success, "fixture must be boundary-valid for this pin to mean anything");
assert.ok(parseQuizInput(boundary.data));
});
it("boundary parity: anything the PROBE schema accepts, the display parse accepts", () => {
const boundary = probeQuizInputSchema.safeParse(PROBER_QUIZ);
assert.ok(boundary.success);
assert.ok(parseQuizInput(boundary.data));
});
it("the live crash shape — options as a JSON string — does not parse", () => {
const stringified = { ...PROBER_QUIZ, options: JSON.stringify(PROBER_QUIZ.options) };
assert.equal(parseQuizInput(stringified), undefined);
});
it("garbage never parses: non-object, missing question, missing correctAnswer", () => {
assert.equal(parseQuizInput("a string"), undefined);
assert.equal(parseQuizInput({ options: [] }), undefined);
assert.equal(parseQuizInput({ ...PROBER_QUIZ, correctAnswer: undefined }), undefined);
});
});
describe("parseAskInput", () => {
it("parses free-text mode (no options) and option mode", () => {
assert.ok(parseAskInput({ question: "What do you want to learn?" }));
assert.ok(parseAskInput({ question: "Which topics?", options: [{ label: "Physics" }, { label: "Logic" }], multiSelect: true }));
});
it("the live crash shape — stringified options — does not parse", () => {
assert.equal(parseAskInput({ question: "q", options: "[]" }), undefined);
});
});
describe("parseProbeDoneInput", () => {
it("parses the valid assessment", () => {
const parsed = parseProbeDoneInput(VALID_ASSESSMENT);
assert.ok(parsed);
assert.equal(parsed.assessment.strands.length, 2);
assert.equal(parsed.assessment.summary, "Bracketed: teach the symbols on the intuition he has.");
});
it("boundary parity: anything the tool schema accepts, the display parse accepts", () => {
const boundary = probeDoneInputSchema.safeParse(VALID_ASSESSMENT);
assert.ok(boundary.success);
assert.ok(parseProbeDoneInput(boundary.data));
});
it("the live crash shape — assessment STRINGIFIED — does not parse", () => {
// The exact double-encoding from the modal-realism run: strands died
// on `.map` because the card cast the shape. The parse kills the class.
const stringified = { assessment: JSON.stringify(VALID_ASSESSMENT.assessment) };
assert.equal(parseProbeDoneInput(stringified), undefined);
const garbage = JSON.stringify(VALID_ASSESSMENT.assessment).slice(0, 30);
assert.equal(parseProbeDoneInput(garbage), undefined);
});
it("a mid-stream partial (no strands yet) does not parse", () => {
assert.equal(parseProbeDoneInput({ assessment: { summary: "…" } }), undefined);
});
});