fix(arch): parse, don't cast — the UI parse seam for model-authored input
The architectural answer to the validation-crash question: validation IS
first at the model-facing boundary (pi validates arguments before
execute; failures return to the model same-turn — all three live
incidents retried successfully). But the same bytes also exist UNVALIDATED
by design in two other places: the live wire streams raw tool-input
deltas DURING generation, and the transcript keeps the raw input of
REJECTED calls (honesty + the replay invariant). The consumers were
supposed to trust part STATES — but that trust was enforced by
TypeScript casts ('part.input as X'), which are compile-time lies with
no runtime check behind them. That is the hole both crashes came
through, and it should not have been possible.
The structural rule now: model-authored INPUT is consumed only through
parsers running the SAME zod schema the model faced — boundary-valid ⇔
consumer-parseable, mechanically. Server/learner-authored OUTPUT stays
trusted (our code authors it).
- src/core/tool-parts.ts (new): parseQuizInput (display parse over the
shared quizInputBase — strips the teacher-only nodeId, accepts both
prober and teacher boundary shapes), parseAskInput, parseProbeDoneInput.
- contracts/probe.ts: probeDoneInputSchema is now core-owned zod — the
probe_done tool declares it (single source of truth), the seam runs
the literal same schema.
- question-dock.tsx: the dock parses its input at entry; unparsable
renders the quiet malformed-question fallback, never a crash.
- probe-done-card.tsx: the input preview goes through the parse; the
old state reasoning and shape guards collapse into it.
Pinned in tests/tool-parts.test.ts: both exact live crash shapes
(stringified quiz options, stringified assessment), boundary-parity
cases for all three schemas, nodeId stripping, mid-stream partials;
render-cards pins the dock fallback markup.
Gates: tsc, lint, build, 328/328.
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
"use client";
|
||||
|
||||
import { ClipboardCheckIcon } from "lucide-react";
|
||||
import type { ProbeDoneResult, ProbeDoneInput } from "@/src/core";
|
||||
import { parseProbeDoneInput, type ProbeDoneResult } from "@/src/core";
|
||||
|
||||
/**
|
||||
* The probe conclusion card (docs/probe-teach-split.md §2): renders the
|
||||
@@ -22,16 +22,18 @@ import type { ProbeDoneResult, ProbeDoneInput } from "@/src/core";
|
||||
* the UI wave; this card just guarantees the data is never invisible.
|
||||
*/
|
||||
export function ProbeDoneCard({ part }: { part: { input?: unknown; output?: unknown; state?: unknown } }) {
|
||||
const input = part.input as ProbeDoneInput | undefined;
|
||||
const output = part.output as ProbeDoneResult | undefined;
|
||||
const state = typeof part.state === "string" ? part.state : "";
|
||||
|
||||
const assessment =
|
||||
output?.ok
|
||||
? output.assessment
|
||||
: state === "output-error"
|
||||
? null // the rejected call — its input is unvalidated garbage
|
||||
: (input?.assessment ?? null);
|
||||
// Parse, don't cast (the seam rule, src/core/tool-parts.ts): the model
|
||||
// authored this input — it is consumer-trustworthy only if the SAME
|
||||
// schema the tool declared says so. The parse subsumes the old state
|
||||
// reasoning and shape guards: an errored call's stringified assessment,
|
||||
// a mid-stream partial — anything that doesn't parse falls to the
|
||||
// fallback branch. Output stays a cast: OUR execute authored it.
|
||||
const input = parseProbeDoneInput(part.input);
|
||||
|
||||
const assessment = output?.ok ? output.assessment : input?.assessment;
|
||||
const strands = assessment && Array.isArray(assessment.strands) ? assessment.strands : null;
|
||||
|
||||
return (
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
resolveCorrect,
|
||||
seededRandom,
|
||||
shuffleOptions,
|
||||
parseAskInput,
|
||||
parseQuizInput,
|
||||
toggleOption,
|
||||
toggleOther,
|
||||
type AnswerRequest,
|
||||
@@ -76,12 +78,26 @@ export function QuestionDock({
|
||||
onSubmit,
|
||||
phase = "probe",
|
||||
}: QuestionDockProps) {
|
||||
// Parse, don't cast (src/core/tool-parts.ts): the model authored this
|
||||
// input, and only the schema the tool declared vouches for its shape.
|
||||
// The dock mounts on input-available parts (already validated at the
|
||||
// bridge), so this is pure defense in depth — but it is STRUCTURAL now:
|
||||
// anything that slips through renders the quiet fallback, never a
|
||||
// crash on `.map` over model-authored bytes.
|
||||
const parsed = toolName === "quiz" ? parseQuizInput(input) : parseAskInput(input);
|
||||
if (!parsed) {
|
||||
return (
|
||||
<div data-testid="malformed-question" className="rounded-xl border border-dashed bg-card/60 p-3 text-sm text-muted-foreground">
|
||||
This question arrived malformed — the transcript keeps the retry.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<>
|
||||
{!expanded && (
|
||||
<QuestionDockBar
|
||||
toolName={toolName}
|
||||
question={input.question}
|
||||
question={parsed.question}
|
||||
onExpand={onExpand}
|
||||
variant={reveal ? "reveal" : "collapsed"}
|
||||
/>
|
||||
@@ -90,7 +106,7 @@ export function QuestionDock({
|
||||
<QuestionDockForm
|
||||
toolName={toolName}
|
||||
toolCallId={toolCallId}
|
||||
input={input}
|
||||
input={parsed}
|
||||
phase={phase}
|
||||
onSubmit={onSubmit}
|
||||
/>
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { z } from "zod";
|
||||
import { lhTool, type LhTool } from "../lh-tool.ts";
|
||||
import type { ProbeDoneInput, ProbeDoneResult } from "../../core/contracts.ts";
|
||||
import { probeDoneInputSchema, type ProbeDoneInput, type ProbeDoneResult } from "../../core/contracts.ts";
|
||||
import type { UIMessage } from "ai";
|
||||
import { findPendingToolCall } from "../../core/pending.ts";
|
||||
import type { ChatStore } from "../../core/ports.ts";
|
||||
@@ -56,30 +55,9 @@ export function probeDoneTool(ctx: ProbeDoneContext): Record<string, LhTool<any,
|
||||
"(floor, ceiling, misconceptions, what to build from). It streams to the learner live and becomes " +
|
||||
"the handoff artifact the teacher plans from. Refused while a question is still open, and after " +
|
||||
"the probe is already concluded. After it succeeds, say your closing sentence and end your turn.",
|
||||
schema: z.object({
|
||||
assessment: z
|
||||
.object({
|
||||
strands: z
|
||||
.array(
|
||||
z.object({
|
||||
strand: z.string().describe("The dependency strand (e.g. 'the derivative as a limit') — a name the plan can hang a node on."),
|
||||
assessment: z
|
||||
.string()
|
||||
.describe(
|
||||
"Where his edge sits on this strand: the floor (what he reliably gets right), the ceiling (where it runs out), " +
|
||||
"misconceptions found, and what the lesson should build from.",
|
||||
),
|
||||
}),
|
||||
)
|
||||
.min(1)
|
||||
.describe("Every strand the planned lesson will depend on, each with your read of his edge on it."),
|
||||
summary: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Optional overall synthesis: his apparent goal and where the teaching should enter."),
|
||||
})
|
||||
.describe("The learner assessment — the probe's distilled output."),
|
||||
}) satisfies z.ZodType<ProbeDoneInput>,
|
||||
// The core-owned schema (contracts/probe.ts) — the SAME schema the UI
|
||||
// parse seam (tool-parts.ts) runs, so boundary-valid ⇔ consumer-parseable.
|
||||
schema: probeDoneInputSchema,
|
||||
execute: async (input): Promise<ProbeDoneResult> => {
|
||||
const record = ctx.chatStore.getChat(ctx.chatId);
|
||||
if (!record) return { ok: false, error: `Chat not found: ${ctx.chatId}` };
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
* the learner's edge sits.
|
||||
*/
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
/** One dependency strand the planned lesson will rest on. */
|
||||
export interface ProbeStrand {
|
||||
/** Name/description of the strand (e.g. "the derivative as a limit"). */
|
||||
@@ -36,6 +38,42 @@ export interface ProbeDoneInput {
|
||||
assessment: ProbeAssessment;
|
||||
}
|
||||
|
||||
// The zod schema of that input — SINGLE SOURCE OF TRUTH, lived by the tool
|
||||
// (src/agent/tools/probe.ts) and by the UI parse seam (src/core/tool-parts.ts):
|
||||
// the boundary validation and the consumer parse run literally the same
|
||||
// schema, so "schema-valid at the boundary" ⇔ "parseable by a consumer" is
|
||||
// structural, not a convention. (Live crash 2026-09-12: the model sent
|
||||
// `assessment` as a STRING; the card cast it and died on `.strands.map`.)
|
||||
export const probeDoneInputSchema = z
|
||||
.object({
|
||||
assessment: z
|
||||
.object({
|
||||
strands: z
|
||||
.array(
|
||||
z.object({
|
||||
strand: z
|
||||
.string()
|
||||
.describe(
|
||||
"The dependency strand (e.g. 'the derivative as a limit') — a name the plan can hang a node on.",
|
||||
),
|
||||
assessment: z
|
||||
.string()
|
||||
.describe(
|
||||
"Where his edge sits on this strand: the floor (what he reliably gets right), the ceiling (where it runs out), " +
|
||||
"misconceptions found, and what the lesson should build from.",
|
||||
),
|
||||
}),
|
||||
)
|
||||
.min(1)
|
||||
.describe("Every strand the planned lesson will depend on, each with your read of his edge on it."),
|
||||
summary: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Optional overall synthesis: his apparent goal and where the teaching should enter."),
|
||||
})
|
||||
.describe("The learner assessment — the probe's distilled output."),
|
||||
}) satisfies z.ZodType<ProbeDoneInput>;
|
||||
|
||||
/** probe_done tool output. */
|
||||
export interface ProbeDoneResult {
|
||||
ok: boolean;
|
||||
|
||||
@@ -24,4 +24,5 @@ export * from "./phases.ts";
|
||||
export * from "./pending.ts";
|
||||
export * from "./math-delims.ts";
|
||||
export * from "./ui-message.ts";
|
||||
export * from "./tool-parts.ts";
|
||||
export * from "./prices.ts";
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* The UI parse seam — parse, don't cast (the rule born of the 2026-09-12
|
||||
* live crashes).
|
||||
*
|
||||
* WHY THIS EXISTS. Validation IS first at the model-facing boundary — pi
|
||||
* validates arguments before execute, and our zod re-run adds the
|
||||
* refinements; a failure returns to the model as an error tool result,
|
||||
* same turn. But the same bytes have two other consumers, and there they
|
||||
* exist UNVALIDATED BY DESIGN:
|
||||
*
|
||||
* - the live wire streams raw tool-input deltas while the model is still
|
||||
* typing the call (validation cannot run before the call is complete);
|
||||
* - the transcript keeps the raw input of REJECTED calls too (transcript
|
||||
* honesty + the replay invariant: every toolCall followed by a result).
|
||||
*
|
||||
* So invalid input is always observable, and part STATES carry the verdict
|
||||
* (input-streaming = partial, input-available = validated, output-error =
|
||||
* garbage). The flaw that made the crashes possible: consumers trusted
|
||||
* those shapes via TypeScript CASTS — `part.input as X` is a compile-time
|
||||
* lie the checker believes, with no runtime check behind it.
|
||||
*
|
||||
* THE RULE (mechanical, not vigilance): model-authored INPUT is consumed
|
||||
* only through these parsers, each running the SAME zod schema the model
|
||||
* faced — boundary-valid ⇔ consumer-parseable, structurally. Server- and
|
||||
* learner-authored OUTPUT stays trusted (our own code authors it: the
|
||||
* execute acks, the learner's buildAnswer). A parse failure renders the
|
||||
* quiet fallback, never a crash.
|
||||
*/
|
||||
|
||||
import { z } from "zod";
|
||||
import { quizInputBase, askInputSchema, type AskToolInput, type QuizToolInput } from "./question-core.ts";
|
||||
import { probeDoneInputSchema, type ProbeDoneInput } from "./contracts/probe.ts";
|
||||
|
||||
/**
|
||||
* Quiz parse for DISPLAY: the shared authoring base, pre-refinement.
|
||||
* Both boundary schemas (the prober's probeQuizInputSchema over the bare
|
||||
* base, the teacher's quizInputSchema over base + optional nodeId) refine
|
||||
* THE SAME quizInputBase, so anything that passed either boundary parses
|
||||
* here: refinements don't weaken on re-run, and the teacher-only `nodeId`
|
||||
* is an unknown key the base strips. This is deliberately the LENIENT
|
||||
* common core — the strict per-agent refinements are the boundary's job.
|
||||
*/
|
||||
export function parseQuizInput(input: unknown): QuizToolInput | undefined {
|
||||
const parsed = quizInputBase.safeParse(input);
|
||||
return parsed.success ? (parsed.data as QuizToolInput) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask parse for DISPLAY: the full ask schema (options optional — free-text
|
||||
* mode is the options-less shape). The only refinement is option sanity
|
||||
* (distinct values), which the boundary already enforced.
|
||||
*/
|
||||
export function parseAskInput(input: unknown): AskToolInput | undefined {
|
||||
const parsed = askInputSchema.safeParse(input);
|
||||
return parsed.success ? parsed.data : undefined;
|
||||
}
|
||||
|
||||
/** probe_done parse — the core-owned schema the tool itself declares. */
|
||||
export function parseProbeDoneInput(input: unknown): ProbeDoneInput | undefined {
|
||||
const parsed = probeDoneInputSchema.safeParse(input);
|
||||
return parsed.success ? parsed.data : undefined;
|
||||
}
|
||||
|
||||
/** Parse helper for schemas this seam doesn't own (kept for future cards). */
|
||||
export function parseWith<T>(schema: z.ZodType<T>, input: unknown): T | undefined {
|
||||
const parsed = schema.safeParse(input);
|
||||
return parsed.success ? parsed.data : undefined;
|
||||
}
|
||||
@@ -120,6 +120,19 @@ describe("render smoke tests", () => {
|
||||
assert.ok(html.includes('data-testid="other-option"'), "Other is still an option row in multi-select");
|
||||
});
|
||||
|
||||
it("a malformed question input renders the quiet fallback, never a crash", () => {
|
||||
// The seam rule (tool-parts.ts): the dock PARSES the model-authored
|
||||
// input instead of casting it — the live crash shape (options as a
|
||||
// JSON string) must render the fallback, not die on .map.
|
||||
const html = renderDock({
|
||||
toolName: "quiz",
|
||||
toolCallId: "bad",
|
||||
input: { question: "q?", options: JSON.stringify([{ label: "a" }, { label: "b" }]) } as never,
|
||||
});
|
||||
assert.ok(html.includes('data-testid="malformed-question"'));
|
||||
assert.match(html, /malformed/);
|
||||
});
|
||||
|
||||
it("phase scales the thinking field: teaching invites depth, probing stays quick", () => {
|
||||
const teach = renderToStaticMarkup(
|
||||
<QuestionDock
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Unit tests — the UI parse seam (src/core/tool-parts.ts).
|
||||
*
|
||||
* The rule born of the 2026-09-12 live crashes: model-authored tool input
|
||||
* is consumed only through these parsers, each running the SAME zod schema
|
||||
* the model faced at the boundary. These pins hold the two exact live
|
||||
* crash shapes (the model's double-encoding habit) and the boundary-parity
|
||||
* property: anything a boundary schema accepts, the display parse accepts.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
parseAskInput,
|
||||
parseProbeDoneInput,
|
||||
parseQuizInput,
|
||||
} from "../src/core/tool-parts.ts";
|
||||
import { quizInputSchema, probeQuizInputSchema } from "../src/core/question-core.ts";
|
||||
import { probeDoneInputSchema } from "../src/core/contracts/probe.ts";
|
||||
|
||||
const TEACHER_QUIZ = {
|
||||
question: "What does Tarski's convention-T define?",
|
||||
options: [
|
||||
{ label: "Truth for a language", value: "truth" },
|
||||
{ label: "Provability", value: "provability" },
|
||||
],
|
||||
correctAnswer: "truth",
|
||||
explanation: "Convention-T schematizes truth for the object language.",
|
||||
nodeId: "convention-t", // teacher-only field — the display parse must accept and strip it
|
||||
};
|
||||
|
||||
const PROBER_QUIZ = {
|
||||
question: "Which planet is closest to the sun?",
|
||||
options: [{ label: "Venus" }, { label: "Mercury" }],
|
||||
correctAnswer: "Mercury",
|
||||
explanation: "Mercury is the innermost planet.",
|
||||
};
|
||||
|
||||
const VALID_ASSESSMENT = {
|
||||
assessment: {
|
||||
strands: [
|
||||
{ strand: "modal logic basics", assessment: "Floor: solid intuition; ceiling: notation." },
|
||||
{ strand: "indexical 'actual'", assessment: "Genuine floor — prior exposure likely." },
|
||||
],
|
||||
summary: "Bracketed: teach the symbols on the intuition he has.",
|
||||
},
|
||||
};
|
||||
|
||||
describe("parseQuizInput", () => {
|
||||
it("parses a boundary-valid TEACHER quiz and strips the teacher-only nodeId", () => {
|
||||
const parsed = parseQuizInput(TEACHER_QUIZ);
|
||||
assert.ok(parsed);
|
||||
assert.equal(parsed.question, TEACHER_QUIZ.question);
|
||||
assert.equal((parsed as Record<string, unknown>).nodeId, undefined, "unknown keys are stripped, not trusted");
|
||||
});
|
||||
|
||||
it("parses a boundary-valid PROBER quiz (no nodeId at all)", () => {
|
||||
assert.ok(parseQuizInput(PROBER_QUIZ));
|
||||
});
|
||||
|
||||
it("boundary parity: anything the TEACH schema accepts, the display parse accepts", () => {
|
||||
const boundary = quizInputSchema.safeParse(TEACHER_QUIZ);
|
||||
assert.ok(boundary.success, "fixture must be boundary-valid for this pin to mean anything");
|
||||
assert.ok(parseQuizInput(boundary.data));
|
||||
});
|
||||
|
||||
it("boundary parity: anything the PROBE schema accepts, the display parse accepts", () => {
|
||||
const boundary = probeQuizInputSchema.safeParse(PROBER_QUIZ);
|
||||
assert.ok(boundary.success);
|
||||
assert.ok(parseQuizInput(boundary.data));
|
||||
});
|
||||
|
||||
it("the live crash shape — options as a JSON string — does not parse", () => {
|
||||
const stringified = { ...PROBER_QUIZ, options: JSON.stringify(PROBER_QUIZ.options) };
|
||||
assert.equal(parseQuizInput(stringified), undefined);
|
||||
});
|
||||
|
||||
it("garbage never parses: non-object, missing question, missing correctAnswer", () => {
|
||||
assert.equal(parseQuizInput("a string"), undefined);
|
||||
assert.equal(parseQuizInput({ options: [] }), undefined);
|
||||
assert.equal(parseQuizInput({ ...PROBER_QUIZ, correctAnswer: undefined }), undefined);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseAskInput", () => {
|
||||
it("parses free-text mode (no options) and option mode", () => {
|
||||
assert.ok(parseAskInput({ question: "What do you want to learn?" }));
|
||||
assert.ok(parseAskInput({ question: "Which topics?", options: [{ label: "Physics" }, { label: "Logic" }], multiSelect: true }));
|
||||
});
|
||||
|
||||
it("the live crash shape — stringified options — does not parse", () => {
|
||||
assert.equal(parseAskInput({ question: "q", options: "[]" }), undefined);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseProbeDoneInput", () => {
|
||||
it("parses the valid assessment", () => {
|
||||
const parsed = parseProbeDoneInput(VALID_ASSESSMENT);
|
||||
assert.ok(parsed);
|
||||
assert.equal(parsed.assessment.strands.length, 2);
|
||||
assert.equal(parsed.assessment.summary, "Bracketed: teach the symbols on the intuition he has.");
|
||||
});
|
||||
|
||||
it("boundary parity: anything the tool schema accepts, the display parse accepts", () => {
|
||||
const boundary = probeDoneInputSchema.safeParse(VALID_ASSESSMENT);
|
||||
assert.ok(boundary.success);
|
||||
assert.ok(parseProbeDoneInput(boundary.data));
|
||||
});
|
||||
|
||||
it("the live crash shape — assessment STRINGIFIED — does not parse", () => {
|
||||
// The exact double-encoding from the modal-realism run: strands died
|
||||
// on `.map` because the card cast the shape. The parse kills the class.
|
||||
const stringified = { assessment: JSON.stringify(VALID_ASSESSMENT.assessment) };
|
||||
assert.equal(parseProbeDoneInput(stringified), undefined);
|
||||
const garbage = JSON.stringify(VALID_ASSESSMENT.assessment).slice(0, 30);
|
||||
assert.equal(parseProbeDoneInput(garbage), undefined);
|
||||
});
|
||||
|
||||
it("a mid-stream partial (no strands yet) does not parse", () => {
|
||||
assert.equal(parseProbeDoneInput({ assessment: { summary: "…" } }), undefined);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user