diff --git a/src/agent/tools.ts b/src/agent/tools.ts index e8a0028..b078c61 100644 --- a/src/agent/tools.ts +++ b/src/agent/tools.ts @@ -26,6 +26,7 @@ export const quizTool = tool({ "Ask the learner a GRADED question with a known correct answer, then instantly grade and give feedback. " + "Use it to (1) assess what the learner already understands before teaching, and (2) run tight practice/retrieval loops after explaining. " + "Options-only: single-select or multi-select, plus an automatic 'I don't know' choice so the learner can signal a genuine gap instead of guessing. " + + "correctAnswer must be the EXACT value (or label) of one of the listed options — a correctAnswer that matches no option fails validation and the call is returned to you to fix. " + "No free-text answers — for non-graded questions use ask_user_question instead. " + "Ask exactly one question per tool call; after calling, end your turn and wait for the graded result.", inputSchema: quizInputSchema, diff --git a/src/core/question-core.ts b/src/core/question-core.ts index 3cf92f9..5756bf4 100644 --- a/src/core/question-core.ts +++ b/src/core/question-core.ts @@ -39,52 +39,141 @@ const optionSchema = z.object({ description: z.string().optional().describe("Optional extra detail shown below the option"), }); -export const quizInputSchema = z.object({ - question: z.string().describe("The single quiz question to ask"), - options: z - .array(optionSchema) - .min(2) - .describe( - "The answer options (2 or more). Options only — no free-text mode. Never add your own 'I don't know' option.", - ), - correctAnswer: z - .union([z.string(), z.array(z.string())]) - .describe( - "The correct answer as the option value(s) — the `value` field of the option you intend. " + - "Single-select: one string. Multi-select: an array; the learner is only correct if their selection matches this set exactly.", - ), - explanation: z - .string() - .describe("Explanation revealed AFTER the learner answers. Always say why the correct answer is correct."), - multiSelect: z.boolean().optional().describe("Set true only when more than one option is correct"), - details: z.string().optional().describe("Optional extra context or instructions shown under the question"), - nodeId: z - .string() - .optional() - .describe( - "Optional plan-tree node id this question assesses. Set whenever the question targets a tree node — " + - "the outcome feeds the learner's knowledge graph. Omit for ad-hoc probes.", - ), -}); +// ─── cross-field authoring validation (tool-schema refinements) ────────── +// Enforced at the tool-call boundary (see the superRefines below): an invalid +// question NEVER reaches the learner — the SDK returns the validation error +// to the teacher as a tool result and the teacher retries. The rules mirror +// the grading path exactly (normalizeOptions / coerceCorrectAnswer / +// resolveCorrect), so schema-valid ⇔ gradeable. -export const askInputSchema = z.object({ - question: z.string().describe("The single question to ask"), - mode: z - .enum(["single-select", "multi-select", "text"]) - .optional() - .describe( - "Answer mode. Omit for single-select when options are given, or text when they are not.", - ), - options: z - .array(optionSchema) - .min(2) - .optional() - .describe( - "Optional multiple-choice options. Omit for free-form text input. The learner can always choose 'Other' and type a custom answer.", - ), - multiSelect: z.boolean().optional().describe("Set true to allow multiple answers to be selected"), - details: z.string().optional().describe("Optional extra context or instructions shown under the question"), -}); +/** Option sanity shared by quiz + ask: labels non-empty, resolved values + * (trimmed `value`, falling back to the trimmed label — exactly how + * normalizeOptions keys them) distinct. Returns the problems plus the + * resolved keys for the correctAnswer check. */ +function optionProblems( + options: Array<{ label: string; value?: string }>, +): { problems: string[]; keys: string[] } { + const problems: string[] = []; + const keys: string[] = []; + options.forEach((option, i) => { + const label = option.label.trim(); + if (label.length === 0) { + problems.push(`options[${i}].label is empty — every option needs a label.`); + return; + } + const key = option.value?.trim() || label; + if (keys.includes(key)) { + problems.push(`Two options resolve to the same value "${key}" — options must be distinct.`); + } + keys.push(key); + }); + return { problems, keys }; +} + +/** ask options: the shared sanity, nothing more (ask is not graded). */ +export function askOptionProblems(options: Array<{ label: string; value?: string }>): string[] { + return optionProblems(options).problems; +} + +/** The full quiz authoring contract: option sanity + correctAnswer must be + * gradeable. Empty result = a quiz the dock can arm and grade faithfully. */ +export function quizInputProblems(input: { + options: Array<{ label: string; value?: string }>; + correctAnswer: string | string[]; + multiSelect?: boolean; +}): string[] { + const { problems, keys } = optionProblems(input.options); + + const entries = coerceCorrectAnswer(input.correctAnswer).map((v) => v.trim()); + const unique = [...new Set(entries)]; + if (unique.length === 0 || unique.some((v) => v.length === 0)) { + problems.push("correctAnswer is required — provide the value (or label) of the correct option(s)."); + return problems; + } + for (const v of unique) { + if (!keys.includes(v)) { + problems.push( + `correctAnswer "${v}" does not match any option value. Valid values: ${keys.map((k) => `"${k}"`).join(", ")}. ` + + "Copy the exact value (or label) of the intended option — never a paraphrase or slug.", + ); + } + } + if (unique.length > 1 && input.multiSelect !== true) { + problems.push( + `correctAnswer lists ${unique.length} values but multiSelect is not true — a single-select question with several ` + + "correct answers can never be answered correctly. Set multiSelect: true, or make exactly one option correct.", + ); + } + return problems; +} + +export const quizInputSchema = z + .object({ + question: z.string().describe("The single quiz question to ask"), + options: z + .array(optionSchema) + .min(2) + .describe( + "The answer options (2 or more). Options only — no free-text mode. Never add your own 'I don't know' option.", + ), + correctAnswer: z + .union([z.string(), z.array(z.string())]) + .describe( + "The correct answer as the option value(s): copy the EXACT `value` — or `label` when `value` is omitted — " + + "of the intended option(s), never a paraphrase or slug; a mismatch fails validation. " + + "Single-select: one string. Multi-select: an array; the learner is only correct if their selection matches this set exactly.", + ), + explanation: z + .string() + .describe("Explanation revealed AFTER the learner answers. Always say why the correct answer is correct."), + multiSelect: z.boolean().optional().describe("Set true only when more than one option is correct"), + details: z.string().optional().describe("Optional extra context or instructions shown under the question"), + nodeId: z + .string() + .optional() + .describe( + "Optional plan-tree node id this question assesses. Set whenever the question targets a tree node — " + + "the outcome feeds the learner's knowledge graph. Omit for ad-hoc probes.", + ), + }) + // The cross-field authoring contract (the Gate-3 hardening): a quiz whose + // correctAnswer can't be graded NEVER reaches the learner. The SDK validates + // client-side tool inputs against this schema at the call boundary and, on + // failure, returns the validation error to the teacher as a tool result — + // the teacher retries with a fixed quiz in the same turn. Schema-valid ⇔ + // gradeable: the rules are exactly what the grading path accepts. + .superRefine((input, ctx) => { + for (const problem of quizInputProblems(input)) { + ctx.addIssue({ code: "custom", message: problem }); + } + }); + +export const askInputSchema = z + .object({ + question: z.string().describe("The single question to ask"), + mode: z + .enum(["single-select", "multi-select", "text"]) + .optional() + .describe( + "Answer mode. Omit for single-select when options are given, or text when they are not.", + ), + options: z + .array(optionSchema) + .min(2) + .optional() + .describe( + "Optional multiple-choice options. Omit for free-form text input. The learner can always choose 'Other' and type a custom answer.", + ), + multiSelect: z.boolean().optional().describe("Set true to allow multiple answers to be selected"), + details: z.string().optional().describe("Optional extra context or instructions shown under the question"), + }) + // Same option sanity as the quiz (labels non-empty, values distinct — ask + // is not graded, but duplicate values break the dock's value keying). + .superRefine((input, ctx) => { + for (const problem of askOptionProblems(input.options ?? [])) { + ctx.addIssue({ code: "custom", message: problem }); + } + }); export type QuizToolInput = z.infer; export type AskToolInput = z.infer; diff --git a/src/server/chat-handler.ts b/src/server/chat-handler.ts index ff3e0ec..b7bf9c2 100644 --- a/src/server/chat-handler.ts +++ b/src/server/chat-handler.ts @@ -1,4 +1,4 @@ -import { createAgentUIStreamResponse, type UIMessage } from "ai"; +import { createAgentUIStreamResponse, InvalidToolInputError, NoSuchToolError, type UIMessage } from "ai"; import { createTeacherAgent, type TeacherAgent } from "../agent/teacher-agent.ts"; import { createJobRunner } from "../agent/subagents/runner.ts"; import { createExtractionPipeline } from "../agent/subagents/extraction.ts"; @@ -45,6 +45,17 @@ export async function handleChatRequest(req: Request, agent?: TeacherAgent): Pro buildTeacherContext(stores.graphStore, chatId), ), uiMessages: body.messages as UIMessage[], + // Tool-call validation failures are messages OUR schemas authored (the + // quiz authoring contract, unknown tools) — safe and useful in the + // transcript: they are exactly what the teacher saw and retried on. The + // tool loop delivers them as plain strings (what the model received) or + // as the original error object; expose both, and keep everything else + // (provider/transport internals) masked, as the SDK defaults. + onError: (error) => { + if (typeof error === "string") return error; + if (InvalidToolInputError.isInstance(error) || NoSuchToolError.isInstance(error)) return error.message; + return "An error occurred."; + }, onFinish: ({ messages, isAborted }) => { persistTurn(stores.chatStore, chatId, messages as UIMessage[], isAborted); spawnTurnJobs(jobRunner, chatId, messages as UIMessage[]); diff --git a/tests/chat-flow.test.ts b/tests/chat-flow.test.ts index b1d4eef..8795aab 100644 --- a/tests/chat-flow.test.ts +++ b/tests/chat-flow.test.ts @@ -229,6 +229,91 @@ describe("teacher chat flow (scripted model)", () => { const saved = chatStore.getChat(chat.id)!; assert.ok(saved.messages.length >= 1, "partial transcript persisted"); }); + + it("an ungradeable quiz is rejected at the call boundary, the error reaches the model, and the model retries", async () => { + // The authoring contract (question-core superRefine): a quiz whose + // correctAnswer matches no option value must NEVER reach the dock — + // the SDK validates client-side tool inputs against the schema, marks + // the call invalid, and returns the validation error to the teacher as + // a tool result. The loop re-prompts the model, which retries in the + // same turn. This pins that whole mechanism (the live Nietzsche bug). + const BAD_QUIZ_INPUT = { + question: "What is Nietzsche's genealogical claim about religion?", + options: [{ label: "A cultural diagnosis" }, { label: "Atheism as a conclusion" }], + correctAnswer: "cultural-diagnosis", // the exact live failure shape: a slug, no match + explanation: "Genealogy diagnoses, it does not convert.", + }; + + const calls: string[][] = []; + let step = 0; + const retryModel = new MockLanguageModelV4({ + doStream: async (options) => { + calls.push(options.prompt.map((m) => JSON.stringify(m.content))); + step++; + const chunks: LanguageModelV4StreamPart[] = + step === 1 + ? [ + { type: "stream-start", warnings: [] }, + { type: "tool-input-start", id: "call-bad", toolName: "quiz" }, + { type: "tool-input-delta", id: "call-bad", delta: JSON.stringify(BAD_QUIZ_INPUT) }, + { type: "tool-input-end", id: "call-bad" }, + { + type: "tool-call", + toolCallId: "call-bad", + toolName: "quiz", + input: JSON.stringify(BAD_QUIZ_INPUT), + }, + { type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() }, + ] + : [ + { type: "stream-start", warnings: [] }, + { type: "tool-input-start", id: "call-good", toolName: "quiz" }, + { type: "tool-input-delta", id: "call-good", delta: JSON.stringify(QUIZ_INPUT) }, + { type: "tool-input-end", id: "call-good" }, + { + type: "tool-call", + toolCallId: "call-good", + toolName: "quiz", + input: JSON.stringify(QUIZ_INPUT), + }, + { type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() }, + ]; + return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) }; + }, + }); + + const agent = createTeacherAgent(retryModel); + const chat = chatStore.createChat("Test: bad quiz retry"); + createdChats.push(chat.id); + const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] }; + await drain(await handleChatRequest(post(chat.id, [firstUser]), agent)); + + // The model was re-prompted (the error did not end the turn)… + assert.equal(calls.length, 2, "the invalid call triggered a second model prompt"); + // …and the second prompt carried the validation error back as a tool result. + assert.match( + calls[1]!.join(" "), + /does not match any option value/, + "the validation error reached the model", + ); + + // The transcript: the failed call persisted as an error part (renders + // nothing client-side), the retry as the pending quiz. + const saved = chatStore.getChat(chat.id)!; + const assistantMsg = saved.messages[1]!; + const quizParts = assistantMsg.parts.filter((p) => p.type === "tool-quiz"); + const failed = quizParts.find( + (p) => p.type === "tool-quiz" && (p as { state?: string }).state === "output-error", + ); + assert.ok(failed, "the invalid call persisted as an error-state part"); + assert.match( + String((failed as { errorText?: string }).errorText), + /does not match any option value/, + ); + const pending = findPendingToolCall(saved.messages); + assert.equal(pending?.toolCallId, "call-good", "the RETRIED quiz is the pending one"); + assert.deepEqual(pending?.input, QUIZ_INPUT); + }); }); describe("chat store", () => { diff --git a/tests/question-core.test.ts b/tests/question-core.test.ts index bfb3f8b..fcb36cb 100644 --- a/tests/question-core.test.ts +++ b/tests/question-core.test.ts @@ -5,6 +5,8 @@ import assert from "node:assert/strict"; import { describe, it } from "node:test"; import { + askInputSchema, + askOptionProblems, buildAnswer, coerceCorrectAnswer, findPendingQuestion, @@ -14,12 +16,124 @@ import { normalizeOptions, QUESTION_ANSWERED_ENTRY, QUESTION_ENTRY, + quizInputProblems, + quizInputSchema, resolveCorrect, shuffleOptions, type QuestionData, toClientQuestion, } from "../src/core/question-core.ts"; +// ─── the authoring contract (tool-schema refinements) ─────────────────── +// Schema-valid ⇔ gradeable: a quiz whose correctAnswer can't be resolved +// dies at the tool-call boundary, and the SDK returns the error to the +// teacher to retry — it must never reach the learner's dock. + +describe("quizInputSchema / quizInputProblems (the authoring contract)", () => { + const valid = { + question: "Which planet is closest to the Sun?", + options: [{ label: "Venus" }, { label: "Mercury" }, { label: "Mars" }], + correctAnswer: "Mercury", + explanation: "Mercury is the innermost planet.", + }; + + it("accepts a valid single-select quiz (label as correctAnswer)", () => { + const parsed = quizInputSchema.safeParse(valid); + assert.equal(parsed.success, true); + assert.deepEqual(quizInputProblems(valid), []); + }); + + it("accepts explicit values and multi-select arrays (incl. JSON-stringified)", () => { + const multi = { + ...valid, + options: [ + { label: "A", value: "a" }, + { label: "B", value: "b" }, + { label: "C", value: "c" }, + ], + correctAnswer: ["a", "b"], + multiSelect: true, + }; + assert.equal(quizInputSchema.safeParse(multi).success, true); + // The union lists String first, so providers sometimes stringify arrays. + const stringified = { ...multi, correctAnswer: JSON.stringify(multi.correctAnswer) }; + assert.equal(quizInputSchema.safeParse(stringified).success, true); + assert.deepEqual(quizInputProblems(stringified), []); + }); + + it("rejects the observed failure: a correctAnswer slug matching no option value, listing the valid values", () => { + // The exact class GLM produced live (Gate 3 + live verification). + const bad = { + question: "What is Nietzsche's genealogical claim about religion?", + options: [ + { label: "A cultural diagnosis" }, + { label: "Atheism as a conclusion" }, + { label: "A call to convert people" }, + ], + correctAnswer: "cultural-diagnosis", + explanation: "...", + }; + const parsed = quizInputSchema.safeParse(bad); + assert.equal(parsed.success, false); + const problems = quizInputProblems(bad); + assert.equal(problems.length, 1); + assert.match(problems[0]!, /does not match any option value/); + assert.match(problems[0]!, /"A cultural diagnosis"/); // the valid values are listed for the retry + // The schema's error message carries the problems (the SDK returns them to the teacher). + const msg = JSON.stringify(parsed.error?.issues); + assert.match(msg, /does not match any option value/); + }); + + it("rejects multiple correct answers without multiSelect (ungradeable single-select)", () => { + const bad = { ...valid, correctAnswer: ["Venus", "Mercury"] }; + assert.equal(quizInputSchema.safeParse(bad).success, false); + assert.match(quizInputProblems(bad).join(" "), /multiSelect/); + }); + + it("rejects duplicate option values", () => { + const bad = { + ...valid, + options: [ + { label: "Mercury", value: "same" }, + { label: "Venus", value: "same" }, + ], + }; + assert.equal(quizInputSchema.safeParse(bad).success, false); + assert.match(quizInputProblems(bad).join(" "), /same value/); + }); + + it("rejects an empty option label", () => { + const bad = { ...valid, options: [{ label: " " }, { label: "Mercury" }, { label: "Mars" }] }; + assert.equal(quizInputSchema.safeParse(bad).success, false); + assert.match(quizInputProblems(bad).join(" "), /label is empty/); + }); + + it("requires a resolvable correctAnswer", () => { + assert.equal(quizInputSchema.safeParse({ ...valid, correctAnswer: "" }).success, false); + assert.equal(quizInputSchema.safeParse({ ...valid, correctAnswer: "[]" }).success, false); + assert.match(quizInputProblems({ ...valid, correctAnswer: "" }).join(" "), /required/); + }); +}); + +describe("askInputSchema / askOptionProblems", () => { + it("accepts a valid ask", () => { + const parsed = askInputSchema.safeParse({ + question: "Which topic first?", + options: [{ label: "Limits" }, { label: "Chain rule" }], + }); + assert.equal(parsed.success, true); + }); + + it("rejects duplicate option values and empty labels (they break the dock's keying)", () => { + const bad = { question: "?", options: [{ label: "A", value: "x" }, { label: "B", value: "x" }] }; + assert.equal(askInputSchema.safeParse(bad).success, false); + assert.match(askOptionProblems(bad.options).join(" "), /same value/); + const emptyLabel = { question: "?", options: [{ label: "" }, { label: "B" }] }; + assert.equal(askInputSchema.safeParse(emptyLabel).success, false); + assert.match(askOptionProblems(emptyLabel.options).join(" "), /label is empty/); + }); +}); + // ─── normalizeOptions ─────────────────────────────────────────────────────── describe("normalizeOptions", () => {