quiz nodeId required (owner ruling 2026-09-26): no untagged teach quizzes — boundary rejects, model retries tagged; off-map checks route to ask_user_question; anti-rush doctrine in schema+prompt; docs+comments updated

This commit is contained in:
amosblomqvist
2026-09-27 02:05:51 +03:00
parent 8c0f0f7fbf
commit d6d7c09e2e
9 changed files with 72 additions and 35 deletions
+15
View File
@@ -55,6 +55,7 @@ const QUIZ_INPUT = {
],
correctAnswer: "Mercury",
explanation: "Mercury is the innermost planet.",
nodeId: "planet-order",
};
/** Turn 1: a bit of reasoning, then the quiz tool call. Turn 2: plain text.
@@ -187,6 +188,18 @@ function post(chatId: string, input: unknown): Request {
function createConfiguredChat(title: string) {
const chat = chatStore.createChat(title);
chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } });
// nodeId is required on teach quizzes (owner ruling 2026-09-26) — the
// scripted lessons quiz against this seeded tree.
chatStore.saveTree(chat.id, {
nodes: [
{ id: "planet-order", title: "Planet order" },
{ id: "genealogy", title: "Genealogy of religion" },
{ id: "indexicals", title: "Indexical 'actual'" },
{ id: "modal-realism", title: "Modal realism" },
],
edges: [],
version: 1,
});
return chat;
}
@@ -560,6 +573,7 @@ describe("teacher chat flow (scripted model)", () => {
// same turn. This pins that whole mechanism (the live Nietzsche bug).
const BAD_QUIZ_INPUT = {
question: "What is Nietzsche's genealogical claim about religion?",
nodeId: "genealogy",
options: [{ label: "A cultural diagnosis" }, { label: "Atheism as a conclusion" }],
correctAnswer: "cultural-diagnosis", // the exact live failure shape: a slug, no match
explanation: "Genealogy diagnoses, it does not convert.",
@@ -635,6 +649,7 @@ describe("teacher chat flow (scripted model)", () => {
// question, and the model's retry must become the pending quiz.
const STRINGY_QUIZ_INPUT = {
question: "Lewis's indexical analysis of 'actual' — which is it?",
nodeId: "indexicals",
options: JSON.stringify([
{ label: "Like 'I'", value: "indexical" },
{ label: "Like a title", value: "unique" },
+1
View File
@@ -73,6 +73,7 @@ describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
options: [{ label: "Venus" }, { label: "Mercury" }, { label: "Mars" }],
correctAnswer: "Mercury",
explanation: "Mercury is the innermost planet.",
nodeId: "planet-order",
};
it("accepts a valid single-select quiz (label as correctAnswer)", () => {
+25 -6
View File
@@ -25,7 +25,7 @@ import {
} from "../src/agent/tools.ts";
import { TOGGLEABLE_TOOLS } from "../src/core/pending.ts";
import { jobStore } from "../src/store/jobs.ts";
import { probeQuizInputSchema } from "../src/core/question-core.ts";
import { probeQuizInputSchema, quizInputObject } from "../src/core/question-core.ts";
import { chatStore } from "../src/store/chats.ts";
import { PATHS } from "../config.ts";
@@ -57,6 +57,7 @@ const validQuiz = {
],
correctAnswer: "4",
explanation: "Arithmetic.",
nodeId: "arith",
};
import type { AgentTool } from "@earendil-works/pi-agent-core";
@@ -119,15 +120,33 @@ describe("teachQuizToolFor: the dynamic plan-nodeId validation (position signal)
assert.match(msg, /arith/);
});
it("no tree yet: ANY nodeId is rejected — omit it for ad-hoc checks", async () => {
it("no tree yet: ANY nodeId is rejected — quizzes wait for plan_init", async () => {
const chatId = newChat("NodeId: no tree");
const tool = teachQuizToolFor({ chatId, chatStore });
const res = await boundaryParse(tool, { ...validQuiz, nodeId: "anything" });
assert.equal(res.success, false);
assert.match(problemLines(res.error).at(-1)!, /No plan map exists yet/);
// …while an untagged ad-hoc check is fine
const plain = await boundaryParse(tool, validQuiz);
assert.equal(plain.success, true);
assert.match(problemLines(res.error).at(-1)!, /No plan map exists yet — call plan_init/);
});
it("missing nodeId is rejected at the boundary (owner ruling 2026-09-26: no untagged quizzes)", async () => {
const chatId = newChat("NodeId: required");
chatStore.saveTree(chatId, {
nodes: [{ id: "arith", title: "Arithmetic" }],
edges: [],
version: 1,
});
const tool = teachQuizToolFor({ chatId, chatStore });
const { nodeId: _dropped, ...untagged } = validQuiz;
const res = await boundaryParse(tool, untagged);
assert.equal(res.success, false, "an untagged quiz must be rejected back to the model");
// The structural pin: the SCHEMA itself names the missing field (pi's
// boundary formats Value.Errors as the tool-input-error the model sees).
const schemaErrors = [...Value.Errors(quizInputObject, untagged)].map((e) => e.message);
assert.match(
schemaErrors.join("\n"),
/nodeId/,
"the schema rejection names nodeId so the model can retry tagged",
);
});
it("reads the LIVE record — a plan_init mutation mid-turn is honored immediately", async () => {