question tools: phase-visible names, hardened teach schemas, context-slim seams
The tools_fails_2 diagnosis (glm-5.3 dropping trailing required fields):
- probe-phase quiz/explain renamed to probe/recall — a shared name carried
the probe call shape into teaching (the first teach quiz dropped nodeId);
display seams accept the old names forever
- teach quiz+explain schemas lead with the required pair (type, nodeId);
explain gains the predict/check type and joins the reveal law (checks
collapse, predictions arrive open; no-type records render check-style)
- plan_init/probe_done results go context-slim (owner, context diet):
content = validated nodeIds + binding-id reminder / fixed ack; details
keeps the tree for the popup and {ok:true} — serializer-verified that
details never reaches the model; failure echoes dropped
- the cards carry the tool's own name (Probe/Recall) and the quiz type
reads on the label (Prediction/Check) in composer and transcript
This commit is contained in:
amosblomqvist
@@ -1107,6 +1107,7 @@ it("explain tool call persists as a pending part; the free-text answer reveals t
|
||||
question: "Explain why Mercury is hard to observe.",
|
||||
correctAnswer: "Its small angular separation from the Sun — it never strays far from the Sun in our sky.",
|
||||
nodeId: "planet-order",
|
||||
type: "check", // the teach explain's role field (owner, 2026-10-09)
|
||||
};
|
||||
const { engine, calls } = scriptedTeacher((cs) => [
|
||||
(context) => {
|
||||
|
||||
@@ -98,8 +98,8 @@ describe("prober graph tools — registration", () => {
|
||||
"list_sessions",
|
||||
"ls",
|
||||
"pdf",
|
||||
"probe",
|
||||
"probe_done",
|
||||
"quiz",
|
||||
"read",
|
||||
"research",
|
||||
]);
|
||||
|
||||
+10
-8
@@ -120,19 +120,20 @@ describe("plan_init — the author-once mermaid tool (execute against the real s
|
||||
});
|
||||
assert.equal(res.ok, true);
|
||||
assert.equal(res.error, undefined);
|
||||
assert.equal(res.tree.version, 1);
|
||||
const tree = res.tree!;
|
||||
assert.equal(tree.version, 1);
|
||||
assert.deepEqual(
|
||||
res.tree.nodes.map((n) => n.id),
|
||||
tree.nodes.map((n) => n.id),
|
||||
["primitives", "operators", "expressions", "detour"],
|
||||
);
|
||||
assert.deepEqual(res.tree.edges, [
|
||||
assert.deepEqual(tree.edges, [
|
||||
{ from: "primitives", to: "operators" },
|
||||
{ from: "operators", to: "expressions" },
|
||||
]);
|
||||
|
||||
const saved = chatStore.getChat(chatId)!.tree;
|
||||
assert.ok(saved, "tree persisted to the chat record");
|
||||
assert.deepEqual(saved, res.tree);
|
||||
assert.deepEqual(saved, tree);
|
||||
});
|
||||
|
||||
it("RE-CALLABLE: a rewrite replaces the tree wholesale (the mutation tools are gone)", async () => {
|
||||
@@ -147,7 +148,7 @@ describe("plan_init — the author-once mermaid tool (execute against the real s
|
||||
});
|
||||
assert.equal(res.ok, true, "re-init is the sanctioned rewrite path now");
|
||||
assert.deepEqual(
|
||||
res.tree.nodes.map((n) => n.id),
|
||||
res.tree!.nodes.map((n) => n.id),
|
||||
["truth", "goal"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
@@ -169,7 +170,8 @@ describe("plan_init — the author-once mermaid tool (execute against the real s
|
||||
assert.match(err, /line 2/);
|
||||
assert.match(err, /line 3/);
|
||||
assert.match(err, /line 4/);
|
||||
assert.deepEqual(res.tree, emptyTree());
|
||||
// Context diet (2026-10-09): failures carry just the error — no tree echo.
|
||||
assert.equal(res.tree, undefined);
|
||||
assert.equal(chatStore.getChat(chatId)!.tree, undefined, "nothing persisted on failure");
|
||||
});
|
||||
|
||||
@@ -181,7 +183,7 @@ describe("plan_init — the author-once mermaid tool (execute against the real s
|
||||
});
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error!, /cycle/);
|
||||
assert.deepEqual(res.tree, emptyTree());
|
||||
assert.equal(res.tree, undefined, "no tree echo on failure (context diet, 2026-10-09)");
|
||||
});
|
||||
|
||||
it("only plan_init exists — the seven mutation/tracking tools are gone by design", () => {
|
||||
@@ -194,7 +196,7 @@ describe("plan_init — the author-once mermaid tool (execute against the real s
|
||||
const res = await exec(tools, "plan_init", { mermaid: "graph TD\n a[A]\n" });
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error!, /Chat not found/);
|
||||
assert.deepEqual(res.tree, emptyTree());
|
||||
assert.equal(res.tree, undefined);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+30
-21
@@ -26,7 +26,7 @@ import {
|
||||
probeQuizInputSchema,
|
||||
quizDisplayInputSchema,
|
||||
normalizeQuizType,
|
||||
quizNeedsReveal,
|
||||
questionNeedsReveal,
|
||||
resolveCorrect,
|
||||
toggleOption,
|
||||
toggleOther,
|
||||
@@ -54,22 +54,22 @@ describe("the schema's argument order (reasoning before the pick)", () => {
|
||||
// Schema property order, so this pin guards the emission end to end.
|
||||
// All consumers read by key — the order is authoring guidance.
|
||||
it("emits explanation immediately before correctAnswer (both agents' schemas)", () => {
|
||||
// The 2026-09-25 ruling stands for the CONTENT fields (reasoning
|
||||
// before the pick); the 2026-10-09 reorder adds its mirror for the
|
||||
// teach schema — the required pair (`type`, `nodeId`) LEADS the
|
||||
// object, because glm-5.3 dropped exactly those fields when they
|
||||
// trailed every long content field (the tools_fails_2 drops).
|
||||
// Models emit tool args in schema order: commit to the role and the
|
||||
// target first, reason, then pick.
|
||||
for (const schema of [quizInputSchema, probeQuizInputSchema]) {
|
||||
const keys = Object.keys((schema as { properties: Record<string, unknown> }).properties);
|
||||
assert.ok(
|
||||
keys.indexOf("explanation") < keys.indexOf("correctAnswer"),
|
||||
`explanation must precede correctAnswer in the emitted schema (got: ${keys.join(", ")})`,
|
||||
);
|
||||
assert.deepEqual(keys, [
|
||||
"question",
|
||||
"options",
|
||||
"multiSelect",
|
||||
"explanation",
|
||||
"correctAnswer",
|
||||
"details",
|
||||
...(keys.includes("type") ? ["type"] : []),
|
||||
...(keys.includes("nodeId") ? ["nodeId"] : []),
|
||||
]);
|
||||
assert.deepEqual(keys, schema === quizInputSchema
|
||||
? ["type", "nodeId", "question", "options", "multiSelect", "explanation", "correctAnswer", "details"]
|
||||
: ["question", "options", "multiSelect", "explanation", "correctAnswer", "details"]);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -567,9 +567,13 @@ describe("explainInputBase / explainInputProblems (the commitment contract)", ()
|
||||
assert.deepEqual(keys, ["question", "correctAnswer", "details"]);
|
||||
});
|
||||
|
||||
it("the teach object adds the required nodeId AFTER the base fields", () => {
|
||||
it("the teach object LEADS with the required pair (type, nodeId) — base fields follow", () => {
|
||||
// Owner, 2026-10-09: glm-5.3 dropped `type`/`nodeId` when they trailed
|
||||
// the content fields — required fields generated FIRST get dropped
|
||||
// least, and the order doubles as the authoring order (role, target,
|
||||
// then the question). Mirrors the quiz reorder (see above).
|
||||
const keys = Object.keys(explainInputObject.properties);
|
||||
assert.deepEqual(keys, [...Object.keys(explainInputBase.properties), "nodeId"]);
|
||||
assert.deepEqual(keys, ["type", "nodeId", ...Object.keys(explainInputBase.properties)]);
|
||||
});
|
||||
|
||||
it("an empty commitment is the one authoring problem", () => {
|
||||
@@ -661,18 +665,23 @@ describe("the quiz role field (predict | check)", () => {
|
||||
assert.equal(normalizeQuizType(42), undefined);
|
||||
});
|
||||
|
||||
it("quizNeedsReveal: checks collapse, predictions never do", () => {
|
||||
assert.equal(quizNeedsReveal("quiz", { type: "check" }, "teach"), true);
|
||||
assert.equal(quizNeedsReveal("quiz", { type: "predict" }, "teach"), false, "predictions arrive OPEN");
|
||||
it("questionNeedsReveal: checks collapse, predictions never do — both teach tools", () => {
|
||||
assert.equal(questionNeedsReveal("quiz", { type: "check" }, "teach"), true);
|
||||
assert.equal(questionNeedsReveal("quiz", { type: "predict" }, "teach"), false, "predictions arrive OPEN");
|
||||
assert.equal(
|
||||
quizNeedsReveal("quiz", undefined, "teach"),
|
||||
questionNeedsReveal("quiz", undefined, "teach"),
|
||||
true,
|
||||
"old records (no type) render check-style",
|
||||
);
|
||||
assert.equal(quizNeedsReveal("quiz", { type: "predict" }, "probe"), false);
|
||||
assert.equal(quizNeedsReveal("quiz", { type: "check" }, "probe"), false);
|
||||
assert.equal(quizNeedsReveal("ask_user_question", {}, "teach"), false);
|
||||
assert.equal(quizNeedsReveal("explain", {}, "teach"), false);
|
||||
assert.equal(questionNeedsReveal("quiz", { type: "predict" }, "probe"), false);
|
||||
assert.equal(questionNeedsReveal("quiz", { type: "check" }, "probe"), false);
|
||||
assert.equal(questionNeedsReveal("ask_user_question", {}, "teach"), false);
|
||||
// Explain joined the law 2026-10-09: teach checks collapse, predictions
|
||||
// don't, old no-type records render check-style — mirroring quiz exactly.
|
||||
assert.equal(questionNeedsReveal("explain", { type: "check" }, "teach"), true);
|
||||
assert.equal(questionNeedsReveal("explain", { type: "predict" }, "teach"), false);
|
||||
assert.equal(questionNeedsReveal("explain", undefined, "teach"), true, "pre-type explain records render check-style");
|
||||
assert.equal(questionNeedsReveal("explain", { type: "check" }, "probe"), false);
|
||||
});
|
||||
|
||||
function quizQd(): QuestionData {
|
||||
|
||||
@@ -202,10 +202,10 @@ describe("the prober's quiz: no nodeId field at all", () => {
|
||||
// Fix wave (2026-09-20): the owner ruling adds research + pdf dispatch
|
||||
// to the probe phase's registry (demo/visual stay teach-only).
|
||||
assert.deepEqual(Object.keys(tools).sort(), [
|
||||
"ask_user_question", "find", "grep", "ls", "pdf", "probe_done", "quiz", "read", "research",
|
||||
"ask_user_question", "find", "grep", "ls", "pdf", "probe", "probe_done", "read", "research",
|
||||
]);
|
||||
const quiz = tools.quiz as AgentTool<any>;
|
||||
assert.equal(Value.Check(quiz.parameters, { ...validQuiz, nodeId: "anything" }), true, "no session check to fail");
|
||||
const probe = tools.probe as AgentTool<any>;
|
||||
assert.equal(Value.Check(probe.parameters, { ...validQuiz, nodeId: "anything" }), true, "no session check to fail");
|
||||
});
|
||||
});
|
||||
// ─── tool availability (2026-09 settings seam) ───────────────────────────────
|
||||
@@ -225,22 +225,22 @@ describe("tool toggles (core/pending.ts + factories)", () => {
|
||||
const chatId = newChat("Prober: toggles");
|
||||
const ctx = { chatId, chatStore, deliver: () => {}, jobStore, spawn: () => {} };
|
||||
assert.deepEqual(Object.keys(createProberTools(ctx)).sort(), [
|
||||
"ask_user_question", "find", "grep", "ls", "pdf", "probe_done", "quiz", "read", "research",
|
||||
"ask_user_question", "find", "grep", "ls", "pdf", "probe", "probe_done", "read", "research",
|
||||
]);
|
||||
|
||||
const noAsk = createProberTools(ctx, { ask_user_question: false });
|
||||
assert.deepEqual(Object.keys(noAsk).sort(), ["find", "grep", "ls", "pdf", "probe_done", "quiz", "read", "research"]);
|
||||
assert.deepEqual(Object.keys(noAsk).sort(), ["find", "grep", "ls", "pdf", "probe", "probe_done", "read", "research"]);
|
||||
|
||||
// Quizzing is mechanics-managed now (owner 2026-09-27): quiz/explain
|
||||
// NEVER pass through the generic filter — hand-written quiz:false is a
|
||||
// no-op (the wedge danger stands: a quiz-less prober collapses its
|
||||
// assessment flow). probe_done is never droppable — it ends the phase.
|
||||
const noQuiz = createProberTools(ctx, { quiz: false });
|
||||
assert.deepEqual(Object.keys(noQuiz).sort(), ["ask_user_question", "find", "grep", "ls", "pdf", "probe_done", "quiz", "read", "research"]);
|
||||
assert.deepEqual(Object.keys(noQuiz).sort(), ["ask_user_question", "find", "grep", "ls", "pdf", "probe", "probe_done", "read", "research"]);
|
||||
// The mechanic switch (owner 2026-09-27): tools.explain === true arms
|
||||
// free-recall — explain in, quiz out, exactly one quizzing tool.
|
||||
const free = createProberTools(ctx, { explain: true });
|
||||
assert.deepEqual(Object.keys(free).sort(), ["ask_user_question", "explain", "find", "grep", "ls", "pdf", "probe_done", "read", "research"]);
|
||||
assert.deepEqual(Object.keys(free).sort(), ["ask_user_question", "find", "grep", "ls", "pdf", "probe_done", "read", "recall", "research"]);
|
||||
assert.ok(createProberTools(ctx, { probe_done: false }).probe_done, "probe_done always-on");
|
||||
});
|
||||
|
||||
@@ -320,6 +320,7 @@ describe("teachExplainToolFor: the explain variant validates nodeId like the qui
|
||||
question: "Explain why the null result matters.",
|
||||
correctAnswer: "It ruled out the aether — a stationary aether predicts a fringe shift, and none was seen.",
|
||||
nodeId: "arith",
|
||||
type: "check",
|
||||
};
|
||||
|
||||
it("accepts a nodeId matching a tree node", async () => {
|
||||
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
coerceCorrectAnswer,
|
||||
normalizeOptions,
|
||||
resolveCorrect,
|
||||
quizNeedsReveal,
|
||||
questionNeedsReveal,
|
||||
type QuestionData,
|
||||
} from "../src/core/question-core.ts";
|
||||
|
||||
@@ -65,7 +65,12 @@ function askQD(): QuestionData {
|
||||
};
|
||||
}
|
||||
|
||||
function renderDock(props: { toolName: "quiz" | "ask_user_question"; toolCallId: string; input: never; onSubmit?: () => void }) {
|
||||
function renderDock(props: {
|
||||
toolName: "quiz" | "probe" | "ask_user_question" | "explain" | "recall";
|
||||
toolCallId: string;
|
||||
input: never;
|
||||
onSubmit?: () => void;
|
||||
}) {
|
||||
// expanded=true renders the full form (the collapsed bar is covered by the
|
||||
// QuestionDockBar test below). No scroll context needed — the DockOverlay
|
||||
// in the chat view owns that.
|
||||
@@ -458,16 +463,23 @@ describe("probe-done card", () => {
|
||||
summary: "Bracketed: teach the symbols on the intuition he has.",
|
||||
};
|
||||
|
||||
it("renders the strand-by-strand assessment from the seam ack", () => {
|
||||
it("renders the strand-by-strand assessment — the CALL is its home (context diet 2026-10-09)", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<ProbeDoneCard
|
||||
part={{ state: "output-available", output: { ok: true, assessment: validAssessment } } as never}
|
||||
part={{ state: "output-available", input: { assessment: validAssessment }, output: { ok: true } } as never}
|
||||
/>,
|
||||
);
|
||||
assert.ok(html.includes('data-testid="probe-done-card"'));
|
||||
assert.match(html, /modal logic basics/);
|
||||
assert.match(html, /indexical 'actual'|indexical 'actual'/);
|
||||
assert.match(html, /Bracketed/);
|
||||
// An ok output alone (the pre-diet echo shape) renders the quiet
|
||||
// fallback, never strands: the assessment no longer rides the result.
|
||||
const echoless = renderToStaticMarkup(
|
||||
<ProbeDoneCard part={{ state: "output-available", output: { ok: true } } as never} />,
|
||||
);
|
||||
assert.match(echoless, /Writing the assessment…/);
|
||||
assert.ok(!echoless.includes("modal logic basics"), "no assessment echo in the result");
|
||||
});
|
||||
|
||||
it("previews the assessment from a PENDING (validated) input", () => {
|
||||
@@ -533,6 +545,70 @@ describe("AnsweredCard: the explain variant", () => {
|
||||
|
||||
});
|
||||
|
||||
// ─── probe-phase naming (owner, 2026-10-09): the probe/recall rename —
|
||||
// both surfaces carry the tool's OWN name ────────────────────────────
|
||||
|
||||
describe("probe-phase naming: Probe/Recall labels", () => {
|
||||
it("the dock labels a probe quiz as Probe (composer surface)", () => {
|
||||
const html = renderDock({ toolName: "probe", toolCallId: "p1", input: quizInput as never });
|
||||
assert.match(html, /Probe/, "the chip carries the tool's own name");
|
||||
assert.ok(!html.includes(">Quiz<"), "not the teach label");
|
||||
assert.match(html, /Which planet is closest to the sun\?/, "still a quiz-shaped card");
|
||||
});
|
||||
|
||||
it("the dock labels a recall question as Recall", () => {
|
||||
const html = renderDock({ toolName: "recall", toolCallId: "r1", input: { question: "Explain the chain rule.", correctAnswer: "Outer function times the derivative of the inner." } as never });
|
||||
assert.match(html, /Recall/);
|
||||
assert.ok(!html.includes(">Explain<"), "not the teach label");
|
||||
});
|
||||
|
||||
it("an answered probe labels Probe — the phase boundary visible in the transcript", () => {
|
||||
const qd: QuestionData = {
|
||||
id: "p2",
|
||||
kind: "quiz",
|
||||
question: "Which planet is closest to the sun?",
|
||||
mode: "single-select",
|
||||
options: [{ label: "Mercury", value: "mercury" }, { label: "Venus", value: "venus" }],
|
||||
correctIndices: [1],
|
||||
committedAnswer: "Mercury — closest orbit.",
|
||||
explanation: "Mercury, always Mercury.",
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
const html = renderToStaticMarkup(<AnsweredCard answer={buildAnswer(qd, { values: ["mercury"] }, "probe")} />);
|
||||
assert.match(html, /Probe/);
|
||||
assert.ok(!html.includes(">Quiz<"), "not the teach label");
|
||||
});
|
||||
|
||||
it("an answered recall labels Recall", () => {
|
||||
const recallQD: QuestionData = {
|
||||
id: "r2",
|
||||
kind: "explain",
|
||||
question: "Explain why the null result matters.",
|
||||
mode: "text",
|
||||
options: [],
|
||||
committedAnswer: "It ruled out the luminiferous aether.",
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
const html = renderToStaticMarkup(<AnsweredCard answer={buildAnswer(recallQD, { text: "A hand-wave." }, "recall")} />);
|
||||
assert.match(html, /Recall/);
|
||||
assert.ok(!html.includes(">Explain<"), "not the teach label");
|
||||
});
|
||||
|
||||
it("old records (no tool) keep the kind labels", () => {
|
||||
const oldQD: QuestionData = {
|
||||
id: "o2",
|
||||
kind: "explain",
|
||||
question: "Explain why the null result matters.",
|
||||
mode: "text",
|
||||
options: [],
|
||||
committedAnswer: "It ruled out the luminiferous aether.",
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
const html = renderToStaticMarkup(<AnsweredCard answer={buildAnswer(oldQD, { text: "A hand-wave." })} />);
|
||||
assert.match(html, /Explain/, "no tool field → the standing kind label");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the two roles in the UI (predict vs check, owner 2026-10-03) ─────────
|
||||
// Predictions arrive OPEN before the teaching; checks arrive collapsed after
|
||||
// it. The answered card tells them apart (eyebrow) and tucks a prediction's
|
||||
@@ -555,7 +631,7 @@ describe("predict vs check rendering", () => {
|
||||
assert.ok(html.includes("Prediction"), `eyebrow missing: ${html.slice(0, 200)}`);
|
||||
});
|
||||
|
||||
it("a check quiz keeps the 'Quiz' eyebrow (old records unchanged)", () => {
|
||||
it("a check quiz reads the 'Check' eyebrow — the type name IS the label (owner, 2026-10-09)", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<QuestionDock
|
||||
{...{
|
||||
@@ -568,7 +644,8 @@ describe("predict vs check rendering", () => {
|
||||
onSubmit={() => {}}
|
||||
/>,
|
||||
);
|
||||
assert.ok(html.includes("Quiz"));
|
||||
assert.ok(html.includes("Check"), "the chip carries the question's type");
|
||||
assert.ok(!html.includes(">Quiz<"), "the generic label is retired");
|
||||
});
|
||||
|
||||
it("a prediction answer SHOWS the verdict but TUCKS the explanation behind a toggle", () => {
|
||||
|
||||
@@ -100,6 +100,19 @@ describe("messageMarkdown", () => {
|
||||
assert.match(md, /The learner answered in his own words: it never strays far/);
|
||||
});
|
||||
|
||||
it("probe-phase questions export under their own names (owner, 2026-10-09)", () => {
|
||||
const probe = messageMarkdown(
|
||||
result("r1", "p1", "probe", { questionText: "What is a derivative?", correct: true, dontKnow: false }),
|
||||
"r1",
|
||||
)!;
|
||||
assert.match(probe, /^### Probe · r1$/m);
|
||||
const recall = messageMarkdown(
|
||||
result("r2", "rr1", "recall", { questionText: "Why does Mercury stay near the Sun?", text: "it never strays far" }),
|
||||
"r2",
|
||||
)!;
|
||||
assert.match(recall, /^### Recall · r2$/m);
|
||||
});
|
||||
|
||||
it("ask blocks: question + chosen options + free text", () => {
|
||||
const chosen = messageMarkdown(
|
||||
result("r1", "a1", "ask_user_question", { questionText: "Which way?", answers: ["second"], other: "also the third" }),
|
||||
|
||||
Reference in New Issue
Block a user