free-recall explain tool: the quizzing-mechanic switch (owner 2026-09-27)
Sycophancy mitigation as a structural commitment: the model writes its full correctAnswer (plus explanation, in that order) BEFORE the learner sees the question; the commitment rides the persisted call args, is revealed on submit, and grades his free text in the model's next reply — the client never grades free recall. - tools.explain setting (segmented 'Quizzing — Multiple choice | Free recall' in Agent settings' Tools group): free arms explain and drops quiz for BOTH agents (prober variant carries no nodeId, same pattern as the probe quiz); mechanics keys never pass filterEnabledTools — quiz stays unwritable (the 2026-09-19 wedge ruling stands) - doctrine lives pi-natively in the tool descriptions (owner ruling — no conditional system-prompt chunks): explain carries the commitment rule; the MC option-building craft moved out of the prompt into the quiz description; shared/20-question-tools.md slims to the quiz-agnostic core - display seams: parseExplainInput (LaTeX-debris restoration included), dock text-mode with an 'I don't know' row (kbd row 0), AnsweredCard reveal (The answer / Why, no verdict chips), [Explain answer] format - phases/plan seams recognize both quizzing tools' nodeId tags; explain answers deliberately feed no ledger evidence (no client grading to key on — a future ruling can parse the model's grade) 1049/1049 tests, typecheck/lint/build green.
This commit is contained in:
@@ -20,6 +20,8 @@ import {
|
||||
createProberTools,
|
||||
createTeacherTools,
|
||||
probeQuizTool,
|
||||
probeExplainTool,
|
||||
teachExplainToolFor,
|
||||
teachQuizToolFor,
|
||||
type TeacherToolContext,
|
||||
} from "../src/agent/tools.ts";
|
||||
@@ -228,10 +230,16 @@ describe("tool toggles (core/pending.ts + factories)", () => {
|
||||
const noAsk = createProberTools(ctx, { ask_user_question: false });
|
||||
assert.deepEqual(Object.keys(noAsk).sort(), ["find", "grep", "ls", "pdf", "probe_done", "quiz", "read", "research"]);
|
||||
|
||||
// quiz off is honored by the generic machinery too (future toggleable),
|
||||
// but probe_done is never droppable — it is how the phase ends.
|
||||
// Quizzing is mechanics-managed now (owner 2026-09-27): quiz/explain
|
||||
// NEVER pass through the generic filter — hand-written quiz:false is a
|
||||
// no-op (the wedge danger stands: a quiz-less prober collapses its
|
||||
// assessment flow). probe_done is never droppable — it ends the phase.
|
||||
const noQuiz = createProberTools(ctx, { quiz: false });
|
||||
assert.deepEqual(Object.keys(noQuiz).sort(), ["ask_user_question", "find", "grep", "ls", "pdf", "probe_done", "read", "research"]);
|
||||
assert.deepEqual(Object.keys(noQuiz).sort(), ["ask_user_question", "find", "grep", "ls", "pdf", "probe_done", "quiz", "read", "research"]);
|
||||
// The mechanic switch (owner 2026-09-27): tools.explain === true arms
|
||||
// free-recall — explain in, quiz out, exactly one quizzing tool.
|
||||
const free = createProberTools(ctx, { explain: true });
|
||||
assert.deepEqual(Object.keys(free).sort(), ["ask_user_question", "explain", "find", "grep", "ls", "pdf", "probe_done", "read", "research"]);
|
||||
assert.ok(createProberTools(ctx, { probe_done: false }).probe_done, "probe_done always-on");
|
||||
});
|
||||
|
||||
@@ -255,6 +263,16 @@ describe("tool toggles (core/pending.ts + factories)", () => {
|
||||
full,
|
||||
"absent keys mean enabled — old records behave exactly as today",
|
||||
);
|
||||
|
||||
// The mechanic switch (owner 2026-09-27): free-recall swaps the teach
|
||||
// quiz for the teach explain — one quizzing tool, whichever is armed.
|
||||
const freeFull = ["ask_user_question", "demo", "explain", "find", "grep", "ls", "pdf", "plan_init", "read", "research", "visual"];
|
||||
assert.deepEqual(Object.keys(createTeacherTools(ctx, { explain: true })).sort(), freeFull);
|
||||
assert.deepEqual(
|
||||
Object.keys(createTeacherTools(ctx, { explain: true, research: false })).sort(),
|
||||
freeFull.filter((t) => t !== "research"),
|
||||
"mechanics and plain toggles compose",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -293,3 +311,78 @@ describe("prepareQuestionArguments (the string-encoded options wobble)", () => {
|
||||
assert.equal(prepare(unparseable), unparseable, "unparseable strings pass through — validation reports honestly");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the explain tool boundary (free-recall, owner 2026-09-27) ─────────────
|
||||
|
||||
describe("teachExplainToolFor: the explain variant validates nodeId like the quiz", () => {
|
||||
const validExplain = {
|
||||
question: "Explain why the null result matters.",
|
||||
explanation: "A stationary aether predicts a fringe shift.",
|
||||
correctAnswer: "It ruled out the aether.",
|
||||
nodeId: "arith",
|
||||
};
|
||||
|
||||
it("accepts a nodeId matching a tree node", async () => {
|
||||
const chatId = newChat("Explain nodeId: valid");
|
||||
chatStore.saveTree(chatId, {
|
||||
nodes: [
|
||||
{ id: "arith", title: "Arithmetic" },
|
||||
{ id: "expressions", title: "Expressions" },
|
||||
],
|
||||
edges: [{ from: "arith", to: "expressions" }],
|
||||
version: 1,
|
||||
});
|
||||
const res = await boundaryParse(teachExplainToolFor({ chatId, chatStore }), {
|
||||
...validExplain,
|
||||
nodeId: "expressions",
|
||||
});
|
||||
assert.equal(res.success, true);
|
||||
});
|
||||
|
||||
it("rejects an off-map nodeId with the slugs listed", async () => {
|
||||
const chatId = newChat("Explain nodeId: typo");
|
||||
chatStore.saveTree(chatId, { nodes: [{ id: "arith", title: "A" }], edges: [], version: 1 });
|
||||
const res = await boundaryParse(teachExplainToolFor({ chatId, chatStore }), {
|
||||
...validExplain,
|
||||
nodeId: "arithmetic",
|
||||
});
|
||||
assert.equal(res.success, false);
|
||||
assert.match(problemLines(res.error).join("\n"), /matches no node/);
|
||||
});
|
||||
|
||||
it("no tree yet: rejected — checks wait for plan_init", async () => {
|
||||
const chatId = newChat("Explain nodeId: no tree");
|
||||
const res = await boundaryParse(teachExplainToolFor({ chatId, chatStore }), validExplain);
|
||||
assert.equal(res.success, false);
|
||||
assert.match(problemLines(res.error).at(-1)!, /No plan map exists yet — call plan_init/);
|
||||
});
|
||||
|
||||
it("an empty commitment is rejected at the boundary", async () => {
|
||||
const chatId = newChat("Explain: empty commitment");
|
||||
chatStore.saveTree(chatId, { nodes: [{ id: "arith", title: "A" }], edges: [], version: 1 });
|
||||
const res = await boundaryParse(teachExplainToolFor({ chatId, chatStore }), {
|
||||
...validExplain,
|
||||
correctAnswer: " ",
|
||||
});
|
||||
assert.equal(res.success, false);
|
||||
assert.match(problemLines(res.error).join("\n"), /correctAnswer is required/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the prober's explain: no nodeId field at all", () => {
|
||||
it("the probe schema carries no nodeId; the commitment contract still binds", async () => {
|
||||
assert.ok(!("nodeId" in probeExplainTool.parameters.properties));
|
||||
const res = await boundaryParse(probeExplainTool, {
|
||||
question: "Explain what a derivative measures.",
|
||||
explanation: "It is instantaneous rate of change.",
|
||||
correctAnswer: "The instantaneous rate of change.",
|
||||
});
|
||||
assert.equal(res.success, true);
|
||||
const empty = await boundaryParse(probeExplainTool, {
|
||||
question: "q",
|
||||
explanation: "e",
|
||||
correctAnswer: " ",
|
||||
});
|
||||
assert.equal(empty.success, false);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user