free-recall explain tool: the quizzing-mechanic switch (owner 2026-09-27)
Sycophancy mitigation as a structural commitment: the model writes its full correctAnswer (plus explanation, in that order) BEFORE the learner sees the question; the commitment rides the persisted call args, is revealed on submit, and grades his free text in the model's next reply — the client never grades free recall. - tools.explain setting (segmented 'Quizzing — Multiple choice | Free recall' in Agent settings' Tools group): free arms explain and drops quiz for BOTH agents (prober variant carries no nodeId, same pattern as the probe quiz); mechanics keys never pass filterEnabledTools — quiz stays unwritable (the 2026-09-19 wedge ruling stands) - doctrine lives pi-natively in the tool descriptions (owner ruling — no conditional system-prompt chunks): explain carries the commitment rule; the MC option-building craft moved out of the prompt into the quiz description; shared/20-question-tools.md slims to the quiz-agnostic core - display seams: parseExplainInput (LaTeX-debris restoration included), dock text-mode with an 'I don't know' row (kbd row 0), AnsweredCard reveal (The answer / Why, no verdict chips), [Explain answer] format - phases/plan seams recognize both quizzing tools' nodeId tags; explain answers deliberately feed no ledger evidence (no client grading to key on — a future ruling can parse the model's grade) 1049/1049 tests, typecheck/lint/build green.
This commit is contained in:
@@ -20,6 +20,9 @@ import {
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
quizInputProblems,
|
||||
explainInputBase,
|
||||
explainInputObject,
|
||||
explainInputProblems,
|
||||
quizInputSchema,
|
||||
probeQuizInputSchema,
|
||||
resolveCorrect,
|
||||
@@ -543,3 +546,71 @@ describe("dock selection (toggleOption / toggleOther)", () => {
|
||||
assert.deepEqual([...sel.selected], [1]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the explain tool: free-recall quizzing (owner 2026-09-27) ─────────────
|
||||
|
||||
describe("explainInputBase / explainInputProblems (the commitment contract)", () => {
|
||||
it("commits the argument order: explanation (reasoning) BEFORE correctAnswer (the pick)", () => {
|
||||
// Same ruling as the quiz (2026-09-25): models emit tool args in
|
||||
// schema order — reasoning first, then the answer it supports.
|
||||
const keys = Object.keys(explainInputBase.properties);
|
||||
assert.ok(keys.indexOf("explanation") < keys.indexOf("correctAnswer"));
|
||||
});
|
||||
|
||||
it("the teach object adds the required nodeId AFTER the base fields", () => {
|
||||
const keys = Object.keys(explainInputObject.properties);
|
||||
assert.deepEqual(keys, [...Object.keys(explainInputBase.properties), "nodeId"]);
|
||||
});
|
||||
|
||||
it("an empty commitment is the one authoring problem", () => {
|
||||
assert.deepEqual(explainInputProblems({ correctAnswer: "Full speed is c." }), []);
|
||||
assert.deepEqual(explainInputProblems({ correctAnswer: " " }), [
|
||||
"correctAnswer is required — commit to the full correct answer BEFORE posing the question.",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildAnswer / formatAnswerMessage: explain", () => {
|
||||
const explainQD = (overrides?: Partial<QuestionData>): QuestionData => ({
|
||||
id: "e1",
|
||||
kind: "explain",
|
||||
question: "Why does the Michelson-Morley null result matter?",
|
||||
mode: "text",
|
||||
options: [],
|
||||
committedAnswer: "It ruled out the luminiferous aether and set up special relativity.",
|
||||
explanation: "A stationary aether would produce a fringe shift; none was observed.",
|
||||
createdAt: new Date().toISOString(),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
it("snapshots the commitment + explanation; the client NEVER grades free text", () => {
|
||||
const ad = buildAnswer(explainQD(), { text: "Something about light speed being constant?" });
|
||||
assert.equal(ad.correct, undefined, "no client verdict — the model grades in its reply");
|
||||
assert.equal(ad.correctIndices, undefined);
|
||||
assert.equal(ad.committed, "It ruled out the luminiferous aether and set up special relativity.");
|
||||
assert.equal(ad.text, "Something about light speed being constant?");
|
||||
});
|
||||
|
||||
it("dontKnow records the gap with no attempted text", () => {
|
||||
// The DOCK never sends text alongside dontKnow (its submit gates it);
|
||||
// buildAnswer stays a pure mapping of the request it receives.
|
||||
const ad = buildAnswer(explainQD(), { dontKnow: true });
|
||||
assert.equal(ad.dontKnow, true);
|
||||
assert.equal(ad.text, undefined);
|
||||
});
|
||||
|
||||
it("formats the model's view: his answer, the immutable commitment, and the grading instruction", () => {
|
||||
const text = formatAnswerMessage(buildAnswer(explainQD(), { text: "No idea about aether." }));
|
||||
assert.ok(text.startsWith("[Explain answer]"));
|
||||
assert.match(text, /The learner answered: No idea about aether\./);
|
||||
assert.match(text, /Your committed correct answer \(immutable/);
|
||||
assert.match(text, /luminiferous aether/);
|
||||
assert.match(text, /Grade his answer strictly/);
|
||||
});
|
||||
|
||||
it("formats dontKnow as a gap, not a guess", () => {
|
||||
const text = formatAnswerMessage(buildAnswer(explainQD(), { dontKnow: true }));
|
||||
assert.match(text, /did NOT attempt/);
|
||||
assert.ok(!text.includes("The learner answered:"), "no phantom answer beside the gap");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user