free-recall explain tool: the quizzing-mechanic switch (owner 2026-09-27)

Sycophancy mitigation as a structural commitment: the model writes its
full correctAnswer (plus explanation, in that order) BEFORE the learner
sees the question; the commitment rides the persisted call args, is
revealed on submit, and grades his free text in the model's next reply —
the client never grades free recall.

- tools.explain setting (segmented 'Quizzing — Multiple choice | Free
  recall' in Agent settings' Tools group): free arms explain and drops
  quiz for BOTH agents (prober variant carries no nodeId, same pattern
  as the probe quiz); mechanics keys never pass filterEnabledTools —
  quiz stays unwritable (the 2026-09-19 wedge ruling stands)
- doctrine lives pi-natively in the tool descriptions (owner ruling —
  no conditional system-prompt chunks): explain carries the commitment
  rule; the MC option-building craft moved out of the prompt into the
  quiz description; shared/20-question-tools.md slims to the
  quiz-agnostic core
- display seams: parseExplainInput (LaTeX-debris restoration included),
  dock text-mode with an 'I don't know' row (kbd row 0), AnsweredCard
  reveal (The answer / Why, no verdict chips), [Explain answer] format
- phases/plan seams recognize both quizzing tools' nodeId tags; explain
  answers deliberately feed no ledger evidence (no client grading to
  key on — a future ruling can parse the model's grade)

1049/1049 tests, typecheck/lint/build green.
This commit is contained in:
amosblomqvist
2026-09-27 08:55:13 +03:00
parent 07d2c84461
commit 2f39766528
25 changed files with 784 additions and 68 deletions
+82
View File
@@ -26,6 +26,7 @@ import {
type FauxResponseStep,
} from "@earendil-works/pi-ai";
import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
import { createTeacherTools } from "../src/agent/tools.ts";
import { handleChatInput, handleChatRequest } from "../src/server/chat-handler.ts";
import { effectiveSettings } from "../src/server/stores.ts";
import { isTurnRunning } from "../src/server/turn-liveness.ts";
@@ -1050,3 +1051,84 @@ describe("background-turn runner (owner ruling 2026-09-25: every turn is a serve
}
});
});
// ─── the explain mechanic end-to-end (free-recall, owner 2026-09-27) ───────
it("explain tool call persists as a pending part; the free-text answer reveals the commitment and reaches the model", async () => {
const EXPLAIN_INPUT = {
question: "Explain why Mercury is hard to observe.",
explanation: "It never strays far from the Sun in our sky.",
correctAnswer: "Its small angular separation from the Sun.",
nodeId: "planet-order",
};
const { engine, calls } = scriptedTeacher((cs) => [
(context) => {
cs.push(context.messages.map((m) => JSON.stringify((m as { content: unknown }).content)));
return fauxAssistantMessage(
[fauxThinking("Free recall this time."), fauxToolCall("explain", EXPLAIN_INPUT, { id: "call-1" })],
{ stopReason: "toolUse" },
);
},
(context) => {
cs.push(context.messages.map((m) => JSON.stringify((m as { content: unknown }).content)));
return fauxAssistantMessage([fauxText("Right — it hugs the Sun.")]);
},
]);
const chat = createConfiguredChat("Test: explain mechanic");
createdChats.push(chat.id);
// Free-recall armed: the mechanic swap is the ONLY change to the registry.
const tools = createTeacherTools(
{ chatId: chat.id, chatStore, jobStore, spawn: () => {} },
{ explain: true },
);
assert.ok(tools.explain && !tools.quiz, "explain armed, quiz dropped — one quizzing tool");
const factory: Parameters<typeof handleChatRequest>[1] = (opts) =>
createTeacherAgent(undefined, { engine, messages: opts.messages, tools });
await drain(await handleChatRequest(post(chat.id, { type: "text", text: "teach me the solar system" }), factory));
const saved1 = chatStore.getChat(chat.id)!;
const savedContent1 = (saved1.messages[1] as { content: unknown }).content as Array<{ type: string; name?: string }>;
const call = savedContent1.find((b) => b.type === "toolCall" && b.name === "explain");
assert.ok(call, "explain tool call persisted");
const pending = danglingQuestionOf(saved1.messages);
assert.equal(pending?.tool, "explain");
assert.deepEqual(pending?.input, EXPLAIN_INPUT);
const view1 = projectView(saved1.messages);
assert.ok(view1[1]!.parts.some((p) => p.type === "tool-explain"), "tool-explain part projected — the dock arms");
// The client answers in his own words (the dock's buildAnswer seam):
const qd: QuestionData = {
id: "call-1",
kind: "explain",
question: EXPLAIN_INPUT.question,
mode: "text",
options: [],
committedAnswer: EXPLAIN_INPUT.correctAnswer,
explanation: EXPLAIN_INPUT.explanation,
createdAt: new Date().toISOString(),
};
const answer = buildAnswer(qd, { text: "It stays near the Sun." });
assert.equal(answer.correct, undefined, "no client grading — the model grades");
await drain(await handleChatRequest(post(chat.id, { type: "answer", toolCallId: "call-1", answer }), factory));
const saved2 = chatStore.getChat(chat.id)!;
const view2 = projectView(saved2.messages);
const part = view2[1]!.parts.find((p) => p.type === "tool-explain");
assert.equal((part as { state?: string }).state, "output-available");
const out = (part as { output?: { committed?: string; explanation?: string } }).output!;
assert.equal(out.committed, EXPLAIN_INPUT.correctAnswer, "the commitment snapshots into the answer record");
assert.equal(out.explanation, EXPLAIN_INPUT.explanation);
// The model's continuation saw the answer JSON (fork 1: the answer IS
// the call's tool result) — his text AND the commitment snapshot ride it.
const seen = calls[1]!.join("\n");
assert.match(seen, /It stays near the Sun\./);
assert.match(seen, /committed/);
assert.match(seen, /small angular separation/);
assert.equal(danglingQuestionOf(saved2.messages), null, "no pending question remains");
});