free-recall explain tool: the quizzing-mechanic switch (owner 2026-09-27)
Sycophancy mitigation as a structural commitment: the model writes its full correctAnswer (plus explanation, in that order) BEFORE the learner sees the question; the commitment rides the persisted call args, is revealed on submit, and grades his free text in the model's next reply — the client never grades free recall. - tools.explain setting (segmented 'Quizzing — Multiple choice | Free recall' in Agent settings' Tools group): free arms explain and drops quiz for BOTH agents (prober variant carries no nodeId, same pattern as the probe quiz); mechanics keys never pass filterEnabledTools — quiz stays unwritable (the 2026-09-19 wedge ruling stands) - doctrine lives pi-natively in the tool descriptions (owner ruling — no conditional system-prompt chunks): explain carries the commitment rule; the MC option-building craft moved out of the prompt into the quiz description; shared/20-question-tools.md slims to the quiz-agnostic core - display seams: parseExplainInput (LaTeX-debris restoration included), dock text-mode with an 'I don't know' row (kbd row 0), AnsweredCard reveal (The answer / Why, no verdict chips), [Explain answer] format - phases/plan seams recognize both quizzing tools' nodeId tags; explain answers deliberately feed no ledger evidence (no client grading to key on — a future ruling can parse the model's grade) 1049/1049 tests, typecheck/lint/build green.
This commit is contained in:
amosblomqvist
@@ -11,6 +11,9 @@ import type { QuestionAnswerData } from "@/src/core";
|
||||
* Quiz answers are GRADED: ✓/✗ marks, the correct answers, and the
|
||||
* explanation. Ask answers are NOT graded — a plain record of what was said.
|
||||
* (The v1 bug this pins: ask answers must never get quiz grading UI.)
|
||||
* Explain answers are graded by the MODEL in its next reply, not by the
|
||||
* client — the card reveals his answer, the committed correct answer, and
|
||||
* the explanation, with NO verdict chips (owner ruling 2026-09-27).
|
||||
*/
|
||||
export function AnsweredCard({ answer }: { answer: QuestionAnswerData }) {
|
||||
if (answer.dismissed) {
|
||||
@@ -22,13 +25,14 @@ export function AnsweredCard({ answer }: { answer: QuestionAnswerData }) {
|
||||
}
|
||||
|
||||
const isQuiz = answer.kind === "quiz";
|
||||
const isExplain = answer.kind === "explain";
|
||||
const selectedIdx = new Set(answer.answers.map((a) => a.index));
|
||||
const correctIdx = new Set(answer.correctIndices ?? []);
|
||||
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="answered-card">
|
||||
<div className="mb-1 flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||
{isQuiz ? "Quiz" : "Question"}
|
||||
{isQuiz ? "Quiz" : isExplain ? "Explain" : "Question"}
|
||||
{isQuiz &&
|
||||
(answer.dontKnow ? (
|
||||
<span className="rounded bg-muted px-1.5 py-0.5">didn't know</span>
|
||||
@@ -97,7 +101,9 @@ export function AnsweredCard({ answer }: { answer: QuestionAnswerData }) {
|
||||
)}
|
||||
|
||||
{answer.text && <div className="lh-prose mt-2 whitespace-pre-wrap">“{answer.text}”</div>}
|
||||
{answer.dontKnow && isQuiz && <div className="mt-2 text-sm text-muted-foreground">Answered “I don't know”.</div>}
|
||||
{answer.dontKnow && (isQuiz || isExplain) && (
|
||||
<div className="mt-2 text-sm text-muted-foreground">Answered “I don't know”.</div>
|
||||
)}
|
||||
|
||||
{answer.note && (
|
||||
<div data-testid="your-thinking" className="mt-3 border-l-2 border-primary/40 pl-3">
|
||||
@@ -119,6 +125,26 @@ export function AnsweredCard({ answer }: { answer: QuestionAnswerData }) {
|
||||
<MessageResponse className="inline [&_p]:inline [&_p]:m-0">{answer.explanation}</MessageResponse>
|
||||
</div>
|
||||
)}
|
||||
{/* The explain reveal (owner ruling 2026-09-27): the committed answer
|
||||
and its explanation, shown on submit — no verdict, the model's next
|
||||
reply IS the grading. */}
|
||||
{isExplain && (answer.committed || answer.explanation) && (
|
||||
<div className="mt-3 rounded-lg bg-muted py-2 text-sm text-muted-foreground">
|
||||
{answer.committed && (
|
||||
<>
|
||||
<span className="font-medium text-foreground">The answer: </span>
|
||||
<MessageResponse className="inline [&_p]:inline [&_p]:m-0">{answer.committed}</MessageResponse>
|
||||
</>
|
||||
)}
|
||||
{answer.committed && answer.explanation && <div className="my-1" />}
|
||||
{answer.explanation && (
|
||||
<>
|
||||
<span className="font-medium text-foreground">Why: </span>
|
||||
<MessageResponse className="inline [&_p]:inline [&_p]:m-0">{answer.explanation}</MessageResponse>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
+3
-2
@@ -1115,7 +1115,8 @@ export function Chat({
|
||||
/>
|
||||
);
|
||||
case "tool-quiz":
|
||||
case "tool-ask_user_question": {
|
||||
case "tool-ask_user_question":
|
||||
case "tool-explain": {
|
||||
// Pending/streaming questions live in the dock, not the
|
||||
// transcript; answered ones render as graded records.
|
||||
if (part.state === "output-available") {
|
||||
@@ -1650,7 +1651,7 @@ function DockOverlay({
|
||||
<div key={mode} className={mode === initialModeRef.current ? undefined : "dock-variant"}>
|
||||
{pending ? (
|
||||
<QuestionDock
|
||||
toolName={pending.tool as "quiz" | "ask_user_question"}
|
||||
toolName={pending.tool as "quiz" | "ask_user_question" | "explain"}
|
||||
toolCallId={pending.toolCallId}
|
||||
input={pending.input as never}
|
||||
expanded={expanded}
|
||||
|
||||
@@ -16,11 +16,13 @@ import {
|
||||
shuffleOptions,
|
||||
parseAskInput,
|
||||
parseQuizInput,
|
||||
parseExplainInput,
|
||||
toggleOption,
|
||||
toggleOther,
|
||||
type AnswerRequest,
|
||||
type AskToolInput,
|
||||
type DockSelection,
|
||||
type ExplainToolInput,
|
||||
type QuestionAnswerData,
|
||||
type QuestionData,
|
||||
type QuestionMode,
|
||||
@@ -50,9 +52,9 @@ import {
|
||||
*/
|
||||
|
||||
export interface QuestionDockProps {
|
||||
toolName: "quiz" | "ask_user_question";
|
||||
toolName: "quiz" | "ask_user_question" | "explain";
|
||||
toolCallId: string;
|
||||
input: QuizToolInput | AskToolInput;
|
||||
input: QuizToolInput | AskToolInput | ExplainToolInput;
|
||||
/** Collapse state, owned by the DockOverlay (it drives the shell's height morph). */
|
||||
expanded: boolean;
|
||||
/**
|
||||
@@ -92,7 +94,8 @@ export function QuestionDock({
|
||||
// bridge), so this is pure defense in depth — but it is STRUCTURAL now:
|
||||
// anything that slips through renders the quiet fallback, never a
|
||||
// crash on `.map` over model-authored bytes.
|
||||
const parsed = toolName === "quiz" ? parseQuizInput(input) : parseAskInput(input);
|
||||
const parsed =
|
||||
toolName === "quiz" ? parseQuizInput(input) : toolName === "explain" ? parseExplainInput(input) : parseAskInput(input);
|
||||
if (!parsed) {
|
||||
return (
|
||||
<div data-testid="malformed-question" className="rounded-xl border border-dashed bg-card/60 p-3 text-sm text-muted-foreground">
|
||||
@@ -148,7 +151,7 @@ export function QuestionDockBar({
|
||||
onExpand,
|
||||
variant = "collapsed",
|
||||
}: {
|
||||
toolName: "quiz" | "ask_user_question";
|
||||
toolName: "quiz" | "ask_user_question" | "explain";
|
||||
question: string;
|
||||
onExpand: () => void;
|
||||
variant?: "collapsed" | "reveal";
|
||||
@@ -182,7 +185,7 @@ export function QuestionDockBar({
|
||||
className="dock-variant flex w-full items-center gap-2.5 px-4 py-2.5 text-left text-sm transition-colors hover:bg-accent/40"
|
||||
>
|
||||
<span className="shrink-0 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||
{toolName === "quiz" ? "Quiz" : "Question"}
|
||||
{toolName === "quiz" ? "Quiz" : toolName === "explain" ? "Explain" : "Question"}
|
||||
</span>
|
||||
<span className="min-w-0 flex-1 truncate text-muted-foreground">{teaserText(question)}</span>
|
||||
<span className="flex shrink-0 items-center gap-1 text-xs font-medium">
|
||||
@@ -202,7 +205,7 @@ function QuestionDockForm({
|
||||
onSubmit,
|
||||
onEscape,
|
||||
}: Omit<QuestionDockProps, "expanded" | "onExpand" | "reveal">) {
|
||||
const kind = toolName === "quiz" ? ("quiz" as const) : ("ask" as const);
|
||||
const kind = toolName === "quiz" ? ("quiz" as const) : toolName === "explain" ? ("explain" as const) : ("ask" as const);
|
||||
|
||||
// Canonical QuestionData, built once per question. Quiz options are
|
||||
// deterministically shuffled (seeded by toolCallId) so position never leaks
|
||||
@@ -211,7 +214,8 @@ function QuestionDockForm({
|
||||
const qd = useMemo<QuestionData>(() => {
|
||||
const quizInput = kind === "quiz" ? (input as QuizToolInput) : null;
|
||||
const askInput = kind === "ask" ? (input as AskToolInput) : null;
|
||||
const multi = input.multiSelect === true;
|
||||
const explainInput = kind === "explain" ? (input as ExplainToolInput) : null;
|
||||
const multi = input instanceof Object && "multiSelect" in input ? input.multiSelect === true : false;
|
||||
const mode: QuestionMode =
|
||||
kind === "quiz"
|
||||
? multi
|
||||
@@ -241,6 +245,12 @@ function QuestionDockForm({
|
||||
q.correctIndices = resolved.indices;
|
||||
q.explanation = quizInput.explanation;
|
||||
}
|
||||
if (explainInput) {
|
||||
// The commitment rides the question data (never sent to the
|
||||
// client before answering) and snapshots into the answer at submit.
|
||||
q.committedAnswer = explainInput.correctAnswer;
|
||||
q.explanation = explainInput.explanation;
|
||||
}
|
||||
return q;
|
||||
}, [toolCallId, kind, input]);
|
||||
|
||||
@@ -275,7 +285,7 @@ function QuestionDockForm({
|
||||
|
||||
const canSubmit =
|
||||
qd.mode === "text"
|
||||
? text.trim().length > 0
|
||||
? sel.dontKnow || text.trim().length > 0 // explain: dontKnow is a first-class answer
|
||||
: sel.dontKnow || (sel.otherSelected && other.trim().length > 0) || sel.selected.size > 0;
|
||||
|
||||
const submit = () => {
|
||||
@@ -314,13 +324,14 @@ function QuestionDockForm({
|
||||
const otherInputRef = useRef<HTMLInputElement>(null);
|
||||
const thinkingRef = useRef<HTMLTextAreaElement>(null);
|
||||
// One row past the options is the trailing row: quiz → don't-know,
|
||||
// ask → Other (a question never has both).
|
||||
// ask → Other (a question never has both). Explain has exactly ONE
|
||||
// row — its don't-know (there are no options to walk).
|
||||
const optionCount = qd.mode === "text" ? 0 : qd.options.length;
|
||||
const rowCount = qd.mode === "text" ? 0 : optionCount + 1;
|
||||
const rowCount = qd.mode === "text" ? (kind === "explain" ? 1 : 0) : optionCount + 1;
|
||||
|
||||
const kbdAct = (idx: number) => {
|
||||
if (idx < optionCount) toggle(idx);
|
||||
else if (kind === "quiz")
|
||||
else if (kind === "quiz" || kind === "explain")
|
||||
setSel((prev) => ({ ...prev, selected: new Set(), otherSelected: false, dontKnow: !prev.dontKnow }));
|
||||
else {
|
||||
// Space on the ask's Other row (owner refinement round,
|
||||
@@ -412,7 +423,7 @@ function QuestionDockForm({
|
||||
{/* Question area — scrolls internally if very tall; the bottom row stays put. */}
|
||||
<div className="max-h-[55dvh] overflow-y-auto px-4 pt-3">
|
||||
<div className="mb-1 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||
{kind === "quiz" ? "Quiz" : "Question"}
|
||||
{kind === "quiz" ? "Quiz" : kind === "explain" ? "Explain" : "Question"}
|
||||
{qd.mode === "multi-select" && " · select all that apply"}
|
||||
</div>
|
||||
<div className="mb-3 font-medium">
|
||||
@@ -424,7 +435,10 @@ function QuestionDockForm({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{qd.mode !== "text" && (
|
||||
{/* Text-mode questions render no options list — EXCEPT explain,
|
||||
whose single "I don't know" row is the one walkable answer
|
||||
(kbd row 0). The answer itself goes in the composer below. */}
|
||||
{(qd.mode !== "text" || kind === "explain") && (
|
||||
<div className="flex flex-col gap-2 pb-2" role={qd.mode === "multi-select" ? "group" : "radiogroup"}>
|
||||
{qd.options.map((opt, i) => {
|
||||
const isSel = sel.selected.has(i);
|
||||
@@ -470,7 +484,7 @@ function QuestionDockForm({
|
||||
);
|
||||
})}
|
||||
|
||||
{kind === "quiz" && (
|
||||
{(kind === "quiz" || kind === "explain") && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() =>
|
||||
|
||||
@@ -66,6 +66,7 @@ import {
|
||||
} from "@/src/core";
|
||||
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import { prettyModel } from "@/components/usage-format";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { SettingsGroup, SettingsRow, Toggle } from "./controls.tsx";
|
||||
import { ModelMenuList, ThinkingMenuList } from "./model-menu";
|
||||
|
||||
@@ -110,6 +111,15 @@ export function composeTool(agent: AgentSettings | undefined, id: string, on: bo
|
||||
return { ...agent, tools: { ...agent?.tools, [id]: on } };
|
||||
}
|
||||
|
||||
/** The quizzing mechanics (owner 2026-09-27): a segmented control, not a
|
||||
* toggle — exactly one mechanic is ever active. `explain: true` arms
|
||||
* free-recall (explain in, quiz out); anything else is multiple choice.
|
||||
* `quiz` itself is never written (mechanics-managed — see quizMechanics
|
||||
* in agent/tools.ts). Pure — exported for tests. */
|
||||
export function composeQuizMechanics(agent: AgentSettings | undefined, mechanics: "choice" | "free"): AgentSettings {
|
||||
return { ...agent, tools: { ...agent?.tools, explain: mechanics === "free" } };
|
||||
}
|
||||
|
||||
/** Compose the prober graph-tools toggle (wave 7): its own key in the
|
||||
* agent section — never in the tool-toggles map (owner ruling). Absent =
|
||||
* ON (the toggle discipline); the write is explicit either way, so an
|
||||
@@ -725,6 +735,46 @@ export function AgentSection({
|
||||
</SettingsRow>
|
||||
);
|
||||
})()}
|
||||
{/* The quizzing mechanic (owner 2026-09-27): segmented — exactly one
|
||||
of Multiple choice / Free recall is active; the choice swaps which
|
||||
quizzing tool BOTH agents carry (prober checks + teach checks).
|
||||
Persisted as agent.tools.explain; absent = Multiple choice. */}
|
||||
{(() => {
|
||||
const free = agent?.tools?.explain === true;
|
||||
return (
|
||||
<SettingsRow
|
||||
label="Quizzing"
|
||||
hint="How both tutors check your understanding"
|
||||
testid="quiz-mechanics-row"
|
||||
>
|
||||
<div
|
||||
role="radiogroup"
|
||||
aria-label="Quizzing mechanics"
|
||||
className="flex rounded-lg border border-border/60 bg-muted/30 p-0.5"
|
||||
>
|
||||
{(["choice", "free"] as const).map((m) => {
|
||||
const active = m === "free" ? free : !free;
|
||||
return (
|
||||
<button
|
||||
key={m}
|
||||
type="button"
|
||||
role="radio"
|
||||
aria-checked={active}
|
||||
data-testid={`quiz-mechanics-${m}`}
|
||||
onClick={() => onSectionChange(composeQuizMechanics(agent, m))}
|
||||
className={cn(
|
||||
"h-8 rounded-md px-3 text-sm font-medium transition-colors",
|
||||
active ? "bg-background text-foreground shadow-sm" : "text-muted-foreground hover:text-foreground",
|
||||
)}
|
||||
>
|
||||
{m === "choice" ? "Multiple choice" : "Free recall"}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</SettingsRow>
|
||||
);
|
||||
})()}
|
||||
{TOGGLE_TOOLS.map((id) => {
|
||||
// Research needs a live search engine; the row + toggle gray
|
||||
// on the same signal the registry drops the tool on (see
|
||||
|
||||
Reference in New Issue
Block a user