From 9b9f6372fada07f028204b73fa543bd9e4adfe72 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Fri, 2 Oct 2026 00:24:05 +0300 Subject: [PATCH] =?UTF-8?q?quiz=20answers=20carry=20a=20confidence=20slide?= =?UTF-8?q?r=20=E2=80=94=20raw=20probability,=20snapped=200/30/50/70/90/10?= =?UTF-8?q?0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner design round 2026-10-01: every multiple-choice quiz answer now carries the learner's stated probability that his selected answer is correct — calibration data for the prober and teacher, the flow itself unchanged. Definition (owner ruling, after weighing chance-normalization): RAW probability. Sub-chance reports are diagnostic gold, not dead space — 'picked an option, believes it wrong' is a real state. The agents get the sophistication: new shared/25-confidence.md composes for both agents (chance level, sub-chance = distrusts the pick, correct+low = fragile, wrong+high = committed misconception, IDK = genuine gap). - question-core: AnswerRequest/QuestionAnswerData.confidence (quiz only, optional — old records valid), buildAnswer clamps and drops it on dontKnow, formatAnswerMessage states the line, CONFIDENCE_STOPS + DEFAULT_CONFIDENCE_INDEX exported. - question-dock: snapped slider below 'I don't know', above 'Your thinking' — native range over the stop INDEX so ←/→ step the uneven grid cleanly, Tab chain rows → slider → thinking, Enter submits from the slider too. Rests at 50 and always sends (owner: 50 must mean a real coin flip, never 'untouched'). 'I don't know' GRAYS it out — layout holds, no value sent. Deliberately untouched: explain free-recall card (quiz-only scope), answered-card rendering of confidence, ask flow. --- components/question-dock.tsx | 79 +++++++++++++++++++++-- src/agent/prompts/shared/25-confidence.md | 11 ++++ src/core/question-core.ts | 20 +++++- tests/prompt-overrides.test.ts | 1 + tests/question-core.test.ts | 32 +++++++++ 5 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 src/agent/prompts/shared/25-confidence.md diff --git a/components/question-dock.tsx b/components/question-dock.tsx index 213a9fd..413f249 100644 --- a/components/question-dock.tsx +++ b/components/question-dock.tsx @@ -9,6 +9,8 @@ import { cn } from "@/lib/utils"; import { buildAnswer, coerceCorrectAnswer, + CONFIDENCE_STOPS, + DEFAULT_CONFIDENCE_INDEX, emptyDockSelection, normalizeOptions, resolveCorrect, @@ -264,6 +266,12 @@ function QuestionDockForm({ const [thinking, setThinking] = useState(""); const [other, setOther] = useState(""); const [text, setText] = useState(""); + // Confidence (owner, 2026-10-01): an index into CONFIDENCE_STOPS, + // resting at 50 — a real coin flip by definition, untouched or + // deliberate (the same number, the same meaning). The dock unmounts + // between questions, so each question starts fresh at 50. + const [confIdx, setConfIdx] = useState(DEFAULT_CONFIDENCE_INDEX); + const confidenceRef = useRef(null); const selectMode: SelectMode = qd.mode === "multi-select" ? "multi-select" : "single-select"; // The PRIMARY dictation input (owner, STT round 1): while a question @@ -297,6 +305,9 @@ function QuestionDockForm({ other: sel.otherSelected ? other : undefined, text: qd.mode === "text" ? text : undefined, note: qd.mode !== "text" ? thinking.trim() || undefined : undefined, + // Always sent on quiz answers (owner: 50 must mean 50, never + // "untouched"); buildAnswer drops it on "I don't know". + confidence: kind === "quiz" ? CONFIDENCE_STOPS[confIdx] : undefined, }; onSubmit(buildAnswer(qd, req)); }; @@ -368,14 +379,19 @@ function QuestionDockForm({ // Alt/Cmd/Ctrl combos are never ours (Alt+arrows cycle sessions // globally — they must not ALSO walk the question rows). const modded = e.altKey || e.metaKey || e.ctrlKey; - // Plain Tab toggles focus: rows → thinking, anything else → the - // walked (or first) row. Shift+Tab falls through (the global - // thinking-level cycle owns it), so the shift check matters. + // Plain Tab walks the dock's focus chain forward: rows → + // confidence slider (quiz, when enabled) → thinking. Shift+Tab + // falls through (the global thinking-level cycle owns it), so the + // shift check matters. if (e.key === "Tab" && !e.shiftKey && !modded) { e.preventDefault(); e.stopPropagation(); - if (t?.closest?.("[data-kbdrow]")) thinkingRef.current?.focus(); - else { + if (t?.closest?.("[data-confidence]")) thinkingRef.current?.focus(); + else if (t?.closest?.("[data-kbdrow]")) { + const slider = confidenceRef.current; + if (slider && !slider.disabled) slider.focus(); + else thinkingRef.current?.focus(); + } else { const idx = kbdIdx ?? 0; setKbdIdx(idx); focusRow(idx); @@ -573,6 +589,59 @@ function QuestionDockForm({ )} + {/* Confidence (owner, 2026-10-01): a snapped slider below the options + and above "Your thinking" — the learner's stated probability + that his selected answer is correct. ←/→ move it (native range + semantics over the stop INDEX, so the uneven grid steps + cleanly). "I don't know" GRAYS it out — the layout holds, no + value is sent, the IDK signal stands alone. */} + {kind === "quiz" && ( +
+
+ + Confidence + + setConfIdx(Number(e.target.value))} + onKeyDown={(e) => { + // Enter submits from here too (dock-wide intuition); + // arrows stay native — ←/→ step the stop index. + if (e.key === "Enter" && canSubmit) { + e.preventDefault(); + submit(); + } + }} + className="h-1.5 min-w-0 flex-1 accent-primary disabled:opacity-40" + /> + + {sel.dontKnow ? "—" : `${CONFIDENCE_STOPS[confIdx]}%`} + +
+
+ )} + {/* The learner's half of the dock — a composer TWIN: a labeled, bordered field with Submit docked inside, exactly like the chat composer's send. Select modes get "Your thinking" (Enter makes a newline — it diff --git a/src/agent/prompts/shared/25-confidence.md b/src/agent/prompts/shared/25-confidence.md new file mode 100644 index 0000000..ad1d57b --- /dev/null +++ b/src/agent/prompts/shared/25-confidence.md @@ -0,0 +1,11 @@ +## Confidence — reading the learner's self-assessment + +Every quiz answer arrives with a `Confidence: N%` line: the learner's stated probability that his selected answer is correct, set on a snapped slider (0, 30, 50, 70, 90, 100). This is calibration data, not decoration. Read it against the verdict: + +- 50 is a genuine coin flip. 90–100 claims certainty; 0–30 claims real doubt. +- For an n-option question, chance is 1/n. Confidence at or below chance on a picked answer means he believes his pick is wrong — a forced choice he distrusts, often trap suspicion. Treat it as honesty, not error. +- Correct + high confidence: solid knowledge — move on. +- Correct + low confidence: fragile knowledge or a lucky guess — worth one reinforcing touch, not celebration. +- Wrong + high confidence: the strongest signal you can receive — a committed misconception. Name the crack, probe it gently, and re-teach around it. +- Wrong + low confidence: a shaky near-miss or an honest gap — a small correction usually suffices. +- "I don't know" carries no confidence line: he declined to guess. A genuine gap, never a wrong answer. diff --git a/src/core/question-core.ts b/src/core/question-core.ts index b78fcdf..4059d0d 100644 --- a/src/core/question-core.ts +++ b/src/core/question-core.ts @@ -350,6 +350,7 @@ export interface QuestionAnswerData { other?: string; // ask: "Other" free text text?: string; // ask/explain: text-mode answer note?: string; + confidence?: number; // quiz only: the learner's stated probability (0-100, raw) that his selected answer is correct — never sent on "I don't know" (owner, 2026-10-01) correct?: boolean; // quiz only correctIndices?: number[]; // quiz only (rendering) committed?: string; // explain only (rendering): the model's committed answer @@ -375,8 +376,17 @@ export interface AnswerRequest { other?: string; // ask "Other" text?: string; // ask text-mode note?: string; + confidence?: number; // quiz only: 0-100 raw probability — 50 is a real coin flip, always sent (owner, 2026-10-01) } +/** The confidence slider's snap stops (owner, 2026-10-01): raw probability + * that the selected answer is correct, uneven on purpose — resolution + * concentrates at the top where the calibration signal lives (90 vs 100 + * separates knows from mastered). 50 is a genuine coin flip and the + * resting default — untouched and deliberate 50 are the same number. */ +export const CONFIDENCE_STOPS: readonly number[] = [0, 30, 50, 70, 90, 100]; +export const DEFAULT_CONFIDENCE_INDEX = 2; + // ─── dock selection (the client state machine, pure for tests) ───────────── /** The question dock's selection state (0-based indices in display order). */ @@ -571,7 +581,6 @@ export function buildAnswer(qd: QuestionData, req: AnswerRequest): QuestionAnswe note: req.note?.trim() || undefined, answeredAt: new Date().toISOString(), }; - // resolve selected option values → OptionAnswer (in display order) if (req.values && req.values.length > 0) { const byValue = new Map(qd.options.map((o, i) => [o.value, { ...o, index: i + 1 }])); @@ -589,6 +598,12 @@ export function buildAnswer(qd: QuestionData, req: AnswerRequest): QuestionAnswe base.correctIndices = qd.correctIndices ?? []; base.explanation = qd.explanation; base.correct = base.dontKnow ? false : isCorrect(base.answers.map((a) => a.index), qd.correctIndices ?? []); + // The confidence slider rides on real quiz answers only — never on + // "I don't know" (no answer, no probability). Clamped defensively; + // the dock snaps to the grid, the record accepts any integer. + if (!base.dontKnow && req.confidence !== undefined) { + base.confidence = Math.max(0, Math.min(100, Math.round(req.confidence))); + } } if (qd.kind === "explain") { // No client grading — the model grades in its reply, against the @@ -620,6 +635,9 @@ export function formatAnswerMessage(ad: QuestionAnswerData): string { const selectedStr = ad.answers.map((a) => `${a.index}. ${a.label}`).join(", "); lines.push(`Selected: ${selectedStr || "(none)"}`); lines.push(`Result: ${verdict}`); + if (ad.confidence !== undefined) { + lines.push(`Confidence: ${ad.confidence}% — the learner's stated probability that his selected answer is correct.`); + } } lines.push(`Correct: ${correctStr}`); } else if (ad.kind === "explain") { diff --git a/tests/prompt-overrides.test.ts b/tests/prompt-overrides.test.ts index 9f6a57c..51b87b4 100644 --- a/tests/prompt-overrides.test.ts +++ b/tests/prompt-overrides.test.ts @@ -179,6 +179,7 @@ describe("the source-derived segment list", () => { "system-prompt.md", "prober/10-probe-doctrine.md", "shared/20-question-tools.md", + "shared/25-confidence.md", "shared/30-learner-files.md", "shared/40-formatting.md", ], diff --git a/tests/question-core.test.ts b/tests/question-core.test.ts index 0efe705..9a77b02 100644 --- a/tests/question-core.test.ts +++ b/tests/question-core.test.ts @@ -12,6 +12,8 @@ import { askOptionProblems, buildAnswer, coerceCorrectAnswer, + CONFIDENCE_STOPS, + DEFAULT_CONFIDENCE_INDEX, findPendingQuestion, formatAnswerMessage, isCorrect, @@ -337,6 +339,13 @@ function quizData(overrides?: Partial): QuestionData { }; } +describe("confidence stops", () => { + it("the owner-decided grid (2026-10-01): resolution concentrates at the top", () => { + assert.deepEqual(CONFIDENCE_STOPS, [0, 30, 50, 70, 90, 100]); + assert.equal(CONFIDENCE_STOPS[DEFAULT_CONFIDENCE_INDEX], 50, "rests at a real coin flip"); + }); +}); + describe("buildAnswer", () => { it("grades a correct selection", () => { const ad = buildAnswer(quizData(), { values: ["mercury"] }); @@ -368,6 +377,22 @@ describe("buildAnswer", () => { assert.deepEqual(ad.answers.map((a) => a.value), ["mercury"]); }); + // Confidence (owner, 2026-10-01): the slider's raw probability rides on + // real quiz answers only — never on "I don't know" (no answer, no + // probability), never on ask/explain. Clamped defensively. + it("carries confidence on quiz answers and clamps out-of-range values", () => { + assert.equal(buildAnswer(quizData(), { values: ["mercury"], confidence: 90 }).confidence, 90); + assert.equal(buildAnswer(quizData(), { values: ["mercury"], confidence: 150 }).confidence, 100); + assert.equal(buildAnswer(quizData(), { values: ["mercury"], confidence: -5 }).confidence, 0); + assert.equal(buildAnswer(quizData(), { values: ["mercury"] }).confidence, undefined); + }); + + it("drops confidence on dontKnow and on non-quiz kinds", () => { + assert.equal(buildAnswer(quizData(), { dontKnow: true, confidence: 70 }).confidence, undefined); + const ask = quizData({ kind: "ask", correctIndices: undefined, explanation: undefined }); + assert.equal(buildAnswer(ask, { other: "x", confidence: 70 }).confidence, undefined); + }); + it("multi-select grades as exact-set", () => { const multi = quizData({ mode: "multi-select", @@ -412,6 +437,13 @@ describe("formatAnswerMessage", () => { assert.match(formatAnswerMessage(ad), /Note: guessed/); }); + it("states the confidence line on answered quizzes, absent on dontKnow", () => { + const ad = buildAnswer(quizData(), { values: ["mercury"], confidence: 70 }); + assert.match(formatAnswerMessage(ad), /Confidence: 70% — the learner's stated probability that his selected answer is correct\./); + const idk = buildAnswer(quizData(), { dontKnow: true, confidence: 70 }); + assert.ok(!formatAnswerMessage(idk).includes("Confidence:")); + }); + it("formats ask answers", () => { const ask = quizData({ kind: "ask", correctIndices: undefined, explanation: undefined }); const ad = buildAnswer(ask, { other: "whatever" });