v1 web interface: dispatch-mode quiz/ask, Fastify+WS server, React frontend, deterministic test suite
- extensions/core/question-core.ts: pure quiz/ask logic (normalize/shuffle/resolve/grade/format) shared by both interfaces - extensions/core/bridge.ts: in-process server<->extension bridge (answer/dismiss/re-arm) - extensions/quiz-web.ts + ask-web.ts: web dispatch mode — tool returns immediately, graded answer arrives as next user message; sessions resumable mid-question via lh_question/lh_question_answered custom entries (never in LLM context) - extensions/quiz.ts: refactored onto shared core (terminal blocking mode unchanged) - src/server/index.ts: Fastify + WebSocket; REST (list/create/resume), entry sync (diff-based, all message kinds), streaming deltas, pending-question push - web/: Vite + React — home view (graph placeholder, session list, new-session input), session view (markdown+KaTeX+mermaid transcript, quiz cards, streaming) - src/testing/scripted-model.ts: deterministic fake LLM (scripted model runtime injected via factory overrides) - tests/: 48 deterministic tests (unit: question-core + bridge; integration: full pi loop with scripted model incl. resume re-arm) - scripts/e2e.ts: live smoke test (manual, real LLM, never a gate) - teacher/system-prompt.md: end-your-turn rule for question tools
This commit is contained in:
Amos
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* ask-web — the ask_user_question tool in WEB DISPATCH mode.
|
||||
*
|
||||
* Same dispatch shape as quiz-web: the tool validates and persists the
|
||||
* question (lh_question custom entry), returns immediately, and the learner's
|
||||
* answer arrives as the next user message once the server calls the bridge.
|
||||
* Not graded — this is the preference/decision sibling of quiz.
|
||||
*/
|
||||
|
||||
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
||||
import { Text } from "@mariozechner/pi-tui";
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import {
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
type QuestionData,
|
||||
normalizeOptions,
|
||||
} from "./core/question-core.ts";
|
||||
import { getBridge } from "./core/bridge.ts";
|
||||
|
||||
const OptionSchema = Type.Object({
|
||||
label: Type.String({
|
||||
description:
|
||||
'Display label for the option. If you recommend an option, place it first and append "(Recommended)" to the label.',
|
||||
}),
|
||||
value: Type.Optional(
|
||||
Type.String({ description: "Optional machine-readable value returned for the option. Defaults to the label." }),
|
||||
),
|
||||
description: Type.Optional(Type.String({ description: "Optional extra detail shown below the option." })),
|
||||
});
|
||||
|
||||
const AskUserQuestionParams = Type.Object({
|
||||
question: Type.String({
|
||||
description: "The single question to ask the user. Ask exactly one question per tool call.",
|
||||
}),
|
||||
details: Type.Optional(
|
||||
Type.String({ description: "Optional extra context or instructions shown under the question." }),
|
||||
),
|
||||
options: Type.Optional(
|
||||
Type.Array(OptionSchema, {
|
||||
description:
|
||||
"Optional multiple-choice options. Omit or pass an empty array for free-form text input. Users will always be able to choose Other and type a custom answer when options are provided.",
|
||||
}),
|
||||
),
|
||||
multiSelect: Type.Optional(
|
||||
Type.Boolean({ description: "Set to true to allow multiple answers to be selected for a question." }),
|
||||
),
|
||||
});
|
||||
|
||||
const PRESENTED_MESSAGE =
|
||||
"Question displayed to the learner. End your turn now and wait: their answer will arrive as the next user message. Do not call any other tool before receiving it.";
|
||||
|
||||
function unavailableResult(question: string, mode: string, message: string, context?: string) {
|
||||
return {
|
||||
content: [{ type: "text" as const, text: message }],
|
||||
details: { status: "unavailable", question, context, mode, message },
|
||||
};
|
||||
}
|
||||
|
||||
export default function askWeb(pi: ExtensionAPI) {
|
||||
const bridge = getBridge();
|
||||
|
||||
let currentCtx: { sessionManager: { getEntries(): Array<{ type: string; customType?: string; data?: unknown }> } } | null = null;
|
||||
|
||||
bridge.registerBackend({
|
||||
kind: "ask",
|
||||
appendAnswered: (data) => pi.appendEntry(QUESTION_ANSWERED_ENTRY, data),
|
||||
sendUserMessage: (text) => pi.sendUserMessage(text, { deliverAs: "followUp" }),
|
||||
getEntries: () => currentCtx?.sessionManager.getEntries() ?? [],
|
||||
});
|
||||
|
||||
// Re-arm pending questions on session (re)load — exit/resume mid-question.
|
||||
pi.on("session_start", (_event, ctx) => {
|
||||
currentCtx = ctx;
|
||||
bridge.clear("ask");
|
||||
bridge.rearm("ask");
|
||||
});
|
||||
|
||||
pi.registerTool({
|
||||
name: "ask_user_question",
|
||||
label: "ask_user_question",
|
||||
description:
|
||||
"Ask the user a single question and pause execution until they answer. Use this when requirements are ambiguous, user preferences are needed, a decision would materially affect implementation, or you need confirmation before proceeding. Ask exactly one question per tool call, and prefer multiple separate tool calls over bundling unrelated questions together.",
|
||||
promptSnippet:
|
||||
"Use this tool to ask exactly one clarifying question, missing-requirement question, preference question, or decision question before continuing.",
|
||||
promptGuidelines: [
|
||||
"Ask exactly one question per tool call.",
|
||||
"If you need answers to multiple questions, make multiple separate ask_user_question tool calls instead of combining them into one prompt.",
|
||||
'Users will always be able to select "Other" to provide custom text input when options are provided.',
|
||||
"Use multiSelect: true only when you need multiple answers to the same question.",
|
||||
'If you recommend a specific option, make it the first option in the list and add "(Recommended)" at the end of the label.',
|
||||
"Prefer this tool over guessing when requirements, preferences, or implementation choices are unclear.",
|
||||
"Use this tool when multiple valid implementation paths exist and the preferred path depends on user choice.",
|
||||
"After calling ask_user_question, END YOUR TURN. The answer arrives as the next user message; do not call any other tool while waiting.",
|
||||
],
|
||||
parameters: AskUserQuestionParams,
|
||||
|
||||
async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
|
||||
const context = params.details?.trim() || undefined;
|
||||
let options;
|
||||
try {
|
||||
options = normalizeOptions(params.options);
|
||||
} catch (e) {
|
||||
return unavailableResult(params.question, "single-select", `ask_user_question ${(e as Error).message}`, context);
|
||||
}
|
||||
const mode = options.length === 0 ? "text" : params.multiSelect ? "multi-select" : "single-select";
|
||||
|
||||
const data: QuestionData = {
|
||||
id: `q_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`,
|
||||
kind: "ask",
|
||||
question: params.question,
|
||||
context,
|
||||
mode,
|
||||
options,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
pi.appendEntry(QUESTION_ENTRY, data);
|
||||
bridge.publish(data);
|
||||
|
||||
return {
|
||||
content: [{ type: "text" as const, text: PRESENTED_MESSAGE }],
|
||||
details: {
|
||||
status: "presented",
|
||||
question: params.question,
|
||||
context,
|
||||
mode,
|
||||
options: options.map((o, i) => ({ index: i + 1, label: o.label })),
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
renderCall(args, theme) {
|
||||
let text = theme.fg("toolTitle", theme.bold("ask_user_question ")) + theme.fg("muted", args.question);
|
||||
if (args.multiSelect) text += theme.fg("dim", " [multi-select]");
|
||||
return new Text(text, 0, 0);
|
||||
},
|
||||
|
||||
renderResult(result, _options, theme) {
|
||||
const details = result.details as { status?: string; message?: string } | undefined;
|
||||
if (details?.status === "unavailable") {
|
||||
return new Text(theme.fg("warning", details.message || "ask_user_question unavailable"), 0, 0);
|
||||
}
|
||||
return new Text(theme.fg("muted", "Question displayed to the learner — awaiting their answer."), 0, 0);
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* bridge — in-process communication between the web server and the question
|
||||
* extensions. Both run in the same Node process (the server embeds pi via the
|
||||
* SDK), so a globalThis registry is the simplest robust channel.
|
||||
*
|
||||
* The server only ever needs ONE operation: `answer()` / `dismiss()` — which
|
||||
* must run inside an extension (it needs the ExtensionAPI: appendEntry +
|
||||
* sendUserMessage). Everything else (new questions, answered questions) the
|
||||
* server observes directly from the session event stream (`entry_appended`),
|
||||
* which is the single source of truth.
|
||||
*
|
||||
* The registry is rebuilt on every `session_start` (resume re-arm) by scanning
|
||||
* the session entries for lh_question / lh_question_answered.
|
||||
*/
|
||||
|
||||
import {
|
||||
type AnswerRequest,
|
||||
type QuestionData,
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
type QuestionKind,
|
||||
buildAnswer,
|
||||
formatAnswerMessage,
|
||||
} from "./question-core.ts";
|
||||
|
||||
interface Answerer {
|
||||
(data: QuestionData, req: AnswerRequest): void;
|
||||
}
|
||||
|
||||
interface Backend {
|
||||
kind: QuestionKind;
|
||||
/** Append a custom entry to the session (not sent to the LLM). */
|
||||
appendAnswered(data: unknown): void;
|
||||
/** Send a user message into the session (triggers an agent turn). */
|
||||
sendUserMessage(text: string): void;
|
||||
/** All entries of the current session branch. */
|
||||
getEntries(): Array<{ type: string; customType?: string; data?: unknown }>;
|
||||
}
|
||||
|
||||
export interface AnswerOutcome {
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
class QuestionBridge {
|
||||
/** kind → backend (registered by each web extension at load) */
|
||||
private backends = new Map<QuestionKind, Backend>();
|
||||
/** questionId → data, insertion-ordered (oldest first) */
|
||||
private pending = new Map<string, QuestionData>();
|
||||
|
||||
registerBackend(backend: Backend): void {
|
||||
this.backends.set(backend.kind, backend);
|
||||
}
|
||||
|
||||
/** Called by a web extension's tool execute(): record + persist the question. */
|
||||
publish(data: QuestionData): void {
|
||||
this.pending.set(data.id, data);
|
||||
}
|
||||
|
||||
/** Clear this kind's pending questions (called on session_start before re-arm). */
|
||||
clear(kind: QuestionKind): void {
|
||||
for (const [id, qd] of this.pending) {
|
||||
if (qd.kind === kind) this.pending.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-arm from session entries (called on session_start after clear). */
|
||||
rearm(kind: QuestionKind): void {
|
||||
const backend = this.backends.get(kind);
|
||||
if (!backend) return;
|
||||
const answered = new Set<string>();
|
||||
for (const e of backend.getEntries()) {
|
||||
if (e.type === "custom" && e.customType === QUESTION_ANSWERED_ENTRY) {
|
||||
const d = e.data as { questionId?: string } | undefined;
|
||||
if (d?.questionId) answered.add(d.questionId);
|
||||
}
|
||||
}
|
||||
for (const e of backend.getEntries()) {
|
||||
if (e.type === "custom" && e.customType === QUESTION_ENTRY) {
|
||||
const d = e.data as QuestionData | undefined;
|
||||
if (d?.kind === kind && !answered.has(d.id)) this.pending.set(d.id, d);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
answer(questionId: string, req: AnswerRequest): AnswerOutcome {
|
||||
const qd = this.pending.get(questionId);
|
||||
if (!qd) return { ok: false, error: "unknown or already-answered question" };
|
||||
const backend = this.backends.get(qd.kind);
|
||||
if (!backend) return { ok: false, error: "no backend registered" };
|
||||
this.pending.delete(questionId);
|
||||
const answerData = buildAnswer(qd, req);
|
||||
backend.appendAnswered(answerData);
|
||||
backend.sendUserMessage(formatAnswerMessage(answerData));
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
dismiss(questionId: string): AnswerOutcome {
|
||||
const qd = this.pending.get(questionId);
|
||||
if (!qd) return { ok: false, error: "unknown or already-answered question" };
|
||||
const backend = this.backends.get(qd.kind);
|
||||
if (!backend) return { ok: false, error: "no backend registered" };
|
||||
this.pending.delete(questionId);
|
||||
const answerData = buildAnswer(qd, { dontKnow: false });
|
||||
answerData.dismissed = true;
|
||||
backend.appendAnswered(answerData);
|
||||
backend.sendUserMessage(formatAnswerMessage(answerData));
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
|
||||
const BRIDGE_KEY = "__learningHarnessQuestionBridge";
|
||||
|
||||
export function getBridge(): QuestionBridge {
|
||||
const g = globalThis as Record<string, unknown>;
|
||||
if (!g[BRIDGE_KEY]) g[BRIDGE_KEY] = new QuestionBridge();
|
||||
return g[BRIDGE_KEY] as QuestionBridge;
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
/**
|
||||
* question-core — pure logic shared by the terminal (blocking) and web
|
||||
* (dispatch) quiz/ask extensions. No pi imports, no UI: everything here is
|
||||
* deterministic and unit-testable.
|
||||
*
|
||||
* The WEB dispatch shape:
|
||||
* 1. Tool execute() validates + shuffles + resolves the correct answer,
|
||||
* appends an `lh_question` custom entry (NOT in LLM context — safe to
|
||||
* store correctAnswer/explanation there) and returns immediately.
|
||||
* 2. The agent ends its turn. The server pushes the question to the client.
|
||||
* 3. The learner answers; the server calls back into the extension bridge.
|
||||
* 4. The extension grades, appends `lh_question_answered` (gold-standard
|
||||
* structured data for the extractor), and sends the graded answer into
|
||||
* the session as a user message so the agent can continue.
|
||||
*
|
||||
* Because nothing blocks, a session can be exited and resumed mid-question:
|
||||
* the pending state lives in the session file itself (lh_question entries
|
||||
* with no matching lh_question_answered).
|
||||
*/
|
||||
|
||||
// ─── shared types ───────────────────────────────────────────────────────────
|
||||
|
||||
export interface QuizOption {
|
||||
label: string;
|
||||
value: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
/** A real selected answer, in the order the options were displayed. */
|
||||
export interface OptionAnswer {
|
||||
label: string;
|
||||
value: string;
|
||||
index: number; // 1-based display index
|
||||
}
|
||||
|
||||
export type QuestionKind = "quiz" | "ask";
|
||||
export type SelectMode = "single-select" | "multi-select";
|
||||
export type QuestionMode = SelectMode | "text";
|
||||
|
||||
/** Full question data — stored in the `lh_question` custom entry. */
|
||||
export interface QuestionData {
|
||||
id: string;
|
||||
kind: QuestionKind;
|
||||
question: string;
|
||||
context?: string;
|
||||
mode: QuestionMode;
|
||||
options: QuizOption[]; // display order (post-shuffle)
|
||||
// quiz only. NEVER send to the client before answering:
|
||||
correctIndices?: number[];
|
||||
explanation?: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** The learner's answer — full structured record, stored in `lh_question_answered`. */
|
||||
export interface QuestionAnswerData {
|
||||
questionId: string;
|
||||
kind: QuestionKind;
|
||||
questionText: string;
|
||||
options?: { index: number; label: string }[]; // snapshot for rendering
|
||||
dontKnow: boolean;
|
||||
answers: OptionAnswer[];
|
||||
other?: string; // ask: "Other" free text
|
||||
text?: string; // ask: text-mode answer
|
||||
note?: string;
|
||||
correct?: boolean; // quiz only
|
||||
correctIndices?: number[]; // quiz only (rendering)
|
||||
explanation?: string; // quiz only (rendering)
|
||||
dismissed?: boolean;
|
||||
answeredAt: string;
|
||||
}
|
||||
|
||||
/** Client-facing question payload — deliberately omits the answer. */
|
||||
export interface ClientQuestion {
|
||||
id: string;
|
||||
kind: QuestionKind;
|
||||
question: string;
|
||||
context?: string;
|
||||
mode: QuestionMode;
|
||||
options: QuizOption[];
|
||||
}
|
||||
|
||||
/** What the server passes to the bridge when the learner answers. */
|
||||
export interface AnswerRequest {
|
||||
dontKnow?: boolean;
|
||||
values?: string[]; // option values
|
||||
other?: string; // ask "Other"
|
||||
text?: string; // ask text-mode
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export const QUESTION_ENTRY = "lh_question";
|
||||
export const QUESTION_ANSWERED_ENTRY = "lh_question_answered";
|
||||
|
||||
// ─── options ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function normalizeOptions(
|
||||
options: Array<{ label: string; value?: string; description?: string }> | undefined,
|
||||
): QuizOption[] {
|
||||
const seen = new Set<string>();
|
||||
return (options || [])
|
||||
.map((option) => ({
|
||||
label: option.label.trim(),
|
||||
value: option.value?.trim() || option.label.trim(),
|
||||
description: option.description?.trim() || undefined,
|
||||
}))
|
||||
.filter((option) => {
|
||||
if (option.label.length === 0) return false;
|
||||
if (seen.has(option.value)) throw new Error(`duplicate option value "${option.value}"`);
|
||||
seen.add(option.value);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
/** Fisher-Yates over a copy. correctAnswer is keyed by value, so display
|
||||
* order can be shuffled safely before resolving indices. */
|
||||
export function shuffleOptions(options: QuizOption[]): QuizOption[] {
|
||||
const out = [...options];
|
||||
for (let i = out.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
const a = out[i]!;
|
||||
const b = out[j]!;
|
||||
out[i] = b;
|
||||
out[j] = a;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── quiz grading ───────────────────────────────────────────────────────────
|
||||
|
||||
/** The harness sometimes delivers a multi-select correctAnswer array as a
|
||||
* JSON-stringified string because the schema union lists String first. */
|
||||
export function coerceCorrectAnswer(correctAnswer: string | string[]): string[] {
|
||||
if (Array.isArray(correctAnswer)) return correctAnswer;
|
||||
const trimmed = correctAnswer.trim();
|
||||
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
if (Array.isArray(parsed)) return parsed.map((v) => String(v));
|
||||
} catch {
|
||||
// not valid JSON — treat as a single literal value
|
||||
}
|
||||
}
|
||||
return [correctAnswer];
|
||||
}
|
||||
|
||||
/** Resolve author-supplied value(s) to 1-based display indices. Keying by
|
||||
* value (not position) makes a typo a hard error instead of a silent wrong
|
||||
* grade. Must run AFTER shuffling. */
|
||||
export function resolveCorrect(
|
||||
correctAnswer: string | string[] | undefined,
|
||||
options: QuizOption[],
|
||||
): { indices: number[]; error?: string } {
|
||||
if (correctAnswer === undefined) return { indices: [], error: "correctAnswer is required" };
|
||||
const arr = coerceCorrectAnswer(correctAnswer);
|
||||
if (arr.length === 0) return { indices: [], error: "correctAnswer is required" };
|
||||
const byValue = new Map(options.map((o, i) => [o.value, i + 1]));
|
||||
const indices: number[] = [];
|
||||
for (const raw of arr) {
|
||||
const v = typeof raw === "string" ? raw.trim() : raw;
|
||||
const idx = byValue.get(v);
|
||||
if (idx === undefined) {
|
||||
const known = options.map((o) => `"${o.value}"`).join(", ");
|
||||
return { indices: [], error: `correctAnswer "${v}" does not match any option value (${known})` };
|
||||
}
|
||||
indices.push(idx);
|
||||
}
|
||||
return { indices: Array.from(new Set(indices)).sort((a, b) => a - b) };
|
||||
}
|
||||
|
||||
/** Exact-set match: same number of selections, same indices. */
|
||||
export function isCorrect(selectedIndices: number[], correctIndices: number[]): boolean {
|
||||
if (selectedIndices.length !== correctIndices.length) return false;
|
||||
const a = [...selectedIndices].sort((x, y) => x - y);
|
||||
const b = [...correctIndices].sort((x, y) => x - y);
|
||||
return a.every((v, i) => v === b[i]);
|
||||
}
|
||||
|
||||
// ─── arming + answering ─────────────────────────────────────────────────────
|
||||
|
||||
export function makeId(): string {
|
||||
return `q_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`;
|
||||
}
|
||||
|
||||
export function toClientQuestion(qd: QuestionData): ClientQuestion {
|
||||
return {
|
||||
id: qd.id,
|
||||
kind: qd.kind,
|
||||
question: qd.question,
|
||||
context: qd.context,
|
||||
mode: qd.mode,
|
||||
options: qd.options,
|
||||
};
|
||||
}
|
||||
|
||||
function formatOptionRef(options: QuizOption[], index: number): string {
|
||||
const opt = options.find((o, i) => i + 1 === index);
|
||||
return `${index}. ${opt ? opt.label : "(unknown)"}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn an AnswerRequest into the full answer record + the user message the
|
||||
* agent will see. Quiz answers are graded here; ask answers are not.
|
||||
*/
|
||||
export function buildAnswer(qd: QuestionData, req: AnswerRequest): QuestionAnswerData {
|
||||
const base: QuestionAnswerData = {
|
||||
questionId: qd.id,
|
||||
kind: qd.kind,
|
||||
questionText: qd.question,
|
||||
options: qd.options.map((o, i) => ({ index: i + 1, label: o.label })),
|
||||
dontKnow: req.dontKnow === true,
|
||||
answers: [],
|
||||
note: req.note?.trim() || undefined,
|
||||
answeredAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
// resolve selected option values → OptionAnswer (in display order)
|
||||
if (req.values && req.values.length > 0) {
|
||||
const byValue = new Map(qd.options.map((o, i) => [o.value, { ...o, index: i + 1 }]));
|
||||
for (const v of req.values) {
|
||||
const opt = byValue.get(v);
|
||||
if (!opt) continue; // unknown value — ignore
|
||||
base.answers.push({ label: opt.label, value: opt.value, index: opt.index });
|
||||
}
|
||||
base.answers.sort((a, b) => a.index - b.index);
|
||||
}
|
||||
if (req.other?.trim()) base.other = req.other.trim();
|
||||
if (req.text?.trim()) base.text = req.text.trim();
|
||||
|
||||
if (qd.kind === "quiz") {
|
||||
base.correctIndices = qd.correctIndices ?? [];
|
||||
base.explanation = qd.explanation;
|
||||
base.correct = base.dontKnow ? false : isCorrect(base.answers.map((a) => a.index), qd.correctIndices ?? []);
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
/** The user message that carries the answer into the agent's context. */
|
||||
export function formatAnswerMessage(ad: QuestionAnswerData): string {
|
||||
const label = ad.kind === "quiz" ? "Quiz answer" : "Answer to your question";
|
||||
const head = `[${label}]`;
|
||||
const lines: string[] = [head];
|
||||
|
||||
if (ad.dismissed) {
|
||||
lines.push("The learner dismissed this question without answering.");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
if (ad.kind === "quiz") {
|
||||
const correctStr = (ad.correctIndices ?? []).map((i) => formatOptionRefSnapshot(ad, i)).join(", ");
|
||||
if (ad.dontKnow) {
|
||||
lines.push(
|
||||
`The learner selected "I don't know" — they did NOT attempt an answer (a genuine knowledge gap, not a wrong guess).`,
|
||||
);
|
||||
} else {
|
||||
const verdict = ad.correct ? "CORRECT" : "INCORRECT";
|
||||
const selectedStr = ad.answers.map((a) => `${a.index}. ${a.label}`).join(", ");
|
||||
lines.push(`Selected: ${selectedStr || "(none)"}`);
|
||||
lines.push(`Result: ${verdict}`);
|
||||
}
|
||||
lines.push(`Correct: ${correctStr}`);
|
||||
} else {
|
||||
if (ad.text) {
|
||||
lines.push(`The learner answered: ${ad.text}`);
|
||||
} else if (ad.other) {
|
||||
lines.push(`The learner answered (Other): ${ad.other}`);
|
||||
} else if (ad.answers.length > 0) {
|
||||
const selectedStr = ad.answers.map((a) => `${a.index}. ${a.label}`).join(", ");
|
||||
lines.push(`Selected: ${selectedStr}`);
|
||||
} else {
|
||||
lines.push("(no answer)");
|
||||
}
|
||||
}
|
||||
|
||||
if (ad.note) lines.push(`Note: ${ad.note}`);
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function formatOptionRefSnapshot(ad: QuestionAnswerData, index: number): string {
|
||||
const opt = (ad.options ?? []).find((o) => o.index === index);
|
||||
return `${index}. ${opt ? opt.label : "(unknown)"}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the pending (oldest unanswered) question from a session's entries.
|
||||
* Used on resume/backfill — pending state lives in the session file itself.
|
||||
*/
|
||||
export function findPendingQuestion(
|
||||
entries: Array<{ type: string; customType?: string; data?: unknown }>,
|
||||
): ClientQuestion | null {
|
||||
const answered = new Set<string>();
|
||||
for (const e of entries) {
|
||||
if (e.type === "custom" && e.customType === QUESTION_ANSWERED_ENTRY) {
|
||||
const d = e.data as QuestionAnswerData | undefined;
|
||||
if (d?.questionId) answered.add(d.questionId);
|
||||
}
|
||||
}
|
||||
for (const e of entries) {
|
||||
if (e.type === "custom" && e.customType === QUESTION_ENTRY) {
|
||||
const d = e.data as QuestionData | undefined;
|
||||
if (d && !answered.has(d.id)) return toClientQuestion(d);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
/**
|
||||
* quiz-web — the quiz tool in WEB DISPATCH mode.
|
||||
*
|
||||
* Unlike the terminal quiz (which blocks on an interactive component), this
|
||||
* variant returns immediately after persisting the question:
|
||||
*
|
||||
* 1. execute() validates + shuffles + resolves correct indices, appends an
|
||||
* `lh_question` custom entry (NOT in LLM context — the correct answer and
|
||||
* explanation are safe there), publishes to the bridge, and returns.
|
||||
* 2. The tool result tells the agent to end its turn — the learner's graded
|
||||
* answer arrives as the next user message.
|
||||
* 3. The server calls bridge.answer() when the learner answers; the
|
||||
* extension grades, appends `lh_question_answered`, and sends the graded
|
||||
* answer into the session as a user message.
|
||||
*
|
||||
* This shape makes exit/resume mid-quiz free: the pending state lives in the
|
||||
* session file itself, and is re-armed on session_start.
|
||||
*/
|
||||
|
||||
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
||||
import { Text } from "@mariozechner/pi-tui";
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import {
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
type QuestionData,
|
||||
normalizeOptions,
|
||||
resolveCorrect,
|
||||
shuffleOptions,
|
||||
} from "./core/question-core.ts";
|
||||
import { getBridge } from "./core/bridge.ts";
|
||||
|
||||
const OptionSchema = Type.Object({
|
||||
label: Type.String({ description: "Display label for the answer option." }),
|
||||
value: Type.Optional(
|
||||
Type.String({ description: "Optional machine-readable value returned for the option. Defaults to the label." }),
|
||||
),
|
||||
description: Type.Optional(Type.String({ description: "Optional extra detail shown below the option." })),
|
||||
});
|
||||
|
||||
const QuizParams = Type.Object({
|
||||
question: Type.String({
|
||||
description: "The single quiz question to ask. Ask exactly one question per tool call.",
|
||||
}),
|
||||
details: Type.Optional(
|
||||
Type.String({ description: "Optional extra context or instructions shown under the question." }),
|
||||
),
|
||||
options: Type.Array(OptionSchema, {
|
||||
description:
|
||||
"The answer options (2 or more). Options only — there is no free-text mode. Give each option a stable `value`; you reference the correct one by that value in correctAnswer.",
|
||||
minItems: 2,
|
||||
}),
|
||||
multiSelect: Type.Optional(
|
||||
Type.Boolean({ description: "Set to true when more than one option is correct and the user must select all of them." }),
|
||||
),
|
||||
correctAnswer: Type.Union([Type.String(), Type.Array(Type.String())], {
|
||||
description:
|
||||
'REQUIRED. The correct answer as the option value(s) — the `value` field of the option you intend. Single-select: a single string (e.g. "mercury"). Multi-select: an array of strings (e.g. ["belize", "niue"]); the user is only correct if their selection matches this set exactly. Always pass the value, not a position number — this is self-checking and prevents miscounting.',
|
||||
}),
|
||||
explanation: Type.String({
|
||||
description:
|
||||
"REQUIRED. Explanation revealed AFTER the user answers (shown whether they got it right or wrong). Use it to reinforce why the correct answer is correct.",
|
||||
}),
|
||||
shuffle: Type.Optional(
|
||||
Type.Boolean({
|
||||
description:
|
||||
"Defaults to true: options are randomly reordered before display so the correct answer isn't always in the same position. Set to false only when option order is meaningful (e.g. ordered numeric values, or an 'All/None of the above' option that must stay last).",
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
const PRESENTED_MESSAGE =
|
||||
"Quiz displayed to the learner. End your turn now and wait: their graded answer will arrive as the next user message. Do not call any other tool before receiving it.";
|
||||
|
||||
function unavailableResult(question: string, mode: string, message: string, context?: string) {
|
||||
return {
|
||||
content: [{ type: "text" as const, text: message }],
|
||||
details: { status: "unavailable", question, context, mode, message },
|
||||
};
|
||||
}
|
||||
|
||||
export default function quizWeb(pi: ExtensionAPI) {
|
||||
const bridge = getBridge();
|
||||
|
||||
// The web server drives answering through the bridge. This extension is
|
||||
// the only place with ExtensionAPI access (appendEntry + sendUserMessage).
|
||||
// ctx (with sessionManager) is captured from event handlers.
|
||||
let currentCtx: { sessionManager: { getEntries(): Array<{ type: string; customType?: string; data?: unknown }> } } | null = null;
|
||||
|
||||
bridge.registerBackend({
|
||||
kind: "quiz",
|
||||
appendAnswered: (data) => pi.appendEntry(QUESTION_ANSWERED_ENTRY, data),
|
||||
sendUserMessage: (text) => pi.sendUserMessage(text, { deliverAs: "followUp" }),
|
||||
getEntries: () => currentCtx?.sessionManager.getEntries() ?? [],
|
||||
});
|
||||
|
||||
// Re-arm pending questions on session (re)load — exit/resume mid-quiz.
|
||||
pi.on("session_start", (_event, ctx) => {
|
||||
currentCtx = ctx;
|
||||
bridge.clear("quiz");
|
||||
bridge.rearm("quiz");
|
||||
});
|
||||
|
||||
pi.registerTool({
|
||||
name: "quiz",
|
||||
label: "quiz",
|
||||
description:
|
||||
"Ask the user a GRADED question with a known correct answer, then instantly grade and give feedback. Unlike ask_user_question (which collects preferences/decisions with no right answer), quiz always has a correct answer supplied by you, marks the user's selection right/wrong (✓/✗), reveals the correct answer, and can show an explanation. Use it to (1) assess what the learner already understands before teaching, and (2) run tight practice/retrieval loops after explaining, or probe understanding whenever you're unsure they've got it. Options-only: single-select or multi-select, plus an automatic 'I don't know' choice so the user can signal a genuine gap instead of guessing. An always-present optional note field lets the user attach a free-text note to ANY answer; it reaches you only when non-empty. No free-text answers — for non-graded questions use ask_user_question instead.",
|
||||
promptSnippet:
|
||||
"Use the quiz tool to test the user with a graded multiple-choice or multi-select question (required correct answer + required explanation). For non-graded questions, use ask_user_question.",
|
||||
promptGuidelines: [
|
||||
"quiz is GRADED; ask_user_question is not. If the question has a correct answer, use quiz. If you just need a preference, decision, or open-ended input, use ask_user_question.",
|
||||
'correctAnswer is REQUIRED and is the option value, not a position number. Single-select: one string (e.g. "mercury"). Multi-select: an array of strings (e.g. ["belice", "niue"]).',
|
||||
"Always pass the option's `value` string as correctAnswer — it is self-checking and prevents miscounting positions. A value that matches no option is a hard error.",
|
||||
"explanation is REQUIRED — always say why the correct answer is correct.",
|
||||
"Multi-select is graded as an exact-set match: the user is correct only if they select every correct option and no incorrect ones.",
|
||||
"There is no free-text mode. An 'I don't know' choice is ALWAYS added automatically — provide ONLY the real, gradable options (at least two). Never add your own uncertainty/opt-out option like 'I don't know', 'I'm not sure', or 'Not sure'; that is handled for you and a manual one would be redundant or gradable-as-wrong.",
|
||||
"If a result comes back as dontKnow, the user honestly did not know and did NOT guess — treat it as a genuine knowledge gap to teach into, not as a wrong answer.",
|
||||
"Any answer (right, wrong, or 'I don't know') may carry an optional free-text `note` the user typed in the always-present note field. When present it reflects what they were thinking or unsure about — read it and let it steer your follow-up. It is omitted entirely when empty.",
|
||||
"Treat each wrong answer (distractor) as a diagnostic probe, not just filler: make it a specific, believable mistake the user might actually hold — a common misconception, or an adjacent/easily-confused concept — so that WHICH wrong answer they pick reveals WHICH nuance of their understanding is off. You learn far more from a targeted wrong choice than from a binary right/wrong, and the choice tells you exactly which gap to teach into next (and what the explanation should address).",
|
||||
"Guardrail: every distractor must be unambiguously wrong on the intended reading — tempting, but a real error, not a defensible alternative. Don't drift into trick questions.",
|
||||
"Anti-guessing hygiene: don't let the correct answer stand out by form (longest, most precise, most hedged, or the only one in the right format). Keep options similar in length, specificity, and phrasing so it can't be picked from shape alone.",
|
||||
"Set multiSelect: true only when more than one option is correct.",
|
||||
"Options are shuffled before display by default, so don't worry about which position you list the correct answer in. Set shuffle: false only when option order is meaningful (ordered values, or an 'All/None of the above' option that must stay last).",
|
||||
"To probe nuance, ask several quick quiz questions and adapt each one based on the previous answers, rather than writing one giant question.",
|
||||
"Don't leak the answer through formatting: keep option phrasing/length even and don't hint which is correct.",
|
||||
"After calling quiz, END YOUR TURN. The graded answer arrives as the next user message; do not call any other tool while waiting.",
|
||||
],
|
||||
parameters: QuizParams,
|
||||
|
||||
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
||||
const context = params.details?.trim() || undefined;
|
||||
const explanation = params.explanation.trim();
|
||||
const mode = params.multiSelect ? "multi-select" : "single-select";
|
||||
|
||||
let options;
|
||||
try {
|
||||
options = normalizeOptions(params.options);
|
||||
} catch (e) {
|
||||
return unavailableResult(params.question, mode, `quiz ${(e as Error).message}`, context);
|
||||
}
|
||||
if (options.length < 2) {
|
||||
return unavailableResult(params.question, mode, "quiz requires at least two options", context);
|
||||
}
|
||||
if (params.shuffle !== false) {
|
||||
options = shuffleOptions(options);
|
||||
}
|
||||
const { indices: correctIndices, error } = resolveCorrect(
|
||||
params.correctAnswer as string | string[],
|
||||
options,
|
||||
);
|
||||
if (error) {
|
||||
return unavailableResult(params.question, mode, `quiz ${error}`, context);
|
||||
}
|
||||
|
||||
const data: QuestionData = {
|
||||
id: `q_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`,
|
||||
kind: "quiz",
|
||||
question: params.question,
|
||||
context,
|
||||
mode,
|
||||
options,
|
||||
correctIndices,
|
||||
explanation,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
// Persist the full question (answer included — custom entries are NOT
|
||||
// sent to the LLM) and register it as pending, then return immediately.
|
||||
pi.appendEntry(QUESTION_ENTRY, data);
|
||||
bridge.publish(data);
|
||||
|
||||
return {
|
||||
content: [{ type: "text" as const, text: PRESENTED_MESSAGE }],
|
||||
details: {
|
||||
status: "presented",
|
||||
question: params.question,
|
||||
context,
|
||||
mode,
|
||||
options: options.map((o, i) => ({ index: i + 1, label: o.label })),
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
renderCall(args, theme) {
|
||||
// Never render correctAnswer/explanation — it would leak the answer.
|
||||
let text = theme.fg("toolTitle", theme.bold("quiz ")) + theme.fg("muted", args.question);
|
||||
if (args.multiSelect) text += theme.fg("dim", " [multi-select]");
|
||||
return new Text(text, 0, 0);
|
||||
},
|
||||
|
||||
renderResult(result, _options, theme) {
|
||||
const details = result.details as { status?: string; message?: string } | undefined;
|
||||
if (details?.status === "unavailable") {
|
||||
return new Text(theme.fg("warning", details.message || "quiz unavailable"), 0, 0);
|
||||
}
|
||||
return new Text(theme.fg("muted", "Quiz displayed to the learner — awaiting their answer."), 0, 0);
|
||||
},
|
||||
});
|
||||
}
|
||||
+33
-114
@@ -9,9 +9,18 @@ import {
|
||||
wrapTextWithAnsi,
|
||||
} from "@mariozechner/pi-tui";
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import {
|
||||
type OptionAnswer,
|
||||
type QuizOption,
|
||||
isCorrect,
|
||||
normalizeOptions,
|
||||
resolveCorrect,
|
||||
shuffleOptions,
|
||||
} from "./core/question-core.ts";
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// quiz — a GRADED sibling of ask_user_question.
|
||||
// quiz — a GRADED sibling of ask_user_question (TERMINAL mode: blocks on an
|
||||
// interactive pi-tui component until the learner answers).
|
||||
//
|
||||
// Where ask_user_question collects a preference/decision with no notion of
|
||||
// right or wrong, `quiz` poses a question that HAS a correct answer, grades the
|
||||
@@ -21,26 +30,17 @@ import { Type } from "@sinclair/typebox";
|
||||
// It is intentionally options-only: single-select or multi-select. There is no
|
||||
// free-text mode and no "Other" option, because a free-text answer can't be
|
||||
// graded against a correct index.
|
||||
//
|
||||
// Pure logic (normalize/shuffle/resolve/grade) lives in ./core/question-core.ts,
|
||||
// shared with the web dispatch variant (quiz-web.ts).
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface QuizOption {
|
||||
label: string;
|
||||
value: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
interface DisplayOption extends QuizOption {
|
||||
id: string;
|
||||
index: number;
|
||||
isSubmit?: boolean;
|
||||
}
|
||||
|
||||
interface OptionAnswer {
|
||||
label: string;
|
||||
value: string;
|
||||
index: number; // 1-based, matches the number shown to the user
|
||||
}
|
||||
|
||||
// The always-present "I don't know" choice. It is NOT a real option: it never
|
||||
// participates in shuffling, has no correct-answer value, and produces a
|
||||
// distinct signal (dontKnow) rather than a right/wrong grade — so an honest
|
||||
@@ -61,26 +61,6 @@ interface QuizResponse {
|
||||
type QuizStatus = "answered" | "cancelled" | "unavailable";
|
||||
type QuizMode = "single-select" | "multi-select";
|
||||
|
||||
interface DisplayedOption {
|
||||
index: number; // 1-based, in the final (possibly shuffled) display order
|
||||
label: string;
|
||||
}
|
||||
|
||||
interface QuizResultDetails {
|
||||
status: QuizStatus;
|
||||
question: string;
|
||||
context?: string;
|
||||
mode: QuizMode;
|
||||
answers: OptionAnswer[];
|
||||
correctIndices: number[];
|
||||
options?: DisplayedOption[]; // full option list in display order, for the transcript
|
||||
correct?: boolean;
|
||||
dontKnow?: boolean; // user selected "I don't know" instead of guessing
|
||||
note?: string; // optional free-text from the always-present note field (any answer)
|
||||
explanation?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
const OptionSchema = Type.Object({
|
||||
label: Type.String({ description: "Display label for the answer option." }),
|
||||
value: Type.Optional(
|
||||
@@ -120,80 +100,6 @@ const QuizParams = Type.Object({
|
||||
),
|
||||
});
|
||||
|
||||
function normalizeOptions(
|
||||
options: Array<{ label: string; value?: string; description?: string }> | undefined,
|
||||
): QuizOption[] {
|
||||
const seen = new Set<string>();
|
||||
return (options || [])
|
||||
.map((option) => ({
|
||||
label: option.label.trim(),
|
||||
value: option.value?.trim() || option.label.trim(),
|
||||
description: option.description?.trim() || undefined,
|
||||
}))
|
||||
.filter((option) => {
|
||||
if (option.label.length === 0) return false;
|
||||
if (seen.has(option.value)) throw new Error(`duplicate option value "${option.value}"`);
|
||||
seen.add(option.value);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
// Fisher-Yates shuffle over a copy. Safe to reorder for display because
|
||||
// correctAnswer is keyed by value, not position — indices are resolved AFTER
|
||||
// shuffling, so grading always matches what the user actually sees.
|
||||
function shuffleOptions(options: QuizOption[]): QuizOption[] {
|
||||
const out = [...options];
|
||||
for (let i = out.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[out[i], out[j]] = [out[j], out[i]];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Resolve author-supplied option value(s) to 1-based indices. Keying by value
|
||||
// (not position) makes the correct answer self-documenting: the author writes
|
||||
// `correctAnswer: "mercury"` and a typo becomes a hard error instead of a
|
||||
// silent wrong grade.
|
||||
// The harness sometimes delivers a multi-select `correctAnswer` array as a
|
||||
// JSON-stringified string (e.g. '["a", "b"]') instead of a real array, because
|
||||
// the schema union lists String first. Detect that case and parse it back into
|
||||
// an array so grading resolves against real option values. A plain single value
|
||||
// is wrapped as-is.
|
||||
function coerceCorrectAnswer(correctAnswer: string | string[]): string[] {
|
||||
if (Array.isArray(correctAnswer)) return correctAnswer;
|
||||
const trimmed = correctAnswer.trim();
|
||||
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
if (Array.isArray(parsed)) return parsed.map((v) => String(v));
|
||||
} catch {
|
||||
// Not valid JSON — fall through and treat as a single literal value.
|
||||
}
|
||||
}
|
||||
return [correctAnswer];
|
||||
}
|
||||
|
||||
function resolveCorrect(
|
||||
correctAnswer: string | string[] | undefined,
|
||||
options: QuizOption[],
|
||||
): { indices: number[]; error?: string } {
|
||||
if (correctAnswer === undefined) return { indices: [], error: "correctAnswer is required" };
|
||||
const arr = coerceCorrectAnswer(correctAnswer);
|
||||
if (arr.length === 0) return { indices: [], error: "correctAnswer is required" };
|
||||
const byValue = new Map(options.map((o, i) => [o.value, i + 1]));
|
||||
const indices: number[] = [];
|
||||
for (const raw of arr) {
|
||||
const v = typeof raw === "string" ? raw.trim() : raw;
|
||||
const idx = byValue.get(v);
|
||||
if (idx === undefined) {
|
||||
const known = options.map((o) => `"${o.value}"`).join(", ");
|
||||
return { indices: [], error: `correctAnswer "${v}" does not match any option value (${known})` };
|
||||
}
|
||||
indices.push(idx);
|
||||
}
|
||||
return { indices: Array.from(new Set(indices)).sort((a, b) => a - b) };
|
||||
}
|
||||
|
||||
function createEditorTheme(theme: any): EditorTheme {
|
||||
return {
|
||||
borderColor: (s) => theme.fg("accent", s),
|
||||
@@ -214,13 +120,6 @@ function addWrapped(lines: string[], text: string, width: number, indent = ""):
|
||||
}
|
||||
}
|
||||
|
||||
function isCorrect(selectedIndices: number[], correctIndices: number[]): boolean {
|
||||
if (selectedIndices.length !== correctIndices.length) return false;
|
||||
const a = [...selectedIndices].sort((x, y) => x - y);
|
||||
const b = [...correctIndices].sort((x, y) => x - y);
|
||||
return a.every((v, i) => v === b[i]);
|
||||
}
|
||||
|
||||
function buildStructuredResult(
|
||||
status: QuizStatus,
|
||||
question: string,
|
||||
@@ -258,6 +157,26 @@ function formatOptionRef(options: QuizOption[], index: number): string {
|
||||
return `${index}. ${opt ? opt.label : "(unknown)"}`;
|
||||
}
|
||||
|
||||
interface DisplayedOption {
|
||||
index: number; // 1-based, in the final (possibly shuffled) display order
|
||||
label: string;
|
||||
}
|
||||
|
||||
interface QuizResultDetails {
|
||||
status: QuizStatus;
|
||||
question: string;
|
||||
context?: string;
|
||||
mode: QuizMode;
|
||||
answers: OptionAnswer[];
|
||||
correctIndices: number[];
|
||||
options?: DisplayedOption[]; // full option list in display order, for the transcript
|
||||
correct?: boolean;
|
||||
dontKnow?: boolean; // user selected "I don't know" instead of guessing
|
||||
note?: string; // optional free-text from the always-present note field (any answer)
|
||||
explanation?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
function buildResult(
|
||||
question: string,
|
||||
context: string | undefined,
|
||||
|
||||
Reference in New Issue
Block a user