v1 web interface: dispatch-mode quiz/ask, Fastify+WS server, React frontend, deterministic test suite
- extensions/core/question-core.ts: pure quiz/ask logic (normalize/shuffle/resolve/grade/format) shared by both interfaces - extensions/core/bridge.ts: in-process server<->extension bridge (answer/dismiss/re-arm) - extensions/quiz-web.ts + ask-web.ts: web dispatch mode — tool returns immediately, graded answer arrives as next user message; sessions resumable mid-question via lh_question/lh_question_answered custom entries (never in LLM context) - extensions/quiz.ts: refactored onto shared core (terminal blocking mode unchanged) - src/server/index.ts: Fastify + WebSocket; REST (list/create/resume), entry sync (diff-based, all message kinds), streaming deltas, pending-question push - web/: Vite + React — home view (graph placeholder, session list, new-session input), session view (markdown+KaTeX+mermaid transcript, quiz cards, streaming) - src/testing/scripted-model.ts: deterministic fake LLM (scripted model runtime injected via factory overrides) - tests/: 48 deterministic tests (unit: question-core + bridge; integration: full pi loop with scripted model incl. resume re-arm) - scripts/e2e.ts: live smoke test (manual, real LLM, never a gate) - teacher/system-prompt.md: end-your-turn rule for question tools
This commit is contained in:
Amos
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Unit tests — the question bridge (state machine, fake backends only).
|
||||
*
|
||||
* The bridge is globalThis-singleton state shared with other tests in the same
|
||||
* process, so every test creates its own bridge instance via the class-free
|
||||
* export path... (it doesn't export the class — we exercise the singleton and
|
||||
* reset it between tests with clear()).
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { beforeEach, describe, it } from "node:test";
|
||||
import {
|
||||
type AnswerRequest,
|
||||
type QuestionData,
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
} from "../extensions/core/question-core.ts";
|
||||
import { getBridge } from "../extensions/core/bridge.ts";
|
||||
|
||||
interface Recorded {
|
||||
answered: unknown[];
|
||||
messages: string[];
|
||||
entries: Array<{ type: string; customType?: string; data?: unknown }>;
|
||||
}
|
||||
|
||||
function fakeBackend(kind: "quiz" | "ask"): { recorded: Recorded } {
|
||||
const recorded: Recorded = { answered: [], messages: [], entries: [] };
|
||||
getBridge().registerBackend({
|
||||
kind,
|
||||
appendAnswered: (data) => recorded.answered.push(data),
|
||||
sendUserMessage: (text) => recorded.messages.push(text),
|
||||
getEntries: () => recorded.entries,
|
||||
});
|
||||
return { recorded };
|
||||
}
|
||||
|
||||
function quizData(id: string): QuestionData {
|
||||
return {
|
||||
id,
|
||||
kind: "quiz",
|
||||
question: `Question ${id}?`,
|
||||
mode: "single-select",
|
||||
options: [
|
||||
{ label: "Right", value: "right" },
|
||||
{ label: "Wrong", value: "wrong" },
|
||||
],
|
||||
correctIndices: [1],
|
||||
explanation: "because",
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("question bridge", () => {
|
||||
let quiz: { recorded: Recorded };
|
||||
let ask: { recorded: Recorded };
|
||||
|
||||
beforeEach(() => {
|
||||
const bridge = getBridge();
|
||||
bridge.clear("quiz");
|
||||
bridge.clear("ask");
|
||||
quiz = fakeBackend("quiz");
|
||||
ask = fakeBackend("ask");
|
||||
});
|
||||
|
||||
it("publish makes a question answerable; answer grades and sends a user message", () => {
|
||||
const bridge = getBridge();
|
||||
const qd = quizData("q1");
|
||||
bridge.publish(qd);
|
||||
|
||||
const outcome = bridge.answer("q1", { values: ["right"] });
|
||||
assert.equal(outcome.ok, true);
|
||||
|
||||
assert.equal(quiz.recorded.answered.length, 1);
|
||||
const ad = quiz.recorded.answered[0] as { correct: boolean; questionId: string };
|
||||
assert.equal(ad.questionId, "q1");
|
||||
assert.equal(ad.correct, true);
|
||||
|
||||
assert.equal(quiz.recorded.messages.length, 1);
|
||||
assert.ok(quiz.recorded.messages[0].startsWith("[Quiz answer]"));
|
||||
// ask backend untouched
|
||||
assert.equal(ask.recorded.messages.length, 0);
|
||||
});
|
||||
|
||||
it("answering a question twice fails the second time", () => {
|
||||
const bridge = getBridge();
|
||||
bridge.publish(quizData("q2"));
|
||||
assert.equal(bridge.answer("q2", { values: ["wrong"] }).ok, true);
|
||||
const second = bridge.answer("q2", { values: ["right"] });
|
||||
assert.equal(second.ok, false);
|
||||
assert.match(second.error ?? "", /already-answered/);
|
||||
});
|
||||
|
||||
it("answering an unknown question fails", () => {
|
||||
const outcome = getBridge().answer("nope", { values: [] });
|
||||
assert.equal(outcome.ok, false);
|
||||
});
|
||||
|
||||
it("dismiss records a dismissed answer and notifies the agent", () => {
|
||||
const bridge = getBridge();
|
||||
bridge.publish(quizData("q3"));
|
||||
assert.equal(bridge.dismiss("q3").ok, true);
|
||||
|
||||
const ad = quiz.recorded.answered[0] as { dismissed: boolean };
|
||||
assert.equal(ad.dismissed, true);
|
||||
assert.match(quiz.recorded.messages[0], /dismissed/);
|
||||
});
|
||||
|
||||
it("dontKnow answers are never correct", () => {
|
||||
const bridge = getBridge();
|
||||
bridge.publish(quizData("q4"));
|
||||
bridge.answer("q4", { dontKnow: true });
|
||||
const ad = quiz.recorded.answered[0] as { dontKnow: boolean; correct: boolean };
|
||||
assert.equal(ad.dontKnow, true);
|
||||
assert.equal(ad.correct, false);
|
||||
});
|
||||
|
||||
it("rearm picks up unanswered questions from session entries", () => {
|
||||
const bridge = getBridge();
|
||||
// Simulate a resumed session whose entries contain an unanswered question.
|
||||
quiz.recorded.entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData("q5") },
|
||||
{
|
||||
type: "custom",
|
||||
customType: QUESTION_ANSWERED_ENTRY,
|
||||
data: { questionId: "other", answers: [] },
|
||||
},
|
||||
];
|
||||
bridge.rearm("quiz");
|
||||
|
||||
// Now answerable through the bridge.
|
||||
assert.equal(bridge.answer("q5", { values: ["right"] }).ok, true);
|
||||
});
|
||||
|
||||
it("rearm ignores already-answered questions", () => {
|
||||
const bridge = getBridge();
|
||||
quiz.recorded.entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData("q6") },
|
||||
{
|
||||
type: "custom",
|
||||
customType: QUESTION_ANSWERED_ENTRY,
|
||||
data: { questionId: "q6", answers: [], dontKnow: true },
|
||||
},
|
||||
];
|
||||
bridge.rearm("quiz");
|
||||
const outcome = bridge.answer("q6", { values: ["right"] });
|
||||
assert.equal(outcome.ok, false);
|
||||
});
|
||||
|
||||
it("clear removes only its kind's pending questions", () => {
|
||||
const bridge = getBridge();
|
||||
bridge.publish(quizData("q7"));
|
||||
const askQd: QuestionData = { ...quizData("q8"), kind: "ask", correctIndices: undefined };
|
||||
bridge.publish(askQd);
|
||||
|
||||
bridge.clear("quiz");
|
||||
assert.equal(bridge.answer("q7", { values: ["right"] }).ok, false);
|
||||
assert.equal(bridge.answer("q8", { other: "x" }).ok, true);
|
||||
});
|
||||
|
||||
it("ask answers route to the ask backend ungraded", () => {
|
||||
const bridge = getBridge();
|
||||
const askQd: QuestionData = { ...quizData("q9"), kind: "ask", correctIndices: undefined };
|
||||
bridge.publish(askQd);
|
||||
|
||||
bridge.answer("q9", { other: "custom answer" });
|
||||
const ad = ask.recorded.answered[0] as { other?: string; correct?: boolean };
|
||||
assert.equal(ad.other, "custom answer");
|
||||
assert.equal(ad.correct, undefined);
|
||||
assert.ok(ask.recorded.messages[0].startsWith("[Answer to your question]"));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* Integration test — the full teaching flow with a SCRIPTED model.
|
||||
*
|
||||
* Real pi agent loop, real web extensions (quiz-web/ask-web), real bridge,
|
||||
* real session persistence — the only fake is the LLM. Verifies the dispatch
|
||||
* contract end to end:
|
||||
*
|
||||
* prompt → scripted turn calls quiz → extension persists lh_question and
|
||||
* returns immediately → agent ends turn → bridge.answer() grades and sends
|
||||
* the [Quiz answer] user message → scripted final turn → settle.
|
||||
*
|
||||
* Then reloads the session file into a NEW runtime (resume) and verifies the
|
||||
* pending-question re-arm from persisted entries.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { after, before, describe, it } from "node:test";
|
||||
import { createAgentSessionRuntime, SessionManager } from "@earendil-works/pi-coding-agent";
|
||||
import { PATHS } from "../config.ts";
|
||||
import { buildTeacherRuntimeFactory } from "../src/teacher/create-session.ts";
|
||||
import {
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
findPendingQuestion,
|
||||
toClientQuestion,
|
||||
type QuestionAnswerData,
|
||||
type QuestionData,
|
||||
} from "../extensions/core/question-core.ts";
|
||||
import { getBridge } from "../extensions/core/bridge.ts";
|
||||
import { createScriptedRuntime } from "../src/testing/scripted-model.ts";
|
||||
|
||||
const QUIZ_ARGS = {
|
||||
question: "Which is closest to the Sun?",
|
||||
options: [
|
||||
{ label: "Mercury", value: "mercury" },
|
||||
{ label: "Venus", value: "venus" },
|
||||
],
|
||||
correctAnswer: "mercury",
|
||||
explanation: "Mercury is the innermost planet.",
|
||||
};
|
||||
|
||||
let tmpDir: string;
|
||||
let sessionDir: string;
|
||||
|
||||
before(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "lh-int-"));
|
||||
sessionDir = path.join(tmpDir, "sessions");
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
});
|
||||
|
||||
after(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function entriesOf(sm: SessionManager) {
|
||||
return sm.getEntries() as Array<{ type: string; customType?: string; data?: unknown; id: string }>;
|
||||
}
|
||||
|
||||
function customEntries(sm: SessionManager, customType: string) {
|
||||
return entriesOf(sm).filter((e) => e.type === "custom" && e.customType === customType);
|
||||
}
|
||||
|
||||
function userMessages(sm: SessionManager) {
|
||||
return entriesOf(sm)
|
||||
.filter((e) => e.type === "message" && (e as { message: { role: string } }).message.role === "user")
|
||||
.map((e) => (e as unknown as { message: { content: Array<{ type: string; text?: string }> } }).message);
|
||||
}
|
||||
|
||||
function waitForIdle(session: { waitForIdle(): Promise<void> }, timeoutMs = 10_000): Promise<void> {
|
||||
return Promise.race([
|
||||
session.waitForIdle(),
|
||||
new Promise<never>((_, reject) => setTimeout(() => reject(new Error("timeout waiting for idle")), timeoutMs)),
|
||||
]);
|
||||
}
|
||||
|
||||
/** Poll until pred() is true (bridge sendUserMessage is fire-and-forget async,
|
||||
* so waitForIdle alone can win a race against the incoming prompt). */
|
||||
function waitFor(pred: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {
|
||||
const start = Date.now();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tick = () => {
|
||||
if (pred()) return resolve();
|
||||
if (Date.now() - start > timeoutMs) return reject(new Error(`timeout waiting for ${what}`));
|
||||
setTimeout(tick, 25);
|
||||
};
|
||||
tick();
|
||||
});
|
||||
}
|
||||
|
||||
describe("scripted teaching flow (web dispatch mode)", () => {
|
||||
it("runs the quiz dispatch cycle and persists gold-standard data", async () => {
|
||||
const scripted = createScriptedRuntime([
|
||||
// Turn 1: call quiz, end turn (toolUse stop reason).
|
||||
{ toolCalls: [{ name: "quiz", arguments: QUIZ_ARGS }] },
|
||||
// Turn 2 (after the tool result, model declines to continue): final text.
|
||||
{ text: "Take a shot." },
|
||||
// Turn 3 (after the [Quiz answer] user message): wrap up.
|
||||
{ text: "Right — Mercury it is. Lesson over." },
|
||||
]);
|
||||
|
||||
const sessionManager = SessionManager.create(PATHS.root, sessionDir);
|
||||
const runtime = await createAgentSessionRuntime(buildTeacherRuntimeFactory("web", {
|
||||
model: scripted.model,
|
||||
modelRuntime: scripted.modelRuntime,
|
||||
}), {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager,
|
||||
});
|
||||
assert.ok(!runtime.diagnostics.some((d) => d.type === "error"), "no fatal diagnostics");
|
||||
|
||||
// ── Turn 1: prompt → quiz presented ────────────────────────────────
|
||||
const promptDone = runtime.session.prompt("Teach me about the solar system.");
|
||||
await waitForIdle(runtime.session);
|
||||
await promptDone;
|
||||
|
||||
const questions = customEntries(sessionManager, QUESTION_ENTRY) as Array<{ data?: QuestionData }>;
|
||||
assert.equal(questions.length, 1, "exactly one lh_question entry");
|
||||
const qd = questions[0]!.data!;
|
||||
assert.equal(qd.kind, "quiz");
|
||||
assert.equal(qd.mode, "single-select");
|
||||
assert.deepEqual(qd.correctIndices, [qd.options.findIndex((o) => o.value === "mercury") + 1], "correct index resolved post-shuffle");
|
||||
assert.equal(qd.explanation, QUIZ_ARGS.explanation);
|
||||
|
||||
// Pending question visible without leaking the answer.
|
||||
const pending = findPendingQuestion(entriesOf(sessionManager));
|
||||
assert.ok(pending, "pending question found");
|
||||
const clientPayload = JSON.stringify(toClientQuestion(qd));
|
||||
assert.ok(!clientPayload.includes("correctIndices"));
|
||||
assert.ok(!clientPayload.includes("Mercury is the innermost"));
|
||||
|
||||
// ── Answer: graded + injected as user message ──────────────────────
|
||||
const outcome = getBridge().answer(qd.id, { values: ["mercury"] });
|
||||
assert.equal(outcome.ok, true);
|
||||
await waitFor(() => userMessages(sessionManager).length === 2, "quiz answer user message");
|
||||
await waitForIdle(runtime.session);
|
||||
|
||||
const answers = customEntries(sessionManager, QUESTION_ANSWERED_ENTRY) as Array<{
|
||||
data?: QuestionAnswerData;
|
||||
}>;
|
||||
assert.equal(answers.length, 1, "exactly one lh_question_answered entry");
|
||||
const ad = answers[0]!.data!;
|
||||
assert.equal(ad.correct, true);
|
||||
assert.equal(ad.questionId, qd.id);
|
||||
assert.equal(ad.answers.length, 1);
|
||||
assert.equal(ad.answers[0]!.value, "mercury");
|
||||
|
||||
const users = userMessages(sessionManager);
|
||||
assert.equal(users.length, 2, "topic prompt + quiz answer");
|
||||
const answerMsg = users[1]!.content.find((c) => c.type === "text")!.text!;
|
||||
assert.ok(answerMsg.startsWith("[Quiz answer]"), "answer arrives as user message");
|
||||
assert.match(answerMsg, /Result: CORRECT/);
|
||||
assert.match(answerMsg, /Selected: \d\. Mercury/);
|
||||
|
||||
// Pending is cleared after answering.
|
||||
assert.equal(findPendingQuestion(entriesOf(sessionManager)), null);
|
||||
|
||||
// The scripted model saw the answer in its context (turn 3).
|
||||
assert.ok(scripted.contextsSeen.length >= 3);
|
||||
const lastContext = scripted.contextsSeen[scripted.contextsSeen.length - 1]!;
|
||||
const lastMessages = JSON.stringify(lastContext.messages);
|
||||
assert.ok(lastMessages.includes("[Quiz answer]"), "answer message reached the LLM context");
|
||||
// …but the tool RESULT must never reveal the answer (the correct answer
|
||||
// and explanation live only in the lh_question custom entry, which is
|
||||
// excluded from LLM context). The toolCall arguments legitimately contain
|
||||
// them — the model wrote those itself.
|
||||
for (const ctx of scripted.contextsSeen) {
|
||||
const messages = ctx.messages as Array<{ role?: string; content?: Array<{ type?: string; text?: string; name?: string }> }>;
|
||||
for (const m of messages) {
|
||||
if (m.role !== "toolResult") continue;
|
||||
for (const c of m.content ?? []) {
|
||||
if (c.type === "text" && c.text) {
|
||||
assert.ok(!c.text.includes("Mercury is the innermost"), "no explanation leak via tool result");
|
||||
assert.ok(!/correct/i.test(c.text), "no correct-answer leak via tool result");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
runtime.session.dispose();
|
||||
|
||||
// ── Resume: pending re-arm from the session file ───────────────────
|
||||
const sessionFile = sessionManager.getSessionFile()!;
|
||||
const sm2 = SessionManager.open(sessionFile, sessionDir, PATHS.root);
|
||||
const scripted2 = createScriptedRuntime([{ text: "resumed" }]);
|
||||
const runtime2 = await createAgentSessionRuntime(buildTeacherRuntimeFactory("web", {
|
||||
model: scripted2.model,
|
||||
modelRuntime: scripted2.modelRuntime,
|
||||
}), {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager: sm2,
|
||||
});
|
||||
// session_start fired during construction → extensions re-armed.
|
||||
assert.equal(findPendingQuestion(entriesOf(sm2)), null, "answered question is NOT re-armed");
|
||||
assert.equal(customEntries(sm2, QUESTION_ENTRY).length, 1, "question entry survived on disk");
|
||||
runtime2.session.dispose();
|
||||
|
||||
// ── Resume with an UNANSWERED question pending ─────────────────────
|
||||
// New session, fresh script: quiz presented but never answered.
|
||||
const scripted3 = createScriptedRuntime([
|
||||
{ toolCalls: [{ name: "quiz", arguments: QUIZ_ARGS }] },
|
||||
{ text: "Waiting for your answer." },
|
||||
]);
|
||||
const sm3 = SessionManager.create(PATHS.root, sessionDir);
|
||||
const runtime3 = await createAgentSessionRuntime(buildTeacherRuntimeFactory("web", {
|
||||
model: scripted3.model,
|
||||
modelRuntime: scripted3.modelRuntime,
|
||||
}), {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager: sm3,
|
||||
});
|
||||
const p3 = runtime3.session.prompt("Teach me something else.");
|
||||
await waitForIdle(runtime3.session);
|
||||
await p3;
|
||||
runtime3.session.dispose();
|
||||
|
||||
const file3 = sm3.getSessionFile()!;
|
||||
const sm4 = SessionManager.open(file3, sessionDir, PATHS.root);
|
||||
const scripted4 = createScriptedRuntime([{ text: "ok" }]);
|
||||
const runtime4 = await createAgentSessionRuntime(buildTeacherRuntimeFactory("web", {
|
||||
model: scripted4.model,
|
||||
modelRuntime: scripted4.modelRuntime,
|
||||
}), {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager: sm4,
|
||||
});
|
||||
const reArmed = findPendingQuestion(entriesOf(sm4));
|
||||
assert.ok(reArmed, "unanswered question re-arms on resume");
|
||||
assert.equal(reArmed!.kind, "quiz");
|
||||
// And it is answerable through the bridge (the re-armed id).
|
||||
const answerOutcome = getBridge().answer(reArmed!.id, { dontKnow: true, note: "forgot" });
|
||||
assert.equal(answerOutcome.ok, true);
|
||||
const answeredOnDisk = customEntries(sm4, QUESTION_ANSWERED_ENTRY) as Array<{
|
||||
data?: QuestionAnswerData;
|
||||
}>;
|
||||
assert.equal(answeredOnDisk.length, 1);
|
||||
assert.equal(answeredOnDisk[0]!.data!.dontKnow, true);
|
||||
runtime4.session.dispose();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* Unit tests — question-core (pure logic, no pi, no server, no LLM).
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
buildAnswer,
|
||||
coerceCorrectAnswer,
|
||||
findPendingQuestion,
|
||||
formatAnswerMessage,
|
||||
isCorrect,
|
||||
makeId,
|
||||
normalizeOptions,
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
resolveCorrect,
|
||||
shuffleOptions,
|
||||
type QuestionData,
|
||||
toClientQuestion,
|
||||
} from "../extensions/core/question-core.ts";
|
||||
|
||||
// ─── normalizeOptions ───────────────────────────────────────────────────────
|
||||
|
||||
describe("normalizeOptions", () => {
|
||||
it("trims labels and defaults value to label", () => {
|
||||
const opts = normalizeOptions([{ label: " Mercury " }, { label: "Venus", value: " v " }]);
|
||||
assert.deepEqual(opts, [
|
||||
{ label: "Mercury", value: "Mercury", description: undefined },
|
||||
{ label: "Venus", value: "v", description: undefined },
|
||||
]);
|
||||
});
|
||||
|
||||
it("drops empty labels", () => {
|
||||
const opts = normalizeOptions([{ label: "" }, { label: "ok" }]);
|
||||
assert.equal(opts.length, 1);
|
||||
});
|
||||
|
||||
it("throws on duplicate option values", () => {
|
||||
assert.throws(() => normalizeOptions([{ label: "a", value: "x" }, { label: "b", value: "x" }]), /duplicate/);
|
||||
});
|
||||
|
||||
it("handles undefined input", () => {
|
||||
assert.deepEqual(normalizeOptions(undefined), []);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── shuffleOptions ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("shuffleOptions", () => {
|
||||
it("permutes without adding, removing, or mutating options", () => {
|
||||
const input = Array.from({ length: 20 }, (_, i) => ({ label: `o${i}`, value: `o${i}` }));
|
||||
const shuffled = shuffleOptions(input);
|
||||
assert.equal(shuffled.length, input.length);
|
||||
assert.deepEqual([...shuffled].sort((a, b) => Number(a.label.slice(1)) - Number(b.label.slice(1))), input);
|
||||
assert.notEqual(input, shuffled); // a copy
|
||||
});
|
||||
|
||||
it("is stable for single/empty inputs", () => {
|
||||
assert.deepEqual(shuffleOptions([]), []);
|
||||
assert.deepEqual(shuffleOptions([{ label: "only", value: "only" }]), [{ label: "only", value: "only" }]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── coerceCorrectAnswer ────────────────────────────────────────────────────
|
||||
|
||||
describe("coerceCorrectAnswer", () => {
|
||||
it("wraps a single string", () => {
|
||||
assert.deepEqual(coerceCorrectAnswer("mercury"), ["mercury"]);
|
||||
});
|
||||
|
||||
it("passes arrays through", () => {
|
||||
assert.deepEqual(coerceCorrectAnswer(["a", "b"]), ["a", "b"]);
|
||||
});
|
||||
|
||||
it("parses a JSON-stringified multi-select array", () => {
|
||||
assert.deepEqual(coerceCorrectAnswer('["a", "b"]'), ["a", "b"]);
|
||||
});
|
||||
|
||||
it("leaves a non-JSON bracketed string alone", () => {
|
||||
assert.deepEqual(coerceCorrectAnswer("[not json"), ["[not json"]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── resolveCorrect ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("resolveCorrect", () => {
|
||||
const options = normalizeOptions([
|
||||
{ label: "Mercury", value: "mercury" },
|
||||
{ label: "Venus", value: "venus" },
|
||||
{ label: "Earth", value: "earth" },
|
||||
]);
|
||||
|
||||
it("resolves a single value to its 1-based display index", () => {
|
||||
assert.deepEqual(resolveCorrect("venus", options), { indices: [2] });
|
||||
});
|
||||
|
||||
it("resolves multi-select values, deduped and sorted", () => {
|
||||
assert.deepEqual(resolveCorrect(["earth", "mercury", "mercury"], options), { indices: [1, 3] });
|
||||
});
|
||||
|
||||
it("errors when the value matches no option", () => {
|
||||
const r = resolveCorrect("mars", options);
|
||||
assert.ok(r.error);
|
||||
assert.match(r.error, /does not match any option value/);
|
||||
});
|
||||
|
||||
it("errors on missing correctAnswer", () => {
|
||||
assert.ok(resolveCorrect(undefined, options).error);
|
||||
});
|
||||
|
||||
it("resolves against POST-shuffle order (indices track display)", () => {
|
||||
const shuffled = shuffleOptions(options);
|
||||
const value = "earth";
|
||||
const r = resolveCorrect(value, shuffled);
|
||||
const displayedIndex = shuffled.findIndex((o) => o.value === value) + 1;
|
||||
assert.deepEqual(r.indices, [displayedIndex]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── isCorrect (exact-set match) ────────────────────────────────────────────
|
||||
|
||||
describe("isCorrect", () => {
|
||||
it("matches identical sets regardless of order", () => {
|
||||
assert.ok(isCorrect([3, 1], [1, 3]));
|
||||
});
|
||||
|
||||
it("rejects partial matches", () => {
|
||||
assert.ok(!isCorrect([1], [1, 3]));
|
||||
assert.ok(!isCorrect([1, 2, 3], [1, 3]));
|
||||
});
|
||||
|
||||
it("rejects wrong selections of the same size", () => {
|
||||
assert.ok(!isCorrect([1, 2], [1, 3]));
|
||||
});
|
||||
|
||||
it("rejects empty selection against a non-empty answer", () => {
|
||||
assert.ok(!isCorrect([], [1]));
|
||||
});
|
||||
});
|
||||
|
||||
// ─── buildAnswer + formatAnswerMessage ──────────────────────────────────────
|
||||
|
||||
function quizData(overrides?: Partial<QuestionData>): QuestionData {
|
||||
return {
|
||||
id: "q_test",
|
||||
kind: "quiz",
|
||||
question: "Which planet is closest to the Sun?",
|
||||
mode: "single-select",
|
||||
options: normalizeOptions([
|
||||
{ label: "Mercury", value: "mercury" },
|
||||
{ label: "Venus", value: "venus" },
|
||||
]),
|
||||
correctIndices: [1],
|
||||
explanation: "Mercury orbits closest.",
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildAnswer", () => {
|
||||
it("grades a correct selection", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["mercury"] });
|
||||
assert.equal(ad.correct, true);
|
||||
assert.equal(ad.answers[0]?.label, "Mercury");
|
||||
assert.equal(ad.answers[0]?.index, 1);
|
||||
assert.equal(ad.dontKnow, false);
|
||||
});
|
||||
|
||||
it("grades a wrong selection", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["venus"] });
|
||||
assert.equal(ad.correct, false);
|
||||
});
|
||||
|
||||
it("dontKnow is never correct and records no answers", () => {
|
||||
const ad = buildAnswer(quizData(), { dontKnow: true });
|
||||
assert.equal(ad.correct, false);
|
||||
assert.equal(ad.dontKnow, true);
|
||||
assert.deepEqual(ad.answers, []);
|
||||
});
|
||||
|
||||
it("keeps the note only when non-empty", () => {
|
||||
assert.equal(buildAnswer(quizData(), { values: ["mercury"], note: " " }).note, undefined);
|
||||
assert.equal(buildAnswer(quizData(), { values: ["mercury"], note: " x " }).note, "x");
|
||||
});
|
||||
|
||||
it("ignores unknown option values", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["mars", "mercury"] });
|
||||
assert.deepEqual(ad.answers.map((a) => a.value), ["mercury"]);
|
||||
});
|
||||
|
||||
it("multi-select grades as exact-set", () => {
|
||||
const multi = quizData({
|
||||
mode: "multi-select",
|
||||
options: normalizeOptions([
|
||||
{ label: "A", value: "a" },
|
||||
{ label: "B", value: "b" },
|
||||
{ label: "C", value: "c" },
|
||||
]),
|
||||
correctIndices: [1, 2],
|
||||
});
|
||||
assert.equal(buildAnswer(multi, { values: ["a", "b"] }).correct, true);
|
||||
assert.equal(buildAnswer(multi, { values: ["a"] }).correct, false);
|
||||
assert.equal(buildAnswer(multi, { values: ["a", "b", "c"] }).correct, false);
|
||||
});
|
||||
|
||||
it("ask questions are not graded", () => {
|
||||
const ask = quizData({ kind: "ask", correctIndices: undefined, explanation: undefined });
|
||||
const ad = buildAnswer(ask, { values: ["mercury"] });
|
||||
assert.equal(ad.correct, undefined);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatAnswerMessage", () => {
|
||||
it("formats a correct quiz answer for the agent", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["mercury"] });
|
||||
const text = formatAnswerMessage(ad);
|
||||
assert.ok(text.startsWith("[Quiz answer]"));
|
||||
assert.match(text, /Selected: 1\. Mercury/);
|
||||
assert.match(text, /Result: CORRECT/);
|
||||
});
|
||||
|
||||
it("formats dontKnow as a gap, not a guess", () => {
|
||||
const ad = buildAnswer(quizData(), { dontKnow: true });
|
||||
const text = formatAnswerMessage(ad);
|
||||
assert.match(text, /did NOT attempt/);
|
||||
assert.match(text, /knowledge gap/);
|
||||
assert.ok(!text.includes("Result:"));
|
||||
});
|
||||
|
||||
it("includes the note when present", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["venus"], note: "guessed" });
|
||||
assert.match(formatAnswerMessage(ad), /Note: guessed/);
|
||||
});
|
||||
|
||||
it("formats ask answers", () => {
|
||||
const ask = quizData({ kind: "ask", correctIndices: undefined, explanation: undefined });
|
||||
const ad = buildAnswer(ask, { other: "whatever" });
|
||||
const text = formatAnswerMessage(ad);
|
||||
assert.ok(text.startsWith("[Answer to your question]"));
|
||||
assert.match(text, /\(Other\): whatever/);
|
||||
});
|
||||
|
||||
it("formats a dismissed question", () => {
|
||||
const ad = buildAnswer(quizData(), {});
|
||||
ad.dismissed = true;
|
||||
assert.match(formatAnswerMessage(ad), /dismissed/);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── toClientQuestion (leak safety) ─────────────────────────────────────────
|
||||
|
||||
describe("toClientQuestion", () => {
|
||||
it("NEVER exposes correctIndices or explanation", () => {
|
||||
const cq = toClientQuestion(quizData());
|
||||
const raw = JSON.stringify(cq);
|
||||
assert.ok(!raw.includes("correctIndices"));
|
||||
assert.ok(!raw.includes("Mercury orbits closest"));
|
||||
assert.equal(cq.id, "q_test");
|
||||
assert.equal(cq.kind, "quiz");
|
||||
assert.equal(cq.options.length, 2);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── findPendingQuestion (resume re-arm) ────────────────────────────────────
|
||||
|
||||
describe("findPendingQuestion", () => {
|
||||
it("finds an unanswered question", () => {
|
||||
const entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
|
||||
];
|
||||
const pq = findPendingQuestion(entries);
|
||||
assert.ok(pq);
|
||||
assert.equal(pq.id, "q_test");
|
||||
});
|
||||
|
||||
it("returns null once answered", () => {
|
||||
const entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
|
||||
{
|
||||
type: "custom",
|
||||
customType: QUESTION_ANSWERED_ENTRY,
|
||||
data: { questionId: "q_test", dontKnow: true, answers: [] },
|
||||
},
|
||||
];
|
||||
assert.equal(findPendingQuestion(entries), null);
|
||||
});
|
||||
|
||||
it("returns the OLDEST unanswered question", () => {
|
||||
const entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData({ id: "q1" }) },
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData({ id: "q2" }) },
|
||||
];
|
||||
assert.equal(findPendingQuestion(entries)?.id, "q1");
|
||||
});
|
||||
|
||||
it("skips a dismissed question", () => {
|
||||
const entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
|
||||
{
|
||||
type: "custom",
|
||||
customType: QUESTION_ANSWERED_ENTRY,
|
||||
data: { questionId: "q_test", dismissed: true, answers: [] },
|
||||
},
|
||||
];
|
||||
assert.equal(findPendingQuestion(entries), null);
|
||||
});
|
||||
|
||||
it("ignores other custom entries", () => {
|
||||
const entries = [{ type: "custom", customType: "md-log", data: { file: null } }];
|
||||
assert.equal(findPendingQuestion(entries), null);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── makeId ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("makeId", () => {
|
||||
it("generates unique ids", () => {
|
||||
const ids = new Set(Array.from({ length: 200 }, () => makeId()));
|
||||
assert.equal(ids.size, 200);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user