tool-result diet: question answers return only learner-added truth (compact JSON for the model, full record in details); dead-code sweep

This commit is contained in:
amosblomqvist
2026-10-02 04:04:04 +03:00
parent 9a2e434748
commit a85087528e
8 changed files with 131 additions and 212 deletions
+2 -2
View File
@@ -282,8 +282,8 @@ describe("teacher chat flow (scripted model)", () => {
// The graded answer rode the record as the call's tool result and
// round-trips through JSON (undefined keys drop) — compare its
// SUBSTANCE, not key-for-key against the in-memory build.
const out = (answeredQuiz as { output?: { questionId?: string; correct?: boolean } }).output!;
assert.equal(out.questionId, "call-1");
const out = (answeredQuiz as { output?: { kind?: string; correct?: boolean } }).output!;
assert.equal(out.kind, "quiz");
assert.equal(out.correct, true);
// The model actually received the graded answer as a tool result.
+2 -2
View File
@@ -196,7 +196,7 @@ function fixtureChat() {
...assistant("a1", [
{ type: "text", text: "The chain rule composes rates of change along a path." },
quizPart("probe-1", { question: "What is a derivative, roughly?" }, {
questionId: "probe-1", kind: "quiz", questionText: "What is a derivative, roughly?",
kind: "quiz", questionText: "What is a derivative, roughly?",
correct: false, dontKnow: false, answeredAt: "2026-08-22T01:00:00.000Z",
}),
]),
@@ -211,7 +211,7 @@ function fixtureChat() {
user("u3", "check me"),
...assistant("a3", [
quizPart("check-1", { question: "Check" }, {
questionId: "check-1", kind: "quiz", questionText: "Check", correct: true, dontKnow: false,
kind: "quiz", questionText: "Check", correct: true, dontKnow: false,
answeredAt: "2026-08-22T02:00:00.000Z",
}),
]),
+10 -11
View File
@@ -143,7 +143,6 @@ function askPart(
function answer(over: Partial<QuestionAnswerData>): QuestionAnswerData {
return {
questionId: "q",
kind: "quiz",
questionText: "question text",
dontKnow: false,
@@ -170,14 +169,14 @@ function fixtureMessages(): LhMessage[] {
quizPart(
"call-1",
{ question: "Q1", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "call-1", questionText: "Q1 text", correct: false, answeredAt: "2026-08-22T01:00:00.000Z" }),
answer({ questionText: "Q1 text", correct: false, answeredAt: "2026-08-22T01:00:00.000Z" }),
),
]),
assistant("a2", [
quizPart(
"call-2",
{ question: "Q2", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "call-2", questionText: "Q2 text", dontKnow: true, correct: false, answeredAt: "2026-08-22T01:01:00.000Z" }),
answer({ questionText: "Q2 text", dontKnow: true, correct: false, answeredAt: "2026-08-22T01:01:00.000Z" }),
),
]),
assistant("a3", [
@@ -186,21 +185,21 @@ function fixtureMessages(): LhMessage[] {
quizPart(
"call-3",
{ question: "Q3", nodeId: "alpha", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "call-3", questionText: "Q3 text", correct: true, note: "recalled the diagram", answeredAt: "2026-08-22T02:00:00.000Z" }),
answer({ questionText: "Q3 text", correct: true, note: "recalled the diagram", answeredAt: "2026-08-22T02:00:00.000Z" }),
),
]),
assistant("a4", [
quizPart(
"call-4",
{ question: "Q4", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "call-4", questionText: "Q4 text", correct: false, answeredAt: "2026-08-22T02:01:00.000Z" }),
answer({ questionText: "Q4 text", correct: false, answeredAt: "2026-08-22T02:01:00.000Z" }),
),
]),
assistant("a5", [
quizPart(
"call-5",
{ question: "Q5", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "call-5", questionText: "Q5 text", dontKnow: true, correct: false, answeredAt: "2026-08-22T02:02:00.000Z" }),
answer({ questionText: "Q5 text", dontKnow: true, correct: false, answeredAt: "2026-08-22T02:02:00.000Z" }),
),
]),
assistant("a6", [
@@ -208,12 +207,12 @@ function fixtureMessages(): LhMessage[] {
quizPart(
"call-6",
{ question: "Q6", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "call-6", questionText: "Q6 text", dismissed: true }),
answer({ questionText: "Q6 text", dismissed: true }),
),
askPart(
"call-7",
{ question: "free text?" },
answer({ questionId: "call-7", kind: "ask", questionText: "free text?", text: "because" }),
answer({ kind: "ask", questionText: "free text?", text: "because" }),
),
]),
].flat();
@@ -354,7 +353,7 @@ describe("the merger (mergeFindings)", () => {
quizPart(
"call-9",
{ question: "Q9", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "call-9", questionText: "Q9 text", correct: true, answeredAt: "2026-08-22T03:00:00.000Z" }),
answer({ questionText: "Q9 text", correct: true, answeredAt: "2026-08-22T03:00:00.000Z" }),
),
]),
].flat());
@@ -624,7 +623,7 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
quizPart(
"d-call-1",
{ question: "DQ1", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "d-call-1", questionText: "DQ1 text", correct: true, answeredAt: "2026-09-23T00:00:00.000Z" }),
answer({ questionText: "DQ1 text", correct: true, answeredAt: "2026-09-23T00:00:00.000Z" }),
),
]),
].flat());
@@ -649,7 +648,7 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
quizPart(
"e-call-1",
{ question: "EQ1", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionId: "e-call-1", questionText: "EQ1 text", correct: false, answeredAt: "2026-09-23T00:01:00.000Z" }),
answer({ questionText: "EQ1 text", correct: false, answeredAt: "2026-09-23T00:01:00.000Z" }),
),
]),
].flat());
+48 -105
View File
@@ -11,16 +11,12 @@ import {
quizInputBase,
askOptionProblems,
buildAnswer,
compactAnswerForModel,
coerceCorrectAnswer,
CONFIDENCE_STOPS,
DEFAULT_CONFIDENCE_INDEX,
findPendingQuestion,
formatAnswerMessage,
isCorrect,
makeId,
normalizeOptions,
QUESTION_ANSWERED_ENTRY,
QUESTION_ENTRY,
quizInputProblems,
explainInputBase,
explainInputObject,
@@ -415,47 +411,50 @@ describe("buildAnswer", () => {
});
});
describe("formatAnswerMessage", () => {
it("formats a correct quiz answer for the agent", () => {
const ad = buildAnswer(quizData(), { values: ["mercury"] });
const text = formatAnswerMessage(ad);
assert.ok(text.startsWith("[Quiz answer]"));
assert.match(text, /Selected: 1\. Mercury/);
assert.match(text, /Result: CORRECT/);
// ─── compactAnswerForModel (the model-visible diet, owner 2026-10-02) ──────
describe("compactAnswerForModel", () => {
it("quiz: the learner's selection, verdict, confidence, note — NOTHING the model authored", () => {
const ad = buildAnswer(quizData(), { values: ["mercury"], confidence: 70, note: "guessed" });
const compact = compactAnswerForModel(ad);
assert.deepEqual(compact, {
kind: "quiz",
answers: [{ index: 1, label: "Mercury" }],
correct: true,
confidence: 70,
note: "guessed",
});
const raw = JSON.stringify(compact);
// every one of these lives in the tool CALL's arguments already
assert.ok(!raw.includes("Which planet"), "no questionText");
assert.ok(!raw.includes("Venus"), "no option-list restatement");
assert.ok(!raw.includes("Mercury orbits closest"), "no explanation");
assert.ok(!("correctIndices" in compact), "correct is the verdict; indices are the call's");
assert.ok(!("questionId" in compact), "questionId was the toolCallId restated");
});
it("formats dontKnow as a gap, not a guess", () => {
const ad = buildAnswer(quizData(), { dontKnow: true });
const text = formatAnswerMessage(ad);
assert.match(text, /did NOT attempt/);
assert.match(text, /knowledge gap/);
assert.ok(!text.includes("Result:"));
it("dontKnow: the gap flag stands alone — no verdict, no confidence", () => {
const compact = compactAnswerForModel(buildAnswer(quizData(), { dontKnow: true }));
assert.deepEqual(compact, { kind: "quiz", dontKnow: true });
});
it("includes the note when present", () => {
const ad = buildAnswer(quizData(), { values: ["venus"], note: "guessed" });
assert.match(formatAnswerMessage(ad), /Note: guessed/);
it("ask: selection or text; never graded", () => {
const askQD: QuestionData = {
...quizData(),
kind: "ask",
mode: "text",
correctIndices: undefined,
explanation: undefined,
};
const compact = compactAnswerForModel(buildAnswer(askQD, { text: "lateral thinking" }));
assert.deepEqual(compact, { kind: "ask", text: "lateral thinking" });
});
it("states the confidence line on answered quizzes, absent on dontKnow", () => {
const ad = buildAnswer(quizData(), { values: ["mercury"], confidence: 70 });
assert.match(formatAnswerMessage(ad), /Confidence: 70% — the learner's stated probability that his selected answer is correct\./);
const idk = buildAnswer(quizData(), { dontKnow: true, confidence: 70 });
assert.ok(!formatAnswerMessage(idk).includes("Confidence:"));
});
it("formats ask answers", () => {
const ask = quizData({ kind: "ask", correctIndices: undefined, explanation: undefined });
const ad = buildAnswer(ask, { other: "whatever" });
const text = formatAnswerMessage(ad);
assert.ok(text.startsWith("[Answer to your question]"));
assert.match(text, /\(Other\): whatever/);
});
it("formats a dismissed question", () => {
const ad = buildAnswer(quizData(), {});
ad.dismissed = true;
assert.match(formatAnswerMessage(ad), /dismissed/);
it("dismissed: a one-field record", () => {
const ad = { ...buildAnswer(quizData(), { dontKnow: true }), dismissed: true };
const compact = compactAnswerForModel(ad);
assert.deepEqual(compact, { kind: "quiz", dismissed: true });
});
});
@@ -473,64 +472,6 @@ describe("toClientQuestion", () => {
});
});
// ─── findPendingQuestion (resume re-arm) ────────────────────────────────────
describe("findPendingQuestion", () => {
it("finds an unanswered question", () => {
const entries = [
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
];
const pq = findPendingQuestion(entries);
assert.ok(pq);
assert.equal(pq.id, "q_test");
});
it("returns null once answered", () => {
const entries = [
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
{
type: "custom",
customType: QUESTION_ANSWERED_ENTRY,
data: { questionId: "q_test", dontKnow: true, answers: [] },
},
];
assert.equal(findPendingQuestion(entries), null);
});
it("returns the OLDEST unanswered question", () => {
const entries = [
{ type: "custom", customType: QUESTION_ENTRY, data: quizData({ id: "q1" }) },
{ type: "custom", customType: QUESTION_ENTRY, data: quizData({ id: "q2" }) },
];
assert.equal(findPendingQuestion(entries)?.id, "q1");
});
it("skips a dismissed question", () => {
const entries = [
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
{
type: "custom",
customType: QUESTION_ANSWERED_ENTRY,
data: { questionId: "q_test", dismissed: true, answers: [] },
},
];
assert.equal(findPendingQuestion(entries), null);
});
it("ignores other custom entries", () => {
const entries = [{ type: "custom", customType: "md-log", data: { file: null } }];
assert.equal(findPendingQuestion(entries), null);
});
});
// ─── makeId ─────────────────────────────────────────────────────────────────
describe("makeId", () => {
it("generates unique ids", () => {
const ids = new Set(Array.from({ length: 200 }, () => makeId()));
assert.equal(ids.size, 200);
});
});
// ─── the dock selection state machine (multi-select Other coexistence) ──
// The live bug: in a multi-select ask, picking the custom "Other…" row
@@ -630,13 +571,15 @@ describe("buildAnswer / formatAnswerMessage: explain", () => {
assert.equal(ad.text, undefined);
});
it("formats the model's view: his answer, the immutable commitment, and the grading instruction", () => {
const text = formatAnswerMessage(buildAnswer(explainQD(), { text: "No idea about aether." }));
assert.ok(text.startsWith("[Explain answer]"));
assert.match(text, /The learner answered: No idea about aether\./);
assert.match(text, /Your committed correct answer \(immutable/);
assert.match(text, /luminiferous aether/);
assert.match(text, /Grade his answer strictly/);
it("compactAnswerForModel (explain): his words ONLY — never the model's own commitment", () => {
const compact = compactAnswerForModel(buildAnswer(explainQD(), { text: "No idea about aether." }));
const raw = JSON.stringify(compact);
assert.equal(compact.text, "No idea about aether.");
assert.equal(compact.kind, "explain");
// the commitment is the model's own call argument — NEVER restated
assert.ok(!raw.includes("luminiferous"), "the commitment must not ride the result");
assert.ok(!("committed" in compact));
assert.ok(!("questionText" in compact));
});
});
+33
View File
@@ -25,6 +25,7 @@ import { absorbed, pendingCallTool, SessionModel } from "../components/session-c
import type { ChatInput } from "../src/core/input-events.ts";
import type { LhMessage } from "../src/core/native-messages.ts";
import { newUserMessage, newAnswerResult, needsContinuation } from "../src/core/native-messages.ts";
import { buildAnswer, type QuestionData } from "../src/core/question-core.ts";
const realFetch = globalThis.fetch;
afterEach(() => {
@@ -825,3 +826,35 @@ describe("pendingCallTool", () => {
assert.equal(pendingCallTool(messages, "call-zz"), null);
});
});
// ─── newAnswerResult: the two-audience split (owner, 2026-10-02) ────────────
describe("newAnswerResult two audiences", () => {
it("MODEL sees the compact learner-only record; details keeps the FULL graded answer", () => {
const qd: QuestionData = {
id: "call-1",
kind: "quiz",
question: "Which planet is closest to the Sun?",
mode: "single-select",
options: [
{ label: "Mercury", value: "mercury" },
{ label: "Venus", value: "venus" },
],
correctIndices: [1],
explanation: "Mercury orbits closest.",
createdAt: "2026-01-01T00:00:00Z",
};
const full = buildAnswer(qd, { values: ["mercury"], confidence: 70, note: "guessed" });
const msg = newAnswerResult("call-1", "quiz", full);
const modelSees = JSON.parse((msg.content as { type: "text"; text: string }[])[0]!.text);
assert.deepEqual(modelSees, {
kind: "quiz",
answers: [{ index: 1, label: "Mercury" }],
correct: true,
confidence: 70,
note: "guessed",
});
assert.equal(msg.details, full, "details is the full QuestionAnswerData verbatim");
assert.equal((msg.details as { questionText?: string }).questionText, "Which planet is closest to the Sun?");
});
});