tool-result diet: question answers return only learner-added truth (compact JSON for the model, full record in details); dead-code sweep
This commit is contained in:
@@ -282,8 +282,8 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
// The graded answer rode the record as the call's tool result and
|
||||
// round-trips through JSON (undefined keys drop) — compare its
|
||||
// SUBSTANCE, not key-for-key against the in-memory build.
|
||||
const out = (answeredQuiz as { output?: { questionId?: string; correct?: boolean } }).output!;
|
||||
assert.equal(out.questionId, "call-1");
|
||||
const out = (answeredQuiz as { output?: { kind?: string; correct?: boolean } }).output!;
|
||||
assert.equal(out.kind, "quiz");
|
||||
assert.equal(out.correct, true);
|
||||
|
||||
// The model actually received the graded answer as a tool result.
|
||||
|
||||
@@ -196,7 +196,7 @@ function fixtureChat() {
|
||||
...assistant("a1", [
|
||||
{ type: "text", text: "The chain rule composes rates of change along a path." },
|
||||
quizPart("probe-1", { question: "What is a derivative, roughly?" }, {
|
||||
questionId: "probe-1", kind: "quiz", questionText: "What is a derivative, roughly?",
|
||||
kind: "quiz", questionText: "What is a derivative, roughly?",
|
||||
correct: false, dontKnow: false, answeredAt: "2026-08-22T01:00:00.000Z",
|
||||
}),
|
||||
]),
|
||||
@@ -211,7 +211,7 @@ function fixtureChat() {
|
||||
user("u3", "check me"),
|
||||
...assistant("a3", [
|
||||
quizPart("check-1", { question: "Check" }, {
|
||||
questionId: "check-1", kind: "quiz", questionText: "Check", correct: true, dontKnow: false,
|
||||
kind: "quiz", questionText: "Check", correct: true, dontKnow: false,
|
||||
answeredAt: "2026-08-22T02:00:00.000Z",
|
||||
}),
|
||||
]),
|
||||
|
||||
+10
-11
@@ -143,7 +143,6 @@ function askPart(
|
||||
|
||||
function answer(over: Partial<QuestionAnswerData>): QuestionAnswerData {
|
||||
return {
|
||||
questionId: "q",
|
||||
kind: "quiz",
|
||||
questionText: "question text",
|
||||
dontKnow: false,
|
||||
@@ -170,14 +169,14 @@ function fixtureMessages(): LhMessage[] {
|
||||
quizPart(
|
||||
"call-1",
|
||||
{ question: "Q1", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-1", questionText: "Q1 text", correct: false, answeredAt: "2026-08-22T01:00:00.000Z" }),
|
||||
answer({ questionText: "Q1 text", correct: false, answeredAt: "2026-08-22T01:00:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
assistant("a2", [
|
||||
quizPart(
|
||||
"call-2",
|
||||
{ question: "Q2", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-2", questionText: "Q2 text", dontKnow: true, correct: false, answeredAt: "2026-08-22T01:01:00.000Z" }),
|
||||
answer({ questionText: "Q2 text", dontKnow: true, correct: false, answeredAt: "2026-08-22T01:01:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
assistant("a3", [
|
||||
@@ -186,21 +185,21 @@ function fixtureMessages(): LhMessage[] {
|
||||
quizPart(
|
||||
"call-3",
|
||||
{ question: "Q3", nodeId: "alpha", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-3", questionText: "Q3 text", correct: true, note: "recalled the diagram", answeredAt: "2026-08-22T02:00:00.000Z" }),
|
||||
answer({ questionText: "Q3 text", correct: true, note: "recalled the diagram", answeredAt: "2026-08-22T02:00:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
assistant("a4", [
|
||||
quizPart(
|
||||
"call-4",
|
||||
{ question: "Q4", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-4", questionText: "Q4 text", correct: false, answeredAt: "2026-08-22T02:01:00.000Z" }),
|
||||
answer({ questionText: "Q4 text", correct: false, answeredAt: "2026-08-22T02:01:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
assistant("a5", [
|
||||
quizPart(
|
||||
"call-5",
|
||||
{ question: "Q5", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-5", questionText: "Q5 text", dontKnow: true, correct: false, answeredAt: "2026-08-22T02:02:00.000Z" }),
|
||||
answer({ questionText: "Q5 text", dontKnow: true, correct: false, answeredAt: "2026-08-22T02:02:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
assistant("a6", [
|
||||
@@ -208,12 +207,12 @@ function fixtureMessages(): LhMessage[] {
|
||||
quizPart(
|
||||
"call-6",
|
||||
{ question: "Q6", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-6", questionText: "Q6 text", dismissed: true }),
|
||||
answer({ questionText: "Q6 text", dismissed: true }),
|
||||
),
|
||||
askPart(
|
||||
"call-7",
|
||||
{ question: "free text?" },
|
||||
answer({ questionId: "call-7", kind: "ask", questionText: "free text?", text: "because" }),
|
||||
answer({ kind: "ask", questionText: "free text?", text: "because" }),
|
||||
),
|
||||
]),
|
||||
].flat();
|
||||
@@ -354,7 +353,7 @@ describe("the merger (mergeFindings)", () => {
|
||||
quizPart(
|
||||
"call-9",
|
||||
{ question: "Q9", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-9", questionText: "Q9 text", correct: true, answeredAt: "2026-08-22T03:00:00.000Z" }),
|
||||
answer({ questionText: "Q9 text", correct: true, answeredAt: "2026-08-22T03:00:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
].flat());
|
||||
@@ -624,7 +623,7 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
|
||||
quizPart(
|
||||
"d-call-1",
|
||||
{ question: "DQ1", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "d-call-1", questionText: "DQ1 text", correct: true, answeredAt: "2026-09-23T00:00:00.000Z" }),
|
||||
answer({ questionText: "DQ1 text", correct: true, answeredAt: "2026-09-23T00:00:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
].flat());
|
||||
@@ -649,7 +648,7 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
|
||||
quizPart(
|
||||
"e-call-1",
|
||||
{ question: "EQ1", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "e-call-1", questionText: "EQ1 text", correct: false, answeredAt: "2026-09-23T00:01:00.000Z" }),
|
||||
answer({ questionText: "EQ1 text", correct: false, answeredAt: "2026-09-23T00:01:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
].flat());
|
||||
|
||||
+48
-105
@@ -11,16 +11,12 @@ import {
|
||||
quizInputBase,
|
||||
askOptionProblems,
|
||||
buildAnswer,
|
||||
compactAnswerForModel,
|
||||
coerceCorrectAnswer,
|
||||
CONFIDENCE_STOPS,
|
||||
DEFAULT_CONFIDENCE_INDEX,
|
||||
findPendingQuestion,
|
||||
formatAnswerMessage,
|
||||
isCorrect,
|
||||
makeId,
|
||||
normalizeOptions,
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
quizInputProblems,
|
||||
explainInputBase,
|
||||
explainInputObject,
|
||||
@@ -415,47 +411,50 @@ describe("buildAnswer", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatAnswerMessage", () => {
|
||||
it("formats a correct quiz answer for the agent", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["mercury"] });
|
||||
const text = formatAnswerMessage(ad);
|
||||
assert.ok(text.startsWith("[Quiz answer]"));
|
||||
assert.match(text, /Selected: 1\. Mercury/);
|
||||
assert.match(text, /Result: CORRECT/);
|
||||
|
||||
// ─── compactAnswerForModel (the model-visible diet, owner 2026-10-02) ──────
|
||||
|
||||
describe("compactAnswerForModel", () => {
|
||||
it("quiz: the learner's selection, verdict, confidence, note — NOTHING the model authored", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["mercury"], confidence: 70, note: "guessed" });
|
||||
const compact = compactAnswerForModel(ad);
|
||||
assert.deepEqual(compact, {
|
||||
kind: "quiz",
|
||||
answers: [{ index: 1, label: "Mercury" }],
|
||||
correct: true,
|
||||
confidence: 70,
|
||||
note: "guessed",
|
||||
});
|
||||
const raw = JSON.stringify(compact);
|
||||
// every one of these lives in the tool CALL's arguments already
|
||||
assert.ok(!raw.includes("Which planet"), "no questionText");
|
||||
assert.ok(!raw.includes("Venus"), "no option-list restatement");
|
||||
assert.ok(!raw.includes("Mercury orbits closest"), "no explanation");
|
||||
assert.ok(!("correctIndices" in compact), "correct is the verdict; indices are the call's");
|
||||
assert.ok(!("questionId" in compact), "questionId was the toolCallId restated");
|
||||
});
|
||||
|
||||
it("formats dontKnow as a gap, not a guess", () => {
|
||||
const ad = buildAnswer(quizData(), { dontKnow: true });
|
||||
const text = formatAnswerMessage(ad);
|
||||
assert.match(text, /did NOT attempt/);
|
||||
assert.match(text, /knowledge gap/);
|
||||
assert.ok(!text.includes("Result:"));
|
||||
it("dontKnow: the gap flag stands alone — no verdict, no confidence", () => {
|
||||
const compact = compactAnswerForModel(buildAnswer(quizData(), { dontKnow: true }));
|
||||
assert.deepEqual(compact, { kind: "quiz", dontKnow: true });
|
||||
});
|
||||
|
||||
it("includes the note when present", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["venus"], note: "guessed" });
|
||||
assert.match(formatAnswerMessage(ad), /Note: guessed/);
|
||||
it("ask: selection or text; never graded", () => {
|
||||
const askQD: QuestionData = {
|
||||
...quizData(),
|
||||
kind: "ask",
|
||||
mode: "text",
|
||||
correctIndices: undefined,
|
||||
explanation: undefined,
|
||||
};
|
||||
const compact = compactAnswerForModel(buildAnswer(askQD, { text: "lateral thinking" }));
|
||||
assert.deepEqual(compact, { kind: "ask", text: "lateral thinking" });
|
||||
});
|
||||
|
||||
it("states the confidence line on answered quizzes, absent on dontKnow", () => {
|
||||
const ad = buildAnswer(quizData(), { values: ["mercury"], confidence: 70 });
|
||||
assert.match(formatAnswerMessage(ad), /Confidence: 70% — the learner's stated probability that his selected answer is correct\./);
|
||||
const idk = buildAnswer(quizData(), { dontKnow: true, confidence: 70 });
|
||||
assert.ok(!formatAnswerMessage(idk).includes("Confidence:"));
|
||||
});
|
||||
|
||||
it("formats ask answers", () => {
|
||||
const ask = quizData({ kind: "ask", correctIndices: undefined, explanation: undefined });
|
||||
const ad = buildAnswer(ask, { other: "whatever" });
|
||||
const text = formatAnswerMessage(ad);
|
||||
assert.ok(text.startsWith("[Answer to your question]"));
|
||||
assert.match(text, /\(Other\): whatever/);
|
||||
});
|
||||
|
||||
it("formats a dismissed question", () => {
|
||||
const ad = buildAnswer(quizData(), {});
|
||||
ad.dismissed = true;
|
||||
assert.match(formatAnswerMessage(ad), /dismissed/);
|
||||
it("dismissed: a one-field record", () => {
|
||||
const ad = { ...buildAnswer(quizData(), { dontKnow: true }), dismissed: true };
|
||||
const compact = compactAnswerForModel(ad);
|
||||
assert.deepEqual(compact, { kind: "quiz", dismissed: true });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -473,64 +472,6 @@ describe("toClientQuestion", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ─── findPendingQuestion (resume re-arm) ────────────────────────────────────
|
||||
|
||||
describe("findPendingQuestion", () => {
|
||||
it("finds an unanswered question", () => {
|
||||
const entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
|
||||
];
|
||||
const pq = findPendingQuestion(entries);
|
||||
assert.ok(pq);
|
||||
assert.equal(pq.id, "q_test");
|
||||
});
|
||||
|
||||
it("returns null once answered", () => {
|
||||
const entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
|
||||
{
|
||||
type: "custom",
|
||||
customType: QUESTION_ANSWERED_ENTRY,
|
||||
data: { questionId: "q_test", dontKnow: true, answers: [] },
|
||||
},
|
||||
];
|
||||
assert.equal(findPendingQuestion(entries), null);
|
||||
});
|
||||
|
||||
it("returns the OLDEST unanswered question", () => {
|
||||
const entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData({ id: "q1" }) },
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData({ id: "q2" }) },
|
||||
];
|
||||
assert.equal(findPendingQuestion(entries)?.id, "q1");
|
||||
});
|
||||
|
||||
it("skips a dismissed question", () => {
|
||||
const entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData() },
|
||||
{
|
||||
type: "custom",
|
||||
customType: QUESTION_ANSWERED_ENTRY,
|
||||
data: { questionId: "q_test", dismissed: true, answers: [] },
|
||||
},
|
||||
];
|
||||
assert.equal(findPendingQuestion(entries), null);
|
||||
});
|
||||
|
||||
it("ignores other custom entries", () => {
|
||||
const entries = [{ type: "custom", customType: "md-log", data: { file: null } }];
|
||||
assert.equal(findPendingQuestion(entries), null);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── makeId ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("makeId", () => {
|
||||
it("generates unique ids", () => {
|
||||
const ids = new Set(Array.from({ length: 200 }, () => makeId()));
|
||||
assert.equal(ids.size, 200);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the dock selection state machine (multi-select Other coexistence) ──
|
||||
// The live bug: in a multi-select ask, picking the custom "Other…" row
|
||||
@@ -630,13 +571,15 @@ describe("buildAnswer / formatAnswerMessage: explain", () => {
|
||||
assert.equal(ad.text, undefined);
|
||||
});
|
||||
|
||||
it("formats the model's view: his answer, the immutable commitment, and the grading instruction", () => {
|
||||
const text = formatAnswerMessage(buildAnswer(explainQD(), { text: "No idea about aether." }));
|
||||
assert.ok(text.startsWith("[Explain answer]"));
|
||||
assert.match(text, /The learner answered: No idea about aether\./);
|
||||
assert.match(text, /Your committed correct answer \(immutable/);
|
||||
assert.match(text, /luminiferous aether/);
|
||||
assert.match(text, /Grade his answer strictly/);
|
||||
it("compactAnswerForModel (explain): his words ONLY — never the model's own commitment", () => {
|
||||
const compact = compactAnswerForModel(buildAnswer(explainQD(), { text: "No idea about aether." }));
|
||||
const raw = JSON.stringify(compact);
|
||||
assert.equal(compact.text, "No idea about aether.");
|
||||
assert.equal(compact.kind, "explain");
|
||||
// the commitment is the model's own call argument — NEVER restated
|
||||
assert.ok(!raw.includes("luminiferous"), "the commitment must not ride the result");
|
||||
assert.ok(!("committed" in compact));
|
||||
assert.ok(!("questionText" in compact));
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@ import { absorbed, pendingCallTool, SessionModel } from "../components/session-c
|
||||
import type { ChatInput } from "../src/core/input-events.ts";
|
||||
import type { LhMessage } from "../src/core/native-messages.ts";
|
||||
import { newUserMessage, newAnswerResult, needsContinuation } from "../src/core/native-messages.ts";
|
||||
import { buildAnswer, type QuestionData } from "../src/core/question-core.ts";
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
afterEach(() => {
|
||||
@@ -825,3 +826,35 @@ describe("pendingCallTool", () => {
|
||||
assert.equal(pendingCallTool(messages, "call-zz"), null);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── newAnswerResult: the two-audience split (owner, 2026-10-02) ────────────
|
||||
|
||||
describe("newAnswerResult two audiences", () => {
|
||||
it("MODEL sees the compact learner-only record; details keeps the FULL graded answer", () => {
|
||||
const qd: QuestionData = {
|
||||
id: "call-1",
|
||||
kind: "quiz",
|
||||
question: "Which planet is closest to the Sun?",
|
||||
mode: "single-select",
|
||||
options: [
|
||||
{ label: "Mercury", value: "mercury" },
|
||||
{ label: "Venus", value: "venus" },
|
||||
],
|
||||
correctIndices: [1],
|
||||
explanation: "Mercury orbits closest.",
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
};
|
||||
const full = buildAnswer(qd, { values: ["mercury"], confidence: 70, note: "guessed" });
|
||||
const msg = newAnswerResult("call-1", "quiz", full);
|
||||
const modelSees = JSON.parse((msg.content as { type: "text"; text: string }[])[0]!.text);
|
||||
assert.deepEqual(modelSees, {
|
||||
kind: "quiz",
|
||||
answers: [{ index: 1, label: "Mercury" }],
|
||||
correct: true,
|
||||
confidence: 70,
|
||||
note: "guessed",
|
||||
});
|
||||
assert.equal(msg.details, full, "details is the full QuestionAnswerData verbatim");
|
||||
assert.equal((msg.details as { questionText?: string }).questionText, "Which planet is closest to the Sun?");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user