quiz argument order: explanation before correctAnswer — reasoning precedes the pick (owner ruling, 2026-09-25)

This commit is contained in:
amosblomqvist
2026-09-25 03:31:53 +03:00
parent b3c07f89f4
commit d1542cdcbf
3 changed files with 44 additions and 7 deletions
@@ -11,7 +11,7 @@
- `quiz` is GRADED; `ask_user_question` is not. If the question has a correct answer, use `quiz`. If you just need a preference, decision, or open-ended input, use `ask_user_question`. - `quiz` is GRADED; `ask_user_question` is not. If the question has a correct answer, use `quiz`. If you just need a preference, decision, or open-ended input, use `ask_user_question`.
- `correctAnswer` is REQUIRED and is the option value, not a position number. Single-select: one string (e.g. "mercury"). Multi-select: an array of strings (e.g. ["belize", "niue"]). - `correctAnswer` is REQUIRED and is the option value, not a position number. Single-select: one string (e.g. "mercury"). Multi-select: an array of strings (e.g. ["belize", "niue"]).
- Always pass the option's `value` string as `correctAnswer` — it is self-checking and prevents miscounting positions. A value that matches no option is a hard error. - Always pass the option's `value` string as `correctAnswer` — it is self-checking and prevents miscounting positions. A value that matches no option is a hard error.
- `explanation` is REQUIRED — always say why the correct answer is correct. - `explanation` is REQUIRED — always say why the correct answer is correct. It comes BEFORE `correctAnswer` in the call by design: reason it out first, then commit the answer the reasoning supports.
- Multi-select is graded as an exact-set match: he is correct only if he selects every correct option and no incorrect ones. - Multi-select is graded as an exact-set match: he is correct only if he selects every correct option and no incorrect ones.
- There is no free-text mode. An "I don't know" choice is ALWAYS added automatically — provide ONLY the real, gradable options (at least two). Never add your own uncertainty/opt-out option like "I don't know", "I'm not sure", or "Not sure"; that is handled for you and a manual one would be redundant or gradable-as-wrong. - There is no free-text mode. An "I don't know" choice is ALWAYS added automatically — provide ONLY the real, gradable options (at least two). Never add your own uncertainty/opt-out option like "I don't know", "I'm not sure", or "Not sure"; that is handled for you and a manual one would be redundant or gradable-as-wrong.
- If a result comes back as dontKnow, he honestly did not know and did NOT guess — treat it as a genuine knowledge gap, not as a wrong answer. - If a result comes back as dontKnow, he honestly did not know and did NOT guess — treat it as a genuine knowledge gap, not as a wrong answer.
+11 -6
View File
@@ -127,6 +127,17 @@ export const quizInputBase = z.object({
.boolean() .boolean()
.optional() .optional()
.describe("Set to true when more than one option is correct and the learner must select all of them."), .describe("Set to true when more than one option is correct and the learner must select all of them."),
// Argument-order ruling (owner, 2026-09-25): the explanation is REASONING,
// and reasoning happens BEFORE the answer is chosen — models emit tool
// args in schema order, so explanation sits immediately BEFORE correctAnswer.
// (All consumers read by key; the order is authoring guidance, nothing else.)
explanation: z
.string()
.describe(
"REQUIRED. Explanation revealed AFTER the learner answers (shown whether they got it right or wrong). " +
"Use it to reinforce why the correct answer is correct. It is written BEFORE correctAnswer by design: " +
"reason it out first, then commit the answer the reasoning supports.",
),
correctAnswer: z correctAnswer: z
.union([z.string(), z.array(z.string())]) .union([z.string(), z.array(z.string())])
.describe( .describe(
@@ -136,12 +147,6 @@ export const quizInputBase = z.object({
"Always pass the value, not a position number — this is self-checking and prevents miscounting. " + "Always pass the value, not a position number — this is self-checking and prevents miscounting. " +
"A value that matches no option fails validation and the call is returned to you to fix.", "A value that matches no option fails validation and the call is returned to you to fix.",
), ),
explanation: z
.string()
.describe(
"REQUIRED. Explanation revealed AFTER the learner answers (shown whether they got it right or wrong). " +
"Use it to reinforce why the correct answer is correct.",
),
details: z.string().optional().describe("Optional extra context or instructions shown under the question."), details: z.string().optional().describe("Optional extra context or instructions shown under the question."),
}); });
+32
View File
@@ -4,6 +4,7 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { describe, it } from "node:test"; import { describe, it } from "node:test";
import { z } from "zod";
import { import {
askInputSchema, askInputSchema,
askOptionProblems, askOptionProblems,
@@ -18,6 +19,7 @@ import {
QUESTION_ENTRY, QUESTION_ENTRY,
quizInputProblems, quizInputProblems,
quizInputSchema, quizInputSchema,
probeQuizInputSchema,
resolveCorrect, resolveCorrect,
toggleOption, toggleOption,
toggleOther, toggleOther,
@@ -32,6 +34,36 @@ import {
// dies at the tool-call boundary, and the SDK returns the error to the // dies at the tool-call boundary, and the SDK returns the error to the
// teacher to retry — it must never reach the learner's dock. // teacher to retry — it must never reach the learner's dock.
describe("the schema's argument order (reasoning before the pick)", () => {
// Argument-order ruling (owner, 2026-09-25): the explanation is
// reasoning, and reasoning happens BEFORE the answer is chosen — models
// emit tool args in schema order, so explanation sits immediately before
// correctAnswer. z.toJSONSchema preserves zod's declaration order into
// the model-facing properties, so this pin guards the emission end to
// end. All consumers read by key — the order is authoring guidance.
it("emits explanation immediately before correctAnswer (both agents' schemas)", () => {
for (const schema of [quizInputSchema, probeQuizInputSchema]) {
const json = z.toJSONSchema(schema as z.ZodType, { io: "input" }) as {
properties: Record<string, unknown>;
};
const keys = Object.keys(json.properties);
assert.ok(
keys.indexOf("explanation") < keys.indexOf("correctAnswer"),
`explanation must precede correctAnswer in the emitted schema (got: ${keys.join(", ")})`,
);
assert.deepEqual(keys, [
"question",
"options",
"multiSelect",
"explanation",
"correctAnswer",
"details",
...(keys.includes("nodeId") ? ["nodeId"] : []),
]);
}
});
});
describe("quizInputSchema / quizInputProblems (the authoring contract)", () => { describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
const valid = { const valid = {
question: "Which planet is closest to the Sun?", question: "Which planet is closest to the Sun?",