quiz argument order: explanation before correctAnswer — reasoning precedes the pick (owner ruling, 2026-09-25)
This commit is contained in:
@@ -11,7 +11,7 @@
|
|||||||
- `quiz` is GRADED; `ask_user_question` is not. If the question has a correct answer, use `quiz`. If you just need a preference, decision, or open-ended input, use `ask_user_question`.
|
- `quiz` is GRADED; `ask_user_question` is not. If the question has a correct answer, use `quiz`. If you just need a preference, decision, or open-ended input, use `ask_user_question`.
|
||||||
- `correctAnswer` is REQUIRED and is the option value, not a position number. Single-select: one string (e.g. "mercury"). Multi-select: an array of strings (e.g. ["belize", "niue"]).
|
- `correctAnswer` is REQUIRED and is the option value, not a position number. Single-select: one string (e.g. "mercury"). Multi-select: an array of strings (e.g. ["belize", "niue"]).
|
||||||
- Always pass the option's `value` string as `correctAnswer` — it is self-checking and prevents miscounting positions. A value that matches no option is a hard error.
|
- Always pass the option's `value` string as `correctAnswer` — it is self-checking and prevents miscounting positions. A value that matches no option is a hard error.
|
||||||
- `explanation` is REQUIRED — always say why the correct answer is correct.
|
- `explanation` is REQUIRED — always say why the correct answer is correct. It comes BEFORE `correctAnswer` in the call by design: reason it out first, then commit the answer the reasoning supports.
|
||||||
- Multi-select is graded as an exact-set match: he is correct only if he selects every correct option and no incorrect ones.
|
- Multi-select is graded as an exact-set match: he is correct only if he selects every correct option and no incorrect ones.
|
||||||
- There is no free-text mode. An "I don't know" choice is ALWAYS added automatically — provide ONLY the real, gradable options (at least two). Never add your own uncertainty/opt-out option like "I don't know", "I'm not sure", or "Not sure"; that is handled for you and a manual one would be redundant or gradable-as-wrong.
|
- There is no free-text mode. An "I don't know" choice is ALWAYS added automatically — provide ONLY the real, gradable options (at least two). Never add your own uncertainty/opt-out option like "I don't know", "I'm not sure", or "Not sure"; that is handled for you and a manual one would be redundant or gradable-as-wrong.
|
||||||
- If a result comes back as dontKnow, he honestly did not know and did NOT guess — treat it as a genuine knowledge gap, not as a wrong answer.
|
- If a result comes back as dontKnow, he honestly did not know and did NOT guess — treat it as a genuine knowledge gap, not as a wrong answer.
|
||||||
|
|||||||
@@ -127,6 +127,17 @@ export const quizInputBase = z.object({
|
|||||||
.boolean()
|
.boolean()
|
||||||
.optional()
|
.optional()
|
||||||
.describe("Set to true when more than one option is correct and the learner must select all of them."),
|
.describe("Set to true when more than one option is correct and the learner must select all of them."),
|
||||||
|
// Argument-order ruling (owner, 2026-09-25): the explanation is REASONING,
|
||||||
|
// and reasoning happens BEFORE the answer is chosen — models emit tool
|
||||||
|
// args in schema order, so explanation sits immediately BEFORE correctAnswer.
|
||||||
|
// (All consumers read by key; the order is authoring guidance, nothing else.)
|
||||||
|
explanation: z
|
||||||
|
.string()
|
||||||
|
.describe(
|
||||||
|
"REQUIRED. Explanation revealed AFTER the learner answers (shown whether they got it right or wrong). " +
|
||||||
|
"Use it to reinforce why the correct answer is correct. It is written BEFORE correctAnswer by design: " +
|
||||||
|
"reason it out first, then commit the answer the reasoning supports.",
|
||||||
|
),
|
||||||
correctAnswer: z
|
correctAnswer: z
|
||||||
.union([z.string(), z.array(z.string())])
|
.union([z.string(), z.array(z.string())])
|
||||||
.describe(
|
.describe(
|
||||||
@@ -136,12 +147,6 @@ export const quizInputBase = z.object({
|
|||||||
"Always pass the value, not a position number — this is self-checking and prevents miscounting. " +
|
"Always pass the value, not a position number — this is self-checking and prevents miscounting. " +
|
||||||
"A value that matches no option fails validation and the call is returned to you to fix.",
|
"A value that matches no option fails validation and the call is returned to you to fix.",
|
||||||
),
|
),
|
||||||
explanation: z
|
|
||||||
.string()
|
|
||||||
.describe(
|
|
||||||
"REQUIRED. Explanation revealed AFTER the learner answers (shown whether they got it right or wrong). " +
|
|
||||||
"Use it to reinforce why the correct answer is correct.",
|
|
||||||
),
|
|
||||||
details: z.string().optional().describe("Optional extra context or instructions shown under the question."),
|
details: z.string().optional().describe("Optional extra context or instructions shown under the question."),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
|
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { describe, it } from "node:test";
|
import { describe, it } from "node:test";
|
||||||
|
import { z } from "zod";
|
||||||
import {
|
import {
|
||||||
askInputSchema,
|
askInputSchema,
|
||||||
askOptionProblems,
|
askOptionProblems,
|
||||||
@@ -18,6 +19,7 @@ import {
|
|||||||
QUESTION_ENTRY,
|
QUESTION_ENTRY,
|
||||||
quizInputProblems,
|
quizInputProblems,
|
||||||
quizInputSchema,
|
quizInputSchema,
|
||||||
|
probeQuizInputSchema,
|
||||||
resolveCorrect,
|
resolveCorrect,
|
||||||
toggleOption,
|
toggleOption,
|
||||||
toggleOther,
|
toggleOther,
|
||||||
@@ -32,6 +34,36 @@ import {
|
|||||||
// dies at the tool-call boundary, and the SDK returns the error to the
|
// dies at the tool-call boundary, and the SDK returns the error to the
|
||||||
// teacher to retry — it must never reach the learner's dock.
|
// teacher to retry — it must never reach the learner's dock.
|
||||||
|
|
||||||
|
describe("the schema's argument order (reasoning before the pick)", () => {
|
||||||
|
// Argument-order ruling (owner, 2026-09-25): the explanation is
|
||||||
|
// reasoning, and reasoning happens BEFORE the answer is chosen — models
|
||||||
|
// emit tool args in schema order, so explanation sits immediately before
|
||||||
|
// correctAnswer. z.toJSONSchema preserves zod's declaration order into
|
||||||
|
// the model-facing properties, so this pin guards the emission end to
|
||||||
|
// end. All consumers read by key — the order is authoring guidance.
|
||||||
|
it("emits explanation immediately before correctAnswer (both agents' schemas)", () => {
|
||||||
|
for (const schema of [quizInputSchema, probeQuizInputSchema]) {
|
||||||
|
const json = z.toJSONSchema(schema as z.ZodType, { io: "input" }) as {
|
||||||
|
properties: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
const keys = Object.keys(json.properties);
|
||||||
|
assert.ok(
|
||||||
|
keys.indexOf("explanation") < keys.indexOf("correctAnswer"),
|
||||||
|
`explanation must precede correctAnswer in the emitted schema (got: ${keys.join(", ")})`,
|
||||||
|
);
|
||||||
|
assert.deepEqual(keys, [
|
||||||
|
"question",
|
||||||
|
"options",
|
||||||
|
"multiSelect",
|
||||||
|
"explanation",
|
||||||
|
"correctAnswer",
|
||||||
|
"details",
|
||||||
|
...(keys.includes("nodeId") ? ["nodeId"] : []),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
|
describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
|
||||||
const valid = {
|
const valid = {
|
||||||
question: "Which planet is closest to the Sun?",
|
question: "Which planet is closest to the Sun?",
|
||||||
|
|||||||
Reference in New Issue
Block a user