From c3cc55c5858e6fcca30fc5dd9693ce81712d9491 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 02:32:20 +0300 Subject: [PATCH] Extraction tool calls: tolerate omitted-empty fields (weak-model trap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis from the real 25-file notes import (user-pasted session logs): almost every run hit 'Validation failed for tool submit_findings/review: edges must have required properties edges' — the model (glm flash, low thinking) omits top-level fields when they're empty, and pi-ai's structural JSON-Schema validation runs BEFORE the zod layer, so each omission cost one of the extractor's maxTurns 4 (full payload regeneration) and occasionally burned out the run entirely (the two FAILED imports). The model is technically wrong, but the schema provokes it: required-but-usually-empty arrays are a known weak-model anti-pattern. - lhTool's model-facing declaration now converts with io: "input" (the declaration describes what the MODEL sends): .default() fields drop out of required. No-op for teacher tools (none use defaults). - findingsSchema.edges / auditSchema.dropped: .default([]) — omission means "none"; attribution: .default({quiz:{}}) for quiz-free sessions. - Prompts hardened in all three agents: include every top-level field, never omit because empty. Structural probe: validateToolArguments now passes the exact pasted failure shape; zod re-parse in execute applies the defaults. 355/355 tests, typecheck, lint, build green. (lh-tool.ts is the shared engine seam — flagged to master: io:"input" is the correct mode for every model-facing declaration, teacher path verified unaffected.) --- src/agent/lh-tool.ts | 10 ++++++++-- src/agent/subagents/extraction.ts | 21 +++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/agent/lh-tool.ts b/src/agent/lh-tool.ts index 3a2a14d..24a7b45 100644 --- a/src/agent/lh-tool.ts +++ b/src/agent/lh-tool.ts @@ -66,7 +66,10 @@ export function toPiTool(def: LhTool): AgentTool { name: def.name, label: def.name, description: def.description, - parameters: z.toJSONSchema(def.schema as z.ZodType) as never, + // io: "input" — the declaration describes what the MODEL must send, + // so .default() fields are optional in the schema (omission = the + // default); required-empty arrays are a known weak-model trap. + parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never, // Question tools opt into SEQUENTIAL execution: pi runs the whole // batch in call order when any called tool opts in, so the // tool-input-available chunks arrive in the model's call order — the @@ -110,6 +113,9 @@ export function piToolDeclarations(registry: Record>) { name, label: name, description: def.description, - parameters: z.toJSONSchema(def.schema as z.ZodType) as never, + // io: "input" — the declaration describes what the MODEL must send, + // so .default() fields are optional in the schema (omission = the + // default); required-empty arrays are a known weak-model trap. + parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never, })); } \ No newline at end of file diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index 2f2b608..14f0da3 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -167,7 +167,9 @@ const findingsSchema = z.object({ .describe("Every distinct proposition the session touched — true statements, misconception-free."), edges: z .array(edgeSchema) - .describe("Relations between propositions: follows (derivable) / supports (grounds without entailing)."), + .describe("Relations between propositions: follows (derivable) / supports (grounds without entailing). Pass [] when there are none — never omit the field.") + .default([]), // weak models omit required-but-empty arrays; omission means "none" + attribution: z .object({ quiz: z @@ -176,7 +178,8 @@ const findingsSchema = z.object({ "toolCallId of EVERY quiz line → the proposition id it was really testing. Attribute RETROACTIVELY — the whole transcript is visible; map each quiz to the proposition it tests (often articulated elsewhere in the session). null when genuinely unattributable.", ), }) - .describe("Quiz attribution — your only event judgment."), + .describe("Quiz attribution — your only event judgment. Pass { quiz: {} } when the session has no quiz lines — never omit the field.") + .default({ quiz: {} }), // weak models omit empty objects too }); const NOTES_EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE source — @@ -215,6 +218,9 @@ There is no quiz attribution for notes — the source has no quiz events, and th Evidence discipline: every quote must appear VERBATIM in the source. The auditor rejects anything unbacked. When in doubt, report; the auditor decides. +Include EVERY top-level field of the call even when empty — "edges": [] when there are no relations. NEVER omit +a field because it is empty. + Your entire deliverable is ONE submit_findings call.`; const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session @@ -259,6 +265,9 @@ quiz outcomes carry that the learner believed the wrong version. Evidence discipline: every quote must appear VERBATIM in the transcript view (a QUIZ line counts as the transcript). The auditor rejects anything unbacked. When in doubt, report; the auditor decides. +Include EVERY top-level field of the call even when empty — "edges": [], "attribution": { quiz: {} } when there +are no quizzes. NEVER omit a field because it is empty. + Your entire deliverable is ONE submit_findings call.`; /** One single-submit agent run — the shared shape of EVERY LLM stage @@ -377,7 +386,8 @@ const auditSchema = findingsSchema.extend({ reason: z.string(), }), ) - .describe("Everything you dropped, with the reason — the audit trail."), + .describe("Everything you dropped, with the reason — the audit trail. Pass [] when you dropped nothing — never omit the field.") + .default([]), // same weak-model tolerance as findings.edges }); // The notes front door's shapes: everything session findings have, minus @@ -424,7 +434,10 @@ Also drop propositions not referenced by anything else ONLY if they have no othe well-evidenced proposition is still knowledge. Never invent new candidates. Your entire deliverable is ONE submit_review call containing the verified findings -(the same shapes you received, with linkTo added) plus the dropped list.`; +(the same shapes you received, with linkTo added) plus the dropped list. + +Include EVERY top-level field of the call even when empty — "edges": [], "dropped": []. NEVER omit a field +because it is empty.`; /** The current graph, compact — the auditor's identity context. */ function graphView(nodes: GraphNode[]): string {