slim the submit schemas: evidence and messages die; stringified-payload heal at the pre-validation seam

OpenRouter log diagnosis (owner, 2026-09-28): extraction time was
dominated by tool-validation retries. The failing call had the whole
propositions array emitted as ONE JSON-STRING ('propositions.0: must
be object'), a truncated placeholder artifact inside it, and
word-salad statements late in the array — payload-size collapse.
Field audit: 'evidence' was never persisted anywhere and its
faithfulness-checking consumer died with the fully-trusted ruling;
'messages' fed messageId granularity that NOTHING downstream reads,
and the model sent [] anyway (chat-level provenance in practice).
'aliases' stays (optional now): the auditor's find tool globs them.

Schema: propositions = {id, name, statement, aliases?}; edges =
{from, to, type}. Prompts lose the evidence/messages doctrine and
[id]-anchor guidance, gain a compact faithfulness line;
transcriptView renders anchorless. The auditor echo instruction
drops the evidence/messages mention, so its payload halves too.

Heal: healStringifiedArrays on the submit tool's prepareArguments
(pi-agent-core's sanctioned pre-TypeBox shim) parses a stringified
propositions/edges back into arrays — the LaTeX-restore doctrine:
heal weak-model wire behavior at the seam. Test drives the real
agent-loop seam with a stringified submit.
This commit is contained in:
amosblomqvist
2026-09-28 19:04:39 +03:00
parent 14a7f0de1d
commit bd9694b0e4
7 changed files with 131 additions and 125 deletions
+38 -25
View File
@@ -221,7 +221,7 @@ after(() => {
});
/** What the extractor would produce for the fixture session — tree-blind,
* free-form, with message-granular provenance. */
* free-form, with chat-level provenance. */
const FINDINGS: SessionFindings = {
propositions: [
{
@@ -229,16 +229,12 @@ const FINDINGS: SessionFindings = {
name: "Derivative",
statement: "The derivative measures how fast a function changes",
aliases: [],
messages: ["a2"],
evidence: ["The derivative measures how fast a function changes."],
},
{
id: "p2",
name: "Chain rule",
statement: "The chain rule differentiates composed functions by multiplying local rates.",
aliases: [],
messages: ["a2"],
evidence: ["teach me the chain rule"],
},
],
edges: [
@@ -246,8 +242,6 @@ const FINDINGS: SessionFindings = {
from: "p1",
to: "p2",
type: "follows",
messages: ["a2"],
evidence: ["teach me the chain rule"],
},
],
};
@@ -322,20 +316,21 @@ function silentModel(): PiEngine {
// ─── transcriptView (pure) ───────────────────────────────────────────────────
describe("transcriptView (pure — the shared markdown engine, owner 2026-09-27)", () => {
it("renders id-anchored speaker blocks + check blocks — tree-blind, plan ops folded", () => {
it("renders anchorless speaker blocks + check blocks — tree-blind, plan ops folded", () => {
const { record } = fixtureChat();
const view = transcriptView(record);
// The document header is the session title.
assert.match(view, /^# Extraction fixture$/m);
// Conversation verbatim, each block headed by its speaker + the
// message-id anchor — the provenance the extractor reports back.
assert.match(view, /### Learner · u1\n\nteach me the chain rule/);
assert.match(view, /### Teacher · a2\n\nThe derivative measures how fast a function changes\./);
// Conversation verbatim, each block headed by its speaker only —
// the [id] anchors are GONE (submit-schema slim, 2026-09-28:
// propositions no longer report message ids).
assert.match(view, /### Learner\n\nteach me the chain rule/);
assert.match(view, /### Teacher\n\nThe derivative measures how fast a function changes\./);
// Checks render as readable blocks (question + outcome).
assert.match(view, /### Quiz · r_probe-1/);
assert.match(view, /### Quiz\n/);
assert.match(view, /\*\*Q:\*\* What is a derivative, roughly\?/);
assert.match(view, /The learner answered incorrectly\./);
assert.match(view, /The learner answered correctly\./);
@@ -373,6 +368,25 @@ describe("transcriptView (pure — the shared markdown engine, owner 2026-09-27)
// ─── the pipeline ───────────────────────────────────────────────────────────
describe("the extraction pipeline (extractor → auditor → merger)", () => {
it("stringified-payload heal (owner, 2026-09-28): propositions arriving as ONE JSON string still validate and merge", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
const verdict: AuditResult = { propositions: FINDINGS.propositions, edges: FINDINGS.edges };
// The weak-model failure from the OpenRouter log: the whole array
// emitted as a JSON-STRING ("propositions.0: must be object").
// pi-agent-core's prepareArguments shim runs BEFORE TypeBox — the
// heal parses it, validation passes, no retry burned.
const extractor = submitModel("submit_findings", {
propositions: JSON.stringify(FINDINGS.propositions),
edges: FINDINGS.edges,
});
const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, extractor, auditor: submitModel("submit_review", verdict) });
const stats = await pipeline.runExtraction(chatId);
assert.deepEqual(stats, { nodes: 2, edges: 1 });
});
it("runs end to end: findings → audit verdict (identity) → deterministic merge + checkpoint", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
@@ -631,7 +645,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const ledgers = memoryLedgers();
const single = (id: string, name: string, statement: string): SessionFindings => ({
propositions: [{ id, name, statement, aliases: [], messages: ["a2"], evidence: ["e"] }],
propositions: [{ id, name, statement, aliases: [], }],
edges: [],
});
const findingsA = single("pa", "Alpha claim", "Alpha statement, full precision");
@@ -688,7 +702,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const ledgers = memoryLedgers();
const single = (id: string, name: string, statement: string): SessionFindings => ({
propositions: [{ id, name, statement, aliases: [], messages: ["a2"], evidence: ["e"] }],
propositions: [{ id, name, statement, aliases: [], }],
edges: [],
});
@@ -938,8 +952,6 @@ const NOTES_FINDINGS = {
name: "Vim normal mode",
statement: "Vim starts in Normal mode, where keys are commands.",
aliases: [],
messages: [],
evidence: ["Vim always starts in Normal mode — keys are commands, not text."],
},
],
edges: [],
@@ -980,11 +992,12 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
assert.equal(source.extractions![0]!.stats?.nodes, 1);
assert.ok(source.extractions![0]!.stages.extractor.steps.length > 0, "faux usage captured");
// File-scoped provenance: the mapping line carries the source id;
// the node carries NO message anchors.
// File-scoped provenance (submit-schema slim, 2026-09-28): the
// node carries ONE chat-level point — the source key itself, no
// message anchors.
const node = ledgers.readNodes()[0]!;
assert.equal(node.statement, "Vim starts in Normal mode, where keys are commands.");
assert.deepEqual(node.provenance, []);
assert.deepEqual(node.provenance, [{ chatId: source.id, messageId: "" }]);
assert.deepEqual(ledgers.readMappings(), [{ chatId: source.id, sourceId: "p1", graphNodeId: node.id }]);
// The checkpoint stamps the import.
assert.equal(ledgers.readCheckpoints()[0]!.chatId, source.id);
@@ -1102,10 +1115,10 @@ describe("the chunked pipeline (extraction coverage)", () => {
// Two batches: the first held messages 1-3, the second 4-5 — a
// message is never split, batches are consecutive oldest-first.
assert.equal(views.length, 2, "one extractor call per batch");
assert.ok(views[0]!.includes("### Learner · u1") && views[0]!.includes("### Learner · u3"));
assert.ok(!views[0]!.includes("### Learner · u4"), "the batch stops at the token budget");
assert.ok(views[1]!.includes("### Learner · u4") && views[1]!.includes("### Learner · u5"));
assert.ok(!views[1]!.includes("### Learner · u3"));
assert.ok(views[0]!.includes("### Learner") && views[0]!.includes("topic 1:"));
assert.ok(!views[0]!.includes("topic 4:"), "the batch stops at the token budget");
assert.ok(views[1]!.includes("### Learner") && views[1]!.includes("topic 4:"));
assert.ok(!views[1]!.includes("topic 3:"));
// The document header is the session title.
assert.ok(views[0]!.startsWith("# Chunky coverage fixture"));
@@ -1184,7 +1197,7 @@ describe("the chunked pipeline (extraction coverage)", () => {
});
await finisher.runExtraction(chatId);
assert.equal(reRunViews.length, 1);
assert.ok(reRunViews[0]!.includes("### Learner · u4") && reRunViews[0]!.includes("### Learner · u5"));
assert.ok(reRunViews[0]!.includes("### Learner") && reRunViews[0]!.includes("topic 4:"));
assert.ok(!reRunViews[0]!.includes("### Learner · u1"), "the covered remainder never re-enters the prompt");
assert.equal(ledgers.readCoverage().length, 5, "the re-run closed the gap");
});