sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

verdicts-only auditor: submit_review is {links, bridging edges} — the findings echo dies

Browse Source 
Owner ruling 2026-09-28: the auditor has no control over the
extractor's output and never alters it, so re-emitting the full
findings per submit was pure payload cost — the retry collapse the
schema slim attacked, halved again. submit_review is now links
(local proposition id -> existing graph node; EQUIVALENCE BIAS
carried in the linkTo description) + bridging edges ONLY; the
merger joins extractor findings + verdicts.

mergeFindings(chatId, findings, verdicts, ...) — propositions come
from findings verbatim, links join by id, edges emit extractor's
then auditor's through one dedup-preserving emitter. The old echo
form hid an accidental drop channel: an auditor that omitted a
proposition from its echo silently dropped it from the merge.
Verdicts-only removes it structurally — extractor output always
fully merges (tests updated to full-merge expectations).

AuditResult/AuditedProposition -> IdentityLink/AuditVerdicts;
audit() logs links/bridges; batch view renders anchorless like
transcriptView; auditor prompt rewritten to the verdicts-only
deliverable; architecture.md division-of-trust updated.
This commit is contained in:
amosblomqvist
2026-09-28 19:22:58 +03:00
parent bd9694b0e4
commit f11b1664dc
6 changed files with 163 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/extraction.test.ts
+34 -36
View File
@@ -30,7 +30,7 @@ import type { LhMessage } from "../src/core/native-messages.ts";
import { newPlanBlockMessage } from "../src/core/native-messages.ts";
import { projectView as projectNative } from "../src/core/view.ts";
import type {
AuditResult,
AuditVerdicts,
CoverageEntry,
ExtractionRunUsage,
GraphEdge,
@@ -371,7 +371,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
it("stringified-payload heal (owner, 2026-09-28): propositions arriving as ONE JSON string still validate and merge", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
const verdict: AuditResult = { propositions: FINDINGS.propositions, edges: FINDINGS.edges };
const verdict: AuditVerdicts = { links: [], edges: [] };
// The weak-model failure from the OpenRouter log: the whole array
// emitted as a JSON-STRING ("propositions.0: must be object").
@@ -402,10 +402,8 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
createdAt: "2026-08-01T00:00:00.000Z",
provenance: [],
});
const verdict: AuditResult = {
propositions: [
{ ...FINDINGS.propositions[0]!, linkTo: existingId },
],
const verdict: AuditVerdicts = {
links: [{ id: FINDINGS.propositions[0]!.id, linkTo: existingId }],
edges: [],
};
@@ -417,13 +415,20 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
});
const stats = await pipeline.runExtraction(chatId);
// The linked proposition created NO new node.
assert.deepEqual(stats, { nodes: 0, edges: 0 });
// VERDICTS-ONLY (2026-09-28): findings merge in FULL — the linked
// p1 creates no node, p2 creates one, and the extractor's edge
// resolves through them. (The old echo form could silently drop
// content by omitting it; that accidental drop channel is gone.)
assert.deepEqual(stats, { nodes: 1, edges: 1 });
// Link-append: the phrasing joined the existing node's aliases and
// the provenance unioned; no second node line defines a new node.
assert.equal(ledgers.readNodes().length, 2); // the pre-seeded line + the link line
assert.deepEqual(ledgers.readMappings(), [{ chatId, sourceId: "p1", graphNodeId: existingId }]);
// the provenance unioned; p2's create line is its own node.
assert.equal(ledgers.readNodes().length, 3); // pre-seeded + link line + p2's create
const mappings = ledgers.readMappings();
assert.equal(mappings.length, 2);
assert.deepEqual(mappings[0], { chatId, sourceId: "p1", graphNodeId: existingId });
assert.equal(mappings[1]!.sourceId, "p2");
assert.match(mappings[1]!.graphNodeId, /^n_/);
// COVERAGE checkpointed per message (not a whole-chat checkpoint):
// one entry per transcript message, fingerprinted content each —
@@ -513,11 +518,8 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
it("the LLM cluster-namer is DEAD (wave 7): a merge mints no naming runs — labels are session names now", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
// Identity-pass verdict: p1 becomes a new node, p2 unlinked.
const verdict: AuditResult = {
propositions: [{ ...FINDINGS.propositions[0]! }],
edges: [],
};
// Empty verdicts: everything creates (verdicts-only, 2026-09-28).
const verdict: AuditVerdicts = { links: [], edges: [] };
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
@@ -525,7 +527,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
auditor: submitModel("submit_review", verdict),
});
const stats = await pipeline.runExtraction(chatId);
assert.deepEqual(stats, { nodes: 1, edges: 0 });
assert.deepEqual(stats, { nodes: 2, edges: 1 }); // full findings merge (verdicts-only)
// Wave 7 (950ddfd): birth-session clusters carry session-name labels,
// so the post-merge naming garnish is gone — no model spend, no
// naming-run records, no label writes.
@@ -536,10 +538,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
it("engineFor fills the stages (model slots wave): the chain replaces config pins", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
const verdict: AuditResult = {
propositions: FINDINGS.propositions,
edges: [],
};
const verdict: AuditVerdicts = { links: [], edges: [] };
const consulted: string[] = [];
const pipeline = createExtractionPipeline({
chatStore,
@@ -566,10 +565,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
it("direct test values still WIN over engineFor (the priority pin)", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
const verdict: AuditResult = {
propositions: FINDINGS.propositions,
edges: [],
};
const verdict: AuditVerdicts = { links: [], edges: [] };
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
@@ -622,8 +618,8 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
});
const seen: (string | undefined)[] = [];
const phaseOf = () => extractionStatuses().find((r) => r.chatId === chatId)?.phase;
const verdict: AuditResult = {
propositions: [{ ...FINDINGS.propositions[0]!, linkTo: existingId }],
const verdict: AuditVerdicts = {
links: [{ id: FINDINGS.propositions[0]!.id, linkTo: existingId }],
edges: [],
};
const pipeline = createExtractionPipeline({
@@ -762,8 +758,8 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
provenance: [],
});
const usageSeen: (number | undefined)[] = [];
const verdict: AuditResult = {
propositions: [{ ...FINDINGS.propositions[0]!, linkTo: existingId }],
const verdict: AuditVerdicts = {
links: [{ id: FINDINGS.propositions[0]!.id, linkTo: existingId }],
edges: [],
};
const pipeline = createExtractionPipeline({
@@ -789,7 +785,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const record = chatStore.getChat(chatId)!;
assert.equal(record.extractions?.length, 1);
const run = record.extractions![0]!;
assert.equal(run.stats?.nodes, 0); // linked, not created
assert.equal(run.stats?.nodes, 1); // p2 created; p1 linked, not created
// the scripted engine's model, provider-qualified (model slots wave)
assert.equal(run.stages.extractor.model, "faux:faux-1");
assert.equal(run.stages.extractor.steps.length, 1); // the submit call — the run STOPS after a clean submit
@@ -1073,7 +1069,7 @@ function passThroughAuditStep(): FauxResponseStep {
fauxToolCall(
"submit_review",
{
propositions: [{ ...FINDINGS.propositions[0]! }],
links: [],
edges: [],
} as unknown as ToolCall["arguments"],
{ id: "review-call" },
@@ -1122,9 +1118,10 @@ describe("the chunked pipeline (extraction coverage)", () => {
// The document header is the session title.
assert.ok(views[0]!.startsWith("# Chunky coverage fixture"));
// p1 was a fresh node in batch 1; batch 2's p1 mapped to the same
// node (chat-scoped mapping).
assert.deepEqual(stats, { nodes: 1, edges: 0 });
// Batch 1 merged the full findings (verdicts-only: the extractor's
// output always fully merges); batch 2's identical ids mapped to
// the same nodes (chat-scoped mapping) — nothing new.
assert.deepEqual(stats, { nodes: 2, edges: 1 });
// Per-batch coverage: every message checkpointed, nothing legacy.
const coverage = ledgers.readCoverage();
@@ -1181,7 +1178,7 @@ describe("the chunked pipeline (extraction coverage)", () => {
const stats = await runPromise;
// Batch 1's emissions are kept and its messages stay covered.
assert.deepEqual(stats, { nodes: 1, edges: 0 });
assert.deepEqual(stats, { nodes: 2, edges: 1 });
assert.equal(ledgers.readCoverage().length, 3, "batch 1 checkpointed its three messages");
const cov = coverageOf(record.messages, ledgers.readCoverage(), chatId);
assert.equal(cov.uncoveredCount, 2, "the in-flight batch's messages stay uncovered");
@@ -1254,7 +1251,8 @@ describe("the chunked pipeline (extraction coverage)", () => {
});
await reOpener.runExtraction(chatId);
assert.equal(reRunViews.length, 1, "one batch: the single re-opened message");
assert.ok(reRunViews[0]!.includes("### Learner · s1") && !reRunViews[0]!.includes("### Learner · s2"));
assert.ok(reRunViews[0]!.includes("the answer merged in late") && !reRunViews[0]!.includes("### Learner · s2"));
assert.ok(reRunViews[0]!.includes("### Learner"), "the re-opened message renders");
assert.equal(coverageOf(messages(), ledgers.readCoverage(), chatId).uncoveredCount, 0, "re-covered");
});
});
tests/graph.test.ts
+27 -15
View File
@@ -19,7 +19,7 @@
import assert from "node:assert/strict";
import * as fs from "node:fs";
import { after, describe, it } from "node:test";
import type { AuditResult } from "../src/core/contracts.ts";
import type { AuditVerdicts, SessionFindings } from "../src/core/contracts.ts";
import type { QuestionAnswerData } from "../src/core/question-core.ts";
import type { LhMessage } from "../src/core/native-messages.ts";
import { projectView as projectNative } from "../src/core/view.ts";
@@ -214,9 +214,9 @@ function fixtureMessages(): LhMessage[] {
].flat();
}
/** The auditor's verdict for the fixture chat: three propositions
* (no linkTo — the graph starts empty), one follows edge. */
function fixtureAudit(): AuditResult {
/** The extractor's findings for the fixture chat: three propositions,
* one follows edge (the graph starts empty — no links). */
function fixtureFindings(): SessionFindings {
return {
propositions: [
{
@@ -248,6 +248,12 @@ function fixtureAudit(): AuditResult {
};
}
/** The auditor's verdicts when nothing links: an empty verdict set is a
* valid full review (verdicts-only, owner 2026-09-28). */
function fixtureVerdicts(): AuditVerdicts {
return { links: [], edges: [] };
}
/** The chat every merger test folds (shared, so ordering is intentional). */
const chat1 = fixtureChat("Graph fixture", fixtureMessages());
@@ -275,7 +281,7 @@ function scopedGraph(chatIds: string[]) {
describe("the merger (mergeFindings)", () => {
it("creates nodes from the verdicts with chat-level provenance", () => {
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
const stats = mergeFindings(chat1.id, fixtureFindings(), fixtureVerdicts(), graphDeps, sessionSource(chat1.id));
assert.deepEqual(stats, { nodes: 3, edges: 1 });
const graph = scopedGraph([chat1.id]);
@@ -309,7 +315,7 @@ describe("the merger (mergeFindings)", () => {
});
it("is idempotent — re-merging the same verdicts appends nothing", () => {
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
const stats = mergeFindings(chat1.id, fixtureFindings(), fixtureVerdicts(), graphDeps, sessionSource(chat1.id));
assert.deepEqual(stats, { nodes: 0, edges: 0 });
const graph = scopedGraph([chat1.id]);
assert.equal(graph.nodes.length, 3);
@@ -342,11 +348,13 @@ describe("the merger (mergeFindings)", () => {
name: "Alpha",
statement: "Alpha feel", // a DIFFERENT phrasing — identity is the verdict, not strings
aliases: ["alpha intuition"],
linkTo: alphaId,
},
],
edges: [],
},
// VERDICTS-ONLY: the identity call is a link entry, not a field
// on the proposition (owner, 2026-09-28).
{ links: [{ id: "q1", linkTo: alphaId }], edges: [] },
graphDeps,
sessionSource(chat2.id),
);
@@ -384,6 +392,7 @@ describe("the merger (mergeFindings)", () => {
],
edges: [],
},
{ links: [], edges: [] },
graphDeps,
{ foldedAt: "2026-09-13T00:00:00.000Z" },
);
@@ -408,7 +417,7 @@ describe("the merger (mergeFindings)", () => {
id: `p${i + 1}`,
name: `Cluster node ${i + 1}`,
statement,
aliases: [], // birth provenance — the message anchors
aliases: [],
})),
edges: [
{ from: "p1", to: "p2", type: "follows", },
@@ -416,6 +425,7 @@ describe("the merger (mergeFindings)", () => {
{ from: "p1", to: "p3", type: "follows", },
],
},
{ links: [], edges: [] },
graphDeps,
{ foldedAt: "2026-09-13T00:00:00.000Z" },
);
@@ -462,11 +472,12 @@ describe("the merger (mergeFindings)", () => {
name: "Alpha",
statement: "Alpha intuition", // IDENTICAL to the existing node's statement
aliases: [],
// no linkTo — identity is the auditor's call alone
},
],
edges: [],
},
// empty verdicts — identity is the auditor's call alone
{ links: [], edges: [] },
graphDeps,
sessionSource(chat3.id),
);
@@ -485,11 +496,12 @@ describe("the merger (mergeFindings)", () => {
id: "p1",
name: "Vim normal mode",
statement: "Vim starts in Normal mode, where keys are commands.",
aliases: [], // notes carry no message anchors — the file is the provenance
aliases: [],
},
],
edges: [],
},
{ links: [], edges: [] },
graphDeps,
{ foldedAt: "2026-09-13T00:00:00.000Z" },
);
@@ -586,14 +598,14 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
),
]),
].flat());
const auditA: AuditResult = {
const findingsA: SessionFindings = {
propositions: [
{ id: "p1", name: "Shared core", statement: "Shared core statement", aliases: ["shared feel"], },
{ id: "p2", name: "Solo fact", statement: "Solo fact statement", aliases: ["solo feel"], },
],
edges: [{ from: "p1", to: "p2", type: "follows", }],
};
mergeFindings(dA.id, auditA, graphDeps, sessionSource(dA.id));
mergeFindings(dA.id, findingsA, { links: [], edges: [] }, graphDeps, sessionSource(dA.id));
// Resolve through THIS session's mappings — the shared ledger holds
// lookalike statements from earlier tests, so statement-lookup would
// link dB to the wrong node.
@@ -612,14 +624,14 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
),
]),
].flat());
const auditB: AuditResult = {
const findingsB: SessionFindings = {
propositions: [
{ id: "q1", name: "Shared again", statement: "Shared core, rephrased", aliases: ["B phrasing"], linkTo: xId },
{ id: "q1", name: "Shared again", statement: "Shared core, rephrased", aliases: ["B phrasing"] },
{ id: "q2", name: "B extra", statement: "B extra statement", aliases: ["b extra feel"], },
],
edges: [{ from: "q1", to: "q2", type: "follows", }],
};
mergeFindings(dB.id, auditB, graphDeps, sessionSource(dB.id));
mergeFindings(dB.id, findingsB, { links: [{ id: "q1", linkTo: xId }], edges: [] }, graphDeps, sessionSource(dB.id));
const zId = graphStore.readMappings().find((m) => m.chatId === dB.id && m.sourceId === "q2")!.graphNodeId;
// Coverage rows (both chats) + positions garnish — the deletion