extraction: message-granular proposition anchors; factory budget back to 4096

This commit is contained in:
amosblomqvist
2026-10-13 08:26:31 +03:00
parent d653ba0e91
commit 664c1bfed2
10 changed files with 560 additions and 112 deletions
+173 -47
View File
@@ -28,7 +28,6 @@ import type { GraphStore } from "../../core/ports.ts";
import { emit } from "../../core/events.ts"; import { emit } from "../../core/events.ts";
import { settleGraphLayout } from "../../core/layout-settle.ts"; import { settleGraphLayout } from "../../core/layout-settle.ts";
import { batchRuns, coverageOf, messageViewLines } from "../../core/coverage.ts"; import { batchRuns, coverageOf, messageViewLines } from "../../core/coverage.ts";
import { teacherProseBlocks } from "../../core/coverage.ts";
import type { LhMessage } from "../../core/native-messages.ts"; import type { LhMessage } from "../../core/native-messages.ts";
import { auditorGraphTools } from "../tools/graph.ts"; import { auditorGraphTools } from "../tools/graph.ts";
import { runSubmitAgent, healStringifiedArrays } from "./submit-agent.ts"; import { runSubmitAgent, healStringifiedArrays } from "./submit-agent.ts";
@@ -42,10 +41,15 @@ import { runSubmitAgent, healStringifiedArrays } from "./submit-agent.ts";
* gain; the extractor itself is the assembler now): * gain; the extractor itself is the assembler now):
* *
* extractor (LLM) the ONE stage that sees the raw source (a session * extractor (LLM) the ONE stage that sees the raw source (a session
* rendered as the teacher's prose, or a notes file * rendered as the teacher's prose — each passage
* verbatim, walked in stretches) → submits NEW core * labeled with its ACTUAL message id, "TEACHER [id]",
* ideas + reasoning chains (owner redesign, * owner 2026-10-13 — or a notes file verbatim, walked
* 2026-09-30). DIGEST-AWARE (2026-09-30 evening): * in stretches) → submits NEW core ideas + reasoning
* chains (owner redesign, 2026-09-30); a session
* submit additionally carries each proposition's
* messageId — the anchor the merger stamps into the
* node's message-granular provenance (2026-10-13).
* DIGEST-AWARE (2026-09-30 evening):
* when the source already yielded material, the * when the source already yielded material, the
* prompt carries THE GRAPH SO FAR (that source's * prompt carries THE GRAPH SO FAR (that source's
* propositions + chains, ids verbatim) and the * propositions + chains, ids verbatim) and the
@@ -104,14 +108,16 @@ import { runSubmitAgent, healStringifiedArrays } from "./submit-agent.ts";
/** /**
* The extraction view: the session as TEACHER PROSE ONLY (owner ruling, * The extraction view: the session as TEACHER PROSE ONLY (owner ruling,
* 2026-09-28 — supersedes the 2026-09-27 whole-session markdown view): * 2026-09-28 — supersedes the 2026-09-27 whole-session markdown view),
* plain prose paragraphs under the session title. The shared markdown * each passage labeled with its actual message id (2026-10-13). The
* engine (core/session-markdown.ts) keeps the full session format for * shared markdown engine (core/session-markdown.ts) keeps the full session
* the future user-facing export; the extractor no longer uses it. * format for the future user-facing export; the extractor no longer uses
* it.
*/ */
/** The whole-record view (used by callers that render a full session; /** The whole-record view (used by callers that render a full session;
* the pipeline itself batches). TEACHER PROSE ONLY (owner, 2026-09-28). * the pipeline itself batches). TEACHER PROSE ONLY (owner, 2026-09-28),
* passages labeled with their message ids (2026-10-13).
*/ */
export function transcriptView(record: ChatRecord): string { export function transcriptView(record: ChatRecord): string {
return proseView(record.messages, record.title); return proseView(record.messages, record.title);
@@ -119,13 +125,17 @@ export function transcriptView(record: ChatRecord): string {
/** The extraction view under the teacher-prose ruling (owner, 2026-09-28): /** The extraction view under the teacher-prose ruling (owner, 2026-09-28):
* ONLY the teacher agent's text blocks — no learner messages, no * ONLY the teacher agent's text blocks — no learner messages, no
* thinking, no tool calls or results, no quiz blocks. Plain paragraphs * thinking, no tool calls or results, no quiz blocks — each labeled with
* under the session title; the shared markdown engine stays reserved for * its ACTUAL message id (owner, 2026-10-13): the view lines ARE
* the future user-facing export. */ * core/coverage.ts messageViewLines ("TEACHER [id] text"), the exact
* lines the coverage fingerprints hash, so the extractor can anchor
* every proposition to the passage it came from (the session submit's
* required messageId). The shared markdown engine stays reserved for the
* future user-facing export. */
function proseView(messages: ReadonlyArray<LhMessage>, title?: string): string { function proseView(messages: ReadonlyArray<LhMessage>, title?: string): string {
const prose = teacherProseBlocks(messages); const lines = messages.flatMap((message) => messageViewLines(message, message.id));
const head = title && title.length > 0 ? `# ${title}\n\n` : ""; const head = title && title.length > 0 ? `# ${title}\n\n` : "";
return head + prose.join("\n\n"); return head + lines.join("\n\n");
} }
// ─── the extractor ────────────────────────────────────────────────────────── // ─── the extractor ──────────────────────────────────────────────────────────
@@ -176,21 +186,45 @@ const chainSchema = Type.Object({
// here + a ??-fill at the submit seam (fillFindings/fillAudit below) — same // here + a ??-fill at the submit seam (fillFindings/fillAudit below) — same
// contract on the wire (omission tolerated, empty never means "invalid"), // contract on the wire (omission tolerated, empty never means "invalid"),
// no zod left in the agent band. // no zod left in the agent band.
const findingsProps = { const propositionsDescription =
"Only the NEW core ideas this stretch contributes — ideas the prompt's THE GRAPH SO FAR already " +
"holds are referenced by id in chains, never restated here.";
const chainsDescription =
"Every argument this stretch makes — including arguments that join new ideas to established " +
"ones; conclusions reusable by id as premises of later chains; bare ideas (no argument) ride " +
"the propositions list alone; pass [] when none — never omit the field.";
const findingsSchema = Type.Object({
propositions: Type.Array(Type.Object(propositionProps), { propositions: Type.Array(Type.Object(propositionProps), {
description: description: propositionsDescription,
"Only the NEW core ideas this stretch contributes — ideas the prompt's THE GRAPH SO FAR already " +
"holds are referenced by id in chains, never restated here.",
}), }),
chains: Type.Array(chainSchema, { chains: Type.Array(chainSchema, {
description: chainsDescription,
}),
});
// THE SESSION EXTENSION (owner, 2026-10-13): every session proposition
// carries a REQUIRED messageId — the anchor of the passage it came from,
// copied from the view's "TEACHER [id]" label (validated at the submit
// seam against the CURRENT batch). The NOTES door has no messages: its
// schema stays the base shape above, prompt and wire unchanged.
const sessionPropositionProps = {
...propositionProps,
messageId: Type.String({
description: description:
"Every argument this stretch makes — including arguments that join new ideas to established " + "The message id of the ONE passage in this stretch that most directly states or explains this " +
"ones; conclusions reusable by id as premises of later chains; bare ideas (no argument) ride " + 'proposition — copy it EXACTLY from that passage\'s "TEACHER [id]" label.',
"the propositions list alone; pass [] when none — never omit the field.",
}), }),
}; };
const findingsSchema = Type.Object(findingsProps); const sessionFindingsSchema = Type.Object({
propositions: Type.Array(Type.Object(sessionPropositionProps), {
description: propositionsDescription,
}),
chains: Type.Array(chainSchema, {
description: chainsDescription,
}),
});
/** What the extractor SUBMITS (the redesign's wire shape) — propositions /** What the extractor SUBMITS (the redesign's wire shape) — propositions
* + chains. fillFindings normalizes it; validateFindingsChains checks * + chains. fillFindings normalizes it; validateFindingsChains checks
@@ -199,6 +233,10 @@ const findingsSchema = Type.Object(findingsProps);
* 2026-09-30: no edge translation at any seam). */ * 2026-09-30: no edge translation at any seam). */
type FindingsSubmit = Static<typeof findingsSchema>; type FindingsSubmit = Static<typeof findingsSchema>;
/** The SESSION wire — the notes shape plus the REQUIRED messageId anchor
* on every proposition (owner, 2026-10-13). */
type SessionFindingsSubmit = Static<typeof sessionFindingsSchema>;
// VERDICTS-ONLY (owner, 2026-09-28): the auditor never alters the // VERDICTS-ONLY (owner, 2026-09-28): the auditor never alters the
// extractor's findings, so it does not restate them — links are its // extractor's findings, so it does not restate them — links are its
// ENTIRE deliverable (owner, 2026-09-28 evening: the edges argument is // ENTIRE deliverable (owner, 2026-09-28 evening: the edges argument is
@@ -217,9 +255,11 @@ const auditSchema = Type.Object({
), ),
}); });
// The notes front door's shapes are IDENTICAL to the session shapes now // The notes front door's wire shape is the session shape MINUS the
// (the only difference was quiz attribution, which no longer exists): // messageId anchor (owner, 2026-10-13: only the SESSION door demands
// one schema, one findings type, both front doors. // passage anchors — notes have no messages, so their prompt and schema
// stay exactly as they were): one base findings type for both doors,
// the anchor riding as an optional field.
type NotesFindings = SessionFindings; type NotesFindings = SessionFindings;
type NotesAuditResult = AuditVerdicts; type NotesAuditResult = AuditVerdicts;
@@ -233,13 +273,30 @@ type NotesAuditResult = AuditVerdicts;
// a pile of nodes with no structure). Only the per-item aliases stay // a pile of nodes with no structure). Only the per-item aliases stay
// optional with a ??-fill. // optional with a ??-fill.
function fillFindings(raw: FindingsSubmit): FindingsSubmit { function fillFindings<T extends { propositions: Array<{ aliases?: string[] }>; chains: unknown }>(
raw: T,
): T {
return { return {
...raw,
propositions: raw.propositions.map((p) => ({ ...p, aliases: p.aliases ?? [] })), propositions: raw.propositions.map((p) => ({ ...p, aliases: p.aliases ?? [] })),
chains: raw.chains,
}; };
} }
/** The notes fill: the shared alias ??-fill PLUS the door's boundary — a
* stray messageId (the notes schema has none; extra properties pass
* schema validation) is stripped so notes provenance stays chat-level
* (the file is the anchor, owner 2026-09-13). */
function fillNotesFindings(raw: FindingsSubmit): FindingsSubmit {
return fillFindings({
...raw,
propositions: raw.propositions.map((p) => {
const stripped = { ...p } as typeof p & { messageId?: unknown };
delete stripped.messageId;
return stripped;
}),
});
}
/** The chain laws (extractor redesign, 2026-09-30; digest-aware /** The chain laws (extractor redesign, 2026-09-30; digest-aware
* 2026-09-30 evening) — the LOUD channel: runSubmitAgent turns a * 2026-09-30 evening) — the LOUD channel: runSubmitAgent turns a
* problem string into an error tool result and the model re-submits * problem string into an error tool result and the model re-submits
@@ -266,12 +323,38 @@ function validateFindingsChains(value: FindingsSubmit, existingIds: ReadonlySet<
return null; return null;
} }
/** The SESSION anchor law (owner, 2026-10-13): every proposition names a
* passage of the CURRENT batch — nonempty, and a teacher-prose message
* id present in the stretch this extractor is reading (the same ids the
* view's TEACHER [id] labels carry). Refused anchors ride the existing
* error-back channel: no merge, no coverage checkpoint, until a valid
* submit lands. The boundary is strict on purpose — an id from a LATER
* or already-covered batch, another chat, or a learner/tool message is
* refused even when it is a real message id of this chat. */
function validateSessionAnchors(value: SessionFindingsSubmit, batchIds: ReadonlySet<string>): string | null {
for (const p of value.propositions) {
if (p.messageId.trim().length === 0) {
return `propositions: "${p.id}" is missing its messageId — copy the message id from the brackets of the passage (TEACHER [id]) that most directly states or explains it.`;
}
if (!batchIds.has(p.messageId)) {
return `propositions: "${p.id}" names messageId "${p.messageId}", which is not a passage of THIS stretch — copy the id from the brackets of one of this stretch's own TEACHER [id] passages.`;
}
}
return null;
}
/** (chains-primary, owner 2026-09-30) the wire IS the pipeline shape /** (chains-primary, owner 2026-09-30) the wire IS the pipeline shape
* now: propositions + chains, no translation — the stores keep the * now: propositions + chains, no translation — the stores keep the
* chains and the derived edge view is recomputed on read. The fill * chains and the derived edge view is recomputed on read. The fill
* already ran inside runSubmitAgent; this only re-asserts the * already ran inside runSubmitAgent; this only re-asserts the
* aliases-filled type. */ * aliases-filled type (and preserves the session submit's messageId
const asSessionFindings = (raw: FindingsSubmit): SessionFindings => ({ * anchor, 2026-10-13 — both doors lift through here). */
const asSessionFindings = (
raw: {
propositions: Array<{ id: string; name: string; statement: string; aliases?: string[]; messageId?: string }>;
chains: Array<{ premises: string[]; conclusion: string }>;
},
): SessionFindings => ({
propositions: raw.propositions.map((p) => ({ ...p, aliases: p.aliases ?? [] })), propositions: raw.propositions.map((p) => ({ ...p, aliases: p.aliases ?? [] })),
chains: raw.chains, chains: raw.chains,
}); });
@@ -293,6 +376,14 @@ function fillAudit(raw: Static<typeof auditSchema>): AuditVerdicts {
// describing Quiz/Recall-check/Question blocks was stale since the // describing Quiz/Recall-check/Question blocks was stale since the
// 2026-09-28 teacher-prose ruling and is simply omitted.) // 2026-09-28 teacher-prose ruling and is simply omitted.)
/** The session-only anchor instruction (owner, 2026-10-13, verbatim):
* passages arrive labeled "TEACHER [id]" and every proposition must
* name the ONE passage it came from — the anchor the merger stamps
* into message-granular provenance. Notes have no passages: their
* contextualization stays untouched. */
export const SESSION_MESSAGE_ID_RULE =
"For each new proposition, copy the message ID of the passage that most directly states or explains it into messageId.";
export const SESSION_PREAMBLE = export const SESSION_PREAMBLE =
"You are the extraction stage of a learning application. You read one stretch of a source — a teaching " + "You are the extraction stage of a learning application. You read one stretch of a source — a teaching " +
"session, rendered as the teacher's prose in reading order — and report what it teaches as findings " + "session, rendered as the teacher's prose in reading order — and report what it teaches as findings " +
@@ -337,7 +428,7 @@ reappear, by id, as a premise of later chains. A bare idea that no argument uses
propositions list alone. Include every field even when empty — "chains": [] when the source makes no propositions list alone. Include every field even when empty — "chains": [] when the source makes no
arguments. The tool's field descriptions carry the details.`; arguments. The tool's field descriptions carry the details.`;
export const EXTRACTOR_INSTRUCTIONS = `${SESSION_PREAMBLE}\n\n${EXTRACTION_CORE}`; export const EXTRACTOR_INSTRUCTIONS = `${SESSION_PREAMBLE}\n\n${EXTRACTION_CORE}\n\n${SESSION_MESSAGE_ID_RULE}`;
export const NOTES_EXTRACTOR_INSTRUCTIONS = `${NOTES_PREAMBLE}\n\n${EXTRACTION_CORE}`; export const NOTES_EXTRACTOR_INSTRUCTIONS = `${NOTES_PREAMBLE}\n\n${EXTRACTION_CORE}`;
// ── the same-source digest (owner, 2026-09-30 evening) ── // ── the same-source digest (owner, 2026-09-30 evening) ──
@@ -368,28 +459,33 @@ export function sourceDigest(graph: DerivedGraph, sourceId: string): string {
return lines.join("\n"); return lines.join("\n");
} }
/** The extractor over ONE session transcript (the chat front door). */ /** The extractor over ONE session transcript (the chat front door).
* `batchIds` — the CURRENT batch's teacher-prose message ids — is the
* anchor boundary: every submitted proposition's messageId must name
* one of them (validateSessionAnchors at the submit seam). */
function runExtractor( function runExtractor(
engine: PiEngine | undefined, engine: PiEngine | undefined,
sourceId: string, sourceId: string,
prompt: string, prompt: string,
existingIds: ReadonlySet<string>, existingIds: ReadonlySet<string>,
batchIds: ReadonlySet<string>,
signal: AbortSignal | undefined, signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<SessionFindings> { ): Promise<SessionFindings> {
// Chains in (the wire), chains out (the store) — no translation seam // Chains in (the wire), chains out (the store) — no translation seam
// since chains-primary. Validation runs at the submit seam (error-back) // since chains-primary. Validation runs at the submit seam (error-back)
// over the MIXED namespace: local ids + existing graph ids. // over the MIXED namespace: local ids + existing graph ids, plus the
return runSubmitAgent<FindingsSubmit>({ // session anchor law (messageId ∈ the current batch, 2026-10-13).
return runSubmitAgent<SessionFindingsSubmit>({
role: "extractor", role: "extractor",
// Debug recorder key: the source id (the run's ledger key — the // Debug recorder key: the source id (the run's ledger key — the
// chat's chatId for chat runs, the src id for notes). // chat's chatId for chat runs, the src id for notes).
chatId: sourceId, chatId: sourceId,
toolName: "submit_findings", toolName: "submit_findings",
toolDescription: "Submit the session findings.", toolDescription: "Submit the session findings.",
parameters: findingsSchema, parameters: sessionFindingsSchema,
fill: fillFindings, fill: fillFindings,
validate: (value) => validateFindingsChains(value, existingIds), validate: (value) => validateSessionAnchors(value, batchIds) ?? validateFindingsChains(value, existingIds),
systemPrompt: EXTRACTOR_INSTRUCTIONS, systemPrompt: EXTRACTOR_INSTRUCTIONS,
prompt, prompt,
engine, engine,
@@ -401,7 +497,9 @@ function runExtractor(
} }
/** The extractor over ONE source of raw personal notes (the format-agnostic /** The extractor over ONE source of raw personal notes (the format-agnostic
* front door): same single-submit shape, notes-mode prompt + schema. */ * front door): same single-submit shape, notes-mode prompt + schema —
* the BASE wire (no messageId: notes have no passages, and a stray one
* is stripped at the fill seam). */
function runNotesExtractor( function runNotesExtractor(
engine: PiEngine | undefined, engine: PiEngine | undefined,
sourceId: string, sourceId: string,
@@ -417,7 +515,7 @@ function runNotesExtractor(
toolName: "submit_findings", toolName: "submit_findings",
toolDescription: "Submit the notes findings.", toolDescription: "Submit the notes findings.",
parameters: findingsSchema, parameters: findingsSchema,
fill: fillFindings, fill: fillNotesFindings,
validate: (value) => validateFindingsChains(value, existingIds), validate: (value) => validateFindingsChains(value, existingIds),
systemPrompt: NOTES_EXTRACTOR_INSTRUCTIONS, systemPrompt: NOTES_EXTRACTOR_INSTRUCTIONS,
prompt, prompt,
@@ -793,7 +891,12 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
} }
} }
async function extract(jobKey: string, sourceId: string, view: string): Promise<SessionFindings> { async function extract(
jobKey: string,
sourceId: string,
view: string,
batchIds: ReadonlySet<string>,
): Promise<SessionFindings> {
const signal = runningExtractions.get(jobKey)?.controller.signal; const signal = runningExtractions.get(jobKey)?.controller.signal;
signal?.throwIfAborted(); // an abort during a sync gap ends the job here signal?.throwIfAborted(); // an abort during a sync gap ends the job here
// THE DIGEST (owner, 2026-09-30 evening): the extractor continues a // THE DIGEST (owner, 2026-09-30 evening): the extractor continues a
@@ -816,6 +919,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
sourceId, sourceId,
prompt, prompt,
new Set(graph.nodes.map((n) => n.id)), new Set(graph.nodes.map((n) => n.id)),
batchIds,
signal, signal,
(usage, model, step, finishReason, tools) => { (usage, model, step, finishReason, tools) => {
recordStep(jobKey, "extractor", callUsage(usage), model); recordStep(jobKey, "extractor", callUsage(usage), model);
@@ -838,12 +942,26 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
// The auditor NEVER sees the source (owner 2026-09-27): findings // The auditor NEVER sees the source (owner 2026-09-27): findings
// JSON + identity instructions, and the graph reached through // JSON + identity instructions, and the graph reached through
// exploration tools instead of an injected dump. // exploration tools instead of an injected dump. The proposition
// projection omits messageId (2026-10-13): the auditor is
// IDENTITY-only and never sees the passages, so anchors are useless
// to it — they add cognitive load, not signal.
const auditResult = await runAuditor( const auditResult = await runAuditor(
stageEngine("auditor", deps.auditor), stageEngine("auditor", deps.auditor),
sourceId, sourceId,
`Proposed findings from the extractor (the source itself is not available to you — the extractor is fully trusted; the propositions only — the reasoning chains are the merger's business, owner 2026-09-30):\n\n` + `Proposed findings from the extractor (the source itself is not available to you — the extractor is fully trusted; the propositions only — the reasoning chains are the merger's business, owner 2026-09-30):\n\n` +
`${JSON.stringify({ propositions: findings.propositions }, null, 2)}\n\n` + `${JSON.stringify(
{
propositions: findings.propositions.map((p) => ({
id: p.id,
name: p.name,
statement: p.statement,
aliases: p.aliases,
})),
},
null,
2,
)}\n\n` +
`The current knowledge graph has ${graph.nodes.length} node(s). Explore it with your tools and link what links.`, `The current knowledge graph has ${graph.nodes.length} node(s). Explore it with your tools and link what links.`,
signal, signal,
(usage, model, step, finishReason, tools) => { (usage, model, step, finishReason, tools) => {
@@ -1010,16 +1128,24 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
status: "fulfilled", status: "fulfilled",
value: await runJob({ id: chatId, kind: "chat", part }, async (jobKey) => { value: await runJob({ id: chatId, kind: "chat", part }, async (jobKey) => {
// The batch's source view: TEACHER PROSE ONLY (owner, // The batch's source view: TEACHER PROSE ONLY (owner,
// 2026-09-28) — plain paragraphs, same selection as the // 2026-09-28), each passage labeled with its ACTUAL message
// coverage fingerprints (messageViewLines parity). // id (2026-10-13) — the view lines ARE the coverage
const view = proseView( // fingerprints' messageViewLines ("TEACHER [id] text"), one
batch.map((slot) => slot.message), // shared seam.
record.title, const messages = batch.map((slot) => slot.message);
const view = proseView(messages, record.title);
// The batch's teacher-prose ids — the ONLY anchors this
// job's submit may name (validateSessionAnchors at the
// submit seam refuses anything else: ids from other
// batches — even real ones of this chat — other chats,
// and non-prose messages).
const batchIds = new Set(
messages.filter((m) => messageViewLines(m, m.id).length > 0).map((m) => m.id),
); );
setPhase(jobKey, "queued", "chat", undefined, part); setPhase(jobKey, "queued", "chat", undefined, part);
const findings = await withExtractSlot(jobKey, async () => { const findings = await withExtractSlot(jobKey, async () => {
setPhase(jobKey, "extracting", "chat", undefined, part); setPhase(jobKey, "extracting", "chat", undefined, part);
return extract(jobKey, chatId, view); return extract(jobKey, chatId, view, batchIds);
}); });
// SERIALIZED AUDIT (owner, 2026-09-28): audit + merge are the // SERIALIZED AUDIT (owner, 2026-09-28): audit + merge are the
// graph-mutating critical section — one job at a time, // graph-mutating critical section — one job at a time,
+14 -5
View File
@@ -231,11 +231,20 @@ export interface ExtractedProposition {
/** Verbatim phrasings the session used. */ /** Verbatim phrasings the session used. */
aliases: string[]; aliases: string[];
/* Submit-schema slim (owner, 2026-09-28): `messages` and `evidence` /* Submit-schema slim (owner, 2026-09-28): `messages` and `evidence`
* are GONE. Provenance is chat-level (the run's source key); the * are GONE. The verbatim-quote fields were never persisted and weak
* verbatim-quote fields were never persisted and weak models burned * models burned their retries on them ("propositions.0: must be
* their retries on them ("propositions.0: must be object" — the * object" — the stringified-payload collapse). The extractor reports
* stringified-payload collapse). The extractor reports propositions * propositions + structure, nothing else. */
* + structure, nothing else. */ /** OPTIONAL, SESSION PATH ONLY (owner, 2026-10-13): the message id of
* the ONE teacher passage this proposition came from — the extractor
* copies it from the view's "TEACHER [id]" label (core/coverage.ts
* messageViewLines) and the submit seam validates it against the
* CURRENT batch. The merger stamps it into the node's provenance
* point {chatId, messageId} (new nodes AND link-appends); chains
* stay chat-level. Notes imports and legacy/core callers have none:
* undefined → messageId "" (chat-level), so NO persisted shape or
* format version changes (GraphNode / ProvenancePoint untouched). */
messageId?: string;
} }
/** One proposed argument (extractor redesign, 2026-09-30). Endpoints are /** One proposed argument (extractor redesign, 2026-09-30). Endpoints are
+17 -8
View File
@@ -11,8 +11,10 @@
* same-source digest (digest-aware, 2026-09-30 * same-source digest (digest-aware, 2026-09-30
* evening: the extractor continues its own source, * evening: the extractor continues its own source,
* references existing ids verbatim, assembles * references existing ids verbatim, assembles
* cross-stretch itself). Fully trusted: no stage * cross-stretch itself). Session propositions carry
* re-checks it. * their passage's messageId (2026-10-13) — the
* merger's provenance anchor. Fully trusted: no
* stage re-checks it.
* auditor (LLM) → AuditVerdicts: identity links (local id → an existing * auditor (LLM) → AuditVerdicts: identity links (local id → an existing
* node, or create) via graph-exploration tools — * node, or create) via graph-exploration tools —
* links ONLY (owner, 2026-09-28 evening: the auditor * links ONLY (owner, 2026-09-28 evening: the auditor
@@ -109,11 +111,14 @@ export interface MergeSource {
coverage?: CoverageEntry[]; coverage?: CoverageEntry[];
} }
/** The run's chat-level provenance point (submit-schema slim, 2026-09-28: /** The run's provenance point. Chat-level by default (submit-schema slim,
* propositions no longer carry message ids — the model never reported * 2026-09-28 — the run's source key is the whole point); since the anchor
* them reliably anyway — so every birth from a run shares one point). */ * restore (owner, 2026-10-13) a SESSION birth may carry its passage's
function pointOf(chatId: string): ProvenancePoint[] { * message id — the extractor-copied messageId, validated at the submit
return [{ chatId, messageId: "" }]; * seam against the current batch. Chains, notes imports, and
* legacy/core callers lacking an anchor keep "" (chat-level). */
function pointOf(chatId: string, messageId?: string): ProvenancePoint[] {
return [{ chatId, messageId: messageId ?? "" }];
} }
const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean => const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean =>
@@ -166,7 +171,11 @@ export function mergeFindings(
// extractor's findings verbatim; the auditor's links join here by id. // extractor's findings verbatim; the auditor's links join here by id.
for (const p of findings.propositions) { for (const p of findings.propositions) {
if (mapping.has(p.id)) continue; // duplicate id WITHIN this submit — first resolution stands if (mapping.has(p.id)) continue; // duplicate id WITHIN this submit — first resolution stands
const points = pointOf(chatId); // The anchor (2026-10-13): a session proposition may carry its
// passage's messageId — it rides BOTH the create and the link-append
// line (each re-encounter anchors to ITS passage); absent (notes,
// legacy/core callers) → "", chat-level as before.
const points = pointOf(chatId, p.messageId);
const linkTo = linkOf.get(p.id); const linkTo = linkOf.get(p.id);
let graphNodeId: string; let graphNodeId: string;
if (linkTo !== undefined && existingIds.has(linkTo)) { if (linkTo !== undefined && existingIds.has(linkTo)) {
+6 -3
View File
@@ -321,8 +321,11 @@ export interface GraphSettings {
* carries the whole graph + instructions, so a smaller budget means * carries the whole graph + instructions, so a smaller budget means
* more, smaller runs — each paying that fixed overhead). * more, smaller runs — each paying that fixed overhead).
* *
* FACTORY DEFAULT 4096 (the pre-setting constant, config.ts's * FACTORY DEFAULT 4096 (owner, 2026-10-13: halved back from the
* EXTRACTION.chunkTokens — vision-extraction-coverage.md, 2026-09-15). * digest-era 8192 — the labeled passages + message anchors made
* smaller batches worth their overhead again; the pre-setting
* constant was 4096, vision-extraction-coverage.md, 2026-09-15).
* Only the FACTORY moves: an explicit user value always wins.
* *
* FLOOR 500 (EXTRACTION_CHUNK_TOKENS_FLOOR, the implementer's call * FLOOR 500 (EXTRACTION_CHUNK_TOKENS_FLOOR, the implementer's call
* per the owner's dispatch): every run pays the graph-carrying * per the owner's dispatch): every run pays the graph-carrying
@@ -402,7 +405,7 @@ export const FACTORY_SETTINGS: FactorySettings = {
sessionNameSerif: true, sessionNameSerif: true,
mdRendering: { headingWeight: 900, boldColor: null, italicColor: null }, mdRendering: { headingWeight: 900, boldColor: null, italicColor: null },
}, },
graph: { enabled: true, rendering: true, extractionChunkTokens: 8192 }, graph: { enabled: true, rendering: true, extractionChunkTokens: 4096 },
compaction: { thresholdTokens: 150_000 }, compaction: { thresholdTokens: 150_000 },
}; };
+271 -29
View File
@@ -49,13 +49,14 @@ import {
sourceDigest, sourceDigest,
EXTRACTION_CORE, EXTRACTION_CORE,
SESSION_PREAMBLE, SESSION_PREAMBLE,
SESSION_MESSAGE_ID_RULE,
NOTES_PREAMBLE, NOTES_PREAMBLE,
EXTRACTOR_INSTRUCTIONS, EXTRACTOR_INSTRUCTIONS,
NOTES_EXTRACTOR_INSTRUCTIONS, NOTES_EXTRACTOR_INSTRUCTIONS,
} from "../src/agent/subagents/extraction.ts"; } from "../src/agent/subagents/extraction.ts";
import { simplifyChains } from "../src/core/graph.ts"; import { getGraph, simplifyChains } from "../src/core/graph.ts";
import { ModelsNotConfiguredError, ProviderKeyMissingError, providerKeyMissing, type ModelSlotName } from "../src/core/settings.ts"; import { ModelsNotConfiguredError, ProviderKeyMissingError, providerKeyMissing, type ModelSlotName } from "../src/core/settings.ts";
import { coverageOf, messageViewLines } from "../src/core/coverage.ts"; import { coverageOf, coverageSlots, messageViewLines } from "../src/core/coverage.ts";
import { chatStore } from "../src/store/chats.ts"; import { chatStore } from "../src/store/chats.ts";
import { subscribe } from "../src/core/events.ts"; import { subscribe } from "../src/core/events.ts";
import { PATHS } from "../config.ts"; import { PATHS } from "../config.ts";
@@ -232,7 +233,9 @@ after(() => {
/** What the extractor SUBMITS for the fixture session (owner redesign, /** What the extractor SUBMITS for the fixture session (owner redesign,
* 2026-09-30): propositions + REASONING CHAINS on the wire — one chain * 2026-09-30): propositions + REASONING CHAINS on the wire — one chain
* (p1 ⟹ p2), expanded to a follows-edge at the seam. */ * (p1 ⟹ p2). Each proposition carries its passage anchor (2026-10-13):
* the fixture's teacher-prose messages are a1 (chain rule prose) and a2
* (derivative prose) — p1 anchors to a2, p2 to a1. */
const FINDINGS = { const FINDINGS = {
propositions: [ propositions: [
{ {
@@ -240,12 +243,14 @@ const FINDINGS = {
name: "Derivative", name: "Derivative",
statement: "The derivative measures how fast a function changes", statement: "The derivative measures how fast a function changes",
aliases: [], aliases: [],
messageId: "a2",
}, },
{ {
id: "p2", id: "p2",
name: "Chain rule", name: "Chain rule",
statement: "The chain rule differentiates composed functions by multiplying local rates.", statement: "The chain rule differentiates composed functions by multiplying local rates.",
aliases: [], aliases: [],
messageId: "a1",
}, },
], ],
chains: [ chains: [
@@ -331,10 +336,22 @@ describe("transcriptView (pure — teacher prose only, owner 2026-09-28)", () =>
const view = transcriptView(record); const view = transcriptView(record);
// The document header is the session title; the body is the // The document header is the session title; the body is the
// teacher's text blocks as plain paragraphs. // teacher's text blocks as labeled passages.
assert.match(view, /^# Extraction fixture$/m); assert.match(view, /^# Extraction fixture$/m);
assert.match(view, /The chain rule composes rates of change along a path\./); assert.match(view, /^TEACHER \[a1\] The chain rule composes rates of change along a path\.$/m);
assert.match(view, /The derivative measures how fast a function changes\./); assert.match(view, /^TEACHER \[a2\] The derivative measures how fast a function changes\.$/m);
// MESSAGE-ID LABELS (owner, 2026-10-13): the passages carry their
// ACTUAL message ids, in exactly messageViewLines' shape — the same
// seam the coverage fingerprints hash. No label without prose: a3
// (quiz only) and the learner messages are not passages.
assert.doesNotMatch(view, /TEACHER \[a3\]/);
assert.doesNotMatch(view, /TEACHER \[u1\]/);
assert.equal(
view.slice(view.indexOf("TEACHER")),
record.messages.flatMap((m) => messageViewLines(m, m.id)).join("\n\n"),
"the view body IS the messageViewLines of the transcript — one shared seam",
);
// EVERYTHING else is gone: learner text, quiz blocks, tool notes, // EVERYTHING else is gone: learner text, quiz blocks, tool notes,
// speaker headers — the ruling strips it all. // speaker headers — the ruling strips it all.
@@ -363,7 +380,7 @@ describe("transcriptView (pure — teacher prose only, owner 2026-09-28)", () =>
]), ]),
].flat()); ].flat());
const view = transcriptView(chatStore.getChat(chat.id)!); const view = transcriptView(chatStore.getChat(chat.id)!);
assert.match(view, /Prose survives\./); assert.match(view, /^TEACHER \[a1\] Prose survives\.$/m, "the surviving passage is labeled with its real id");
assert.doesNotMatch(view, /research/); assert.doesNotMatch(view, /research/);
assert.doesNotMatch(view, /long brief/); assert.doesNotMatch(view, /long brief/);
assert.doesNotMatch(view, /### /); assert.doesNotMatch(view, /### /);
@@ -479,6 +496,14 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
linkedLines.some((line) => line.provenance.some((pt) => pt.chatId === chatId)), linkedLines.some((line) => line.provenance.some((pt) => pt.chatId === chatId)),
"the link-append unions provenance", "the link-append unions provenance",
); );
// ANCHORS (2026-10-13): both lines carry their proposition's passage
// id — p1's link-append anchors to a2 (its passage), p2's create to a1.
assert.ok(
linkedLines.some((line) => line.provenance.some((pt) => pt.chatId === chatId && pt.messageId === "a2")),
"the link-append line carries p1's passage id",
);
const created = ledgers.readNodes().find((n) => n.name === "Chain rule")!;
assert.deepEqual(created.provenance, [{ chatId, messageId: "a1" }], "the created node anchors to p2's passage");
// COVERAGE checkpointed per message (not a whole-chat checkpoint): // COVERAGE checkpointed per message (not a whole-chat checkpoint):
// one entry per transcript message, fingerprinted content each — // one entry per transcript message, fingerprinted content each —
@@ -672,7 +697,8 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
chatStore, chatStore,
graphStore: ledgers, graphStore: ledgers,
// 4096 budget: the fixture is sized to split in two (the factory // 4096 budget: the fixture is sized to split in two (the factory
// default doubled to 8192, 2026-09-30). // default again — halved back from the digest-era 8192, owner
// 2026-10-13).
getChunkTokens: () => 4096, getChunkTokens: () => 4096,
engineFor: (role) => engineFor: (role) =>
role === "extractor" role === "extractor"
@@ -725,7 +751,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const ledgers = memoryLedgers(); const ledgers = memoryLedgers();
const single = (id: string, name: string, statement: string) => ({ const single = (id: string, name: string, statement: string) => ({
propositions: [{ id, name, statement, aliases: [], }], propositions: [{ id, name, statement, aliases: [], messageId: "a1" }],
chains: [], chains: [],
}); });
const findingsA = single("pa", "Alpha claim", "Alpha statement, full precision"); const findingsA = single("pa", "Alpha claim", "Alpha statement, full precision");
@@ -782,7 +808,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const ledgers = memoryLedgers(); const ledgers = memoryLedgers();
const single = (id: string, name: string, statement: string) => ({ const single = (id: string, name: string, statement: string) => ({
propositions: [{ id, name, statement, aliases: [], }], propositions: [{ id, name, statement, aliases: [], messageId: "a1" }],
chains: [], chains: [],
}); });
@@ -1016,6 +1042,185 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
}); });
}); });
// ─── message anchors (messageId — owner, 2026-10-13) ────────────────────────
describe("message anchors (session propositions carry their passage's id)", () => {
/** A single-proposition submit with an explicit anchor. */
const anchored = (messageId: string) => ({
propositions: [
{ id: "p1", name: "Anchor claim", statement: "Anchor claim, full precision.", aliases: [], messageId },
],
chains: [],
});
/** One scripted extractor turn: a submit_findings tool call. */
const submitStep = (payload: unknown): FauxResponseStep => () =>
fauxAssistantMessage(
[fauxToolCall("submit_findings", payload as ToolCall["arguments"], { id: "anchor-call" })],
{ stopReason: "toolUse" },
);
it("a submit MISSING messageId never passes: the schema rejects every turn — nothing merges, nothing checkpoints", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
// The degraded shape: propositions without anchors, chains present —
// only the REQUIRED field is missing, and the schema refuses it on
// every turn until the budget is gone (the required-field doctrine,
// 2026-09-28, now covering the anchor too).
const bare = () =>
fauxEngine(
Array.from({ length: 4 }, () =>
submitStep({
propositions: [{ id: "p1", name: "Anchor claim", statement: "Anchor claim, full precision.", aliases: [] }],
chains: [],
}),
),
);
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
engineFor: (role) => (role === "extractor" ? bare() : submitModel("submit_review", { links: [] })),
});
assert.deepEqual(await pipeline.runExtraction(chatId), { nodes: 0, chains: 0 });
assert.equal(ledgers.readNodes().length, 0, "nothing merged");
assert.equal(ledgers.readCoverage().length, 0, "nothing checkpointed — the batch stays re-runnable");
});
it("an EMPTY or UNKNOWN anchor rides the error-back channel: refused, the model re-anchors clean, then merges", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
// Three bad anchors, all schema-valid (messageId is a string) — only
// the anchor law refuses them: empty, a LEARNER message of this very
// chat (real id, not a teacher passage), and pure fiction.
const extractor = fauxEngine([
submitStep(anchored("")),
submitStep(anchored("u1")),
submitStep(anchored("made_up")),
submitStep(anchored("a2")),
]);
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
extractor,
auditor: submitModel("submit_review", { links: [] }),
});
assert.deepEqual(await pipeline.runExtraction(chatId), { nodes: 1, chains: 0 }, "the retry anchored and merged");
const node = ledgers.readNodes()[0]!;
assert.deepEqual(node.provenance, [{ chatId, messageId: "a2" }], "the node anchors to its passage");
});
it("the boundary is the CURRENT BATCH: a real teacher message id from another batch is refused, the in-batch retry merges", async () => {
const { chatId } = chunkyChat(); // deterministic batches [a1,a2,a3] + [a4,a5] at the 4096 budget
const ledgers = memoryLedgers();
// Batch 1: "a5" is a REAL teacher-prose id of this very chat — but it
// lives in batch 2, so this stretch refuses it; the re-anchor to "a2"
// merges. Batch 2 anchors to its own "a4".
const extractor = fauxEngine([
submitStep(anchored("a5")),
submitStep(anchored("a2")),
submitStep(anchored("a4")),
]);
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
getChunkTokens: () => 4096,
extractor,
// One review per batch job (the retry lives in the extractor's
// own turn budget, not the auditor's).
auditor: fauxEngine([passThroughAuditStep(), passThroughAuditStep()]),
});
assert.deepEqual(await pipeline.runExtraction(chatId), { nodes: 2, chains: 0 });
const anchors = ledgers.readNodes().map((n) => n.provenance[0]!.messageId).sort();
assert.deepEqual(anchors, ["a2", "a4"], "each job's node anchors INSIDE its own batch — never a5");
assert.equal(ledgers.readCoverage().length, 5, "both batches checkpointed");
});
it("a LINK-APPEND carries its proposition's anchor into the existing node's provenance", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
const existingId = "n_anchor_existing";
ledgers.appendNode({
id: existingId,
name: "Derivative",
statement: "Derivatives measure rates of change",
aliases: [],
createdAt: "2026-08-01T00:00:00.000Z",
provenance: [],
});
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
extractor: submitModel("submit_findings", anchored("a2")),
auditor: submitModel("submit_review", { links: [{ id: "p1", linkTo: existingId }] }),
});
await pipeline.runExtraction(chatId);
// The link-append line — the re-encounter — carries the passage id.
const appended = ledgers
.readNodes()
.filter((n) => n.id === existingId)
.find((line) => line.provenance.some((pt) => pt.chatId === chatId))!;
assert.deepEqual(appended.provenance, [{ chatId, messageId: "a2" }]);
// And the derived graph unions it under the same point identity.
const merged = getGraph(ledgers).nodes.find((n) => n.id === existingId)!;
assert.ok(merged.provenance.some((pt) => pt.chatId === chatId && pt.messageId === "a2"));
});
it("the notes door stays anchor-free: no messageId required — and a stray one never reaches provenance", async () => {
const ledgers = memoryLedgers();
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
// NOTES_FINDINGS carries no messageId (the notes schema never
// asks for one) — the submit validates exactly as before.
extractor: submitModel("submit_findings", NOTES_FINDINGS),
auditor: submitModel("submit_review", { links: [] }),
});
assert.deepEqual(await pipeline.importNotes({ filePath: "/vault/anchor-free.md", text: NOTES_TEXT }), { nodes: 1, chains: 0 });
assert.deepEqual(
ledgers.readNodes()[0]!.provenance,
[{ chatId: ledgers.readSources()[0]!.id, messageId: "" }],
"notes provenance stays chat-level — the file is the anchor",
);
// The boundary: even a hallucinated messageId (extra properties pass
// schema validation) is stripped at the notes fill seam.
const stray = {
propositions: [{ ...NOTES_FINDINGS.propositions[0], messageId: "made_up" }],
chains: [],
};
const second = createExtractionPipeline({
chatStore,
graphStore: ledgers,
extractor: submitModel("submit_findings", stray),
auditor: submitModel("submit_review", { links: [] }),
});
assert.deepEqual(await second.importNotes({ filePath: "/vault/stray-anchor.md", text: NOTES_TEXT }), { nodes: 1, chains: 0 });
const secondSource = ledgers.readSources()[1]!;
const strayNode = ledgers.readNodes().find((n) => n.provenance.some((pt) => pt.chatId === secondSource.id))!;
assert.deepEqual(strayNode.provenance, [{ chatId: secondSource.id, messageId: "" }], "the stray anchor was stripped before the merger");
});
it("coverage fingerprints are UNCHANGED by the anchors: the stored fps equal the plain coverage-slot hashes", async () => {
const { chatId, record } = chunkyChat();
const ledgers = memoryLedgers();
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
getChunkTokens: () => 4096,
extractor: fauxEngine([findingsStep(), findingsStep()]),
auditor: fauxEngine([passThroughAuditStep(), passThroughAuditStep()]),
});
await pipeline.runExtraction(chatId);
// The labeled view reuses messageViewLines; the fingerprint seam is
// the same lines it always was — same hashes, same coverage.
assert.deepEqual(
ledgers.readCoverage().map((c) => c.fp).sort(),
coverageSlots(record.messages).map((slot) => slot.fp).sort(),
);
assert.equal(coverageOf(record.messages, ledgers.readCoverage(), chatId).uncoveredCount, 0);
});
});
// ── the notes front door (format-agnostic import) ──────────────────────────── // ── the notes front door (format-agnostic import) ────────────────────────────
/** Raw personal notes — arbitrary format, no session, no message ids. */ /** Raw personal notes — arbitrary format, no session, no message ids. */
@@ -1069,9 +1274,9 @@ describe("the extractor redesign (owner, 2026-09-30): chains are the deliverable
const ledgers = memoryLedgers(); const ledgers = memoryLedgers();
const extractor = submitModel("submit_findings", { const extractor = submitModel("submit_findings", {
propositions: [ propositions: [
{ id: "p1", name: "A", statement: "A holds.", aliases: [] }, { id: "p1", name: "A", statement: "A holds.", aliases: [], messageId: "a1" },
{ id: "p2", name: "B", statement: "B holds.", aliases: [] }, { id: "p2", name: "B", statement: "B holds.", aliases: [], messageId: "a2" },
{ id: "p3", name: "C", statement: "A and B together entail C.", aliases: [] }, { id: "p3", name: "C", statement: "A and B together entail C.", aliases: [], messageId: "a1" },
], ],
chains: [{ premises: ["p1", "p2"], conclusion: "p3" }], chains: [{ premises: ["p1", "p2"], conclusion: "p3" }],
}); });
@@ -1100,10 +1305,10 @@ describe("the extractor redesign (owner, 2026-09-30): chains are the deliverable
const ledgers = memoryLedgers(); const ledgers = memoryLedgers();
const extractor = submitModel("submit_findings", { const extractor = submitModel("submit_findings", {
propositions: [ propositions: [
{ id: "p1", name: "A", statement: "A holds.", aliases: [] }, { id: "p1", name: "A", statement: "A holds.", aliases: [], messageId: "a1" },
{ id: "p2", name: "B", statement: "B holds.", aliases: [] }, { id: "p2", name: "B", statement: "B holds.", aliases: [], messageId: "a2" },
{ id: "p3", name: "C", statement: "C follows.", aliases: [] }, { id: "p3", name: "C", statement: "C follows.", aliases: [], messageId: "a1" },
{ id: "p4", name: "Lonely", statement: "A bare proposition no argument uses.", aliases: [] }, { id: "p4", name: "Lonely", statement: "A bare proposition no argument uses.", aliases: [], messageId: "a2" },
], ],
chains: [ chains: [
{ premises: ["p1"], conclusion: "p3" }, { premises: ["p1"], conclusion: "p3" },
@@ -1238,6 +1443,16 @@ describe("the extractor redesign (owner, 2026-09-30): chains are the deliverable
assert.ok(NOTES_EXTRACTOR_INSTRUCTIONS.startsWith(NOTES_PREAMBLE)); assert.ok(NOTES_EXTRACTOR_INSTRUCTIONS.startsWith(NOTES_PREAMBLE));
assert.ok(NOTES_EXTRACTOR_INSTRUCTIONS.includes(EXTRACTION_CORE), "the notes extractor shares the core by construction"); assert.ok(NOTES_EXTRACTOR_INSTRUCTIONS.includes(EXTRACTION_CORE), "the notes extractor shares the core by construction");
// THE ANCHOR RULE (owner, 2026-10-13) is SESSION-ONLY: the exact
// sentence rides the session instructions; the notes
// contextualization never mentions message ids at all.
assert.equal(
SESSION_MESSAGE_ID_RULE,
"For each new proposition, copy the message ID of the passage that most directly states or explains it into messageId.",
);
assert.ok(EXTRACTOR_INSTRUCTIONS.includes(SESSION_MESSAGE_ID_RULE), "the session instructions carry the rule");
assert.ok(!NOTES_EXTRACTOR_INSTRUCTIONS.includes("messageId"), "the notes prompt never mentions message ids");
// The preambles differ in exactly their source description. // The preambles differ in exactly their source description.
assert.notEqual(SESSION_PREAMBLE, NOTES_PREAMBLE); assert.notEqual(SESSION_PREAMBLE, NOTES_PREAMBLE);
assert.match(SESSION_PREAMBLE, /a teaching\s+session, rendered as the teacher's prose in reading order/); assert.match(SESSION_PREAMBLE, /a teaching\s+session, rendered as the teacher's prose in reading order/);
@@ -1431,12 +1646,22 @@ function capturedView(context: unknown): string {
return typeof last?.content === "string" ? last.content : ""; return typeof last?.content === "string" ? last.content : "";
} }
/** An extractor step that submits FINDINGS, optionally capturing its view. */ /** An extractor step that submits FINDINGS, optionally capturing its view.
* The step plays by the anchor law it runs under (2026-10-13): every
* proposition's messageId is copied from a TEACHER [id] label of the
* CURRENT stretch, so multi-batch fixtures reuse this step across batches
* without ever naming a passage their batch does not hold. */
function findingsStep(capture?: (view: string) => void): FauxResponseStep { function findingsStep(capture?: (view: string) => void): FauxResponseStep {
return (context) => { return (context) => {
capture?.(capturedView(context)); const view = capturedView(context);
capture?.(view);
const anchor = /TEACHER \[([^\]]+)\]/.exec(view)?.[1] ?? "a1";
const submit = {
...FINDINGS,
propositions: FINDINGS.propositions.map((p) => ({ ...p, messageId: anchor })),
};
return fauxAssistantMessage( return fauxAssistantMessage(
[fauxToolCall("submit_findings", FINDINGS as unknown as ToolCall["arguments"], { id: "findings-call" })], [fauxToolCall("submit_findings", submit as unknown as ToolCall["arguments"], { id: "findings-call" })],
{ stopReason: "toolUse" }, { stopReason: "toolUse" },
); );
}; };
@@ -1487,7 +1712,8 @@ describe("the chunked pipeline (extraction coverage)", () => {
chatStore, chatStore,
graphStore: ledgers, graphStore: ledgers,
// 4096 budget: the fixture is sized to split in two (the factory // 4096 budget: the fixture is sized to split in two (the factory
// default doubled to 8192, 2026-09-30). // default again — halved back from the digest-era 8192, owner
// 2026-10-13).
getChunkTokens: () => 4096, getChunkTokens: () => 4096,
extractor: fauxEngine([findingsStep((v) => views.push(v)), findingsStep((v) => views.push(v))]), extractor: fauxEngine([findingsStep((v) => views.push(v)), findingsStep((v) => views.push(v))]),
auditor: fauxEngine([passThroughAuditStep(), passThroughAuditStep()]), auditor: fauxEngine([passThroughAuditStep(), passThroughAuditStep()]),
@@ -1501,9 +1727,17 @@ describe("the chunked pipeline (extraction coverage)", () => {
assert.ok(!views[0]!.includes("topic 4:"), "the batch stops at the token budget"); assert.ok(!views[0]!.includes("topic 4:"), "the batch stops at the token budget");
assert.ok(views[1]!.includes("topic 4:") && views[1]!.includes("topic 5:")); assert.ok(views[1]!.includes("topic 4:") && views[1]!.includes("topic 5:"));
assert.ok(!views[1]!.includes("topic 3:")); assert.ok(!views[1]!.includes("topic 3:"));
assert.doesNotMatch(views[0]!, /### /, "plain paragraphs — no speaker headers"); assert.doesNotMatch(views[0]!, /### /, "no speaker headers — passages are TEACHER [id] lines");
// The document header is the session title. // The document header is the session title.
assert.ok(views[0]!.startsWith("# Chunky coverage fixture")); assert.ok(views[0]!.startsWith("# Chunky coverage fixture"));
// MESSAGE-ID LABELS (2026-10-13): every passage carries its REAL
// message id — batch 1 holds a1-a3, batch 2 a4-a5, and the labels
// follow the split exactly.
assert.match(views[0]!, /^TEACHER \[a1\] topic 1:/m);
assert.match(views[0]!, /^TEACHER \[a3\] topic 3:/m);
assert.doesNotMatch(views[0]!, /TEACHER \[a4\]/);
assert.match(views[1]!, /^TEACHER \[a4\] topic 4:/m);
assert.match(views[1]!, /^TEACHER \[a5\] topic 5:/m);
// Batch 1 merged the full findings (verdicts-only: the extractor's // Batch 1 merged the full findings (verdicts-only: the extractor's
// output always fully merges); batch 2's identical ids are a // output always fully merges); batch 2's identical ids are a
@@ -1538,7 +1772,8 @@ describe("the chunked pipeline (extraction coverage)", () => {
chatStore, chatStore,
graphStore: ledgers, graphStore: ledgers,
// 4096 budget: the fixture is sized to split in two (the factory // 4096 budget: the fixture is sized to split in two (the factory
// default doubled to 8192, 2026-09-30). // default again — halved back from the digest-era 8192, owner
// 2026-10-13).
getChunkTokens: () => 4096, getChunkTokens: () => 4096,
// Batch 1 submits and merges; batch 2's extractor call HANGS // Batch 1 submits and merges; batch 2's extractor call HANGS
// until the abort fires — the dock's abort button, mid-run. // until the abort fires — the dock's abort button, mid-run.
@@ -1859,7 +2094,7 @@ describe("digest-aware serialized extraction (the extractor is the assembler)",
}); });
ledgers.appendNode(seed("n_a", "Self-ownership")); ledgers.appendNode(seed("n_a", "Self-ownership"));
const FINDINGS_B: SessionFindings = { const FINDINGS_B: SessionFindings = {
propositions: [{ id: "p1", name: "Appropriation axiom", statement: "c", aliases: [] }], propositions: [{ id: "p1", name: "Appropriation axiom", statement: "c", aliases: [], messageId: "a1" }],
chains: [{ premises: ["n_a", "p1"], conclusion: "p1", }].slice(0, 0), // chains live in the engine below chains: [{ premises: ["n_a", "p1"], conclusion: "p1", }].slice(0, 0), // chains live in the engine below
}; };
const views: string[] = []; const views: string[] = [];
@@ -1871,7 +2106,7 @@ describe("digest-aware serialized extraction (the extractor is the assembler)",
chains: [{ premises: ["n_a", "p1"], conclusion: "p1b" }], chains: [{ premises: ["n_a", "p1"], conclusion: "p1b" }],
}; };
// conclusion p1b must exist — real submit: // conclusion p1b must exist — real submit:
submit.propositions.push({ id: "p1b", name: "Property rights", statement: "d", aliases: [] }); submit.propositions.push({ id: "p1b", name: "Property rights", statement: "d", aliases: [], messageId: "a2" });
return fauxAssistantMessage([fauxToolCall("submit_findings", submit as unknown as ToolCall["arguments"], { id: "f1" })], { stopReason: "toolUse" }); return fauxAssistantMessage([fauxToolCall("submit_findings", submit as unknown as ToolCall["arguments"], { id: "f1" })], { stopReason: "toolUse" });
}, },
]); ]);
@@ -1918,7 +2153,7 @@ describe("digest-aware serialized extraction (the extractor is the assembler)",
fauxToolCall( fauxToolCall(
"submit_findings", "submit_findings",
{ {
propositions: [{ id: "p1", name: "Axiom", statement: "s", aliases: [] }], propositions: [{ id: "p1", name: "Axiom", statement: "s", aliases: [], messageId: "a1" }],
chains: [], chains: [],
} as unknown as ToolCall["arguments"], } as unknown as ToolCall["arguments"],
{ id: "good" }, { id: "good" },
@@ -1944,11 +2179,18 @@ describe("digest-aware serialized extraction (the extractor is the assembler)",
let extractCalls = 0; let extractCalls = 0;
let auditCalls = 0; let auditCalls = 0;
// One step per engine call (faux engines consume steps): two batches. // One step per engine call (faux engines consume steps): two batches.
const extractStep = (): FauxResponseStep => () => { const extractStep = (): FauxResponseStep => (context) => {
extractCalls++; extractCalls++;
events.push(`extract${extractCalls}`); events.push(`extract${extractCalls}`);
// The anchor is copied from the CURRENT stretch's first TEACHER
// label (a1 in batch 1, a4 in batch 2) — batch-local by construction.
const anchor = /TEACHER \[([^\]]+)\]/.exec(capturedView(context))?.[1] ?? "a1";
const submit = {
...FINDINGS,
propositions: FINDINGS.propositions.map((p) => ({ ...p, messageId: anchor })),
};
return fauxAssistantMessage( return fauxAssistantMessage(
[fauxToolCall("submit_findings", FINDINGS as unknown as ToolCall["arguments"], { id: `f${extractCalls}` })], [fauxToolCall("submit_findings", submit as unknown as ToolCall["arguments"], { id: `f${extractCalls}` })],
{ stopReason: "toolUse" }, { stopReason: "toolUse" },
); );
}; };
@@ -1967,7 +2209,7 @@ describe("digest-aware serialized extraction (the extractor is the assembler)",
graphStore: ledgers, graphStore: ledgers,
extractor, extractor,
auditor, auditor,
getChunkTokens: () => 4096, // two batches over the chunky fixture (factory default doubled to 8192, 2026-09-30) getChunkTokens: () => 4096, // two batches over the chunky fixture (the factory default, 4096 — halved back 2026-10-13)
}); });
const stats = await pipeline.runExtraction(chatId); const stats = await pipeline.runExtraction(chatId);
assert.deepEqual(events, ["extract1", "audit1", "extract2", "audit2"], "strict per-source order: extract → audit → merge → NEXT extract"); assert.deepEqual(events, ["extract1", "audit1", "extract2", "audit2"], "strict per-source order: extract → audit → merge → NEXT extract");
+58 -3
View File
@@ -290,9 +290,10 @@ describe("the merger (mergeFindings)", () => {
assert.equal("kind" in node, false, "no kind field"); assert.equal("kind" in node, false, "no kind field");
assert.ok(Array.isArray(node.provenance), "provenance attached"); assert.ok(Array.isArray(node.provenance), "provenance attached");
} }
// Provenance is CHAT-LEVEL (submit-schema slim, 2026-09-28): // Provenance without an anchor stays CHAT-LEVEL: this caller submits
// propositions no longer carry message ids — the run's source key // no messageId (the legacy/core + notes shape — anchors are the
// is the whole provenance. // session door's alone, 2026-10-13), so the run's source key is the
// whole provenance point.
const alpha = graph.nodes.find((n) => n.statement === "Alpha intuition")!; const alpha = graph.nodes.find((n) => n.statement === "Alpha intuition")!;
assert.deepEqual(alpha.provenance, [{ chatId: chat1.id, messageId: "" }]); assert.deepEqual(alpha.provenance, [{ chatId: chat1.id, messageId: "" }]);
@@ -306,6 +307,60 @@ describe("the merger (mergeFindings)", () => {
}); });
it("messageId anchors (2026-10-13): creates AND link-appends stamp the passage id; chains stay chat-level", () => {
// The session submit's anchor (owner, 2026-10-13): a proposition may
// carry its passage's message id — the merger stamps it into the
// node's provenance point for BOTH births and re-encounters. A
// caller without one (notes, legacy/core — pinned above) keeps "".
const chat = fixtureChat(
"Anchor fixture",
[
...user("u1", "teach anchors"),
...teacher("t1", "First anchored passage."),
...teacher("t2", "Second anchored passage."),
].flat(),
);
const first = mergeFindings(
chat.id,
{
propositions: [{ id: "p1", name: "Anchored", statement: "Anchored statement", aliases: [], messageId: "t1" }],
chains: [],
},
{ links: [] },
graphDeps,
sessionSource(chat.id),
);
assert.equal(first.nodes, 1);
const created = scopedGraph([chat.id]).nodes.find((n) => n.statement === "Anchored statement")!;
assert.deepEqual(created.provenance, [{ chatId: chat.id, messageId: "t1" }], "the created node anchors to its passage");
// The re-encounter: link-appended, anchored to ITS passage — while
// the chain the same job asserts stays chat-level.
const second = mergeFindings(
chat.id,
{
propositions: [
{ id: "q1", name: "Anchored again", statement: "Anchored statement, rephrased", aliases: [], messageId: "t2" },
{ id: "q2", name: "Follows", statement: "Follows the anchored claim", aliases: [], messageId: "t2" },
],
chains: [{ premises: ["q1"], conclusion: "q2" }],
},
{ links: [{ id: "q1", linkTo: created.id }] },
graphDeps,
sessionSource(chat.id),
);
assert.equal(second.nodes, 1, "q1 linked (no create), q2 created");
const linkLine = graphStore
.readNodes()
.filter((n) => n.id === created.id)
.find((line) => line.provenance.some((pt) => pt.chatId === chat.id && pt.messageId === "t2"));
assert.ok(linkLine, "the link-append line carries the re-encounter's passage id");
assert.equal(linkLine!.name, "Anchored again", "the appended line is the re-encounter (reads merge it into aliases)");
const chains = graphStore.readChains().filter((c) => c.provenance.some((pt) => pt.chatId === chat.id));
assert.equal(chains.length, 1);
assert.deepEqual(chains[0]!.provenance, [{ chatId: chat.id, messageId: "" }], "chains stay chat-level — never anchored");
});
it("merges INDEPENDENTLY per call — no cross-run identity (owner, 2026-09-29)", () => { it("merges INDEPENDENTLY per call — no cross-run identity (owner, 2026-09-29)", () => {
// The mappings ledger is gone: resolution is a job-local in-memory // The mappings ledger is gone: resolution is a job-local in-memory
// map, so a second merge of the same verdicts knows nothing of the // map, so a second merge of the same verdicts knows nothing of the
+9 -7
View File
@@ -40,8 +40,10 @@ describe("fetchGraphRead — one truth with the extraction trigger (2026-10-02)"
// effectiveSettings gate waits for. The old hand-rolled fallback // effectiveSettings gate waits for. The old hand-rolled fallback
// read the raw delta and guessed `?? 4096` — a stale pre-setting-era // read the raw delta and guessed `?? 4096` — a stale pre-setting-era
// constant — so an unset budget filled the ring at half the truth // constant — so an unset budget filled the ring at half the truth
// (owner repro: 6k shown full, extraction correctly silent at 8192). // (owner repro: 6k shown full, extraction correctly silent at the then-
// Now the raw delta resolves through the SAME core resolver. // factory 8192). The 2026-10-13 ruling restored the factory to 4096 (more
// frequent graph updates); the LAW is unchanged — the denominator is the
// factory the same core resolver serves, never a hand-rolled constant.
const stub = (payload: unknown, ok = true) => { const stub = (payload: unknown, ok = true) => {
const real = globalThis.fetch; const real = globalThis.fetch;
globalThis.fetch = (async () => globalThis.fetch = (async () =>
@@ -51,25 +53,25 @@ describe("fetchGraphRead — one truth with the extraction trigger (2026-10-02)"
}; };
}; };
it("an absent graph section denominates at the factory 8192 — never 4096, never a missing ring", async () => { it("an absent graph section denominates at the factory 4096 — never a hand-rolled constant, never a missing ring", async () => {
const restore = stub({ settings: { version: 1 } }); const restore = stub({ settings: { version: 1 } });
const read = await fetchGraphRead(); const read = await fetchGraphRead();
restore(); restore();
assert.deepEqual(read, { enabled: true, chunkTokens: 8192 }); assert.deepEqual(read, { enabled: true, chunkTokens: 4096 });
}); });
it("the owner repro shape — graph section present, budget never written — is 8192", async () => { it("the owner repro shape — graph section present, budget never written — is the factory", async () => {
const restore = stub({ settings: { version: 1, graph: { enabled: true } } }); const restore = stub({ settings: { version: 1, graph: { enabled: true } } });
const read = await fetchGraphRead(); const read = await fetchGraphRead();
restore(); restore();
assert.equal(read?.chunkTokens, 8192, "the record lacks the key: factory, not the stale 4096"); assert.equal(read?.chunkTokens, 4096, "the record lacks the key: factory, not a stale constant");
}); });
it("first boot (settings null) resolves factory — the ring shows, at the true budget", async () => { it("first boot (settings null) resolves factory — the ring shows, at the true budget", async () => {
const restore = stub({ settings: null }); const restore = stub({ settings: null });
const read = await fetchGraphRead(); const read = await fetchGraphRead();
restore(); restore();
assert.deepEqual(read, { enabled: true, chunkTokens: 8192 }); assert.deepEqual(read, { enabled: true, chunkTokens: 4096 });
}); });
it("a custom budget rides through verbatim", async () => { it("a custom budget rides through verbatim", async () => {
+3 -3
View File
@@ -203,9 +203,9 @@ describe("GraphSection — master switch + chunk budget (wave 7 + 2026-10-02)",
it("the chunk-budget row: unset shows EMPTY with the factory placeholder + hint — one truth with the trigger", () => { it("the chunk-budget row: unset shows EMPTY with the factory placeholder + hint — one truth with the trigger", () => {
const html = renderToStaticMarkup(<GraphSection graph={undefined} onSectionChange={() => {}} />); const html = renderToStaticMarkup(<GraphSection graph={undefined} onSectionChange={() => {}} />);
const input = tagOf(html, "input-chunk-tokens"); const input = tagOf(html, "input-chunk-tokens");
assert.ok(input.includes('placeholder="8192"'), "the placeholder IS the resolved factory default"); assert.ok(input.includes('placeholder="4096"'), "the placeholder IS the resolved factory default");
assert.ok(!/value="\d/.test(input), "unset renders EMPTY, not a materialized 8192"); assert.ok(!/value="\d/.test(input), "unset renders EMPTY, not a materialized 4096");
assert.ok(html.includes(">8192 tok</"), "the hint reports the resolved truth"); assert.ok(html.includes(">4096 tok</"), "the hint reports the resolved truth");
}); });
it("a stored budget shows its value and denominates the hint", () => { it("a stored budget shows its value and denominates the hint", () => {
+5 -4
View File
@@ -43,8 +43,9 @@ describe("resolveSettings", () => {
assert.equal(eff.appearance.sessionNameSerif, true); assert.equal(eff.appearance.sessionNameSerif, true);
assert.equal(eff.appearance.mdRendering.headingWeight, 900); assert.equal(eff.appearance.mdRendering.headingWeight, 900);
// wave 7 collapse: the master switch ONLY + the redesign's chunk // wave 7 collapse: the master switch ONLY + the redesign's chunk
// budget — absent record resolves the factory 8192 (the 2026-09-30 bump) // budget — absent record resolves the factory 4096 (halved back
assert.deepEqual(eff.graph, { enabled: true, rendering: true, extractionChunkTokens: 8192 }); // from the digest-era 8192, owner, 2026-10-13)
assert.deepEqual(eff.graph, { enabled: true, rendering: true, extractionChunkTokens: 4096 });
}); });
it("explicit record values win over every factory default", () => { it("explicit record values win over every factory default", () => {
@@ -76,8 +77,8 @@ describe("resolveSettings", () => {
assert.equal(eff.appearance.proseFontSize, 20); assert.equal(eff.appearance.proseFontSize, 20);
assert.equal(eff.appearance.mdRendering.headingWeight, 600); assert.equal(eff.appearance.mdRendering.headingWeight, 600);
// wave 7 collapse: the section IS the master switch (absent budget // wave 7 collapse: the section IS the master switch (absent budget
// field = factory 8192 — a pre-redesign record resolves cleanly). // field = factory 4096 — a pre-redesign record resolves cleanly).
assert.deepEqual(eff.graph, { enabled: false, rendering: true, extractionChunkTokens: 8192 }); assert.deepEqual(eff.graph, { enabled: false, rendering: true, extractionChunkTokens: 4096 });
// The DISPLAY split (owner, 2026-10-06): extraction off leaves the // The DISPLAY split (owner, 2026-10-06): extraction off leaves the
// rendering switch ON — the data channel and the view are separate. // rendering switch ON — the data channel and the view are separate.
assert.equal(renderingEnabled(eff.graph), true); assert.equal(renderingEnabled(eff.graph), true);
+4 -3
View File
@@ -22,9 +22,10 @@ describe("FACTORY_SETTINGS (the compiled factory layer)", () => {
assert.equal(CURRENT_SETTINGS_VERSION, 1); assert.equal(CURRENT_SETTINGS_VERSION, 1);
assert.deepEqual(FACTORY_SETTINGS.general, { autoUpdate: true }); assert.deepEqual(FACTORY_SETTINGS.general, { autoUpdate: true });
// wave 7 collapse: the master switch ONLY + the extraction-UX // wave 7 collapse: the master switch ONLY + the extraction-UX
// redesign's chunk budget (2024-09-24); default doubled 8192 with // redesign's chunk budget (2026-09-24); doubled to 8192 with the
// the digest-aware redesign (owner, 2026-09-30) // digest-aware redesign (2026-09-30), then HALVED BACK to 4096
assert.deepEqual(FACTORY_SETTINGS.graph, { enabled: true, rendering: true, extractionChunkTokens: 8192 }); // with the message anchors (owner, 2026-10-13)
assert.deepEqual(FACTORY_SETTINGS.graph, { enabled: true, rendering: true, extractionChunkTokens: 4096 });
const a = FACTORY_SETTINGS.appearance; const a = FACTORY_SETTINGS.appearance;
assert.equal(a.mode, "dark"); assert.equal(a.mode, "dark");
assert.equal(a.background, null); // the mode's own default, not a factory hex assert.equal(a.background, null); // the mode's own default, not a factory hex