feat: extraction coverage — message-granular ledger + token-batched runs (d132373 vision)
THE MESSAGE IS THE UNIT OF COVERAGE (vision-extraction-coverage.md): a per-chat append-only ledger (coverage.jsonl) records which messages the pipeline has extracted, by content fingerprint over exactly what the extraction view sees (one shared line builder — core/coverage.ts); the uncovered remainder is DERIVED at read time, never dual-tracked. A quiz answer merging into the asking message re-opens only that message; plan-block data-part attach never re-opens anything. - The trigger (turn-service) replaces every-3-quiz-answers with uncovered-tokens >= EXTRACTION.chunkTokens (4096) — quiz-less sessions extract too, content size is the honest signal. - runExtraction is now chunked: token-budgeted batches of the oldest uncovered consecutive messages (a tool call + result never split), each merged batch checkpointing its messages' coverage atomically — an abort or error loses at most the in-flight batch; merged batches stay covered. - Observation refs are batch-tagged (obs:<batchTag>:<sourceId>): re-runs of the same batch dedupe; a re-opened message's fresh understanding lands as a new event. Fixes the old obs:p1 position-collision. - New API: GET /api/graph/coverage (exact per-session derived coverage + running flags, most-uncovered first) and POST /api/graph/coverage/extract (manual run for one session, 409 while in flight). - Graph view chrome: the coverage button opens a panel with every session's covered/uncovered tokens and a per-session Extract button — the manual answer to aborted runs, errors, and dormant tails. Renders even when the graph is empty: an empty graph with uncovered sessions is exactly when it matters. Polls only while open. - Notes imports keep the whole-file checkpoint path (coverage undefined). - Tests: pure-core coverage suite (11), chunked-pipeline suite (3: multi-batch + no-op, abort-keeps-earlier-batches + remainder re-run, re-open alone + data-part invisibility), panel render suite (5); pendingQuizAnswers and its tests deleted as dead code. 486/486. Live-smoked on real GLM: manual extract of "teach me modal realism" ran one batch, wrote 4 coverage entries, panel flipped to "1.3k covered · complete" with the button disabled, graph refetched (nodes 11), usage persisted with real stats.
This commit is contained in:
amosblomqvist
@@ -18,6 +18,8 @@ import type {
|
||||
import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
|
||||
import type { GraphStore } from "../../core/ports.ts";
|
||||
import { settleGraphLayout } from "../../core/layout-settle.ts";
|
||||
import { batchRuns, batchTag, coverageOf, messageViewLines } from "../../core/coverage.ts";
|
||||
import { EXTRACTION } from "../../../config.ts";
|
||||
import { refreshClusterLabels } from "./cluster-namer.ts";
|
||||
|
||||
/**
|
||||
@@ -71,15 +73,6 @@ import { refreshClusterLabels } from "./cluster-namer.ts";
|
||||
* (question text + what the learner did — kind/correct are rebuilt by the
|
||||
* merger; the extractor only needs enough to attribute.)
|
||||
*/
|
||||
function quizLine(part: { toolCallId: string; input: unknown; output: unknown }): string {
|
||||
const input = part.input as { question?: unknown } | undefined;
|
||||
const out = part.output as Partial<{ questionText: string; correct: boolean; dontKnow: boolean; note: string }> | undefined;
|
||||
const question = typeof input?.question === "string" ? input.question : out?.questionText ?? "(no question)";
|
||||
const outcome = out?.dontKnow === true ? "did not know" : out?.correct === true ? "answered correctly" : "answered incorrectly";
|
||||
const note = typeof out?.note === "string" && out.note ? ` (note: ${out.note})` : "";
|
||||
return `QUIZ [${part.toolCallId}] ${question} — the learner ${outcome}${note}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The compact transcript both LLM stages see: user/teacher text VERBATIM,
|
||||
* each tagged with its message id (provenance anchors — the extractor
|
||||
@@ -90,25 +83,12 @@ function quizLine(part: { toolCallId: string; input: unknown; output: unknown })
|
||||
* are summarized as one line each.
|
||||
*/
|
||||
export function transcriptView(record: ChatRecord): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`SESSION: ${record.title}`);
|
||||
|
||||
// The shared seam (core/coverage.ts messageViewLines): the LLM's prompt
|
||||
// view and the coverage fingerprint use ONE line builder, so "covered"
|
||||
// means — exactly — what the extractor saw.
|
||||
const lines = [`SESSION: ${record.title}`];
|
||||
record.messages.forEach((message, index) => {
|
||||
const who = message.role === "user" ? "LEARNER" : "TEACHER";
|
||||
const mid = (message as UIMessage).id || `m${index}`;
|
||||
for (const part of (message as UIMessage).parts) {
|
||||
if (part.type === "text" && part.text.trim()) {
|
||||
lines.push(`${who} [${mid}] ${part.text.trim()}`);
|
||||
} else if (part.type.startsWith("tool-")) {
|
||||
const name = part.type.slice("tool-".length);
|
||||
const p = part as unknown as { toolCallId: string; state: string; input: unknown; output: unknown };
|
||||
if (name === "quiz" && p.state === "output-available") {
|
||||
lines.push(quizLine(p));
|
||||
} else if (p.state === "output-available") {
|
||||
lines.push(`(${name} tool result delivered)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
lines.push(...messageViewLines(message as UIMessage, (message as UIMessage).id || `m${index}`));
|
||||
});
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -623,12 +603,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
};
|
||||
}
|
||||
|
||||
async function extract(chatId: string): Promise<SessionFindings> {
|
||||
const record = chatStore.getChat(chatId);
|
||||
if (!record) throw new Error(`Chat not found: ${chatId}`);
|
||||
async function extract(chatId: string, view: string): Promise<SessionFindings> {
|
||||
const signal = runningExtractions.get(chatId)?.controller.signal;
|
||||
signal?.throwIfAborted(); // an abort during a sync gap ends the run here
|
||||
const view = transcriptView(record);
|
||||
console.info(`[extraction ${chatId}] extractor starting (${view.split("\n").length} transcript lines)`);
|
||||
const t0 = performance.now();
|
||||
|
||||
@@ -645,12 +622,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
return findings;
|
||||
}
|
||||
|
||||
async function audit(chatId: string, findings: SessionFindings): Promise<AuditResult> {
|
||||
const record = chatStore.getChat(chatId);
|
||||
if (!record) throw new Error(`Chat not found: ${chatId}`);
|
||||
async function audit(chatId: string, findings: SessionFindings, view: string): Promise<AuditResult> {
|
||||
const signal = runningExtractions.get(chatId)?.controller.signal;
|
||||
signal?.throwIfAborted(); // an abort during the extract→audit gap ends here
|
||||
const view = transcriptView(record);
|
||||
const graph = getGraph(graphStore);
|
||||
console.info(
|
||||
`[extraction ${chatId}] auditor starting (${findings.propositions.length} propositions, ${findings.edges.length} edges, ` +
|
||||
@@ -708,21 +682,66 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
if (inFlight.has(chatId)) return { nodes: 0, edges: 0, events: 0 };
|
||||
inFlight.add(chatId);
|
||||
const t0 = performance.now();
|
||||
// COVERAGE (vision-extraction-coverage.md, 2026-09-15): extract ONLY
|
||||
// what the per-chat coverage ledger doesn't already hold — never the
|
||||
// whole session. Batches are token-budgeted runs of the oldest
|
||||
// uncovered messages; each merged batch checkpoints its messages'
|
||||
// coverage ATOMICALLY (inside the merger's step 4), so an abort or
|
||||
// error below loses at most the in-flight batch — every earlier
|
||||
// batch stays covered, the rest stays visibly uncovered.
|
||||
let batchesDone = 0;
|
||||
const total: EmissionStats = { nodes: 0, edges: 0, events: 0 };
|
||||
try {
|
||||
console.info(`[extraction ${chatId}] pipeline start`);
|
||||
setPhase(chatId, "extracting", "chat");
|
||||
const findings = await extract(chatId);
|
||||
setPhase(chatId, "auditing", "chat");
|
||||
const auditResult = await audit(chatId, findings);
|
||||
setPhase(chatId, "merging", "chat");
|
||||
// The merger merges from the CURRENT record — events are rebuilt
|
||||
// from the live transcript, never from stale copies.
|
||||
const record = chatStore.getChat(chatId);
|
||||
if (!record) throw new Error(`Chat not found: ${chatId}`);
|
||||
const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }, {
|
||||
messages: record.messages as UIMessage[],
|
||||
foldedAt: record.updatedAt,
|
||||
});
|
||||
const covered = coverageOf(record.messages as UIMessage[], graphStore.readCoverage(), chatId);
|
||||
if (covered.uncovered.length === 0) {
|
||||
console.info(`[extraction ${chatId}] nothing uncovered — no-op`);
|
||||
return { nodes: 0, edges: 0, events: 0 };
|
||||
}
|
||||
const batches = batchRuns(covered.uncovered, EXTRACTION.chunkTokens);
|
||||
console.info(
|
||||
`[extraction ${chatId}] pipeline start — ${covered.uncoveredTokens} uncovered tok, ` +
|
||||
`${covered.uncoveredCount} messages, ${batches.length} batch(es) of ≤${EXTRACTION.chunkTokens} tok`,
|
||||
);
|
||||
for (const batch of batches) {
|
||||
// The batch's prompt view: SESSION header + the batch's lines,
|
||||
// same provenance [id] tags the fingerprints used (coverage
|
||||
// and prompt share ONE line builder — core/coverage.ts).
|
||||
const mids = record.messages.map((m, i) => ((m as UIMessage).id || `m${i}`) as string);
|
||||
const view = [
|
||||
`SESSION: ${record.title}`,
|
||||
...batch.flatMap((slot) => {
|
||||
const idx = record.messages.indexOf(slot.message);
|
||||
return messageViewLines(slot.message, mids[idx] ?? slot.message.id);
|
||||
}),
|
||||
].join("\n");
|
||||
setPhase(chatId, "extracting", "chat");
|
||||
const findings = await extract(chatId, view);
|
||||
setPhase(chatId, "auditing", "chat");
|
||||
const auditResult = await audit(chatId, findings, view);
|
||||
setPhase(chatId, "merging", "chat");
|
||||
// The batch's messages, snapshotted when the run computed
|
||||
// coverage — if a message mutates mid-run (a quiz answer
|
||||
// merges in), the stored fingerprint won't match its new
|
||||
// one and it re-opens: honest by construction.
|
||||
const foldedAt = new Date().toISOString();
|
||||
const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }, {
|
||||
messages: batch.map((slot) => slot.message),
|
||||
foldedAt,
|
||||
coverage: batch.map((slot) => ({ chatId, fp: slot.fp, tokens: slot.tokens, foldedAt })),
|
||||
batchTag: batchTag(batch),
|
||||
});
|
||||
total.nodes += stats.nodes;
|
||||
total.edges += stats.edges;
|
||||
total.events += stats.events;
|
||||
batchesDone++;
|
||||
console.info(
|
||||
`[extraction ${chatId}] batch ${batchesDone}/${batches.length} merged: ` +
|
||||
`${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events`,
|
||||
);
|
||||
}
|
||||
const stats = total;
|
||||
// Semantic embeddings (DECIDED 2026-09-13): topics come from
|
||||
// MEANING, so embed the new propositions BEFORE the settle and
|
||||
// the namer look at clusters. Failure is non-fatal — clustering
|
||||
@@ -747,18 +766,23 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
}).catch(() => {});
|
||||
return stats;
|
||||
} catch (err) {
|
||||
persistRunUsage(chatId, Math.round(performance.now() - t0), undefined);
|
||||
persistRunUsage(chatId, Math.round(performance.now() - t0), batchesDone > 0 ? total : undefined);
|
||||
// Aborted (the dock's abort button): a deliberate stop, not a
|
||||
// failure — same ledger discipline as a failure (nothing merged,
|
||||
// nothing checkpointed, partial usage persisted), but the run
|
||||
// reports zero emissions instead of throwing.
|
||||
// failure. Per-batch coverage (above) makes the stop CHEAP: every
|
||||
// merged batch stays covered; only the in-flight one doesn't — it
|
||||
// stays uncovered, and the next trigger or manual run picks it
|
||||
// up. Partial usage is persisted either way.
|
||||
if (err instanceof Error && err.name === "AbortError") {
|
||||
console.info(`[extraction ${chatId}] aborted (${Math.round(performance.now() - t0)}ms) — nothing merged, nothing checkpointed`);
|
||||
return { nodes: 0, edges: 0, events: 0 };
|
||||
console.info(
|
||||
`[extraction ${chatId}] aborted (${Math.round(performance.now() - t0)}ms) — ` +
|
||||
`${batchesDone} merged batch(es) stay covered, the in-flight one does not`,
|
||||
);
|
||||
return total;
|
||||
}
|
||||
// The throw travels to the caller; this line is what makes the
|
||||
// run's log legible end-to-end (no hanging "pipeline start").
|
||||
console.error(`[extraction ${chatId}] pipeline failed (${Math.round(performance.now() - t0)}ms):`, err);
|
||||
// A failure keeps the same discipline: merged batches stay
|
||||
// covered, nothing else is checkpointed, and the throw travels
|
||||
// to the caller (this line keeps the run's log legible).
|
||||
console.error(`[extraction ${chatId}] pipeline failed (${Math.round(performance.now() - t0)}ms) after ${batchesDone} batch(es):`, err);
|
||||
throw err;
|
||||
} finally {
|
||||
inFlight.delete(chatId);
|
||||
|
||||
@@ -116,6 +116,24 @@ export interface SessionCheckpoint {
|
||||
foldedAt: string;
|
||||
}
|
||||
|
||||
/** coverage.jsonl line (vision-extraction-coverage.md, settled 2026-09-15):
|
||||
* ONE MESSAGE of one chat has been extracted — by content fingerprint over
|
||||
* exactly what the extraction view sees. The MESSAGE is the unit of
|
||||
* coverage; "uncovered" is derived (fingerprint the current transcript,
|
||||
* diff) — never dual-tracked. A quiz answer merging into the asking
|
||||
* message changes its fingerprint: the message re-opens itself, only
|
||||
* itself. Chat extractions write these instead of whole-chat checkpoints;
|
||||
* notes imports keep the checkpoint (a file is one shot). */
|
||||
export interface CoverageEntry {
|
||||
chatId: string;
|
||||
/** sha256 of the message's view lines — "covered" means exactly this. */
|
||||
fp: string;
|
||||
/** chars/4 estimate of the message's view — display + budgeting. */
|
||||
tokens: number;
|
||||
/** ISO — when this message's last extraction run merged. */
|
||||
foldedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Event kinds (DECIDED 2026-09-02): quiz outcomes only — correct / miss /
|
||||
* dont-know — plus `observation`, the extractor's per-proposition
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Extraction coverage (vision-extraction-coverage.md, settled 2026-09-15) —
|
||||
* the pure core. THE MESSAGE IS THE UNIT OF COVERAGE: a per-chat append-only
|
||||
* ledger records which messages have been extracted, by CONTENT FINGERPRINT
|
||||
* over exactly what the extraction view sees; "uncovered" is DERIVED at
|
||||
* read time (fingerprint the current transcript, diff against the ledger) —
|
||||
* never dual-tracked. Runs are token-budgeted batches of the oldest
|
||||
* uncovered messages, so a tool call and its result (which share a message)
|
||||
* can never be separated.
|
||||
*
|
||||
* The fingerprint and the LLM prompt share ONE line builder —
|
||||
* messageViewLines — so "covered" and "what the model saw" cannot drift
|
||||
* apart. The line logic is tree-blind, identical to the extractor's view:
|
||||
* user/teacher text verbatim, quiz outcomes as one-line summaries, other
|
||||
* tool results as one generic line each, data parts invisible (plan-block
|
||||
* attach does not re-open a covered message).
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import type { UIMessage } from "ai";
|
||||
import { isToolUIPart } from "ai";
|
||||
|
||||
import type { CoverageEntry } from "./contracts/graph.ts";
|
||||
|
||||
export type { CoverageEntry };
|
||||
|
||||
/** The quiz outcome line — both the fingerprint and the LLM view see the
|
||||
* quiz answer, so an answered quiz re-opens the message that asked it. */
|
||||
function quizLine(part: { toolCallId: string; input: unknown; output: unknown }): string {
|
||||
const input = part.input as { question?: unknown } | undefined;
|
||||
const out = part.output as Partial<{ questionText: string; correct: boolean; dontKnow: boolean; note: string }> | undefined;
|
||||
const question = typeof input?.question === "string" ? input.question : out?.questionText ?? "(no question)";
|
||||
const outcome = out?.dontKnow === true ? "did not know" : out?.correct === true ? "answered correctly" : "answered incorrectly";
|
||||
const note = typeof out?.note === "string" && out.note ? ` (note: ${out.note})` : "";
|
||||
return `QUIZ [${part.toolCallId}] ${question} — the learner ${outcome}${note}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The extraction-relevant lines of ONE message — the shared seam between
|
||||
* transcriptView (the LLM's prompt) and messageFingerprint (coverage). Same
|
||||
* logic, same order, so a covered message is by construction exactly what
|
||||
* the extractor saw. `mid` is the provenance tag (the message id; the
|
||||
* transcript view passes an index fallback for legacy id-less messages,
|
||||
* coverage passes the id — persisted messages always have ids).
|
||||
*/
|
||||
export function messageViewLines(message: UIMessage, mid: string): string[] {
|
||||
const lines: string[] = [];
|
||||
const who = message.role === "user" ? "LEARNER" : "TEACHER";
|
||||
for (const part of message.parts) {
|
||||
if (part.type === "text" && part.text.trim()) {
|
||||
lines.push(`${who} [${mid}] ${part.text.trim()}`);
|
||||
} else if (isToolUIPart(part)) {
|
||||
const name = part.type.slice("tool-".length);
|
||||
if (name === "quiz" && part.state === "output-available") {
|
||||
lines.push(quizLine(part as unknown as { toolCallId: string; input: unknown; output: unknown }));
|
||||
} else if (part.state === "output-available") {
|
||||
lines.push(`(${name} tool result delivered)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Tokens of a message's view — chars/4, ceil, the app-wide estimate. */
|
||||
export function viewTokens(lines: string[]): number {
|
||||
return Math.ceil(lines.join("\n").length / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* A message's coverage fingerprint: sha256 of role + its view lines (with
|
||||
* the message's real id as the provenance tag). Changes exactly when the
|
||||
* extraction view of the message changes: an answered quiz merges into the
|
||||
* asking message → new line → the message re-opens itself, and only itself.
|
||||
* Plan-block part attach changes data parts (invisible to the view) → the
|
||||
* fingerprint is stable → still covered.
|
||||
*/
|
||||
export function messageFingerprint(message: UIMessage): string {
|
||||
const h = createHash("sha256");
|
||||
h.update(JSON.stringify([message.role, messageViewLines(message, message.id)]));
|
||||
return h.digest("hex");
|
||||
}
|
||||
|
||||
/** One message's coverage-relevant projection. */
|
||||
export interface CoverageSlot {
|
||||
message: UIMessage;
|
||||
fp: string;
|
||||
tokens: number;
|
||||
}
|
||||
|
||||
/** Fingerprint a whole transcript, in order (used by the trigger, the run,
|
||||
* and the coverage view — one derivation everywhere). */
|
||||
export function coverageSlots(messages: ReadonlyArray<UIMessage>): CoverageSlot[] {
|
||||
return messages.map((message) => {
|
||||
const lines = messageViewLines(message, message.id);
|
||||
const h = createHash("sha256");
|
||||
h.update(JSON.stringify([message.role, lines]));
|
||||
return { message, fp: h.digest("hex"), tokens: viewTokens(lines) };
|
||||
});
|
||||
}
|
||||
|
||||
/** The covered-fingerprint set for one chat, from the ledger. */
|
||||
export function coveredSet(entries: ReadonlyArray<CoverageEntry>, chatId: string): Set<string> {
|
||||
const out = new Set<string>();
|
||||
for (const e of entries) if (e.chatId === chatId) out.add(e.fp);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The derived state: which slots are uncovered, plus the token totals.
|
||||
* "Uncovered" needs no ledger of its own — this diff IS the coverage view. */
|
||||
export function coverageOf(
|
||||
messages: ReadonlyArray<UIMessage>,
|
||||
entries: ReadonlyArray<CoverageEntry>,
|
||||
chatId: string,
|
||||
): {
|
||||
slots: CoverageSlot[];
|
||||
uncovered: CoverageSlot[];
|
||||
coveredTokens: number;
|
||||
uncoveredTokens: number;
|
||||
coveredCount: number;
|
||||
uncoveredCount: number;
|
||||
} {
|
||||
const covered = coveredSet(entries, chatId);
|
||||
const slots = coverageSlots(messages);
|
||||
const uncovered = slots.filter((s) => !covered.has(s.fp));
|
||||
const coveredCount = slots.length - uncovered.length;
|
||||
let coveredTokens = 0;
|
||||
let uncoveredTokens = 0;
|
||||
for (const s of slots) {
|
||||
if (covered.has(s.fp)) coveredTokens += s.tokens;
|
||||
else uncoveredTokens += s.tokens;
|
||||
}
|
||||
return { slots, uncovered, coveredTokens, uncoveredTokens, coveredCount, uncoveredCount: uncovered.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch the uncovered messages into token-budgeted RUNS: consecutive slots,
|
||||
* oldest first, greedy fill — a message is never split (it is the unit), so
|
||||
* a tool call and its result always travel together. A single message larger
|
||||
* than the budget becomes its own (over-budget) run rather than being
|
||||
* dropped. Runs are ephemeral — the durable record is per-message coverage,
|
||||
* checkpointed after each run merges.
|
||||
*/
|
||||
export function batchRuns(uncovered: ReadonlyArray<CoverageSlot>, budget: number): CoverageSlot[][] {
|
||||
if (budget <= 0 || uncovered.length === 0) return [];
|
||||
const runs: CoverageSlot[][] = [];
|
||||
let current: CoverageSlot[] = [];
|
||||
let currentTokens = 0;
|
||||
for (const slot of uncovered) {
|
||||
if (current.length > 0 && currentTokens + slot.tokens > budget) {
|
||||
runs.push(current);
|
||||
current = [];
|
||||
currentTokens = 0;
|
||||
}
|
||||
current.push(slot);
|
||||
currentTokens += slot.tokens;
|
||||
}
|
||||
if (current.length > 0) runs.push(current);
|
||||
return runs;
|
||||
}
|
||||
|
||||
/**
|
||||
* A batch's deterministic tag: hash of its member fingerprints. This is the
|
||||
* observation-ref prefix for the run (obs:<tag>:<sourceId> in the merger):
|
||||
* re-running the SAME batch (crash recovery) produces the same refs and
|
||||
* dedupes; a re-opened (mutated) message changes the tag, so its fresh
|
||||
* understanding observations land as new events — the most-recent fold
|
||||
* sees the update. Run-local source ids ("p1"…) alone could never do this:
|
||||
* their positions shift between runs.
|
||||
*/
|
||||
export function batchTag(batch: ReadonlyArray<CoverageSlot>): string {
|
||||
const h = createHash("sha256");
|
||||
for (const slot of batch) h.update(slot.fp);
|
||||
return h.digest("hex").slice(0, 12);
|
||||
}
|
||||
+37
-38
@@ -40,6 +40,7 @@
|
||||
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { isToolUIPart, type UIMessage } from "ai";
|
||||
import type { CoverageEntry } from "./contracts/graph.ts";
|
||||
import type {
|
||||
AuditResult,
|
||||
Cluster,
|
||||
@@ -67,14 +68,29 @@ function newGraphNodeId(): string {
|
||||
}
|
||||
|
||||
/** What one merge run merges FROM (the caller supplies it — the merger never
|
||||
* reads the chat store): a session's transcript + its `updatedAt`
|
||||
* checkpoint, or — for the format-agnostic notes front door (2026-09-13) —
|
||||
* `messages: null`: raw notes carry no tool parts, so no quiz events are
|
||||
* rebuilt (quiz outcomes inside notes surface only through the extractor's
|
||||
* understanding observations), and the checkpoint stamps the import. */
|
||||
* reads the chat store): a session's transcript + its coverage checkpoint,
|
||||
* or — for the format-agnostic notes front door (2026-09-13) — `messages:
|
||||
* null`: raw notes carry no tool parts, so no quiz events are rebuilt (quiz
|
||||
* outcomes inside notes surface only through the extractor's understanding
|
||||
* observations), and the whole-file checkpoint stamps the import.
|
||||
*
|
||||
* Coverage (vision-extraction-coverage.md, settled 2026-09-15): a chat merge
|
||||
* passes the BATCH it extracted — the caller computed the fingerprints
|
||||
* before the run, from the same snapshot the merger sees — and step 4
|
||||
* appends one CoverageEntry per message on success ("covered" means
|
||||
* exactly this merge happened). `batchTag` prefixes the observation refs so
|
||||
* re-runs of the SAME batch dedupe while a re-opened (mutated) message's
|
||||
* fresh understanding observations land as new events; the notes path
|
||||
* passes neither and keeps the whole-file checkpoint. */
|
||||
export interface MergeSource {
|
||||
messages: UIMessage[] | null;
|
||||
foldedAt: string;
|
||||
/** The batch being merged (chat path): one entry per message in
|
||||
* `messages`, same order — undefined (notes path / coverage-less
|
||||
* callers) → the legacy whole-chat checkpoint is written instead. */
|
||||
coverage?: CoverageEntry[];
|
||||
/** The batch's deterministic tag (core/coverage.ts batchTag) — obs refs. */
|
||||
batchTag?: string;
|
||||
}
|
||||
|
||||
// ─── transcript scanning (shared by the merger + trigger) ─────────────────────
|
||||
@@ -241,10 +257,13 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
|
||||
}
|
||||
}
|
||||
// Understanding observations: one per proposition carrying a note,
|
||||
// resolved to the node the verdict mapped it to.
|
||||
// resolved to the node the verdict mapped it to. Refs carry the batch
|
||||
// tag when there is one: run-local source ids ("p1"…) shift position
|
||||
// between runs, so a bare obs:p1 could never both dedupe a re-run of
|
||||
// the same batch AND let a re-opened message's fresh level through.
|
||||
for (const p of audit.propositions) {
|
||||
if (p.understanding === undefined || p.understanding.length === 0) continue;
|
||||
const ref = `obs:${p.id}`;
|
||||
const ref = source.batchTag !== undefined ? `obs:${source.batchTag}:${p.id}` : `obs:${p.id}`;
|
||||
if (seenRefs.has(ref)) continue;
|
||||
ledgers.appendEvent({
|
||||
nodeId: mapping.get(p.id) ?? null,
|
||||
@@ -258,9 +277,17 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
|
||||
stats.events++;
|
||||
}
|
||||
|
||||
// 4. Checkpoint: this source is extracted through its foldedAt (the
|
||||
// chat's updatedAt, or the import time for notes).
|
||||
ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt });
|
||||
// 4. Checkpoint — THE COVERAGE DISCIPLINE (vision-extraction-coverage.md):
|
||||
// only now, after every append succeeded, is the batch covered. The
|
||||
// chat path appends one CoverageEntry per message (per-batch atomic —
|
||||
// an abort or failure mid-run leaves earlier batches covered, this
|
||||
// batch's messages uncovered); the notes path keeps the whole-file
|
||||
// checkpoint (a file is one shot).
|
||||
if (source.coverage !== undefined) {
|
||||
for (const entry of source.coverage) ledgers.appendCoverage(entry);
|
||||
} else {
|
||||
ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt });
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
@@ -302,34 +329,6 @@ function quizEvent(
|
||||
return event;
|
||||
}
|
||||
|
||||
// ─── the trigger helper (v1: every N quiz answers — swappable seam) ──────────
|
||||
|
||||
/**
|
||||
* How many ANSWERED quiz parts in this chat are not yet in the ledger — i.e.
|
||||
* new evidence since the last extraction. The v1 trigger (turn-service)
|
||||
* fires the pipeline when this crosses EXTRACTION.everyNQuizAnswers. Every
|
||||
* answered quiz always becomes an event (attribution only decides nodeId),
|
||||
* so ref-not-in-ledger is a correct "not yet processed" marker.
|
||||
*/
|
||||
export function pendingQuizAnswers(chatId: string, deps: GraphDeps): number {
|
||||
const record = deps.chatStore.getChat(chatId);
|
||||
if (!record) return 0;
|
||||
const seenRefs = new Set<string>();
|
||||
for (const event of deps.graphStore.readEvents()) {
|
||||
if (event.chatId === chatId && event.ref) seenRefs.add(event.ref);
|
||||
}
|
||||
let count = 0;
|
||||
for (const part of toolParts(record.messages)) {
|
||||
if (part.name !== "quiz" || part.state !== "output-available") continue;
|
||||
const out = part.output;
|
||||
if (typeof out !== "object" || out === null) continue;
|
||||
const answer = out as Partial<QuestionAnswerData>;
|
||||
if (answer.kind !== "quiz" || answer.dismissed === true) continue; // dismissed = not answered
|
||||
if (!seenRefs.has(part.toolCallId)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// ─── derivation (reads only — everything below is recomputable) ─────────────
|
||||
|
||||
/** The content identity of a community: "clu_" + sha256 of the sorted
|
||||
|
||||
@@ -34,4 +34,5 @@ export * from "./ui-message.ts";
|
||||
export * from "./tool-parts.ts";
|
||||
export * from "./prices.ts";
|
||||
export * from "./fuzzy.ts";
|
||||
export * from "./coverage.ts";
|
||||
export * from "./graph-context.ts";
|
||||
|
||||
@@ -23,6 +23,7 @@ import type {
|
||||
NodeMapping,
|
||||
PlanTree,
|
||||
Position,
|
||||
CoverageEntry,
|
||||
SessionCheckpoint,
|
||||
TurnUsage,
|
||||
NotesSource,
|
||||
@@ -84,6 +85,11 @@ export interface GraphStore {
|
||||
/** Append one emission checkpoint. */
|
||||
appendCheckpoint(checkpoint: SessionCheckpoint): void;
|
||||
readCheckpoints(): SessionCheckpoint[];
|
||||
/** Message-granular extraction coverage (vision-extraction-coverage.md):
|
||||
* append-only; one line per covered message. The chat-extraction
|
||||
* checkpoint — "uncovered" is derived from these, never tracked. */
|
||||
appendCoverage(entry: CoverageEntry): void;
|
||||
readCoverage(): CoverageEntry[];
|
||||
|
||||
/** Append one cluster labeling — idempotent by id (the id hashes the
|
||||
* sorted core, so identical member sets dedupe). */
|
||||
|
||||
+15
-10
@@ -17,7 +17,7 @@
|
||||
|
||||
import type { UIMessage } from "ai";
|
||||
import { withPlanBlocks } from "../core/plan-blocks.ts";
|
||||
import { pendingQuizAnswers } from "../core/graph.ts";
|
||||
import { coverageOf } from "../core/coverage.ts";
|
||||
import type { ChatStore, GraphStore } from "../core/ports.ts";
|
||||
import { EXTRACTION } from "../../config.ts";
|
||||
import type { ExtractionPipeline } from "../agent/subagents/extraction.ts";
|
||||
@@ -74,15 +74,17 @@ export function persistTurn(chatStore: ChatStore, chatId: string, messages: UIMe
|
||||
/**
|
||||
* The graph write channel (vision §1 construction): the extraction pipeline
|
||||
* (extractor → auditor → merger — the sole graph writer) runs ASYNC once a
|
||||
* chat accumulates EXTRACTION.everyNQuizAnswers new answered quizzes since
|
||||
* its last checkpoint. Fire-and-forget — never blocks the turn; failures
|
||||
* are logged by the pipeline and leave the checkpoint untouched, so the
|
||||
* next trigger re-runs the whole thing.
|
||||
* chat's UNCOVERED transcript tokens reach EXTRACTION.chunkTokens
|
||||
* (vision-extraction-coverage.md, 2026-09-15 — replacing the old
|
||||
* every-3-quiz-answers count: content size is the honest signal, and
|
||||
* quiz-less sessions extract too). The pipeline itself extracts only the
|
||||
* uncovered batches. Fire-and-forget — never blocks the turn; failures are
|
||||
* logged by the pipeline and leave coverage untouched, so the next trigger
|
||||
* (or the manual extract button) re-runs what's left.
|
||||
*
|
||||
* v1 placeholder trigger, deliberately isolated in this one seam: swap
|
||||
* here when a better mechanism lands — the pipeline is trigger-agnostic.
|
||||
* Skipped under tests: scripted chats are fixtures, not knowledge (the
|
||||
* pipeline itself is tested directly with mock models).
|
||||
* Deliberately isolated in this one seam — the pipeline is
|
||||
* trigger-agnostic. Skipped under tests: scripted chats are fixtures, not
|
||||
* knowledge (the pipeline itself is tested directly with mock models).
|
||||
*/
|
||||
export function triggerExtraction(
|
||||
deps: { chatStore: ChatStore; graphStore: GraphStore },
|
||||
@@ -91,7 +93,10 @@ export function triggerExtraction(
|
||||
): void {
|
||||
if (process.env.NODE_ENV === "test") return;
|
||||
try {
|
||||
if (pendingQuizAnswers(chatId, deps) < EXTRACTION.everyNQuizAnswers) return;
|
||||
const record = deps.chatStore.getChat(chatId);
|
||||
if (!record) return;
|
||||
const covered = coverageOf(record.messages as UIMessage[], deps.graphStore.readCoverage(), chatId);
|
||||
if (covered.uncoveredTokens < EXTRACTION.chunkTokens) return;
|
||||
void extraction.runExtraction(chatId).catch((err) => {
|
||||
console.error(`[chat ${chatId}] extraction pipeline failed:`, err);
|
||||
});
|
||||
|
||||
@@ -42,6 +42,7 @@ import type {
|
||||
NodeMapping,
|
||||
Position,
|
||||
ProvenancePoint,
|
||||
CoverageEntry,
|
||||
SessionCheckpoint,
|
||||
} from "../core/contracts.ts";
|
||||
import type { GraphStore } from "../core/ports.ts";
|
||||
@@ -133,6 +134,9 @@ const isNodeMapping = (v: unknown): v is NodeMapping =>
|
||||
const isCheckpoint = (v: unknown): v is SessionCheckpoint =>
|
||||
isRecord(v) && isString(v.chatId) && isString(v.foldedAt);
|
||||
|
||||
const isCoverageEntry = (v: unknown): v is CoverageEntry =>
|
||||
isRecord(v) && isString(v.chatId) && isString(v.fp) && typeof v.tokens === "number" && isString(v.foldedAt);
|
||||
|
||||
// ─── the port implementation ────────────────────────────────────────────────
|
||||
|
||||
export const graphStore: GraphStore = {
|
||||
@@ -175,6 +179,13 @@ export const graphStore: GraphStore = {
|
||||
return readValid(LEDGER_FILES.sessions, isCheckpoint);
|
||||
},
|
||||
|
||||
appendCoverage(entry: CoverageEntry): void {
|
||||
appendLine(LEDGER_FILES.coverage, entry);
|
||||
},
|
||||
readCoverage(): CoverageEntry[] {
|
||||
return readValid(LEDGER_FILES.coverage, isCoverageEntry);
|
||||
},
|
||||
|
||||
// ── cluster labelings (config.LEDGER_FILES.clusterLabels) ──
|
||||
|
||||
/** Idempotent by id: the id is a hash of the sorted core, so re-appending
|
||||
|
||||
Reference in New Issue
Block a user