feat: extraction coverage — message-granular ledger + token-batched runs (d132373 vision)
THE MESSAGE IS THE UNIT OF COVERAGE (vision-extraction-coverage.md): a per-chat append-only ledger (coverage.jsonl) records which messages the pipeline has extracted, by content fingerprint over exactly what the extraction view sees (one shared line builder — core/coverage.ts); the uncovered remainder is DERIVED at read time, never dual-tracked. A quiz answer merging into the asking message re-opens only that message; plan-block data-part attach never re-opens anything. - The trigger (turn-service) replaces every-3-quiz-answers with uncovered-tokens >= EXTRACTION.chunkTokens (4096) — quiz-less sessions extract too, content size is the honest signal. - runExtraction is now chunked: token-budgeted batches of the oldest uncovered consecutive messages (a tool call + result never split), each merged batch checkpointing its messages' coverage atomically — an abort or error loses at most the in-flight batch; merged batches stay covered. - Observation refs are batch-tagged (obs:<batchTag>:<sourceId>): re-runs of the same batch dedupe; a re-opened message's fresh understanding lands as a new event. Fixes the old obs:p1 position-collision. - New API: GET /api/graph/coverage (exact per-session derived coverage + running flags, most-uncovered first) and POST /api/graph/coverage/extract (manual run for one session, 409 while in flight). - Graph view chrome: the coverage button opens a panel with every session's covered/uncovered tokens and a per-session Extract button — the manual answer to aborted runs, errors, and dormant tails. Renders even when the graph is empty: an empty graph with uncovered sessions is exactly when it matters. Polls only while open. - Notes imports keep the whole-file checkpoint path (coverage undefined). - Tests: pure-core coverage suite (11), chunked-pipeline suite (3: multi-batch + no-op, abort-keeps-earlier-batches + remainder re-run, re-open alone + data-part invisibility), panel render suite (5); pendingQuizAnswers and its tests deleted as dead code. 486/486. Live-smoked on real GLM: manual extract of "teach me modal realism" ran one batch, wrote 4 coverage entries, panel flipped to "1.3k covered · complete" with the button disabled, graph refetched (nodes 11), usage persisted with real stats.
This commit is contained in:
amosblomqvist
@@ -0,0 +1,159 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import type { UIMessage } from "ai";
|
||||
|
||||
import {
|
||||
batchRuns,
|
||||
batchTag,
|
||||
coverageOf,
|
||||
coverageSlots,
|
||||
messageFingerprint,
|
||||
messageViewLines,
|
||||
} from "../src/core/coverage.ts";
|
||||
import type { CoverageEntry } from "../src/core/contracts.ts";
|
||||
|
||||
/**
|
||||
* Extraction coverage — the pure core (vision-extraction-coverage.md,
|
||||
* settled 2026-09-15). THE MESSAGE IS THE UNIT OF COVERAGE; the fingerprint
|
||||
* and the LLM prompt share ONE line builder (messageViewLines), so "covered"
|
||||
* is by construction exactly "what the extractor saw".
|
||||
*/
|
||||
|
||||
function user(id: string, text: string): UIMessage {
|
||||
return { id, role: "user", parts: [{ type: "text", text }] } as unknown as UIMessage;
|
||||
}
|
||||
|
||||
function slot(messages: UIMessage[], id: string) {
|
||||
const found = coverageSlots(messages).find((s) => s.message.id === id);
|
||||
assert.ok(found, `slot ${id}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
describe("messageFingerprint — what changes it, what never does", () => {
|
||||
it("a text edit changes the fingerprint; role participates", () => {
|
||||
const a = messageFingerprint(user("m1", "hello world"));
|
||||
const b = messageFingerprint(user("m1", "hello worlds"));
|
||||
const same = messageFingerprint(user("m1", "hello world"));
|
||||
assert.notEqual(a, b);
|
||||
assert.equal(a, same);
|
||||
// Role is hashed too — a user and an assistant line never collide.
|
||||
const teacher = { id: "m1", role: "assistant", parts: [{ type: "text", text: "hello world" }] } as unknown as UIMessage;
|
||||
assert.notEqual(a, messageFingerprint(teacher));
|
||||
});
|
||||
|
||||
it("the message id is the provenance tag — part of exactly what the extractor saw", () => {
|
||||
// The view lines carry the [id] tag, so identical content under a
|
||||
// different id is a DIFFERENT fingerprint: coverage matches the
|
||||
// extraction view byte-for-byte. (Ids are stable in practice — the
|
||||
// write-ahead persist owns them.)
|
||||
assert.notEqual(messageFingerprint(user("m1", "same")), messageFingerprint(user("m2", "same")));
|
||||
// The id is the ONLY tag: the same id + content re-fingerprints equal.
|
||||
assert.equal(messageFingerprint(user("m1", "same")), messageFingerprint(user("m1", "same")));
|
||||
});
|
||||
|
||||
it("data parts are invisible — plan-block attach never re-opens a covered message", () => {
|
||||
const plain = user("m1", "teach me the chain rule");
|
||||
const withData = {
|
||||
...plain,
|
||||
parts: [...plain.parts, { type: "data-plan-blocks", planBlocks: [] }],
|
||||
} as unknown as UIMessage;
|
||||
assert.equal(messageFingerprint(plain), messageFingerprint(withData));
|
||||
});
|
||||
|
||||
it("a quiz answer merging in changes the fingerprint — and only that message's", () => {
|
||||
const asking = {
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "tool-quiz", toolCallId: "q1", state: "input-available", input: { question: "Q?" } }],
|
||||
} as unknown as UIMessage;
|
||||
const answered = {
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [
|
||||
{ type: "tool-quiz", toolCallId: "q1", state: "output-available", input: { question: "Q?" }, output: { questionText: "Q?", correct: true } },
|
||||
],
|
||||
} as unknown as UIMessage;
|
||||
const fpBefore = messageFingerprint(asking);
|
||||
assert.notEqual(fpBefore, messageFingerprint(answered));
|
||||
// An output-bearing line's mass counts toward coverage tokens.
|
||||
assert.ok(
|
||||
messageViewLines(answered, answered.id).join("").length >
|
||||
messageViewLines(asking, asking.id).join("").length,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("coverageOf — uncovered is DERIVED, never dual-tracked", () => {
|
||||
it("diffs fingerprints: covered stays, uncovered counts, tokens split", () => {
|
||||
const messages = [user("m1", "aaaa"), user("m2", "bbbb"), user("m3", "cccc")];
|
||||
const fp1 = slot(messages, "m1").fp;
|
||||
const fp2 = slot(messages, "m2").fp;
|
||||
const entries: CoverageEntry[] = [
|
||||
{ chatId: "c1", fp: fp1, tokens: 1, foldedAt: "2026-09-15T00:00:00.000Z" },
|
||||
{ chatId: "c1", fp: fp2, tokens: 1, foldedAt: "2026-09-15T00:00:00.000Z" },
|
||||
// Another chat's entries are irrelevant to c1's coverage.
|
||||
{ chatId: "c2", fp: slot(messages, "m3").fp, tokens: 1, foldedAt: "2026-09-15T00:00:00.000Z" },
|
||||
];
|
||||
const cov = coverageOf(messages, entries, "c1");
|
||||
assert.deepEqual(cov.uncovered.map((s) => s.message.id), ["m3"]);
|
||||
assert.equal(cov.coveredCount, 2);
|
||||
assert.equal(cov.uncoveredCount, 1);
|
||||
assert.equal(cov.uncoveredTokens, slot(messages, "m3").tokens);
|
||||
assert.equal(cov.coveredTokens, slot(messages, "m1").tokens + slot(messages, "m2").tokens);
|
||||
});
|
||||
|
||||
it("a mutated covered message re-opens itself, and only itself", () => {
|
||||
const messages = [user("m1", "aaaa"), user("m2", "bbbb")];
|
||||
const entries: CoverageEntry[] = coverageSlots(messages).map((s) => ({
|
||||
chatId: "c1",
|
||||
fp: s.fp,
|
||||
tokens: s.tokens,
|
||||
foldedAt: "2026-09-15T00:00:00.000Z",
|
||||
}));
|
||||
assert.equal(coverageOf(messages, entries, "c1").uncoveredCount, 0);
|
||||
const mutated = [user("m1", "aaaa!"), messages[1]!];
|
||||
const cov = coverageOf(mutated, entries, "c1");
|
||||
assert.deepEqual(cov.uncovered.map((s) => s.message.id), ["m1"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("batchRuns — token-budgeted, consecutive, never splits a message", () => {
|
||||
it("greedy oldest-first fill; the overflowing message starts the next run", () => {
|
||||
// Each message's view line is ~14 tok (chars/4) → budget 45 fits
|
||||
// three, the fourth overflows: [m1,m2,m3] + [m4].
|
||||
const slots = coverageSlots([
|
||||
user("m1", "a".repeat(40)),
|
||||
user("m2", "b".repeat(40)),
|
||||
user("m3", "c".repeat(40)),
|
||||
user("m4", "d".repeat(40)),
|
||||
]);
|
||||
const runs = batchRuns(slots, 45);
|
||||
assert.deepEqual(runs.map((r) => r.map((s) => s.message.id)), [["m1", "m2", "m3"], ["m4"]]);
|
||||
});
|
||||
|
||||
it("a single message larger than the budget becomes its own over-budget run", () => {
|
||||
const slots = coverageSlots([user("m1", "x".repeat(400)), user("m2", "y".repeat(20))]);
|
||||
const runs = batchRuns(slots, 10);
|
||||
assert.deepEqual(runs.map((r) => r.map((s) => s.message.id)), [["m1"], ["m2"]]);
|
||||
});
|
||||
|
||||
it("an empty or non-positive budget never drops messages into a void", () => {
|
||||
assert.deepEqual(batchRuns(coverageSlots([user("m1", "hi")]), 0), []);
|
||||
assert.deepEqual(batchRuns([], 4096), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("batchTag — deterministic, ref-stable, mutation-sensitive", () => {
|
||||
it("the same members in the same order → the same 12-hex tag; a different order is a different batch", () => {
|
||||
const slots = coverageSlots([user("m1", "aaaa"), user("m2", "bbbb")]);
|
||||
assert.equal(batchTag(slots), batchTag(coverageSlots([user("m1", "aaaa"), user("m2", "bbbb")])));
|
||||
assert.notEqual(batchTag(slots), batchTag([...slots].reverse()), "batches are ordered runs");
|
||||
assert.match(batchTag(slots), /^[0-9a-f]{12}$/);
|
||||
});
|
||||
|
||||
it("a re-opened member changes the tag — its observations land as new events", () => {
|
||||
const slots = coverageSlots([user("m1", "aaaa"), user("m2", "bbbb")]);
|
||||
const mutated = coverageSlots([user("m1", "aaaa!"), user("m2", "bbbb")]);
|
||||
assert.notEqual(batchTag(slots), batchTag(mutated));
|
||||
});
|
||||
});
|
||||
+247
-3
@@ -28,6 +28,7 @@ import type { PiEngine } from "../src/agent/models.ts";
|
||||
import type {
|
||||
AuditResult,
|
||||
ClusterLabeling,
|
||||
CoverageEntry,
|
||||
ExtractionRunUsage,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
@@ -39,6 +40,7 @@ import type {
|
||||
} from "../src/core/contracts.ts";
|
||||
import type { GraphStore } from "../src/core/ports.ts";
|
||||
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
|
||||
import { batchTag, coverageOf, coverageSlots } from "../src/core/coverage.ts";
|
||||
import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
|
||||
import { semanticClusters } from "../src/core/semantic-clusters.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
@@ -65,6 +67,7 @@ function memoryLedgers() {
|
||||
const events: LedgerEvent[] = [];
|
||||
const mappings: NodeMapping[] = [];
|
||||
const checkpoints: SessionCheckpoint[] = [];
|
||||
const coverage: CoverageEntry[] = [];
|
||||
const sources = new Map<string, NotesSource>();
|
||||
const clusterLabels: ClusterLabeling[] = [];
|
||||
const positions = new Map<string, { x: number; y: number }>();
|
||||
@@ -86,6 +89,10 @@ function memoryLedgers() {
|
||||
readMappings: () => mappings.map((m) => ({ ...m })),
|
||||
appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c),
|
||||
readCheckpoints: () => checkpoints.map((c) => ({ ...c })),
|
||||
// Extraction coverage (vision-extraction-coverage.md) — append-only;
|
||||
// the chat pipeline checkpoints one entry per covered message.
|
||||
appendCoverage: (e) => void coverage.push({ ...e }),
|
||||
readCoverage: () => coverage.map((e) => ({ ...e })),
|
||||
upsertSource: (s: NotesSource) => {
|
||||
// Mirrors the real store: an upsert KEEPS the existing usage log.
|
||||
const prev = sources.get(s.id);
|
||||
@@ -356,20 +363,35 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
// Events rebuilt deterministically: quizzes in transcript order, then
|
||||
// the observation. check-1's attribution pointed at the DROPPED p2 →
|
||||
// unresolvable → null (the question text is kept for later attribution).
|
||||
// The observation ref is BATCH-TAGGED (obs:<tag>:p1) so a re-run of
|
||||
// the same batch dedupes while a re-opened message's update lands as
|
||||
// a new event (vision-extraction-coverage.md).
|
||||
const events = ledgers.readEvents();
|
||||
const record = chatStore.getChat(chatId)!;
|
||||
const expectedTag = batchTag(coverageSlots(record.messages as UIMessage[]));
|
||||
assert.deepEqual(
|
||||
events.map((e) => [e.kind, e.ref, e.nodeId]),
|
||||
[
|
||||
["miss", "probe-1", existingId],
|
||||
["correct", "check-1", null],
|
||||
["observation", "obs:p1", existingId],
|
||||
["observation", `obs:${expectedTag}:p1`, existingId],
|
||||
],
|
||||
);
|
||||
assert.equal(events[0]!.nodeId, existingId, "probe attributed via the verdict's mapping");
|
||||
assert.equal(events[2]!.note, "solid", "the understanding note joins the ledger");
|
||||
|
||||
// Checkpointed through the record's updatedAt — never the same turns twice.
|
||||
assert.equal(ledgers.readCheckpoints().length, 1);
|
||||
// COVERAGE checkpointed per message (not a whole-chat checkpoint):
|
||||
// one entry per transcript message, fingerprinted content each —
|
||||
// the same turns are never extracted twice.
|
||||
const covered = ledgers.readCoverage();
|
||||
assert.equal(covered.length, record.messages.length);
|
||||
assert.ok(covered.every((c) => c.chatId === chatId && c.fp.length === 64 && c.tokens > 0));
|
||||
assert.equal(ledgers.readCheckpoints().length, 0, "the chat path writes coverage, not the legacy checkpoint");
|
||||
assert.equal(
|
||||
coverageOf(record.messages as UIMessage[], covered, chatId).uncoveredCount,
|
||||
0,
|
||||
"every message is now covered",
|
||||
);
|
||||
});
|
||||
|
||||
it("checkpoints nothing when a stage fails", async () => {
|
||||
@@ -791,3 +813,225 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
|
||||
assert.equal(ledgers.readSources()[0]!.extractions![0]!.stats, undefined, "never merged → no stats");
|
||||
});
|
||||
});
|
||||
|
||||
// ── the chunked pipeline (extraction coverage — vision-extraction-coverage.md) ──
|
||||
|
||||
/** Pull the pipeline's prompt view out of a model call: the pipeline sends
|
||||
* `Transcript:\n\n<view>` as the user turn, so the LAST user message's
|
||||
* text IS what the extractor saw. */
|
||||
function capturedView(context: unknown): string {
|
||||
const messages = (context as { messages?: { role: string; content: string }[] }).messages ?? [];
|
||||
const last = [...messages].reverse().find((m) => m.role === "user");
|
||||
return typeof last?.content === "string" ? last.content : "";
|
||||
}
|
||||
|
||||
/** An extractor step that submits FINDINGS, optionally capturing its view. */
|
||||
function findingsStep(capture?: (view: string) => void): FauxResponseStep {
|
||||
return (context) => {
|
||||
capture?.(capturedView(context));
|
||||
return fauxAssistantMessage(
|
||||
[fauxToolCall("submit_findings", FINDINGS as unknown as Record<string, unknown>, { id: "findings-call" })],
|
||||
{ stopReason: "toolUse" },
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/** An auditor step that passes p1 through (no linkTo — a fresh node), drops p2. */
|
||||
function passThroughAuditStep(): FauxResponseStep {
|
||||
return () =>
|
||||
fauxAssistantMessage(
|
||||
[
|
||||
fauxToolCall(
|
||||
"submit_review",
|
||||
{
|
||||
propositions: [{ ...FINDINGS.propositions[0]! }],
|
||||
edges: [],
|
||||
attribution: { quiz: {} },
|
||||
dropped: [{ ref: "p2", reason: "evidence not found" }],
|
||||
} as unknown as Record<string, unknown>,
|
||||
{ id: "review-call" },
|
||||
),
|
||||
],
|
||||
{ stopReason: "toolUse" },
|
||||
);
|
||||
}
|
||||
|
||||
/** A session with UNCOVERED-multiple-batch mass: five long distinct messages
|
||||
* (~1200 tok each — 3600+1200 > EXTRACTION.chunkTokens 4096 → batches of
|
||||
* [m1,m2,m3] and [m4,m5]). */
|
||||
function chunkyChat() {
|
||||
const chat = chatStore.createChat("Chunky coverage fixture");
|
||||
createdChats.push(chat.id);
|
||||
// ~1230 tok per message (chars/4): three fill the 4096 budget, the
|
||||
// fourth would overflow — deterministic batches [m1,m2,m3] + [m4,m5].
|
||||
chatStore.saveMessages(
|
||||
chat.id,
|
||||
[1, 2, 3, 4, 5].map((n) => user(`u${n}`, `topic ${n}: ${"detail ".repeat(700)}${n}`)),
|
||||
);
|
||||
return { chatId: chat.id, record: chatStore.getChat(chat.id)! };
|
||||
}
|
||||
|
||||
describe("the chunked pipeline (extraction coverage)", () => {
|
||||
it("extracts uncovered messages in token-budgeted batches, checkpoints coverage per batch, and no-ops once covered", async () => {
|
||||
const { chatId, record } = chunkyChat();
|
||||
const ledgers = memoryLedgers();
|
||||
const views: string[] = [];
|
||||
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: fauxEngine([findingsStep((v) => views.push(v)), findingsStep((v) => views.push(v))]),
|
||||
auditor: fauxEngine([passThroughAuditStep(), passThroughAuditStep()]),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const stats = await pipeline.runExtraction(chatId);
|
||||
|
||||
// Two batches: the first held messages 1-3, the second 4-5 — a
|
||||
// message is never split, batches are consecutive oldest-first.
|
||||
assert.equal(views.length, 2, "one extractor call per batch");
|
||||
assert.ok(views[0]!.includes("LEARNER [u1]") && views[0]!.includes("LEARNER [u3]"));
|
||||
assert.ok(!views[0]!.includes("LEARNER [u4]"), "the batch stops at the token budget");
|
||||
assert.ok(views[1]!.includes("LEARNER [u4]") && views[1]!.includes("LEARNER [u5]"));
|
||||
assert.ok(!views[1]!.includes("LEARNER [u3]"));
|
||||
|
||||
// p1 was a fresh node in batch 1; batch 2's p1 mapped to the same
|
||||
// node (chat-scoped mapping) — but its understanding note is a NEW
|
||||
// batch-tagged observation.
|
||||
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 2 });
|
||||
|
||||
// Per-batch coverage: every message checkpointed, nothing legacy.
|
||||
const coverage = ledgers.readCoverage();
|
||||
assert.equal(coverage.length, record.messages.length);
|
||||
assert.equal(ledgers.readCheckpoints().length, 0);
|
||||
assert.equal(coverageOf(record.messages as UIMessage[], coverage, chatId).uncoveredCount, 0);
|
||||
|
||||
// A second run is a NO-OP: nothing uncovered → no model calls, no
|
||||
// new usage entry (no money spent on covered ground).
|
||||
const before = views.length;
|
||||
const again = await pipeline.runExtraction(chatId);
|
||||
assert.deepEqual(again, { nodes: 0, edges: 0, events: 0 });
|
||||
assert.equal(views.length, before);
|
||||
assert.equal(chatStore.getChat(chatId)!.extractions?.length, 1, "the no-op run persists no usage");
|
||||
});
|
||||
|
||||
it("aborting after a merged batch keeps it covered; the re-run extracts only the remainder", async () => {
|
||||
const { chatId, record } = chunkyChat();
|
||||
const ledgers = memoryLedgers();
|
||||
const views: string[] = [];
|
||||
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
// Batch 1 submits and merges; batch 2's extractor call HANGS
|
||||
// until the abort fires — the dock's abort button, mid-run.
|
||||
extractor: fauxEngine([
|
||||
findingsStep((v) => views.push(v)),
|
||||
(_context, options) =>
|
||||
new Promise((resolve) => {
|
||||
const sig = options?.signal;
|
||||
if (sig?.aborted) {
|
||||
resolve(fauxAssistantMessage("late"));
|
||||
return;
|
||||
}
|
||||
sig?.addEventListener("abort", () => resolve(fauxAssistantMessage("late")), { once: true });
|
||||
}),
|
||||
]),
|
||||
auditor: fauxEngine([passThroughAuditStep()]),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const runPromise = pipeline.runExtraction(chatId);
|
||||
// The abort must land while batch 2's extractor hangs. The faux
|
||||
// stream only resolves on abort, so wait for the batch-1 merge to
|
||||
// show in coverage, then abort.
|
||||
await new Promise<void>((resolve) => {
|
||||
const check = setInterval(() => {
|
||||
if (ledgers.readCoverage().length > 0) {
|
||||
clearInterval(check);
|
||||
resolve();
|
||||
}
|
||||
}, 5);
|
||||
});
|
||||
assert.equal(abortExtraction(chatId), true);
|
||||
const stats = await runPromise;
|
||||
|
||||
// Batch 1's emissions are kept and its messages stay covered.
|
||||
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
|
||||
assert.equal(ledgers.readCoverage().length, 3, "batch 1 checkpointed its three messages");
|
||||
const cov = coverageOf(record.messages as UIMessage[], ledgers.readCoverage(), chatId);
|
||||
assert.equal(cov.uncoveredCount, 2, "the in-flight batch's messages stay uncovered");
|
||||
|
||||
// The re-run — the manual extract button — picks up ONLY the
|
||||
// remainder: one batch, messages 4-5 alone in the prompt.
|
||||
const reRunViews: string[] = [];
|
||||
const finisher = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: fauxEngine([findingsStep((v) => reRunViews.push(v))]),
|
||||
auditor: fauxEngine([passThroughAuditStep()]),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await finisher.runExtraction(chatId);
|
||||
assert.equal(reRunViews.length, 1);
|
||||
assert.ok(reRunViews[0]!.includes("LEARNER [u4]") && reRunViews[0]!.includes("LEARNER [u5]"));
|
||||
assert.ok(!reRunViews[0]!.includes("LEARNER [u1]"), "the covered remainder never re-enters the prompt");
|
||||
assert.equal(ledgers.readCoverage().length, 5, "the re-run closed the gap");
|
||||
});
|
||||
|
||||
it("a re-opened message re-extracts alone; a data-part attach never re-opens it", async () => {
|
||||
// A small session: one batch covers everything, so re-openings are
|
||||
// the ONLY thing later runs can see.
|
||||
const chat = chatStore.createChat("Re-open fixture");
|
||||
createdChats.push(chat.id);
|
||||
chatStore.saveMessages(chat.id, [user("s1", "short topic one"), user("s2", "short topic two")]);
|
||||
const chatId = chat.id;
|
||||
const ledgers = memoryLedgers();
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: fauxEngine([findingsStep()]),
|
||||
auditor: fauxEngine([passThroughAuditStep()]),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await pipeline.runExtraction(chatId);
|
||||
const messages = () => chatStore.getChat(chatId)!.messages as UIMessage[];
|
||||
assert.equal(coverageOf(messages(), ledgers.readCoverage(), chatId).uncoveredCount, 0);
|
||||
|
||||
// Data parts are invisible to the extraction view — plan-block
|
||||
// attach does NOT re-open a covered message.
|
||||
const withDataPart = messages().map((m) =>
|
||||
m.id === "u2"
|
||||
? { ...m, parts: [...m.parts, { type: "data-plan-blocks", planBlocks: [] } as unknown as UIMessage["parts"][number]] }
|
||||
: m,
|
||||
);
|
||||
chatStore.saveMessages(chatId, withDataPart);
|
||||
const afterAttach = coverageOf(messages(), ledgers.readCoverage(), chatId);
|
||||
assert.equal(afterAttach.uncoveredCount, 0, "data parts don't change the fingerprint");
|
||||
|
||||
// A quiz answer merging into an ASKING message changes its view →
|
||||
// exactly that message re-opens, and the next run extracts it alone.
|
||||
const mutated = messages().map((m) =>
|
||||
m.id === "s1" ? { ...m, parts: [...m.parts, { type: "text", text: "the answer merged in late" }] } : m,
|
||||
) as unknown as UIMessage[];
|
||||
chatStore.saveMessages(chatId, mutated);
|
||||
const afterMerge = coverageOf(messages(), ledgers.readCoverage(), chatId);
|
||||
assert.equal(afterMerge.uncoveredCount, 1, "only the mutated message re-opens");
|
||||
|
||||
const reRunViews: string[] = [];
|
||||
const reOpener = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: fauxEngine([findingsStep((v) => reRunViews.push(v))]),
|
||||
auditor: fauxEngine([passThroughAuditStep()]),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await reOpener.runExtraction(chatId);
|
||||
assert.equal(reRunViews.length, 1, "one batch: the single re-opened message");
|
||||
assert.ok(reRunViews[0]!.includes("LEARNER [s1]") && !reRunViews[0]!.includes("LEARNER [s2]"));
|
||||
assert.equal(coverageOf(messages(), ledgers.readCoverage(), chatId).uncoveredCount, 0, "re-covered");
|
||||
});
|
||||
});
|
||||
|
||||
+1
-50
@@ -20,7 +20,7 @@ import { after, describe, it } from "node:test";
|
||||
import type { UIMessage } from "ai";
|
||||
import type { AuditResult } from "../src/core/contracts.ts";
|
||||
import type { QuestionAnswerData } from "../src/core/question-core.ts";
|
||||
import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
|
||||
import { clusterContentId, getGraph, graphSnapshot, mergeFindings, type GraphDeps } from "../src/core/graph.ts";
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { LEDGER_FILES, PATHS, SOURCES_FILE, EMBEDDINGS_FILE } from "../config.ts";
|
||||
@@ -564,55 +564,6 @@ describe("the merger (mergeFindings)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the trigger helper ──────────────────────────────────────────────────────
|
||||
|
||||
describe("the trigger helper (pendingQuizAnswers)", () => {
|
||||
it("counts answered quizzes not yet in the ledger — dismissed/ask never count", () => {
|
||||
const chat = fixtureChat("Trigger fixture", [
|
||||
user("u1", "go"),
|
||||
assistant("a1", [
|
||||
quizPart(
|
||||
"call-s",
|
||||
{ question: "QS", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-s", questionText: "QS text", correct: true, answeredAt: "2026-08-22T03:00:00.000Z" }),
|
||||
),
|
||||
quizPart(
|
||||
"call-s2",
|
||||
{ question: "QS2", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-s2", questionText: "QS2 text", dismissed: true }),
|
||||
),
|
||||
{
|
||||
type: "tool-ask_user_question",
|
||||
toolCallId: "call-s3",
|
||||
state: "output-available",
|
||||
input: { question: "free?" },
|
||||
output: answer({ questionId: "call-s3", kind: "ask", questionText: "free?", text: "y" }),
|
||||
},
|
||||
]),
|
||||
]);
|
||||
|
||||
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending");
|
||||
assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0);
|
||||
|
||||
mergeFindings(chat.id, fixtureAudit(), graphDeps, sessionSource(chat.id));
|
||||
// call-s is unattributed here (fixtureAudit has no call-s attribution)
|
||||
// — but it still becomes an event, so nothing stays pending.
|
||||
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending");
|
||||
|
||||
// A NEW answered quiz raises the count again (the re-trigger path).
|
||||
chatStore.saveMessages(chat.id, [
|
||||
...chatStore.getChat(chat.id)!.messages,
|
||||
assistant("a3", [
|
||||
quizPart(
|
||||
"call-s4",
|
||||
{ question: "QS4", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
|
||||
answer({ questionId: "call-s4", questionText: "QS4 text", correct: false, answeredAt: "2026-08-22T03:01:00.000Z" }),
|
||||
),
|
||||
]),
|
||||
]);
|
||||
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── store primitives ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { CoveragePanel, GraphCoverageControl } from "../components/graph-coverage.tsx";
|
||||
|
||||
// The extraction-coverage panel (vision-extraction-coverage.md, settled
|
||||
// 2026-09-15): the graph view's window into WHAT the pipeline has covered,
|
||||
// with a per-session "Extract" button for the uncovered remainder. Static
|
||||
// markup can't click — the closed control and the panel body are pinned
|
||||
// separately. (Interaction — the fetch + POST flow — is covered by the live
|
||||
// smoke.)
|
||||
|
||||
describe("GraphCoverageControl", () => {
|
||||
it("renders the coverage button in the chrome, closed by default", () => {
|
||||
const html = renderToStaticMarkup(<GraphCoverageControl />);
|
||||
assert.ok(html.includes("<button"));
|
||||
assert.ok(html.includes("Extraction coverage"));
|
||||
assert.ok(!html.includes('data-testid="coverage-panel"'), "the panel starts closed");
|
||||
});
|
||||
});
|
||||
|
||||
describe("CoveragePanel — exact per-session coverage data", () => {
|
||||
const base = {
|
||||
chatId: "chat_a",
|
||||
title: "Modal logic",
|
||||
updatedAt: "2026-09-15T00:00:00.000Z",
|
||||
totalMessages: 12,
|
||||
coveredMessages: 10,
|
||||
uncoveredMessages: 2,
|
||||
coveredTokens: 1536,
|
||||
uncoveredTokens: 384,
|
||||
running: false,
|
||||
};
|
||||
|
||||
it("shows each session's covered/uncovered tokens and the Extract button", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<CoveragePanel chats={[base]} loaded isBusy={() => false} onExtract={() => {}} />,
|
||||
);
|
||||
assert.ok(html.includes("Modal logic"));
|
||||
assert.ok(html.includes("1.5k covered"));
|
||||
assert.ok(html.includes("384 uncovered"));
|
||||
assert.ok(html.includes("(2 msgs)"));
|
||||
assert.ok(html.includes(">Extract<"), "the uncovered session offers a manual extract");
|
||||
});
|
||||
|
||||
it("a fully covered session shows · complete with a disabled Extract button", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<CoveragePanel
|
||||
chats={[{ ...base, uncoveredMessages: 0, uncoveredTokens: 0, coveredMessages: 12, coveredTokens: 1920 }]}
|
||||
loaded
|
||||
isBusy={() => false}
|
||||
onExtract={() => {}}
|
||||
/>,
|
||||
);
|
||||
assert.ok(html.includes("· complete"));
|
||||
assert.ok(html.includes("disabled"), "nothing to extract — the button is honest");
|
||||
assert.ok(!html.includes(">Extract<") || html.includes("disabled"));
|
||||
});
|
||||
|
||||
it("a running session shows extracting… and no clickable duplicate start", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<CoveragePanel
|
||||
chats={[{ ...base, running: true }]}
|
||||
loaded
|
||||
isBusy={(c) => c.running}
|
||||
onExtract={() => {}}
|
||||
/>,
|
||||
);
|
||||
assert.ok(html.includes("extracting…"));
|
||||
assert.ok(html.includes(">Running<"));
|
||||
assert.ok(html.includes("disabled"), "single-flight: the in-flight run can't be double-started");
|
||||
});
|
||||
|
||||
it("all-covered panel: the nothing-pending line, not a row list", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<CoveragePanel
|
||||
chats={[{ ...base, uncoveredMessages: 0, uncoveredTokens: 0 }]}
|
||||
loaded
|
||||
isBusy={() => false}
|
||||
onExtract={() => {}}
|
||||
/>,
|
||||
);
|
||||
assert.ok(html.includes("Every session is fully extracted — nothing pending."));
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user