Clusters phases B+C: labeling records, the naming call, snapshot payload

Core-survival labelings (vision section 1, DECIDED 2026-09-13): ClusterLabeling records in a new append-only ledger (data/graph/cluster-labels.jsonl), immutable, id = clu_ + sha256 of the sorted core so re-naming races and re-runs dedupe at the store seam. matchClusterLabelings (core/clusters.ts, pure): a community keeps a record's label while at least half its core survives in it; both sides single-successor (a split keeps one name; greedy by overlap, ties to the newer record). The namer (src/agent/subagents/cluster-namer.ts): ONE singular cheap LLM call (new 'labeler' role, glm-flash/low) names every new/changed community at once, given member names + statements (capped at 12 members per cluster). refreshClusterLabels runs POST-EXTRACTION (user-decided) from both front doors, fire-and-forget — naming failure never fails the run; refreshes serialize behind a chain so parallel imports never mint duplicate labels. import-notes.ts awaits clusterNamingSettled before exit. Payload: GraphSnapshot.clusters (>= MIN_CLUSTER_SIZE 3) joins detection with labelings at read time — claimed clusters carry the record's label and id; unlabeled ones carry a member-hash id and null label. Detection is deterministic so no memo in v1 (ms at current scale; revisit at vault scale). Tests (372/372): matcher half-rule/split/merge/greedy/tie cases; naming end-to-end through importNotes (one call, one record, claim on re-refresh), naming failure non-fatal; snapshot derivation with labeling join; 11 test pipelines now pin an explicit labeler seam.
2026-09-13 03:59:48 +03:00
parent 95a4477283
commit f0267a5fd9
12 changed files with 562 additions and 8 deletions
@@ -6,8 +6,9 @@
 */
 import { strict as assert } from "node:assert";
 import { describe, test } from "node:test";
-import { detectCommunities } from "../src/core/clusters.ts";
+import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "../src/core/clusters.ts";
 import type { Community } from "../src/core/clusters.ts";
+import type { ClusterLabeling } from "../src/core/contracts.ts";

 const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`);

@@ -15,6 +16,62 @@ const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${pref
 const clique = (nodeIds: string[]) =>
 	nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b })));

+const labeling = (id: string, core: string[], createdAt = "2026-09-01T00:00:00.000Z"): ClusterLabeling => ({
+	id,
+	label: `label-${id}`,
+	core,
+	createdAt,
+});
+
+describe("matchClusterLabelings", () => {
+	test("claims a labeling while at least half its core survives in the community", () => {
+		const community: Community = { members: ["a", "b", "c", "d", "e"] };
+		const records = [labeling("clu_1", ["a", "b", "x", "y"])]; // 2 of 4 survive — exactly half
+		const [claim] = matchClusterLabelings([community], records);
+		assert.equal(claim?.id, "clu_1");
+		// One fewer survivor: under half → relabel.
+		const [dead] = matchClusterLabelings([{ members: ["a", "c", "d", "e", "f"] }], records);
+		assert.equal(dead, null);
+	});
+
+	test("single successor: a split core keeps its label in only one community", () => {
+		const records = [labeling("clu_1", ["a", "b", "c", "d"])];
+		const left: Community = { members: ["a", "b", "z"] }; // 2 of 4 — half
+		const right: Community = { members: ["c", "d", "w"] }; // 2 of 4 — half
+		// Tie: larger overlap wins; equal → both eligible, greedy takes first come.
+		const claims = matchClusterLabelings([left, right], records);
+		const claimed = claims.filter((c) => c !== null);
+		assert.equal(claimed.length, 1);
+		assert.equal(claimed[0]?.id, "clu_1");
+	});
+
+	test("greedy: the best-overlapping community wins the record", () => {
+		const records = [labeling("clu_1", ["a", "b", "c", "d"])];
+		const weak: Community = { members: ["a", "b", "q", "r", "s"] }; // 2/4
+		const strong: Community = { members: ["a", "b", "c", "x", "y"] }; // 3/4
+		const claims = matchClusterLabelings([weak, strong], records);
+		assert.equal(claims[0], null);
+		assert.equal(claims[1]?.id, "clu_1");
+	});
+
+	test("one record per community; newest record breaks ties", () => {
+		const oldR = labeling("clu_old", ["a", "b"], "2026-09-01T00:00:00.000Z");
+		const newR = labeling("clu_new", ["a", "b"], "2026-09-02T00:00:00.000Z");
+		const community: Community = { members: ["a", "b", "c"] };
+		const [claim] = matchClusterLabelings([community], [oldR, newR]);
+		assert.equal(claim?.id, "clu_new");
+	});
+
+	test("no labelings: every community unclaimed; empty communities array ok", () => {
+		assert.deepEqual(matchClusterLabelings([{ members: ["a", "b"] }], []), [null]);
+		assert.deepEqual(matchClusterLabelings([], [labeling("clu_1", ["a"])]), []);
+	});
+
+	test("MIN_CLUSTER_SIZE is 3 (the render/naming threshold)", () => {
+		assert.equal(MIN_CLUSTER_SIZE, 3);
+	});
+});
+
 describe("detectCommunities", () => {
 	test("two dense cliques joined by a single bridge split into two communities", () => {
 		const left = ids(4, "L");
@@ -57,7 +114,7 @@ describe("detectCommunities", () => {
 			{ from: "b", to: "c" },
 		]);
 		assert.equal(result.communities.length, 1);
-		assert.deepEqual(result.communities[0].members.sort(), ["a", "b", "c"]);
+		assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b", "c"]);
 	});

 	test("self-loops and unknown-endpoint edges are skipped defensively", () => {
@@ -69,7 +126,7 @@ describe("detectCommunities", () => {
 			{ from: "a", to: "b" }, // the one real edge
 		]);
 		assert.equal(result.communities.length, 1);
-		assert.deepEqual(result.communities[0].members.sort(), ["a", "b"]);
+		assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b"]);
 	});

 	test("members are sorted; communities ordered by size then id", () => {
@@ -84,7 +141,7 @@ describe("detectCommunities", () => {
 			assert.deepEqual(c.members, sorted);
 		}
 		// iso (isolated) is last: size 1 < 4.
-		const last = result.communities[result.communities.length - 1];
+		const last = result.communities[result.communities.length - 1]!;
 		assert.deepEqual(last.members, ["iso"]);
 	});

@@ -27,6 +27,7 @@ import type { UIMessage } from "ai";
 import type { PiEngine } from "../src/agent/models.ts";
 import type {
 	AuditResult,
+	ClusterLabeling,
 	ExtractionRunUsage,
 	GraphEdge,
 	GraphNode,
@@ -38,6 +39,7 @@ import type {
 } from "../src/core/contracts.ts";
 import type { GraphStore } from "../src/core/ports.ts";
 import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
+import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
 import { chatStore } from "../src/store/chats.ts";
 import { PATHS } from "../config.ts";

@@ -63,6 +65,7 @@ function memoryLedgers() {
 	const mappings: NodeMapping[] = [];
 	const checkpoints: SessionCheckpoint[] = [];
 	const sources = new Map<string, NotesSource>();
+	const clusterLabels: ClusterLabeling[] = [];
 	return {
 		appendNode: (n: GraphNode) => void nodes.push(n),
 		readNodes: () => nodes.map((n) => ({ ...n })),
@@ -87,6 +90,11 @@ function memoryLedgers() {
 				source.extractions = [...(source.extractions ?? []), entry];
 			}
 		},
+		// Cluster labelings — mirrors the real store: append dedupes by id.
+		appendClusterLabel: (l: ClusterLabeling) => {
+			if (!clusterLabels.some((x) => x.id === l.id)) clusterLabels.push(l);
+		},
+		readClusterLabels: () => clusterLabels.map((l) => ({ ...l, core: [...l.core] })),
 	} satisfies GraphStore;
 }

@@ -306,6 +314,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 			graphStore: ledgers,
 			extractor: submitModel("submit_findings", FINDINGS),
 			auditor: submitModel("submit_review", verdict),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		const stats = await pipeline.runExtraction(chatId);

@@ -346,6 +355,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 			graphStore: ledgers,
 			extractor: submitModel("submit_findings", FINDINGS),
 			auditor: silentModel(), // never submits the review
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/);
 		assert.equal(ledgers.readNodes().length, 0);
@@ -356,6 +366,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 			graphStore: ledgers,
 			extractor: silentModel(),
 			auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/);
 		assert.equal(ledgers.readCheckpoints().length, 0);
@@ -388,6 +399,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 			// registry must already carry this chat at the right phase.
 			extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())),
 			auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		await pipeline.runExtraction(chatId);
 		assert.deepEqual(seen, ["extracting", "auditing"]);
@@ -422,6 +434,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 			auditor: submitModel("submit_review", verdict, () =>
 				usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens),
 			),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		await pipeline.runExtraction(chatId);

@@ -458,6 +471,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 			graphStore: ledgers,
 			extractor: hangingModel(), // hangs until aborted — no step finishes
 			auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		const runPromise = pipeline.runExtraction(chatId);

@@ -508,6 +522,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 			graphStore: ledgers,
 			extractor: gatedExtractor,
 			auditor: silentModel(),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});

 		const first = pipeline.runExtraction(chatId);
@@ -525,6 +540,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 			graphStore: ledgers,
 			extractor: silentModel(),
 			auditor: silentModel(),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
 	});
@@ -562,6 +578,70 @@ const NOTES_VERDICT = {
 	dropped: [],
 } as const;

+const CLUSTER_FINDINGS = {
+	propositions: [
+		{ id: "p1", name: "Modal editing", statement: "Vim edits text through modal command modes.", aliases: [], messages: [], evidence: ["Normal mode keys are commands."], understanding: "solid" },
+		{ id: "p2", name: "Verb-object grammar", statement: "Vim commands compose as verb and object.", aliases: [], messages: [], evidence: ["d then motion deletes."], understanding: "solid" },
+		{ id: "p3", name: "Operator-pending", statement: "Vim enters operator-pending mode between verb and object.", aliases: [], messages: [], evidence: ["After d, a motion is expected."], understanding: "solid" },
+	],
+	edges: [
+		{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
+		{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
+	],
+} as const;
+
+const CLUSTER_VERDICT = {
+	propositions: CLUSTER_FINDINGS.propositions,
+	edges: CLUSTER_FINDINGS.edges,
+	dropped: [],
+} as const;
+
+describe("cluster naming (the post-extraction garnish)", () => {
+	it("names a new community after a notes merge — one record, claimed on the next refresh", async () => {
+		const ledgers = memoryLedgers();
+		const namerCalls: number[] = [];
+		const labeler = submitModel("submit_labels", { labels: [{ key: "c0", label: "Vim modal grammar" }] }, () => namerCalls.push(1));
+		const pipeline = createExtractionPipeline({
+			chatStore,
+			graphStore: ledgers,
+			extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
+			auditor: submitModel("submit_review", CLUSTER_VERDICT),
+			labeler,
+		});
+		const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes, any format" });
+		assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 });
+		// The refresh is fire-and-forget inside the pipeline — settle it.
+		await clusterNamingSettled();
+		assert.equal(namerCalls.length, 1, "ONE singular naming call");
+		const records = ledgers.readClusterLabels();
+		assert.equal(records.length, 1);
+		assert.equal(records[0]!.label, "Vim modal grammar");
+		assert.equal(records[0]!.core.length, 3);
+		assert.match(records[0]!.id, /^clu_/);
+		// Second refresh: core-survival claims it — nothing new to name.
+		const again = await refreshClusterLabels({ graphStore: ledgers, engine: labeler });
+		assert.equal(again.named, 0);
+		assert.equal(again.appended, 0);
+		assert.ok(again.claimed >= 1);
+		assert.equal(ledgers.readClusterLabels().length, 1, "no duplicate records");
+	});
+
+	it("naming failure never fails the run", async () => {
+		const ledgers = memoryLedgers();
+		const pipeline = createExtractionPipeline({
+			chatStore,
+			graphStore: ledgers,
+			extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
+			auditor: submitModel("submit_review", CLUSTER_VERDICT),
+			labeler: silentModel(), // never submits — the namer throws
+		});
+		const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes" });
+		assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }, "the merge itself succeeded");
+		await clusterNamingSettled();
+		assert.equal(ledgers.readClusterLabels().length, 0, "nothing named, nothing appended");
+	});
+});
+
 describe("the notes front door (importNotes — format-agnostic)", () => {
 	it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
 		const ledgers = memoryLedgers();
@@ -575,6 +655,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
 				const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
 				seen.push({ kind: run?.kind, label: run?.label });
 			}),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });

@@ -618,6 +699,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
 				graphStore: ledgers,
 				extractor: submitModel("submit_findings", NOTES_FINDINGS),
 				auditor: submitModel("submit_review", NOTES_VERDICT),
+				labeler: submitModel("submit_labels", { labels: [] }),
 			});
 		const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
 		assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
@@ -637,6 +719,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
 			graphStore: ledgers,
 			extractor: hangingModel(),
 			auditor: submitModel("submit_review", NOTES_VERDICT),
+			labeler: submitModel("submit_labels", { labels: [] }),
 		});
 		const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });

@@ -20,7 +20,7 @@ import { after, describe, it } from "node:test";
 import type { UIMessage } from "ai";
 import type { AuditResult } from "../src/core/contracts.ts";
 import type { QuestionAnswerData } from "../src/core/question-core.ts";
-import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
+import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
 import { graphStore } from "../src/store/graph.ts";
 import { chatStore } from "../src/store/chats.ts";
 import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";
@@ -407,6 +407,53 @@ describe("the merger (mergeFindings)", () => {
 		assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
 	});

+	it("derives structural clusters in graphSnapshot, joined with persisted labelings", () => {
+		// A self-contained 3-clique: three propositions, all mutually linked.
+		const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const;
+		const merge = mergeFindings(
+			"src_clusterfixture",
+			{
+				propositions: tri.map((statement, i) => ({
+					id: `p${i + 1}`,
+					name: `Cluster node ${i + 1}`,
+					statement,
+					aliases: [],
+					messages: [],
+					evidence: ["fixture evidence"],
+				})),
+				edges: [
+					{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
+					{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
+					{ from: "p1", to: "p3", type: "follows", messages: [], evidence: ["e"] },
+				],
+				attribution: { quiz: {} },
+				dropped: [],
+			},
+			graphDeps,
+			{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
+		);
+		assert.equal(merge.nodes, 3);
+		const snap = graphSnapshot(graphDeps);
+		// Presentation threshold: every cluster is >= 3 members.
+		assert.ok(snap.clusters.every((c) => c.members.length >= 3));
+		// Our trio is exactly one cluster, unlabeled until the namer runs.
+		const mine = snap.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap.nodes.find((n) => n.statement === s)!.id)));
+		assert.ok(mine, "the 3-clique is a cluster");
+		assert.equal(mine!.label, null);
+		assert.match(mine!.id, /^clu_/);
+		// Mint the labeling the namer would append — the next read joins it.
+		graphStore.appendClusterLabel({
+			id: clusterContentId(mine!.members),
+			label: "Test region",
+			core: mine!.members,
+			createdAt: "2026-09-13T00:00:00.000Z",
+		});
+		const snap2 = graphSnapshot(graphDeps);
+		const claimed = snap2.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap2.nodes.find((n) => n.statement === s)!.id)))!;
+		assert.equal(claimed.label, "Test region");
+		assert.equal(claimed.id, clusterContentId(mine!.members), "the claimed cluster carries the record's id");
+	});
+
 	it("derives sessions and per-session understanding observations in graphSnapshot", () => {
 		const snap = graphSnapshot(graphDeps);
 		const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));