Semantic topic clusters: local embeddings replace Louvain

Louvain only ever found the reasoning chains (the graph is disconnected
by nature — every connected component became its own cluster) and
edge-orphans could never join anything. Clusters are now RELATED TOPICS:
propositions grouped by meaning.

- Embedder (src/server/embeddings.ts): in-process transformers.js v4
  running EmbeddingGemma-300M q8 (MTEB 68.4, ~300MB one-time download),
  MRL-truncated to 512d, "title: none | text:" prefix on 100% of inputs,
  L2-normalized so cosine = dot. Model/dim mismatch voids the ledger
  (full re-embed). Injected at the composition root — the agent layer
  never imports the server layer (architecture law).
- Ledger (data/graph/embeddings.json): model + dim + vectors + the
  persisted cluster partition. Reclustering runs at the END of every
  embedding batch (post-merge hook, both front doors) — never per read
  (the agglomerative pass is ~1.5s at 338 nodes).
- Clustering (src/core/semantic-clusters.ts): hand-rolled average-linkage
  agglomerative over cosine (npm HDBSCAN is an abandoned alpha). Stragglers
  stay singletons by default; attachStragglers is the all-in toggle.
  Threshold 0.60 swept on the real graph: 26 topics, 74/90 edge-orphans
  land in real topics, 10 stragglers.
- Consumers (snapshot, settle, namer, replay) read the persisted
  partition — one cluster concept everywhere; core-survival label
  matching unchanged (20/26 topics kept labels).
- Louvain retired: detectCommunities + graphology* deps deleted
  (graphology-louvain had leaked into the client bundle since the
  clusters wave); scripts/bench-graph.ts (dead FA2 bench) removed.
- scripts/embed-graph.ts: backfill/cold-start (338 nodes in 150s
  including the one-time model download).

tests/clusters.test.ts

@@ -6,9 +6,8 @@
  */
 import { strict as assert } from "node:assert";
 import { describe, test } from "node:test";
-import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "../src/core/clusters.ts";
+import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../src/core/clusters.ts";
 import { clusterHue } from "../components/cluster-geometry.ts";
-import type { Community } from "../src/core/clusters.ts";
 import type { ClusterLabeling } from "../src/core/contracts.ts";
 
 const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`);
@@ -84,89 +83,3 @@ describe("clusterHue", () => {
 	});
 });
 
-describe("detectCommunities", () => {
-	test("two dense cliques joined by a single bridge split into two communities", () => {
-		const left = ids(4, "L");
-		const right = ids(4, "R");
-		const bridge = { from: "L0", to: "R0" };
-		const result = detectCommunities([...left, ...right].map((id) => ({ id })), [...clique(left), ...clique(right), bridge]);
-		assert.equal(result.communities.length, 2);
-		assert.equal(result.modularity > 0.3, true);
-		// Member sets, not order: the two big communities are exactly the cliques.
-		const big = result.communities.filter((c) => c.members.length === 4);
-		assert.deepEqual(big.map((c) => c.members.sort()).sort(), [left.slice().sort(), right.slice().sort()]);
-	});
-
-	test("deterministic: same graph, same seed → identical partition", () => {
-		const nodes = ids(10).map((id) => ({ id }));
-		const edges = [...clique(ids(5)), ...clique(ids(5, "n").slice().reverse().slice(0, 5)), { from: "n0", to: "n9" }];
-		const a = detectCommunities(nodes, edges);
-		const b = detectCommunities(nodes, edges);
-		assert.deepEqual(a.communities, b.communities);
-		assert.equal(a.modularity, b.modularity);
-	});
-
-	test("no edges: every node is its own singleton community", () => {
-		const result = detectCommunities(ids(3).map((id) => ({ id })), []);
-		assert.equal(result.communities.length, 3);
-		for (const c of result.communities) assert.equal(c.members.length, 1);
-		assert.equal(result.modularity, 0);
-	});
-
-	test("empty graph: zero communities", () => {
-		assert.deepEqual(detectCommunities([], []), { communities: [], modularity: 0 });
-	});
-
-	test("duplicate and reversed pairs collapse to one edge (undirected, unweighted)", () => {
-		const nodes = ["a", "b", "c"].map((id) => ({ id }));
-		// follows a→b AND supports b→a — same pair twice, both directions.
-		const result = detectCommunities(nodes, [
-			{ from: "a", to: "b" },
-			{ from: "b", to: "a" },
-			{ from: "b", to: "c" },
-		]);
-		assert.equal(result.communities.length, 1);
-		assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b", "c"]);
-	});
-
-	test("self-loops and unknown-endpoint edges are skipped defensively", () => {
-		const nodes = ["a", "b"].map((id) => ({ id }));
-		const result = detectCommunities(nodes, [
-			{ from: "a", to: "a" }, // self-loop
-			{ from: "a", to: "ghost" }, // dangling
-			{ from: "ghost1", to: "ghost2" },
-			{ from: "a", to: "b" }, // the one real edge
-		]);
-		assert.equal(result.communities.length, 1);
-		assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b"]);
-	});
-
-	test("members are sorted; communities ordered by size then id", () => {
-		const left = ids(4, "L");
-		const right = ids(4, "R");
-		const result = detectCommunities(
-			[...left, ...right, "iso"].map((id) => ({ id })),
-			[...clique(left), ...clique(right), { from: "L0", to: "R0" }],
-		);
-		for (const c of result.communities as Community[]) {
-			const sorted = c.members.slice().sort();
-			assert.deepEqual(c.members, sorted);
-		}
-		// iso (isolated) is last: size 1 < 4.
-		const last = result.communities[result.communities.length - 1]!;
-		assert.deepEqual(last.members, ["iso"]);
-	});
-
-	test("seed override changes nothing structurally but stays deterministic", () => {
-		const nodes = ids(8).map((id) => ({ id }));
-		const edges = [...clique(["n0", "n1", "n2", "n3"]), ...clique(["n4", "n5", "n6", "n7"]), { from: "n0", to: "n4" }];
-		const a = detectCommunities(nodes, edges, { seed: "one" });
-		const b = detectCommunities(nodes, edges, { seed: "one" });
-		const c = detectCommunities(nodes, edges, { seed: "two" });
-		assert.deepEqual(a.communities, b.communities);
-		// A different seed partitions the same graph into member sets that
-		// may or may not coincide — but both runs are 2 communities here,
-		// because the structure dominates.
-		assert.equal(c.communities.length, 2);
-	});
-});

tests/extraction.test.ts

@@ -40,6 +40,7 @@ import type {
 import type { GraphStore } from "../src/core/ports.ts";
 import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
 import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
+import { semanticClusters } from "../src/core/semantic-clusters.ts";
 import { chatStore } from "../src/store/chats.ts";
 import { PATHS } from "../config.ts";
 
@@ -67,6 +68,13 @@ function memoryLedgers() {
 	const sources = new Map<string, NotesSource>();
 	const clusterLabels: ClusterLabeling[] = [];
 	const positions = new Map<string, { x: number; y: number }>();
+	const embeddings = {
+		model: "",
+		dim: 0,
+		updatedAt: "",
+		vectors: new Map<string, number[]>(),
+		clusters: new Array<Array<string>>(),
+	};
 	return {
 		appendNode: (n: GraphNode) => void nodes.push(n),
 		readNodes: () => nodes.map((n) => ({ ...n })),
@@ -102,6 +110,16 @@ function memoryLedgers() {
 			positions.clear();
 			for (const [id, p] of Object.entries(next)) positions.set(id, { x: p.x, y: p.y });
 		},
+		// The embedding ledger — the embedder writes it after a merge.
+		readEmbeddings: () => ({ model: embeddings.model, dim: embeddings.dim, vectors: Object.fromEntries([...embeddings.vectors].map(([id, v]) => [id, [...v]])), clusters: [...embeddings.clusters.map((c) => ({ members: [...c] }))], updatedAt: embeddings.updatedAt }),
+		writeEmbeddings: (next) => {
+			embeddings.model = next.model;
+			embeddings.dim = next.dim;
+			embeddings.updatedAt = next.updatedAt;
+			embeddings.vectors.clear();
+			for (const [id, v] of Object.entries(next.vectors)) embeddings.vectors.set(id, [...v]);
+			embeddings.clusters = next.clusters.map((c) => [...c.members]);
+		},
 	} satisfies GraphStore;
 }
 
@@ -319,6 +337,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: submitModel("submit_findings", FINDINGS),
 			auditor: submitModel("submit_review", verdict),
 			labeler: submitModel("submit_labels", { labels: [] }),
@@ -360,6 +379,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: submitModel("submit_findings", FINDINGS),
 			auditor: silentModel(), // never submits the review
 			labeler: submitModel("submit_labels", { labels: [] }),
@@ -371,6 +391,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		const failingExtractor = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: silentModel(),
 			auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }),
 			labeler: submitModel("submit_labels", { labels: [] }),
@@ -402,6 +423,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			// Each stage's first generate is the observation point: the
 			// registry must already carry this chat at the right phase.
 			extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())),
@@ -435,6 +457,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			// During the auditor's generate, the registry must already carry
 			// the extractor's fully accumulated live usage (2 steps × 10/5).
 			extractor: submitModel("submit_findings", FINDINGS),
@@ -476,6 +499,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: hangingModel(), // hangs until aborted — no step finishes
 			auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }),
 			labeler: submitModel("submit_labels", { labels: [] }),
@@ -527,6 +551,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: gatedExtractor,
 			auditor: silentModel(),
 			labeler: submitModel("submit_labels", { labels: [] }),
@@ -545,6 +570,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: silentModel(),
 			auditor: silentModel(),
 			labeler: submitModel("submit_labels", { labels: [] }),
@@ -603,6 +629,27 @@ const CLUSTER_VERDICT = {
 	dropped: [],
 } as const;
 
+/** Test seam for the embedder: fakes the post-merge embedding batch —
+ *  one identical unit vector per node → semanticClusters merges them all
+ *  into ONE topic — then persists the partition like the real embedder. */
+const fakeEmbedder = async (store: GraphStore) => {
+	const seen = new Set<string>();
+	const ids: string[] = [];
+	for (const n of store.readNodes()) {
+		if (seen.has(n.id)) continue;
+		seen.add(n.id);
+		ids.push(n.id);
+	}
+	const vectors = Object.fromEntries(ids.map((id) => [id, [1, 0, 0]]));
+	store.writeEmbeddings({
+		model: "test/embedder",
+		dim: 3,
+		vectors,
+		clusters: semanticClusters(vectors),
+		updatedAt: "2026-09-13T00:00:00.000Z",
+	});
+};
+
 describe("cluster naming (the post-extraction garnish)", () => {
 	it("names a new community after a notes merge — one record, claimed on the next refresh", async () => {
 		const ledgers = memoryLedgers();
@@ -611,6 +658,7 @@ describe("cluster naming (the post-extraction garnish)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
 			auditor: submitModel("submit_review", CLUSTER_VERDICT),
 			labeler,
@@ -638,6 +686,7 @@ describe("cluster naming (the post-extraction garnish)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
 			auditor: submitModel("submit_review", CLUSTER_VERDICT),
 			labeler: silentModel(), // never submits — the namer throws
@@ -656,6 +705,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: submitModel("submit_findings", NOTES_FINDINGS),
 			// During the auditor's call the registry must show the notes run.
 			auditor: submitModel("submit_review", NOTES_VERDICT, () => {
@@ -704,6 +754,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
 			createExtractionPipeline({
 				chatStore,
 				graphStore: ledgers,
+				embedder: fakeEmbedder,
 				extractor: submitModel("submit_findings", NOTES_FINDINGS),
 				auditor: submitModel("submit_review", NOTES_VERDICT),
 				labeler: submitModel("submit_labels", { labels: [] }),
@@ -724,6 +775,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
 		const pipeline = createExtractionPipeline({
 			chatStore,
 			graphStore: ledgers,
+			embedder: fakeEmbedder,
 			extractor: hangingModel(),
 			auditor: submitModel("submit_review", NOTES_VERDICT),
 			labeler: submitModel("submit_labels", { labels: [] }),

tests/graph.test.ts

@@ -23,7 +23,7 @@ import type { QuestionAnswerData } from "../src/core/question-core.ts";
 import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
 import { graphStore } from "../src/store/graph.ts";
 import { chatStore } from "../src/store/chats.ts";
-import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";
+import { LEDGER_FILES, PATHS, SOURCES_FILE, EMBEDDINGS_FILE } from "../config.ts";
 
 /** The wired fs adapters, exactly what src/server/stores.ts passes in production. */
 const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps;
@@ -38,7 +38,9 @@ function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: strin
 
 // ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ──────────────
 
-const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE];
+// EMBEDDINGS_FILE too: the semantic-cluster source (read by graphSnapshot)
+// — isolated like the rest so this suite never touches the real vectors.
+const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE, EMBEDDINGS_FILE];
 const backups: Array<{ file: string; bak: string }> = [];
 const createdChats: string[] = [];
 
@@ -407,8 +409,10 @@ describe("the merger (mergeFindings)", () => {
 		assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
 	});
 
-	it("derives structural clusters in graphSnapshot, joined with persisted labelings", () => {
-		// A self-contained 3-clique: three propositions, all mutually linked.
+	it("derives semantic clusters in graphSnapshot, joined with persisted labelings", () => {
+		// Three propositions; the cluster comes from the EMBEDDING ledger
+		// (semantic topics, DECIDED 2026-09-13) — the embedder persists the
+		// partition, the snapshot just reads it. Here we play the embedder.
 		const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const;
 		const merge = mergeFindings(
 			"src_clusterfixture",
@@ -433,6 +437,16 @@ describe("the merger (mergeFindings)", () => {
 			{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
 		);
 		assert.equal(merge.nodes, 3);
+		// The embedder's contract: recluster at the end of the batch and
+		// PERSIST the partition in the embeddings ledger.
+		const ids = tri.map((s) => getGraph(graphStore).nodes.find((n) => n.statement === s)!.id);
+		graphStore.writeEmbeddings({
+			model: "test/embedder",
+			dim: 2,
+			vectors: {},
+			clusters: [{ members: ids }],
+			updatedAt: "2026-09-13T00:00:00.000Z",
+		});
 		const snap = graphSnapshot(graphDeps);
 		// Presentation threshold: every cluster is >= 3 members.
 		assert.ok(snap.clusters.every((c) => c.members.length >= 3));

tests/layout.test.ts

@@ -110,6 +110,7 @@ describe("settleGraphLayout", () => {
 			readEdges: () => [edge("a", "b")],
 			readPositions: () => ({} as Record<string, Position>),
 			writePositions: (p: Record<string, Position>) => void written.push(p),
+			readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }),
 		};
 		settleGraphLayout(store);
 		assert.equal(written.length, 1);
@@ -123,6 +124,7 @@ describe("settleGraphLayout", () => {
 			readEdges: () => [],
 			readPositions: () => ({} as Record<string, Position>),
 			writePositions: (p) => void written.push(p),
+			readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }),
 		});
 		assert.equal(written.length, 0);
 	});

tests/semantic-clusters.test.ts

@@ -0,0 +1,118 @@
+/**
+ * Semantic topic clustering (the Louvain REPLACEMENT, DECIDED
+ * 2026-09-13): average-linkage agglomerative clustering over the
+ * embedding ledger — related TOPICS, not reasoning chains. Pure and
+ * deterministic, so these tests are exact.
+ */
+import { strict as assert } from "node:assert";
+import { describe, test } from "node:test";
+import { semanticClusters, SEMANTIC_SIM_THRESHOLD, cosineSimilarity } from "../src/core/semantic-clusters.ts";
+
+/** Two clearly-related one-hot-ish vectors (high cosine similarity). */
+function near(vec: number[], target: number[], amount = 0.2): number[] {
+	let dot = 0;
+	for (let i = 0; i < vec.length; i++) dot += vec[i]! * target[i]!;
+	const mixed = vec.map((x, i) => x + amount * target[i]!);
+	let mag = 0;
+	for (const x of mixed) mag += x * x;
+	mag = Math.sqrt(mag);
+	return mixed.map((x) => x / mag);
+}
+
+function axis(dim: number, k: number): number[] {
+	return Array.from({ length: dim }, (_, i) => (i === k ? 1 : 0));
+}
+
+describe("cosineSimilarity", () => {
+	test("identical unit vectors → 1; orthogonal → 0; defensive normalization", () => {
+		const a = axis(4, 0);
+		const b = axis(4, 1);
+		assert.ok(cosineSimilarity(a, a) > 0.999);
+		assert.ok(Math.abs(cosineSimilarity(a, b)) < 1e-9);
+		// Unnormalized twins still match.
+		assert.ok(cosineSimilarity([3, 0], [5, 0]) > 0.999);
+	});
+});
+
+describe("semanticClusters", () => {
+	test("empty ledger: zero clusters (nothing embedded yet)", () => {
+		assert.deepEqual(semanticClusters({}), []);
+	});
+
+	test("related vectors merge; distant ones stay separate (threshold)", () => {
+		const dim = 8;
+		const topicA = axis(dim, 0);
+		const topicB = axis(dim, 1);
+		const vectors = {
+			a1: near(topicA, topicB, 0.1),
+			a2: near(topicA, topicB, 0.15),
+			a3: near(topicA, topicB, 0.05),
+			b1: near(topicB, topicA, 0.1),
+			b2: near(topicB, topicA, 0.12),
+		};
+		const clusters = semanticClusters(vectors);
+		// Both topics form; average-linkage similarity within each is well
+		// above the default threshold, across is well below.
+		const big = clusters.filter((c) => c.members.length >= 2);
+		assert.equal(big.length, 2);
+		assert.deepEqual(
+			big.map((c) => c.members.sort()).sort((x, y) => x[0]!.localeCompare(y[0]!)),
+			[["a1", "a2", "a3"], ["b1", "b2"]],
+		);
+	});
+
+	test("threshold is the granularity knob: higher splits, lower merges", () => {
+		const dim = 6;
+		const vectors = {
+			x: near(axis(dim, 0), axis(dim, 1), 0.3),
+			y: near(axis(dim, 1), axis(dim, 0), 0.3),
+		};
+		// Similarity ~0.72: merges below that, splits above.
+		const sim = cosineSimilarity(vectors.x!, vectors.y!);
+		assert.equal(semanticClusters(vectors, { similarityThreshold: sim - 0.01 }).length, 1);
+		assert.equal(semanticClusters(vectors, { similarityThreshold: sim + 0.01 }).length, 2);
+	});
+
+	test("stragglers stay singletons by default; attachStragglers forces membership", () => {
+		const dim = 6;
+		const topic = axis(dim, 0);
+		const vectors = {
+			t1: near(topic, axis(dim, 1), 0.1),
+			t2: near(topic, axis(dim, 2), 0.1),
+			t3: near(topic, axis(dim, 3), 0.1),
+			loner: axis(dim, 5), // orthogonal: fits nothing
+		};
+		const loose = semanticClusters(vectors);
+		const looseLoner = loose.find((c) => c.members.includes("loner"));
+		assert.equal(looseLoner?.members.length, 1, "default: loner is a straggler");
+		assert.equal(loose.find((c) => c.members.length >= 3)?.members.length, 3);
+
+		const allIn = semanticClusters(vectors, { attachStragglers: true });
+		assert.equal(allIn.length, 1, "attach: the loner joins the nearest topic");
+		assert.ok(allIn[0]!.members.includes("loner"));
+	});
+
+	test("members sorted; clusters ordered by size then id (deterministic)", () => {
+		const dim = 4;
+		const vectors = {
+			solo: axis(dim, 3),
+			p1: near(axis(dim, 0), axis(dim, 1), 0.05),
+			p2: near(axis(dim, 0), axis(dim, 2), 0.05),
+			q1: near(axis(dim, 1), axis(dim, 0), 0.05),
+			q2: near(axis(dim, 1), axis(dim, 2), 0.05),
+		};
+		const a = semanticClusters(vectors);
+		const b = semanticClusters(vectors);
+		assert.deepEqual(a, b, "same input → identical partition");
+		for (let i = 1; i < a.length; i++) {
+			assert.ok(a[i - 1]!.members.length >= a[i]!.members.length, "size-descending");
+		}
+		for (const c of a) {
+			assert.deepEqual(c.members, [...c.members].sort(), "members sorted");
+		}
+	});
+
+	test("default threshold is the swept real-graph value", () => {
+		assert.equal(SEMANTIC_SIM_THRESHOLD, 0.6);
+	});
+});