Semantic topic clusters: local embeddings replace Louvain

Louvain only ever found the reasoning chains (the graph is disconnected
by nature — every connected component became its own cluster) and
edge-orphans could never join anything. Clusters are now RELATED TOPICS:
propositions grouped by meaning.

- Embedder (src/server/embeddings.ts): in-process transformers.js v4
  running EmbeddingGemma-300M q8 (MTEB 68.4, ~300MB one-time download),
  MRL-truncated to 512d, "title: none | text:" prefix on 100% of inputs,
  L2-normalized so cosine = dot. Model/dim mismatch voids the ledger
  (full re-embed). Injected at the composition root — the agent layer
  never imports the server layer (architecture law).
- Ledger (data/graph/embeddings.json): model + dim + vectors + the
  persisted cluster partition. Reclustering runs at the END of every
  embedding batch (post-merge hook, both front doors) — never per read
  (the agglomerative pass is ~1.5s at 338 nodes).
- Clustering (src/core/semantic-clusters.ts): hand-rolled average-linkage
  agglomerative over cosine (npm HDBSCAN is an abandoned alpha). Stragglers
  stay singletons by default; attachStragglers is the all-in toggle.
  Threshold 0.60 swept on the real graph: 26 topics, 74/90 edge-orphans
  land in real topics, 10 stragglers.
- Consumers (snapshot, settle, namer, replay) read the persisted
  partition — one cluster concept everywhere; core-survival label
  matching unchanged (20/26 topics kept labels).
- Louvain retired: detectCommunities + graphology* deps deleted
  (graphology-louvain had leaked into the client bundle since the
  clusters wave); scripts/bench-graph.ts (dead FA2 bench) removed.
- scripts/embed-graph.ts: backfill/cold-start (338 nodes in 150s
  including the one-time model download).

src/agent/subagents/cluster-namer.ts

@@ -21,12 +21,7 @@ import type { PiEngine } from "../models.ts";
 import { runSubagent } from "./pi-run.ts";
 import type { GraphNode } from "../../core/contracts.ts";
 import { getGraph, clusterContentId } from "../../core/graph.ts";
-import {
-	detectCommunities,
-	matchClusterLabelings,
-	MIN_CLUSTER_SIZE,
-	type Community,
-} from "../../core/clusters.ts";
+import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../../core/clusters.ts";
 import type { GraphStore } from "../../core/ports.ts";
 
 /** How many member propositions the namer sees per cluster — enough to
@@ -138,14 +133,17 @@ async function doRefresh(deps: {
 	signal?: AbortSignal;
 }): Promise<ClusterRefreshStats> {
 	const { graphStore, engine, signal } = deps;
-	const { nodes, edges } = getGraph(graphStore);
-	const detection = detectCommunities(nodes, edges);
+	const { nodes } = getGraph(graphStore);
+	// Semantic topics: the partition the embedder persisted in the
+	// embedding ledger (same source the snapshot renders — never
+	// recomputed here; the agglomerative pass is ~1.5s at scale).
+	const communities = graphStore.readEmbeddings().clusters;
 	const labelings = graphStore.readClusterLabels();
-	const claims = matchClusterLabelings(detection.communities, labelings);
+	const claims = matchClusterLabelings(communities, labelings);
 	const byId = new Map(nodes.map((n) => [n.id, n]));
 
 	const claimed = claims.filter((c) => c !== null).length;
-	const unnamed = detection.communities
+	const unnamed = communities
 		.filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE)
 		.map((c: Community, i) => {
 			const nodesFor = c.members
@@ -156,7 +154,7 @@ async function doRefresh(deps: {
 			return { key: `c${i}`, members: c.members, nodes: nodesFor };
 		});
 
-	if (unnamed.length === 0) return { detected: detection.communities.length, claimed, named: 0, appended: 0 };
+	if (unnamed.length === 0) return { detected: communities.length, claimed, named: 0, appended: 0 };
 
 	const labels = await nameClusters(engine, signal, unnamed);
 	let appended = 0;
@@ -174,7 +172,7 @@ async function doRefresh(deps: {
 		});
 		appended++;
 	}
-	return { detected: detection.communities.length, claimed, named: unnamed.length, appended };
+	return { detected: communities.length, claimed, named: unnamed.length, appended };
 }
 
 let namingTail: Promise<unknown> = Promise.resolve();

src/agent/subagents/extraction.ts

@@ -16,6 +16,7 @@ import type {
 	SessionFindings,
 } from "../../core/contracts.ts";
 import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
+import type { GraphStore } from "../../core/ports.ts";
 import { settleGraphLayout } from "../../core/layout-settle.ts";
 import { refreshClusterLabels } from "./cluster-namer.ts";
 
@@ -554,7 +555,14 @@ export interface ExtractionModels {
 	labeler?: PiEngine;
 }
 
-export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
+export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {
+	/** The post-merge embedding batch (semantic clusters run on it).
+	 *  INJECTED by the composition root (the agent layer never imports
+	 *  the server layer — architecture law): production passes the real
+	 *  local-model embedder (server/embeddings.ts embedNewNodes), tests
+	 *  fake it. Failure inside it is non-fatal by contract. */
+	embedder: (graphStore: GraphStore) => Promise<unknown>;
+}
 
 export interface NotesImportInput {
 	/** The source's file path — its identity (deterministic src id) AND its
@@ -576,7 +584,7 @@ export interface ExtractionPipeline {
 }
 
 export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline {
-	const { chatStore, graphStore } = deps;
+	const { chatStore, graphStore, embedder: embed } = deps;
 	const inFlight = new Set<string>();
 
 	function setPhase(chatId: string, phase: ExtractionPhase, kind: "chat" | "notes", label?: string): void {
@@ -715,6 +723,13 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
 				messages: record.messages as UIMessage[],
 				foldedAt: record.updatedAt,
 			});
+			// Semantic embeddings (DECIDED 2026-09-13): topics come from
+			// MEANING, so embed the new propositions BEFORE the settle and
+			// the namer look at clusters. Failure is non-fatal — clustering
+			// runs without the new nodes until the next merge retries.
+			await embed(graphStore).catch((err) => {
+				console.error(`[extraction ${chatId}] embedding failed (non-fatal):`, err);
+			});
 			// Option A: the layout ledger evolves with the merge (sync — the
 			// next snapshot fetch must already reflect the new nodes).
 			settleGraphLayout(graphStore);
@@ -828,6 +843,11 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
 			// (messages: null) — nothing to attribute, nothing to build.
 			const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } };
 			const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt });
+			// Same post-merge chain as the chat path: embeddings first
+			// (topics from meaning), then the settle, then the namer.
+			await embed(graphStore).catch((err) => {
+				console.error(`[extraction ${id}] embedding failed (non-fatal):`, err);
+			});
 			// Option A: same post-merge settle as the chat path.
 			settleGraphLayout(graphStore);
 			persistRunUsage(id, Math.round(performance.now() - t0), stats);