Semantic topic clusters: local embeddings replace Louvain
Louvain only ever found the reasoning chains (the graph is disconnected by nature — every connected component became its own cluster) and edge-orphans could never join anything. Clusters are now RELATED TOPICS: propositions grouped by meaning. - Embedder (src/server/embeddings.ts): in-process transformers.js v4 running EmbeddingGemma-300M q8 (MTEB 68.4, ~300MB one-time download), MRL-truncated to 512d, "title: none | text:" prefix on 100% of inputs, L2-normalized so cosine = dot. Model/dim mismatch voids the ledger (full re-embed). Injected at the composition root — the agent layer never imports the server layer (architecture law). - Ledger (data/graph/embeddings.json): model + dim + vectors + the persisted cluster partition. Reclustering runs at the END of every embedding batch (post-merge hook, both front doors) — never per read (the agglomerative pass is ~1.5s at 338 nodes). - Clustering (src/core/semantic-clusters.ts): hand-rolled average-linkage agglomerative over cosine (npm HDBSCAN is an abandoned alpha). Stragglers stay singletons by default; attachStragglers is the all-in toggle. Threshold 0.60 swept on the real graph: 26 topics, 74/90 edge-orphans land in real topics, 10 stragglers. - Consumers (snapshot, settle, namer, replay) read the persisted partition — one cluster concept everywhere; core-survival label matching unchanged (20/26 topics kept labels). - Louvain retired: detectCommunities + graphology* deps deleted (graphology-louvain had leaked into the client bundle since the clusters wave); scripts/bench-graph.ts (dead FA2 bench) removed. - scripts/embed-graph.ts: backfill/cold-start (338 nodes in 150s including the one-time model download).
This commit is contained in:
amosblomqvist
@@ -21,12 +21,7 @@ import type { PiEngine } from "../models.ts";
|
||||
import { runSubagent } from "./pi-run.ts";
|
||||
import type { GraphNode } from "../../core/contracts.ts";
|
||||
import { getGraph, clusterContentId } from "../../core/graph.ts";
|
||||
import {
|
||||
detectCommunities,
|
||||
matchClusterLabelings,
|
||||
MIN_CLUSTER_SIZE,
|
||||
type Community,
|
||||
} from "../../core/clusters.ts";
|
||||
import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../../core/clusters.ts";
|
||||
import type { GraphStore } from "../../core/ports.ts";
|
||||
|
||||
/** How many member propositions the namer sees per cluster — enough to
|
||||
@@ -138,14 +133,17 @@ async function doRefresh(deps: {
|
||||
signal?: AbortSignal;
|
||||
}): Promise<ClusterRefreshStats> {
|
||||
const { graphStore, engine, signal } = deps;
|
||||
const { nodes, edges } = getGraph(graphStore);
|
||||
const detection = detectCommunities(nodes, edges);
|
||||
const { nodes } = getGraph(graphStore);
|
||||
// Semantic topics: the partition the embedder persisted in the
|
||||
// embedding ledger (same source the snapshot renders — never
|
||||
// recomputed here; the agglomerative pass is ~1.5s at scale).
|
||||
const communities = graphStore.readEmbeddings().clusters;
|
||||
const labelings = graphStore.readClusterLabels();
|
||||
const claims = matchClusterLabelings(detection.communities, labelings);
|
||||
const claims = matchClusterLabelings(communities, labelings);
|
||||
const byId = new Map(nodes.map((n) => [n.id, n]));
|
||||
|
||||
const claimed = claims.filter((c) => c !== null).length;
|
||||
const unnamed = detection.communities
|
||||
const unnamed = communities
|
||||
.filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE)
|
||||
.map((c: Community, i) => {
|
||||
const nodesFor = c.members
|
||||
@@ -156,7 +154,7 @@ async function doRefresh(deps: {
|
||||
return { key: `c${i}`, members: c.members, nodes: nodesFor };
|
||||
});
|
||||
|
||||
if (unnamed.length === 0) return { detected: detection.communities.length, claimed, named: 0, appended: 0 };
|
||||
if (unnamed.length === 0) return { detected: communities.length, claimed, named: 0, appended: 0 };
|
||||
|
||||
const labels = await nameClusters(engine, signal, unnamed);
|
||||
let appended = 0;
|
||||
@@ -174,7 +172,7 @@ async function doRefresh(deps: {
|
||||
});
|
||||
appended++;
|
||||
}
|
||||
return { detected: detection.communities.length, claimed, named: unnamed.length, appended };
|
||||
return { detected: communities.length, claimed, named: unnamed.length, appended };
|
||||
}
|
||||
|
||||
let namingTail: Promise<unknown> = Promise.resolve();
|
||||
|
||||
@@ -16,6 +16,7 @@ import type {
|
||||
SessionFindings,
|
||||
} from "../../core/contracts.ts";
|
||||
import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
|
||||
import type { GraphStore } from "../../core/ports.ts";
|
||||
import { settleGraphLayout } from "../../core/layout-settle.ts";
|
||||
import { refreshClusterLabels } from "./cluster-namer.ts";
|
||||
|
||||
@@ -554,7 +555,14 @@ export interface ExtractionModels {
|
||||
labeler?: PiEngine;
|
||||
}
|
||||
|
||||
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
|
||||
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {
|
||||
/** The post-merge embedding batch (semantic clusters run on it).
|
||||
* INJECTED by the composition root (the agent layer never imports
|
||||
* the server layer — architecture law): production passes the real
|
||||
* local-model embedder (server/embeddings.ts embedNewNodes), tests
|
||||
* fake it. Failure inside it is non-fatal by contract. */
|
||||
embedder: (graphStore: GraphStore) => Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface NotesImportInput {
|
||||
/** The source's file path — its identity (deterministic src id) AND its
|
||||
@@ -576,7 +584,7 @@ export interface ExtractionPipeline {
|
||||
}
|
||||
|
||||
export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline {
|
||||
const { chatStore, graphStore } = deps;
|
||||
const { chatStore, graphStore, embedder: embed } = deps;
|
||||
const inFlight = new Set<string>();
|
||||
|
||||
function setPhase(chatId: string, phase: ExtractionPhase, kind: "chat" | "notes", label?: string): void {
|
||||
@@ -715,6 +723,13 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
messages: record.messages as UIMessage[],
|
||||
foldedAt: record.updatedAt,
|
||||
});
|
||||
// Semantic embeddings (DECIDED 2026-09-13): topics come from
|
||||
// MEANING, so embed the new propositions BEFORE the settle and
|
||||
// the namer look at clusters. Failure is non-fatal — clustering
|
||||
// runs without the new nodes until the next merge retries.
|
||||
await embed(graphStore).catch((err) => {
|
||||
console.error(`[extraction ${chatId}] embedding failed (non-fatal):`, err);
|
||||
});
|
||||
// Option A: the layout ledger evolves with the merge (sync — the
|
||||
// next snapshot fetch must already reflect the new nodes).
|
||||
settleGraphLayout(graphStore);
|
||||
@@ -828,6 +843,11 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
// (messages: null) — nothing to attribute, nothing to build.
|
||||
const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } };
|
||||
const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt });
|
||||
// Same post-merge chain as the chat path: embeddings first
|
||||
// (topics from meaning), then the settle, then the namer.
|
||||
await embed(graphStore).catch((err) => {
|
||||
console.error(`[extraction ${id}] embedding failed (non-fatal):`, err);
|
||||
});
|
||||
// Option A: same post-merge settle as the chat path.
|
||||
settleGraphLayout(graphStore);
|
||||
persistRunUsage(id, Math.round(performance.now() - t0), stats);
|
||||
|
||||
+12
-108
@@ -1,31 +1,19 @@
|
||||
/**
|
||||
* COMMUNITY DETECTION for the knowledge graph (vision §1, DECIDED
|
||||
* 2026-09-13): Louvain modularity over the undirected, unweighted
|
||||
* follows+supports structure — purely structural, so it never depends on
|
||||
* layout, and the extraction pipeline never knows clusters exist.
|
||||
* CLUSTER LABEL MATCHING (vision §1). DETECTION moved to semantic topic
|
||||
* clustering over embeddings (core/semantic-clusters.ts, DECIDED
|
||||
* 2026-09-13) — Louvain over the edge structure is retired: it only ever
|
||||
* found the reasoning chains (the graph is disconnected by nature), and
|
||||
* edge-orphans could never join anything. What REMAINS here is the
|
||||
* cluster contract + core-survival label matching, which is
|
||||
* detection-agnostic: it consumes the same { members } shape semantic
|
||||
* clustering emits.
|
||||
*
|
||||
* Determinism: Louvain's random-walk traversal is driven by a SEEDED
|
||||
* mulberry32 RNG (default seed constant below), so the same graph always
|
||||
* yields the same partition. Community INDICES are still not stable
|
||||
* across graph edits — which is why labelings (Phase B) key to sorted
|
||||
* MEMBER SETS, never to indices.
|
||||
* Labelings key to sorted MEMBER SETS, never to detection indices —
|
||||
* a labeling record outlives any reshuffle of the partition.
|
||||
*
|
||||
* Undirected + unweighted (user-decided 2026-09-13): both edge types are
|
||||
* equally real relations between propositions — direction is
|
||||
* derivational-vs-grounding, not stronger-vs-weaker — and a follows AND
|
||||
* a supports between the same pair collapse to one edge.
|
||||
*
|
||||
* This module is browser-safe pure logic (no node builtins, no fs): the
|
||||
* server runs it for the snapshot payload, the renderer for nothing
|
||||
* (clusters arrive precomputed), and tests run it directly.
|
||||
*
|
||||
* NOTE: no minimum-size filter here — detection returns every community,
|
||||
* singletons included. The ≥3-node render threshold (decided) is a
|
||||
* PRESENTATION rule; it lives in the payload derivation and the
|
||||
* renderer, not in the algorithm.
|
||||
* Browser-safe pure logic (no node builtins, no fs): the server runs
|
||||
* matching for the snapshot payload, tests run it directly.
|
||||
*/
|
||||
import Graph from "graphology";
|
||||
import louvain from "graphology-communities-louvain";
|
||||
import type { ClusterLabeling } from "./contracts.ts";
|
||||
|
||||
/** A detected community. Members are SORTED node ids — the member set IS
|
||||
@@ -39,90 +27,6 @@ export interface Community {
|
||||
* least three propositions to be worth a hull and a label. */
|
||||
export const MIN_CLUSTER_SIZE = 3;
|
||||
|
||||
export interface CommunityDetection {
|
||||
communities: Community[];
|
||||
/** Modularity of the found partition — the one quality number. */
|
||||
modularity: number;
|
||||
}
|
||||
|
||||
export interface DetectOptions {
|
||||
/** Louvain resolution (higher → more, smaller communities). Default 1.
|
||||
* Parked for the multi-scale work; exposed now so that dial has a home. */
|
||||
resolution?: number;
|
||||
/** RNG seed. Default "lh-clusters". Same seed + same graph → same
|
||||
* partition, always. */
|
||||
seed?: string;
|
||||
}
|
||||
|
||||
/** mulberry32 seeded by a string (FNV-1a) — the same recipe as
|
||||
* question-core's seededRandom, inlined so this module has no
|
||||
* cross-domain dependency. */
|
||||
function seededRng(seed: string): () => number {
|
||||
let h = 2166136261;
|
||||
for (let i = 0; i < seed.length; i++) {
|
||||
h ^= seed.charCodeAt(i);
|
||||
h = Math.imul(h, 16777619);
|
||||
}
|
||||
let a = h >>> 0;
|
||||
return () => {
|
||||
a |= 0;
|
||||
a = (a + 0x6d2b79f5) | 0;
|
||||
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
||||
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect communities over a graph's structure. Inputs are the minimal
|
||||
* structural shapes of the snapshot contracts (GraphNode.id,
|
||||
* GraphEdge.from/to), so both the real snapshot and test fixtures feed it.
|
||||
*
|
||||
* Edges referencing unknown node ids are skipped defensively (the ledgers
|
||||
* are append-only; a mid-merge read should never crash detection), and
|
||||
* duplicate pairs (follows+supports, or reverse direction) collapse —
|
||||
* mergeEdge keeps the first.
|
||||
*/
|
||||
export function detectCommunities(
|
||||
nodes: readonly { id: string }[],
|
||||
edges: readonly { from: string; to: string }[],
|
||||
opts?: DetectOptions,
|
||||
): CommunityDetection {
|
||||
const graph = new Graph({ type: "undirected" });
|
||||
for (const node of nodes) graph.addNode(node.id);
|
||||
for (const edge of edges) {
|
||||
if (!graph.hasNode(edge.from) || !graph.hasNode(edge.to)) continue;
|
||||
if (edge.from === edge.to) continue; // self-loops: no community signal
|
||||
graph.mergeEdge(edge.from, edge.to);
|
||||
}
|
||||
// Louvain on a 0-edge graph is degenerate; every node is trivially its
|
||||
// own community (and louvain on an empty graph returns nothing useful).
|
||||
if (graph.size === 0) {
|
||||
return {
|
||||
communities: nodes.map((n) => ({ members: [n.id] })).sort((a, b) => (a.members[0] ?? "").localeCompare(b.members[0] ?? "")),
|
||||
modularity: 0,
|
||||
};
|
||||
}
|
||||
const details = louvain.detailed(graph, {
|
||||
rng: seededRng(opts?.seed ?? "lh-clusters"),
|
||||
resolution: opts?.resolution ?? 1,
|
||||
});
|
||||
const byIndex = new Map<number, string[]>();
|
||||
for (const [id, index] of Object.entries(details.communities)) {
|
||||
const bucket = byIndex.get(index) ?? [];
|
||||
bucket.push(id);
|
||||
byIndex.set(index, bucket);
|
||||
}
|
||||
const communities = [...byIndex.values()]
|
||||
.map((ids) => ({ members: ids.sort() }))
|
||||
.sort(
|
||||
(a, b) =>
|
||||
b.members.length - a.members.length ||
|
||||
(a.members[0] ?? "").localeCompare(b.members[0] ?? ""),
|
||||
);
|
||||
return { communities, modularity: details.modularity };
|
||||
}
|
||||
|
||||
/**
|
||||
* Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED
|
||||
* 2026-09-13): a community keeps a record's label while AT LEAST HALF of
|
||||
|
||||
@@ -278,6 +278,26 @@ export interface AuditResult {
|
||||
* least half its core survives in some single community (core-survival;
|
||||
* under half → the community is renamed, splits can keep only one
|
||||
* successor). */
|
||||
/** The embedding ledger shape (config.EMBEDDINGS_FILE): vectors for
|
||||
* EVERY embedded node, all produced by one model at one dimensionality. */
|
||||
export interface EmbeddingsLedger {
|
||||
/** The embedding model id (e.g. the HF repo) — mismatch = stale
|
||||
* vectors, the writer re-embeds everything. */
|
||||
model: string;
|
||||
/** Vector dimensionality (MRL-truncated on write). */
|
||||
dim: number;
|
||||
/** nodeId → L2-normalized vector (cosine similarity = dot product). */
|
||||
vectors: Record<string, number[]>;
|
||||
/** The topic partition over those vectors, computed by the SAME
|
||||
* embedder run (the user-decided trigger: recluster at the end of
|
||||
* every embedding batch, never per read — the agglomerative pass is
|
||||
* ~1.5s at current scale). Consumers (snapshot, settle, namer) read
|
||||
* this instead of recomputing. Empty when nothing is embedded. */
|
||||
clusters: Array<{ members: string[] }>;
|
||||
/** ISO timestamp of the last write. */
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ClusterLabeling {
|
||||
/** "clu_" + content hash of the sorted core. */
|
||||
id: string;
|
||||
|
||||
+13
-9
@@ -51,7 +51,7 @@ import type {
|
||||
LedgerEvent,
|
||||
LedgerEventKind,
|
||||
} from "./contracts.ts";
|
||||
import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts";
|
||||
import { matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts";
|
||||
import type { QuestionAnswerData } from "./question-core.ts";
|
||||
import type { ChatStore, GraphStore } from "./ports.ts";
|
||||
|
||||
@@ -497,14 +497,18 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
|
||||
(nodeObservations[nodeId] ??= {})[chatId] = note;
|
||||
}
|
||||
|
||||
// Structural communities (vision §1): Louvain over the undirected
|
||||
// follows+supports structure, joined with persisted labelings —
|
||||
// core-survival matching, size-filtered at the presentation threshold.
|
||||
// Detection is DETERMINISTIC (seeded), so no memo in v1: a snapshot
|
||||
// read recomputes it (ms at current scale; revisit at vault scale).
|
||||
const detection = detectCommunities(nodes, edges);
|
||||
const claims = matchClusterLabelings(detection.communities, deps.graphStore.readClusterLabels());
|
||||
const clusters: Cluster[] = detection.communities
|
||||
// Semantic topic communities (DECIDED 2026-09-13, replacing structural
|
||||
// Louvain): average-linkage clustering over embeddings — related
|
||||
// topics, not reasoning chains. Edge-orphans join topics by meaning;
|
||||
// stragglers stay loose (filtered at the presentation threshold
|
||||
// below). The partition is computed ONCE per embedding batch by the
|
||||
// embedder (the agglomerative pass is ~1.5s at current scale) and
|
||||
// PERSISTED in the embedding ledger — reads just load it. Nodes whose
|
||||
// embedding hasn't landed yet simply sit unclustered until the next
|
||||
// read after the embedder runs.
|
||||
const communities = deps.graphStore.readEmbeddings().clusters;
|
||||
const claims = matchClusterLabelings(communities, deps.graphStore.readClusterLabels());
|
||||
const clusters: Cluster[] = communities
|
||||
.map((community, index) => ({ community, claim: claims[index] }))
|
||||
.filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE)
|
||||
.map(({ community, claim }) => ({
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ export * from "./render-settings.ts";
|
||||
export * from "./question-core.ts";
|
||||
// clusters.ts (Louvain) is deliberately NOT re-exported: nothing in the
|
||||
// shell runs community detection — the snapshot carries the partition
|
||||
// from the server. Keeping it out keeps graphology-louvain out of the
|
||||
// from the server. Keeping it out keeps the heavy clustering deps out of the
|
||||
// client bundle (verified 2026-09-13: chunks contained it while this
|
||||
// line existed). Same logic as layout-settle.ts below.
|
||||
export * from "./layout.ts";
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { GraphEdge, GraphNode } from "./contracts.ts";
|
||||
import { detectCommunities } from "./clusters.ts";
|
||||
import type { EmbeddingsLedger, GraphEdge, GraphNode } from "./contracts.ts";
|
||||
import { incrementalLayout } from "./layout.ts";
|
||||
import type { Position } from "./contracts.ts";
|
||||
|
||||
@@ -13,7 +12,7 @@ import type { Position } from "./contracts.ts";
|
||||
*
|
||||
* SPLIT FROM layout.ts deliberately (2026-09-13): layout.ts is in the
|
||||
* browser barrel, and the settle needs the Louvain partition for the
|
||||
* cluster centroid pull — graphology-louvain must not ship in the client
|
||||
* cluster centroid pull — the heavy deps must not ship in the client
|
||||
* bundle. Server code imports this module DIRECTLY; it is never
|
||||
* re-exported through src/core/index.ts.
|
||||
*/
|
||||
@@ -22,6 +21,7 @@ export function settleGraphLayout(graphStore: {
|
||||
readEdges(): GraphEdge[];
|
||||
readPositions(): Record<string, Position>;
|
||||
writePositions(positions: Record<string, Position>): void;
|
||||
readEmbeddings(): EmbeddingsLedger;
|
||||
}): void {
|
||||
const nodes = graphStore.readNodes();
|
||||
if (nodes.length === 0) return;
|
||||
@@ -34,10 +34,11 @@ export function settleGraphLayout(graphStore: {
|
||||
});
|
||||
const edges = graphStore.readEdges();
|
||||
const previous = new Map(Object.entries(graphStore.readPositions()));
|
||||
// The cluster partition for the centroid pull: same Louvain the
|
||||
// snapshot uses (same nodes/edges = same communities). 338 nodes:
|
||||
// ~1-2ms, noise next to the settle itself.
|
||||
const { communities } = detectCommunities(unique, edges);
|
||||
// The cluster partition for the centroid pull: the SAME persisted
|
||||
// semantic topics the snapshot renders (computed by the embedder
|
||||
// once per batch, stored in the embedding ledger) — one cluster
|
||||
// concept everywhere.
|
||||
const communities = graphStore.readEmbeddings().clusters;
|
||||
const next = incrementalLayout(unique, edges, previous, { clusters: communities });
|
||||
graphStore.writePositions(Object.fromEntries(next));
|
||||
}
|
||||
+1
-1
@@ -386,7 +386,7 @@ export interface IncrementalLayoutOptions {
|
||||
gravityStrength?: number;
|
||||
/** Cluster partition (Louvain output shape). Supplied by callers so
|
||||
* this module stays louvain-free (the browser bundle must not carry
|
||||
* graphology-louvain); absent = no cluster pull. */
|
||||
* embedding ledger; absent = no cluster pull. */
|
||||
clusters?: ReadonlyArray<{ members: ReadonlyArray<string> }>;
|
||||
/** Cluster centroid pull strength (default CLUSTER_PULL_STRENGTH). */
|
||||
clusterPullStrength?: number;
|
||||
|
||||
@@ -27,6 +27,7 @@ import type {
|
||||
TurnUsage,
|
||||
NotesSource,
|
||||
ClusterLabeling,
|
||||
EmbeddingsLedger,
|
||||
} from "./contracts.ts";
|
||||
|
||||
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
|
||||
@@ -104,6 +105,13 @@ export interface GraphStore {
|
||||
readPositions(): Record<string, Position>;
|
||||
/** Replace the whole ledger (the settle is a full-graph computation). */
|
||||
writePositions(positions: Record<string, Position>): void;
|
||||
|
||||
/** The embedding ledger (config.EMBEDDINGS_FILE). WRITTEN ONLY by
|
||||
* the embedder (server/embeddings.ts) after a merge — read-only for
|
||||
* clustering consumers (snapshot, settle, namer). */
|
||||
readEmbeddings(): EmbeddingsLedger;
|
||||
/** Replace the whole ledger (embedding is a full-batch computation). */
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void;
|
||||
}
|
||||
|
||||
// ─── JobStore — background sub-agent jobs (data/jobs) ─────────────────────────
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
// Semantic topic clustering — the REPLACEMENT for Louvain community
|
||||
// detection (DECIDED 2026-09-13): structural clusters were just the
|
||||
// reasoning chains (the graph is disconnected by nature — every connected
|
||||
// component became its own "cluster"), and edge-orphans could never join
|
||||
// anything. Clusters are now RELATED TOPICS: nodes whose proposition
|
||||
// MEANING is close, computed over the embedding ledger
|
||||
// (config.EMBEDDINGS_FILE). This pulls orphans into topics by meaning
|
||||
// and groups small chains with the large chains they discuss.
|
||||
//
|
||||
// Algorithm: threshold AGGLOMERATIVE (average linkage over cosine
|
||||
// similarity). No JS HDBSCAN is trustworthy (the npm package is an
|
||||
// abandoned alpha), and at proposition scale (~100s-1000s of short texts)
|
||||
// brute-force pairwise cosine is sub-second — so the honest ~60-line
|
||||
// hand-roll wins. "Stragglers" (nodes that fit nothing) fall out
|
||||
// naturally: they simply never merge, and stay singletons.
|
||||
|
||||
/** The default similarity threshold, SWEPT ON THE REAL GRAPH
|
||||
* (2026-09-13): 0.60 → 26 topics of ~20 propositions, 74 of the 90
|
||||
* edge-orphans land in real topics, stragglers 10. Higher (0.65-0.68)
|
||||
* gives tighter micro-topics with far more stragglers; lower (0.55)
|
||||
* collapses into ~13 mega-topics. */
|
||||
export const SEMANTIC_SIM_THRESHOLD = 0.6;
|
||||
|
||||
/** Cosine similarity of two vectors (defensively normalizes inputs;
|
||||
* ledger vectors are already L2-normalized, so this is a dot product). */
|
||||
export function cosineSimilarity(a: readonly number[], b: readonly number[]): number {
|
||||
let dot = 0;
|
||||
let ma = 0;
|
||||
let mb = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i]! * b[i]!;
|
||||
ma += a[i]! * a[i]!;
|
||||
mb += b[i]! * b[i]!;
|
||||
}
|
||||
if (ma === 0 || mb === 0) return 0;
|
||||
return dot / (Math.sqrt(ma) * Math.sqrt(mb));
|
||||
}
|
||||
|
||||
export interface SemanticClusterOptions {
|
||||
/** Merge while the best cluster pair's average-linkage similarity is
|
||||
* >= this (default SEMANTIC_SIM_THRESHOLD). */
|
||||
similarityThreshold?: number;
|
||||
/** After the cut, force every singleton into its nearest topic
|
||||
* ("all-in" mode) instead of leaving stragglers loose. Default
|
||||
* false: some propositions genuinely fit nothing. */
|
||||
attachStragglers?: boolean;
|
||||
}
|
||||
|
||||
/** Average-linkage agglomerative clustering over embedding vectors.
|
||||
* Pure and deterministic (stable Map iteration, greedy by best
|
||||
* similarity, first-found wins ties). Returns clusters size-desc — the
|
||||
* same { members } shape Louvain produced, so label matching and the
|
||||
* renderer contract are unchanged. An empty/missing ledger returns []
|
||||
* (nothing embedded yet: no clusters, every node loose — the snapshot
|
||||
* degrades gracefully until the first embed). */
|
||||
export function semanticClusters(
|
||||
vectors: Readonly<Record<string, readonly number[]>>,
|
||||
options: SemanticClusterOptions = {},
|
||||
): Array<{ members: string[] }> {
|
||||
const ids = Object.keys(vectors);
|
||||
if (ids.length === 0) return [];
|
||||
const simThreshold = options.similarityThreshold ?? SEMANTIC_SIM_THRESHOLD;
|
||||
const attach = options.attachStragglers ?? false;
|
||||
|
||||
// Sum vectors per cluster: average-linkage similarity between clusters
|
||||
// A and B is dot(sumA, sumB) / (|A|·|B|) — the exact mean of pairwise
|
||||
// dots — so the merge loop never revisits the raw O(n²) matrix.
|
||||
const sums = new Map<number, number[]>();
|
||||
const members = new Map<number, string[]>();
|
||||
ids.forEach((id, i) => {
|
||||
sums.set(i, [...vectors[id]!]);
|
||||
members.set(i, [id]);
|
||||
});
|
||||
|
||||
// Greedy merge: repeatedly join the pair with the highest
|
||||
// average-linkage similarity until it drops below the threshold.
|
||||
// O(k²) pair scan per merge — trivial at proposition scale.
|
||||
for (;;) {
|
||||
let bestA = -1;
|
||||
let bestB = -1;
|
||||
let bestSim = -Infinity;
|
||||
for (const [a, sumA] of sums) {
|
||||
for (const [b, sumB] of sums) {
|
||||
if (a >= b) continue;
|
||||
let dot = 0;
|
||||
for (let i = 0; i < sumA.length; i++) dot += sumA[i]! * sumB[i]!;
|
||||
const sim = dot / (members.get(a)!.length * members.get(b)!.length);
|
||||
if (sim > bestSim) {
|
||||
bestSim = sim;
|
||||
bestA = a;
|
||||
bestB = b;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bestA < 0 || bestSim < simThreshold) break;
|
||||
// Merge B into A (deterministic direction: lower cluster index).
|
||||
const sumA = sums.get(bestA)!;
|
||||
const sumB = sums.get(bestB)!;
|
||||
for (let i = 0; i < sumA.length; i++) sumA[i] = sumA[i]! + sumB[i]!;
|
||||
members.get(bestA)!.push(...members.get(bestB)!);
|
||||
sums.delete(bestB);
|
||||
members.delete(bestB);
|
||||
}
|
||||
|
||||
// Optional all-in pass: singletons join their nearest topic regardless
|
||||
// of threshold — exactly the "forced membership" mode. The difference
|
||||
// from stragglers mode is this one pass. Singleton-vs-cluster
|
||||
// average-linkage similarity is dot(sumS, sumK) / |K|.
|
||||
if (attach) {
|
||||
const singletons = [...members.entries()].filter(([, m]) => m.length === 1).map(([k]) => k);
|
||||
for (const s of singletons) {
|
||||
if (!members.has(s)) continue; // absorbed by an earlier attach
|
||||
let bestTarget = -1;
|
||||
let bestSim = -Infinity;
|
||||
for (const [k, m] of members) {
|
||||
if (k === s || m.length < 2) continue;
|
||||
let dot = 0;
|
||||
const sumS = sums.get(s)!;
|
||||
const sumK = sums.get(k)!;
|
||||
for (let i = 0; i < sumS.length; i++) dot += sumS[i]! * sumK[i]!;
|
||||
const sim = dot / m.length;
|
||||
if (sim > bestSim) {
|
||||
bestSim = sim;
|
||||
bestTarget = k;
|
||||
}
|
||||
}
|
||||
if (bestTarget >= 0) {
|
||||
const sumT = sums.get(bestTarget)!;
|
||||
const sumS = sums.get(s)!;
|
||||
for (let i = 0; i < sumT.length; i++) sumT[i] = sumT[i]! + sumS[i]!;
|
||||
members.get(bestTarget)!.push(...members.get(s)!);
|
||||
sums.delete(s);
|
||||
members.delete(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...members.values()]
|
||||
.map((m) => ({ members: [...m].sort() }))
|
||||
.sort((a, b) => b.members.length - a.members.length || (a.members[0] ?? "").localeCompare(b.members[0] ?? ""));
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { createProberAgent, createTeacherAgent, type TeachingAgent } from "../ag
|
||||
import { piEngineFor } from "../agent/models.ts";
|
||||
import { createJobRunner } from "../agent/subagents/runner.ts";
|
||||
import { createExtractionPipeline } from "../agent/subagents/extraction.ts";
|
||||
import { embedNewNodes } from "./embeddings.ts";
|
||||
import type { ModelRole } from "../../config.ts";
|
||||
import { claimSteers, deliverJobResult, setJobRespawn, setTurnRunner } from "./jobs.ts";
|
||||
import { recordToPiMessages } from "./pi/record-adapter.ts";
|
||||
@@ -40,7 +41,12 @@ const jobRunner = createJobRunner({
|
||||
/** The app's ONE extraction pipeline instance — both front doors share it:
|
||||
* quiz-triggered session runs (turn-service) and notes imports
|
||||
* (/api/graph/import) hit the same single-flight registry. */
|
||||
export const extraction = createExtractionPipeline(stores);
|
||||
export const extraction = createExtractionPipeline({
|
||||
...stores,
|
||||
// The REAL embedder (local transformer model): post-merge embedding
|
||||
// batch — semantic clusters are computed and persisted here.
|
||||
embedder: embedNewNodes,
|
||||
});
|
||||
|
||||
// Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline
|
||||
// fires its server-side teacher turns through THIS composition root, and
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* THE EMBEDDER (DECIDED 2026-09-13): semantic topic clustering needs
|
||||
* vectors, and vectors need a model running ON the user's machine — no
|
||||
* external services, no API bills, no network dependency for re-embedding.
|
||||
*
|
||||
* Runtime: in-process transformers.js (@huggingface/transformers v4 over
|
||||
* onnxruntime-node — native Apple Silicon acceleration). The model
|
||||
* weights download ONCE to the local HF cache on first run (~300MB).
|
||||
*
|
||||
* Model: EmbeddingGemma-300M (q8) — best quality-per-megabyte small
|
||||
* embedding model as of late 2025 (MTEB-en 68.36), built for on-device
|
||||
* use, with MRL truncation so we store 512 of its 768 dims. Every input
|
||||
* uses the model card's document format ("title: none | text: …"),
|
||||
* consistently — mixing prefixed and unprefixed inputs degrades quality.
|
||||
*
|
||||
* Model swaps re-embed EVERYTHING: vectors from different models live in
|
||||
* incompatible spaces. The ledger records model + dim; a mismatch marks
|
||||
* the whole ledger stale and this module re-embeds from scratch (minutes
|
||||
* at current scale).
|
||||
*/
|
||||
import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers";
|
||||
import type { EmbeddingsLedger, GraphNode } from "../core/contracts.ts";
|
||||
import { semanticClusters, SEMANTIC_SIM_THRESHOLD } from "../core/semantic-clusters.ts";
|
||||
|
||||
export const EMBEDDING_MODEL = "onnx-community/embeddinggemma-300m-ONNX";
|
||||
/** MRL truncation of the native 768 — negligible quality loss, half the
|
||||
* ledger size (2026-09-13 researcher sweep). */
|
||||
export const EMBEDDING_DIM = 512;
|
||||
/** Batch size bounds peak memory; propositions are tiny so 32 per call
|
||||
* keeps the pipeline warm without thrashing. */
|
||||
const EMBED_BATCH = 32;
|
||||
|
||||
let pipelinePromise: Promise<FeatureExtractionPipeline> | null = null;
|
||||
|
||||
function getPipeline(): Promise<FeatureExtractionPipeline> {
|
||||
pipelinePromise ??= pipeline("feature-extraction", EMBEDDING_MODEL, { dtype: "q8" });
|
||||
return pipelinePromise;
|
||||
}
|
||||
|
||||
/** EmbeddingGemma's required document format (model card). */
|
||||
function embedInput(statement: string): string {
|
||||
return `title: none | text: ${statement}`;
|
||||
}
|
||||
|
||||
/** Truncate an L2-normalized vector to EMBEDDING_DIM (MRL: leading dims
|
||||
* carry most of the signal) and re-normalize so cosine = dot product. */
|
||||
function truncateNormalize(vec: readonly number[]): number[] {
|
||||
const out = vec.slice(0, EMBEDDING_DIM);
|
||||
let mag = 0;
|
||||
for (const x of out) mag += x * x;
|
||||
mag = Math.sqrt(mag);
|
||||
return mag > 0 ? out.map((x) => x / mag) : out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed every node the ledger is missing — the post-merge hook (new
|
||||
* propositions only, typically a handful) AND the cold-start backfill
|
||||
* (empty ledger = every node) in one function. Model/dim mismatch voids
|
||||
* the ledger first (full re-embed). Also prunes vectors for nodes that
|
||||
* no longer exist. Writes the whole ledger and returns counts for logs.
|
||||
* Non-fatal by contract: the caller treats failure as "clustering runs
|
||||
* without the new nodes until the next merge retries".
|
||||
*/
|
||||
export async function embedNewNodes(
|
||||
graphStore: {
|
||||
readNodes(): GraphNode[];
|
||||
readEmbeddings(): EmbeddingsLedger;
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void;
|
||||
},
|
||||
): Promise<{ embedded: number; total: number }> {
|
||||
const ledger = graphStore.readEmbeddings();
|
||||
const stale = ledger.model !== EMBEDDING_MODEL || ledger.dim !== EMBEDDING_DIM;
|
||||
const vectors: Record<string, number[]> = stale ? {} : { ...ledger.vectors };
|
||||
|
||||
// Node LINES may repeat an id — dedupe, first defines (getGraph rule).
|
||||
const seen = new Set<string>();
|
||||
const todo: GraphNode[] = [];
|
||||
for (const node of graphStore.readNodes()) {
|
||||
if (seen.has(node.id)) continue;
|
||||
seen.add(node.id);
|
||||
if (!vectors[node.id]) todo.push(node);
|
||||
}
|
||||
|
||||
if (todo.length > 0) {
|
||||
const extractor = await getPipeline();
|
||||
for (let i = 0; i < todo.length; i += EMBED_BATCH) {
|
||||
const batch = todo.slice(i, i + EMBED_BATCH);
|
||||
const output = await extractor(batch.map((n) => embedInput(n.statement)), {
|
||||
pooling: "mean",
|
||||
normalize: true,
|
||||
});
|
||||
const rows = output.tolist() as number[][];
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
vectors[batch[j]!.id] = truncateNormalize(rows[j]!);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prune ghosts: vectors for node ids that no longer exist.
|
||||
let pruned = 0;
|
||||
for (const id of Object.keys(vectors)) {
|
||||
if (!seen.has(id)) {
|
||||
delete vectors[id];
|
||||
pruned++;
|
||||
}
|
||||
}
|
||||
|
||||
// RECLUSTER AT THE END OF THE BATCH (user-decided 2026-09-13): the
|
||||
// agglomerative pass is ~1.5s at current scale, so it runs HERE —
|
||||
// once per embedding batch — and consumers (snapshot, settle, namer)
|
||||
// read the persisted partition instead of recomputing per read.
|
||||
const clusters = semanticClusters(vectors, { similarityThreshold: SEMANTIC_SIM_THRESHOLD });
|
||||
|
||||
const wrote = todo.length > 0 || pruned > 0 || stale || ledger.clusters.length !== clusters.length;
|
||||
if (wrote) {
|
||||
graphStore.writeEmbeddings({
|
||||
model: EMBEDDING_MODEL,
|
||||
dim: EMBEDDING_DIM,
|
||||
vectors,
|
||||
clusters,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
return { embedded: todo.length, total: Object.keys(vectors).length };
|
||||
}
|
||||
+59
-1
@@ -29,9 +29,10 @@
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts";
|
||||
import { EMBEDDINGS_FILE, LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts";
|
||||
import type {
|
||||
ClusterLabeling,
|
||||
EmbeddingsLedger,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
ExtractionRunUsage,
|
||||
@@ -216,6 +217,12 @@ export const graphStore: GraphStore = {
|
||||
writePositions(positions: Record<string, Position>): void {
|
||||
writePositionMap(positions);
|
||||
},
|
||||
readEmbeddings(): EmbeddingsLedger {
|
||||
return readEmbeddingsLedger();
|
||||
},
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void {
|
||||
writeEmbeddingsLedger(ledger);
|
||||
},
|
||||
};
|
||||
|
||||
/** positions.json — { positions, updatedAt }. Read-modify-write, NOT
|
||||
@@ -252,6 +259,57 @@ function writePositionMap(positions: Record<string, Position>): void {
|
||||
);
|
||||
}
|
||||
|
||||
/** embeddings.json — the whole EmbeddingsLedger at once. Missing/corrupt
|
||||
* file reads as an EMPTY ledger (model "") so consumers see "nothing
|
||||
* embedded yet" rather than an error; the embedder backfills. */
|
||||
function readEmbeddingsLedger(): EmbeddingsLedger {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(EMBEDDINGS_FILE, "utf-8");
|
||||
} catch {
|
||||
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!isRecord(parsed)) return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
const model = typeof parsed.model === "string" ? parsed.model : "";
|
||||
const dim = typeof parsed.dim === "number" && Number.isFinite(parsed.dim) ? parsed.dim : 0;
|
||||
const vectors: Record<string, number[]> = {};
|
||||
if (isRecord(parsed.vectors)) {
|
||||
for (const [id, value] of Object.entries(parsed.vectors)) {
|
||||
if (
|
||||
Array.isArray(value) &&
|
||||
value.length > 0 &&
|
||||
value.every((x) => typeof x === "number" && Number.isFinite(x))
|
||||
) {
|
||||
vectors[id] = value as number[];
|
||||
}
|
||||
}
|
||||
}
|
||||
const clusters: Array<{ members: string[] }> = [];
|
||||
if (Array.isArray(parsed.clusters)) {
|
||||
for (const c of parsed.clusters) {
|
||||
if (
|
||||
isRecord(c) &&
|
||||
Array.isArray(c.members) &&
|
||||
c.members.length > 0 &&
|
||||
c.members.every((m) => typeof m === "string")
|
||||
) {
|
||||
clusters.push({ members: [...(c.members as string[])] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { model, dim, vectors, clusters, updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "" };
|
||||
} catch {
|
||||
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
}
|
||||
}
|
||||
|
||||
function writeEmbeddingsLedger(ledger: EmbeddingsLedger): void {
|
||||
fs.mkdirSync(path.dirname(EMBEDDINGS_FILE), { recursive: true });
|
||||
fs.writeFileSync(EMBEDDINGS_FILE, JSON.stringify(ledger, null, "\t"), "utf-8");
|
||||
}
|
||||
|
||||
const isPosition = (v: unknown): v is Position =>
|
||||
isRecord(v) && typeof v.x === "number" && Number.isFinite(v.x) && typeof v.y === "number" && Number.isFinite(v.y);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user