sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Semantic topic clusters: local embeddings replace Louvain

Browse Source 
Louvain only ever found the reasoning chains (the graph is disconnected
by nature — every connected component became its own cluster) and
edge-orphans could never join anything. Clusters are now RELATED TOPICS:
propositions grouped by meaning.

- Embedder (src/server/embeddings.ts): in-process transformers.js v4
  running EmbeddingGemma-300M q8 (MTEB 68.4, ~300MB one-time download),
  MRL-truncated to 512d, "title: none | text:" prefix on 100% of inputs,
  L2-normalized so cosine = dot. Model/dim mismatch voids the ledger
  (full re-embed). Injected at the composition root — the agent layer
  never imports the server layer (architecture law).
- Ledger (data/graph/embeddings.json): model + dim + vectors + the
  persisted cluster partition. Reclustering runs at the END of every
  embedding batch (post-merge hook, both front doors) — never per read
  (the agglomerative pass is ~1.5s at 338 nodes).
- Clustering (src/core/semantic-clusters.ts): hand-rolled average-linkage
  agglomerative over cosine (npm HDBSCAN is an abandoned alpha). Stragglers
  stay singletons by default; attachStragglers is the all-in toggle.
  Threshold 0.60 swept on the real graph: 26 topics, 74/90 edge-orphans
  land in real topics, 10 stragglers.
- Consumers (snapshot, settle, namer, replay) read the persisted
  partition — one cluster concept everywhere; core-survival label
  matching unchanged (20/26 topics kept labels).
- Louvain retired: detectCommunities + graphology* deps deleted
  (graphology-louvain had leaked into the client bundle since the
  clusters wave); scripts/bench-graph.ts (dead FA2 bench) removed.
- scripts/embed-graph.ts: backfill/cold-start (338 nodes in 150s
  including the one-time model download).
This commit is contained in:
amosblomqvist
2026-09-13 14:29:16 +03:00
parent 3a1c18b9de
commit f5edc7f0a5
28 changed files with 1443 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/backfill-graph.ts
+2 -1
View File
@@ -8,13 +8,14 @@
import { chatStore } from "../src/store/chats.ts";
import { graphStore } from "../src/store/graph.ts";
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
import { embedNewNodes } from "../src/server/embeddings.ts";
import { getGraph } from "../src/core/graph.ts";
async function main() {
const only = process.argv[2]; // optional: extract just one chat id
const chats = only ? chatStore.listChats().filter((c) => c.id === only) : chatStore.listChats();
console.log(`[backfill] ${chats.length} chats to extract`);
const pipeline = createExtractionPipeline({ chatStore, graphStore });
const pipeline = createExtractionPipeline({ chatStore, graphStore, embedder: embedNewNodes });
for (const chat of chats) {
const t0 = Date.now();
try {
scripts/bench-graph.ts
-70
View File
@@ -1,70 +0,0 @@
/**
* Layout benchmark (W4; vision §1 "Rendering (DECIDED)" — "Pre-implementation
* validation: 2k-node synthetic benchmark to confirm layout time").
*
* npx tsx scripts/bench-graph.ts
*
* Two scales: the vision's target regime (hundreds of nodes, <150 ms budget
* with a synchronous run-then-freeze pass) and the stress case (2,000 nodes,
* ~4,000 edges), the latter timed with and without barnesHutOptimize.
*
* VERDICT (2026-08-22, Apple silicon, Node 24): hundreds of nodes are
* comfortably inside the budget (see the measured lines — ~25 ms at 100,
* ~90 ms at 300). At 2k nodes the naive O(n²) pass takes seconds, and
* barnesHutOptimize cuts it roughly in half — still over budget, which is
* exactly the vision's escalation ladder: Barnes-Hut is already switched on
* automatically above ~500 nodes in components/graph-view.tsx, and the
* bundled Web Worker / server-cached positions stay available as later
* rungs if real graphs ever get there. v1 learner graphs are dozens of
* nodes; the stress case is preparation, not expectation.
*/
import Graph from "graphology";
import forceAtlas2 from "graphology-layout-forceatlas2";
const ITERATIONS = 100;
/** A plausible learning graph: mostly-local chains plus long-range cross-links. */
function synthesize(nodeCount: number, edgeCount: number): Graph {
const graph = new Graph();
for (let i = 0; i < nodeCount; i++) {
graph.addNode(`n${i}`, {
label: `node ${i}`,
size: 6,
x: Math.random() * 100,
y: Math.random() * 100,
});
}
let added = 0;
for (let i = 0; added < edgeCount && i < edgeCount * 3; i++) {
const from = Math.floor(Math.random() * nodeCount);
const span = Math.random() < 0.8 ? 1 + Math.floor(Math.random() * 10) : Math.floor(Math.random() * nodeCount);
const to = (from + span) % nodeCount;
if (from === to || graph.hasEdge(`n${from}`, `n${to}`)) continue;
graph.addEdge(`n${from}`, `n${to}`, { type: "arrow", size: 1 });
added++;
}
return graph;
}
function bench(label: string, nodeCount: number, edgeCount: number, settings: Record<string, unknown>): void {
const graph = synthesize(nodeCount, edgeCount);
const start = performance.now();
forceAtlas2.assign(graph, {
iterations: ITERATIONS,
settings: { ...forceAtlas2.inferSettings(graph), ...settings },
});
const elapsed = performance.now() - start;
console.log(
`${label}: ${elapsed.toFixed(1)} ms (${graph.order} nodes, ${graph.size} edges, ${ITERATIONS} iterations)`,
);
}
console.log(`Target regime — synchronous run-then-freeze (budget: <150 ms):\n`);
bench(" 100 nodes, ~200 edges", 100, 200, {});
bench(" 300 nodes, ~600 edges", 300, 600, {});
bench(" 500 nodes, ~1000 edges", 500, 1000, {});
console.log(`\nStress case — 2k nodes (~4000 edges), the vision's escalation ladder:\n`);
bench(" naive repulsion (O(n²)) ", 2000, 4000, {});
bench(" barnesHutOptimize ", 2000, 4000, { barnesHutOptimize: true });
bench(" barnesHutOptimize (warmed JIT)", 2000, 4000, { barnesHutOptimize: true });
scripts/embed-graph.ts
+23
View File
@@ -0,0 +1,23 @@
/**
* Embed the whole graph (backfill / cold start): one-time pass that
* fills the embedding ledger for every node — the same embedNewNodes the
* post-merge hook runs, just with an empty (or stale-model) ledger so
* every node is "new". First run downloads the model weights (~300MB)
* to the local HF cache; subsequent runs embed only what's missing.
*
* npx tsx scripts/embed-graph.ts
*/
import { graphStore } from "../src/store/graph.ts";
import { embedNewNodes, EMBEDDING_MODEL, EMBEDDING_DIM } from "../src/server/embeddings.ts";
async function main() {
const t0 = Date.now();
console.log(`[embed] model: ${EMBEDDING_MODEL} (dim ${EMBEDDING_DIM}, q8)`);
const { embedded, total } = await embedNewNodes(graphStore);
console.log(`[embed] embedded ${embedded} nodes → ledger holds ${total} vectors (${((Date.now() - t0) / 1000).toFixed(1)}s)`);
}
main().catch((err) => {
console.error("[embed] failed:", err);
process.exit(1);
});
scripts/import-notes.ts
+2 -1
View File
@@ -15,6 +15,7 @@
import * as fs from "node:fs";
import * as path from "node:path";
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
import { embedNewNodes } from "../src/server/embeddings.ts";
import { clusterNamingSettled } from "../src/agent/subagents/cluster-namer.ts";
import { getGraph } from "../src/core/graph.ts";
import { stores } from "../src/server/stores.ts";
@@ -28,7 +29,7 @@ async function main() {
.sort();
console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`);
const pipeline = createExtractionPipeline(stores);
const pipeline = createExtractionPipeline({ ...stores, embedder: embedNewNodes });
const queue = [...files];
const totals = { nodes: 0, edges: 0, events: 0, failed: 0 };
const workers = Array.from({ length: Math.min(limit, files.length) }, async () => {
scripts/replay-layout.ts
+12 -3
View File
@@ -13,7 +13,6 @@
* playground mode baked into the rest state); omit it for the free layout.
*/
import { graphStore } from "../src/store/graph.ts";
import { detectCommunities } from "../src/core/clusters.ts";
// --depth=vertical|radial: bake the playground depth mode into the ledger.
const depthArg = process.argv.find((a) => a.startsWith("--depth="));
const depthMode: "off" | "vertical" | "radial" =
@@ -33,6 +32,12 @@ function main() {
nodes.push(line);
}
const edges = graphStore.readEdges();
// The semantic-topic source: the embedding ledger (run
// scripts/embed-graph.ts first if missing/stale).
const embeddings = graphStore.readEmbeddings();
if (Object.keys(embeddings.vectors).length === 0) {
console.warn("[replay] embedding ledger is empty — clusters will be absent (run scripts/embed-graph.ts)");
}
// Batches: each imported file, in real import order (sources.json
// importedAt). A node belongs to the file whose mapping first claims it
@@ -79,8 +84,12 @@ function main() {
const before = new Map(positions);
const batchNodes = nodes.filter((n) => delivered.has(n.id));
positions = incrementalLayout(batchNodes, activeEdges, positions, {
// The same partition the real settle would see for this batch.
clusters: detectCommunities(batchNodes, activeEdges).communities,
// The same partition the real settle would see for this batch:
// the persisted semantic topics, filtered to delivered nodes
// (never recomputed — the agglomerative pass is ~1.5s at scale).
clusters: embeddings.clusters
.map((c) => ({ members: c.members.filter((id) => delivered.has(id)) }))
.filter((c) => c.members.length > 0),
...(depthMode === "off"
? {}
: {