sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Semantic topic clusters: local embeddings replace Louvain

Browse Source 
Louvain only ever found the reasoning chains (the graph is disconnected
by nature — every connected component became its own cluster) and
edge-orphans could never join anything. Clusters are now RELATED TOPICS:
propositions grouped by meaning.

- Embedder (src/server/embeddings.ts): in-process transformers.js v4
  running EmbeddingGemma-300M q8 (MTEB 68.4, ~300MB one-time download),
  MRL-truncated to 512d, "title: none | text:" prefix on 100% of inputs,
  L2-normalized so cosine = dot. Model/dim mismatch voids the ledger
  (full re-embed). Injected at the composition root — the agent layer
  never imports the server layer (architecture law).
- Ledger (data/graph/embeddings.json): model + dim + vectors + the
  persisted cluster partition. Reclustering runs at the END of every
  embedding batch (post-merge hook, both front doors) — never per read
  (the agglomerative pass is ~1.5s at 338 nodes).
- Clustering (src/core/semantic-clusters.ts): hand-rolled average-linkage
  agglomerative over cosine (npm HDBSCAN is an abandoned alpha). Stragglers
  stay singletons by default; attachStragglers is the all-in toggle.
  Threshold 0.60 swept on the real graph: 26 topics, 74/90 edge-orphans
  land in real topics, 10 stragglers.
- Consumers (snapshot, settle, namer, replay) read the persisted
  partition — one cluster concept everywhere; core-survival label
  matching unchanged (20/26 topics kept labels).
- Louvain retired: detectCommunities + graphology* deps deleted
  (graphology-louvain had leaked into the client bundle since the
  clusters wave); scripts/bench-graph.ts (dead FA2 bench) removed.
- scripts/embed-graph.ts: backfill/cold-start (338 nodes in 150s
  including the one-time model download).
This commit is contained in:
amosblomqvist
2026-09-13 14:29:16 +03:00
parent 3a1c18b9de
commit f5edc7f0a5
28 changed files with 1443 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/clusters.test.ts
+1 -88
View File
@@ -6,9 +6,8 @@
*/
import { strict as assert } from "node:assert";
import { describe, test } from "node:test";
import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "../src/core/clusters.ts";
import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../src/core/clusters.ts";
import { clusterHue } from "../components/cluster-geometry.ts";
import type { Community } from "../src/core/clusters.ts";
import type { ClusterLabeling } from "../src/core/contracts.ts";
const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`);
@@ -84,89 +83,3 @@ describe("clusterHue", () => {
});
});
describe("detectCommunities", () => {
test("two dense cliques joined by a single bridge split into two communities", () => {
const left = ids(4, "L");
const right = ids(4, "R");
const bridge = { from: "L0", to: "R0" };
const result = detectCommunities([...left, ...right].map((id) => ({ id })), [...clique(left), ...clique(right), bridge]);
assert.equal(result.communities.length, 2);
assert.equal(result.modularity > 0.3, true);
// Member sets, not order: the two big communities are exactly the cliques.
const big = result.communities.filter((c) => c.members.length === 4);
assert.deepEqual(big.map((c) => c.members.sort()).sort(), [left.slice().sort(), right.slice().sort()]);
});
test("deterministic: same graph, same seed → identical partition", () => {
const nodes = ids(10).map((id) => ({ id }));
const edges = [...clique(ids(5)), ...clique(ids(5, "n").slice().reverse().slice(0, 5)), { from: "n0", to: "n9" }];
const a = detectCommunities(nodes, edges);
const b = detectCommunities(nodes, edges);
assert.deepEqual(a.communities, b.communities);
assert.equal(a.modularity, b.modularity);
});
test("no edges: every node is its own singleton community", () => {
const result = detectCommunities(ids(3).map((id) => ({ id })), []);
assert.equal(result.communities.length, 3);
for (const c of result.communities) assert.equal(c.members.length, 1);
assert.equal(result.modularity, 0);
});
test("empty graph: zero communities", () => {
assert.deepEqual(detectCommunities([], []), { communities: [], modularity: 0 });
});
test("duplicate and reversed pairs collapse to one edge (undirected, unweighted)", () => {
const nodes = ["a", "b", "c"].map((id) => ({ id }));
// follows a→b AND supports b→a — same pair twice, both directions.
const result = detectCommunities(nodes, [
{ from: "a", to: "b" },
{ from: "b", to: "a" },
{ from: "b", to: "c" },
]);
assert.equal(result.communities.length, 1);
assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b", "c"]);
});
test("self-loops and unknown-endpoint edges are skipped defensively", () => {
const nodes = ["a", "b"].map((id) => ({ id }));
const result = detectCommunities(nodes, [
{ from: "a", to: "a" }, // self-loop
{ from: "a", to: "ghost" }, // dangling
{ from: "ghost1", to: "ghost2" },
{ from: "a", to: "b" }, // the one real edge
]);
assert.equal(result.communities.length, 1);
assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b"]);
});
test("members are sorted; communities ordered by size then id", () => {
const left = ids(4, "L");
const right = ids(4, "R");
const result = detectCommunities(
[...left, ...right, "iso"].map((id) => ({ id })),
[...clique(left), ...clique(right), { from: "L0", to: "R0" }],
);
for (const c of result.communities as Community[]) {
const sorted = c.members.slice().sort();
assert.deepEqual(c.members, sorted);
}
// iso (isolated) is last: size 1 < 4.
const last = result.communities[result.communities.length - 1]!;
assert.deepEqual(last.members, ["iso"]);
});
test("seed override changes nothing structurally but stays deterministic", () => {
const nodes = ids(8).map((id) => ({ id }));
const edges = [...clique(["n0", "n1", "n2", "n3"]), ...clique(["n4", "n5", "n6", "n7"]), { from: "n0", to: "n4" }];
const a = detectCommunities(nodes, edges, { seed: "one" });
const b = detectCommunities(nodes, edges, { seed: "one" });
const c = detectCommunities(nodes, edges, { seed: "two" });
assert.deepEqual(a.communities, b.communities);
// A different seed partitions the same graph into member sets that
// may or may not coincide — but both runs are 2 communities here,
// because the structure dominates.
assert.equal(c.communities.length, 2);
});
});