Clusters phases B+C: labeling records, the naming call, snapshot payload

Core-survival labelings (vision section 1, DECIDED 2026-09-13):
ClusterLabeling records in a new append-only ledger
(data/graph/cluster-labels.jsonl), immutable, id = clu_ + sha256 of the
sorted core so re-naming races and re-runs dedupe at the store seam.
matchClusterLabelings (core/clusters.ts, pure): a community keeps a
record's label while at least half its core survives in it; both sides
single-successor (a split keeps one name; greedy by overlap, ties to
the newer record).

The namer (src/agent/subagents/cluster-namer.ts): ONE singular cheap LLM
call (new 'labeler' role, glm-flash/low) names every new/changed
community at once, given member names + statements (capped at 12
members per cluster). refreshClusterLabels runs POST-EXTRACTION
(user-decided) from both front doors, fire-and-forget — naming failure
never fails the run; refreshes serialize behind a chain so parallel
imports never mint duplicate labels. import-notes.ts awaits
clusterNamingSettled before exit.

Payload: GraphSnapshot.clusters (>= MIN_CLUSTER_SIZE 3) joins detection
with labelings at read time — claimed clusters carry the record's label
and id; unlabeled ones carry a member-hash id and null label. Detection
is deterministic so no memo in v1 (ms at current scale; revisit at
vault scale).

Tests (372/372): matcher half-rule/split/merge/greedy/tie cases; naming
end-to-end through importNotes (one call, one record, claim on
re-refresh), naming failure non-fatal; snapshot derivation with
labeling join; 11 test pipelines now pin an explicit labeler seam.
This commit is contained in:
amosblomqvist
2026-09-13 03:59:48 +03:00
parent 95a4477283
commit f0267a5fd9
12 changed files with 562 additions and 8 deletions
+5
View File
@@ -47,6 +47,9 @@ export const MODELS = {
extractor: { model: "z-ai/glm-5.3-flash", effort: "low" },
/** The auditor verifies evidence exists; keep it cheap. */
auditor: { model: "z-ai/glm-5.3-flash", effort: "low" },
/** Names detected communities — one singular call, given member names
* + statements. Purely a labeling flourish; keep it cheap. */
labeler: { model: "z-ai/glm-5.3-flash", effort: "low" },
} satisfies Record<string, ModelSpec>;
export type ModelRole = keyof typeof MODELS;
@@ -71,6 +74,8 @@ export const LEDGER_FILES = {
events: path.join(PATHS.graph, "events.jsonl"),
mappings: path.join(PATHS.graph, "mappings.jsonl"),
sessions: path.join(PATHS.graph, "sessions.jsonl"),
/** LLM-named community records (contracts/graph.ts ClusterLabeling). */
clusterLabels: path.join(PATHS.graph, "cluster-labels.jsonl"),
} as const;
/**
+8 -1
View File
@@ -15,6 +15,7 @@
import * as fs from "node:fs";
import * as path from "node:path";
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
import { clusterNamingSettled } from "../src/agent/subagents/cluster-namer.ts";
import { getGraph } from "../src/core/graph.ts";
import { stores } from "../src/server/stores.ts";
@@ -52,10 +53,16 @@ async function main() {
});
await Promise.all(workers);
// The post-merge cluster naming is fire-and-forget inside the pipeline;
// settle it before the process exits so labels actually land.
await clusterNamingSettled();
const graph = getGraph(stores.graphStore);
const labels = stores.graphStore.readClusterLabels().length;
console.log(
`[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` +
`graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events`,
`graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events, ` +
`${labels} named clusters`,
);
}
+200
View File
@@ -0,0 +1,200 @@
/**
* CLUSTER NAMING (vision §1, DECIDED 2026-09-13): the garnish layer over
* community detection. Detection is purely structural and deterministic
* (core/clusters.ts); naming is ONE singular cheap LLM call that labels
* every NEW/CHANGED community at once, given member names + statements.
*
* Core-survival: a community keeps an existing labeling while at least
* half its core survives in it (matchClusterLabelings); only communities
* with no surviving label AND at least MIN_CLUSTER_SIZE members are sent
* to the namer. Records are append-only and idempotent (content-hashed
* ids dedupe at the store seam), so a naming race or a re-run appends
* nothing new.
*
* Runs POST-EXTRACTION (user-decided 2026-09-13): both front doors call
* refreshClusterLabels after a successful merge. Failure is NON-FATAL —
* the graph and hulls render fine unlabeled; the next run retries.
*/
import { z } from "zod";
import { lhTool, toPiTools } from "../lh-tool.ts";
import type { PiEngine } from "../models.ts";
import { runSubagent } from "./pi-run.ts";
import type { GraphNode } from "../../core/contracts.ts";
import { getGraph, clusterContentId } from "../../core/graph.ts";
import {
detectCommunities,
matchClusterLabelings,
MIN_CLUSTER_SIZE,
type Community,
} from "../../core/clusters.ts";
import type { GraphStore } from "../../core/ports.ts";
/** How many member propositions the namer sees per cluster — enough to
* recognize the domain without paying for the whole region. */
const MAX_MEMBERS_PER_CLUSTER = 12;
const CLUSTER_NAMER_INSTRUCTIONS = `You name communities of propositions in a personal knowledge graph.
You are given clusters. Each cluster is a set of related propositions
(theory fragments the author is learning) plus, for context, a sketch of
every OTHER cluster already named. For each UNNAMED cluster, mint ONE
short DOMAIN LABEL: 1-4 words naming the region of the world model those
propositions belong to together — like a section heading in a textbook.
Use the members' own vocabulary ("Differential forms", "Sales
psychology", "Server architecture"), not generic words ("Topics",
"Various concepts"). If the propositions are unrelated, that is fine:
name the best common denominator anyway — a reader should recognize the
region at a glance.
Your entire deliverable is ONE submit_labels call. Include EVERY
top-level field even when empty (labels: [] when nothing to name).`;
const labelsSchema = z.object({
labels: z
.array(
z.object({
/** The cluster key from the prompt ("c1", "c2", ...). */
key: z.string(),
label: z.string(),
}),
)
.default([]),
});
export interface UnnamedCluster {
key: string;
members: string[];
nodes: Pick<GraphNode, "name" | "statement">[];
}
/** One single-submit namer run (the same shape as every extraction stage):
* the whole deliverable is ONE zod-validated submit call. */
async function nameClusters(
engine: PiEngine | undefined,
signal: AbortSignal | undefined,
unnamed: UnnamedCluster[],
): Promise<{ key: string; label: string }[]> {
let submitted: { key: string; label: string }[] | undefined;
await runSubagent(
toPiTools({
submit_labels: lhTool({
name: "submit_labels",
description: "Submit the cluster labels.",
schema: labelsSchema,
execute: async (labels) => {
submitted = labels.labels;
return labels;
},
}),
}),
{
role: "labeler",
engine,
systemPrompt: CLUSTER_NAMER_INSTRUCTIONS,
prompt: JSON.stringify(
{
unnamedClusters: unnamed.map((c) => ({
key: c.key,
members: c.nodes.map((n) => ({ name: n.name, statement: n.statement })),
})),
},
null,
2,
),
maxTurns: 4,
abortSignal: signal,
shouldStop: (_message, toolResults) =>
toolResults.some((r) => r.toolName === "submit_labels" && !r.isError),
},
);
if (!submitted) throw new Error("the cluster namer never submitted labels");
return submitted;
}
export interface ClusterRefreshStats {
/** Communities the current graph divides into (all sizes). */
detected: number;
/** Communities that kept a label via core-survival. */
claimed: number;
/** Communities sent to the namer (unnamed, size >= MIN_CLUSTER_SIZE). */
named: number;
/** Labeling records actually appended (idempotent dedupe may shrink this). */
appended: number;
}
/**
* The post-extraction garnish: detect communities on the CURRENT graph,
* join persisted labelings (core-survival), and name what is new — one
* singular LLM call. Appends ClusterLabeling records; the NEXT snapshot
* read joins them (the payload is derived, never stored).
*
* Serialized: concurrent merges (parallel notes imports) chain here, so
* two runs never mint duplicate labels for the same community.
*/
async function doRefresh(deps: {
graphStore: GraphStore;
/** Test seam: a scripted engine (production uses the labeler role). */
engine?: PiEngine;
signal?: AbortSignal;
}): Promise<ClusterRefreshStats> {
const { graphStore, engine, signal } = deps;
const { nodes, edges } = getGraph(graphStore);
const detection = detectCommunities(nodes, edges);
const labelings = graphStore.readClusterLabels();
const claims = matchClusterLabelings(detection.communities, labelings);
const byId = new Map(nodes.map((n) => [n.id, n]));
const claimed = claims.filter((c) => c !== null).length;
const unnamed = detection.communities
.filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE)
.map((c: Community, i) => {
const nodesFor = c.members
.map((id) => byId.get(id))
.filter((n): n is GraphNode => n !== undefined)
.slice(0, MAX_MEMBERS_PER_CLUSTER)
.map((n) => ({ name: n.name, statement: n.statement }));
return { key: `c${i}`, members: c.members, nodes: nodesFor };
});
if (unnamed.length === 0) return { detected: detection.communities.length, claimed, named: 0, appended: 0 };
const labels = await nameClusters(engine, signal, unnamed);
let appended = 0;
const seen = new Set<string>();
for (const { key, members } of unnamed) {
if (seen.has(key)) continue;
const label = labels.find((l) => l.key === key)?.label?.trim();
if (!label) continue;
seen.add(key);
graphStore.appendClusterLabel({
id: clusterContentId(members),
label,
core: members,
createdAt: new Date().toISOString(),
});
appended++;
}
return { detected: detection.communities.length, claimed, named: unnamed.length, appended };
}
let namingTail: Promise<unknown> = Promise.resolve();
/**
* Fire the refresh, serialized behind any refresh already running (a
* parallel import's merge waits its turn; each run reads fresh ledgers).
* Waivable: call sites may fire-and-forget; scripts await the return.
*/
export function refreshClusterLabels(deps: {
graphStore: GraphStore;
engine?: PiEngine;
signal?: AbortSignal;
}): Promise<ClusterRefreshStats> {
const run = namingTail.then(() => doRefresh(deps));
namingTail = run.catch(() => {});
return run;
}
/** Await every refresh fired so far (scripts: don't exit mid-naming). */
export async function clusterNamingSettled(): Promise<void> {
await namingTail;
}
+15
View File
@@ -16,6 +16,7 @@ import type {
SessionFindings,
} from "../../core/contracts.ts";
import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
import { refreshClusterLabels } from "./cluster-namer.ts";
/**
* The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the
@@ -548,6 +549,8 @@ export interface ExtractionModels {
/** Test seam: a scripted engine per stage (production uses the roles). */
extractor?: PiEngine;
auditor?: PiEngine;
/** Test seam for the cluster namer (post-extraction garnish). */
labeler?: PiEngine;
}
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
@@ -716,6 +719,13 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
`[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
`(${Math.round(performance.now() - t0)}ms total)`,
);
// Post-extraction garnish (DECIDED 2026-09-13): name new/changed
// clusters. Fire-and-forget — naming never fails the run.
refreshClusterLabels({
graphStore,
engine: deps.labeler,
signal: runningExtractions.get(chatId)?.controller.signal,
}).catch(() => {});
return stats;
} catch (err) {
persistRunUsage(chatId, Math.round(performance.now() - t0), undefined);
@@ -819,6 +829,11 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
`[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
`(${Math.round(performance.now() - t0)}ms total)`,
);
refreshClusterLabels({
graphStore,
engine: deps.labeler,
signal: runningExtractions.get(id)?.controller.signal,
}).catch(() => {});
return stats;
} catch (err) {
persistRunUsage(id, Math.round(performance.now() - t0), undefined);
+50
View File
@@ -26,6 +26,7 @@
*/
import Graph from "graphology";
import louvain from "graphology-communities-louvain";
import type { ClusterLabeling } from "./contracts.ts";
/** A detected community. Members are SORTED node ids — the member set IS
* the identity a labeling record can outlive a reshuffle with. */
@@ -33,6 +34,11 @@ export interface Community {
members: string[];
}
/** Communities below this size are not rendered, not named — presentation
* rule (user-decided 2026-09-13): a region of the world model needs at
* least three propositions to be worth a hull and a label. */
export const MIN_CLUSTER_SIZE = 3;
export interface CommunityDetection {
communities: Community[];
/** Modularity of the found partition — the one quality number. */
@@ -116,3 +122,47 @@ export function detectCommunities(
);
return { communities, modularity: details.modularity };
}
/**
* Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED
* 2026-09-13): a community keeps a record's label while AT LEAST HALF of
* the record's core survives in it (overlap * 2 >= core.length; exactly
* half survives). Both sides are single-successor: one community holds at
* most one label, one record is claimed by at most one community — on a
* split, only the branch carrying the majority of the core keeps the name;
* on a merge, the best-overlapping record wins.
*
* Greedy assignment by overlap size (largest first; ties broken by newer
* record). Returns a claim (or null) per community, in input order —
* unclaimed communities are the namer's input. Pure: same inputs, same
* claims.
*/
export function matchClusterLabelings(
communities: readonly Community[],
labelings: readonly ClusterLabeling[],
): (ClusterLabeling | null)[] {
interface Pair {
community: number;
labeling: ClusterLabeling;
overlap: number;
}
const pairs: Pair[] = [];
communities.forEach((community, ci) => {
const members = new Set(community.members);
for (const labeling of labelings) {
const overlap = labeling.core.reduce((n, id) => n + (members.has(id) ? 1 : 0), 0);
if (overlap * 2 >= labeling.core.length && overlap > 0) {
pairs.push({ community: ci, labeling, overlap });
}
}
});
pairs.sort((a, b) => b.overlap - a.overlap || (a.labeling.createdAt < b.labeling.createdAt ? 1 : -1));
const claims: (ClusterLabeling | null)[] = communities.map(() => null);
const takenRecords = new Set<string>();
for (const pair of pairs) {
if (claims[pair.community] || takenRecords.has(pair.labeling.id)) continue;
claims[pair.community] = pair.labeling;
takenRecords.add(pair.labeling.id);
}
return claims;
}
+35
View File
@@ -270,6 +270,37 @@ export interface AuditResult {
}
/** The GET /api/graph payload — everything the home view's PURE renderer needs. */
/** cluster-labels.jsonl line — one LLM-named community (vision §1,
* DECIDED 2026-09-13). Append-only history: records are immutable, never
* edited or deleted; dead labelings remain as history. IDENTITY is the
* sorted member set ("core"), hashed into the id, so re-naming the same
* community dedupes and a labeling survives graph edits as long as at
* least half its core survives in some single community (core-survival;
* under half → the community is renamed, splits can keep only one
* successor). */
export interface ClusterLabeling {
/** "clu_" + content hash of the sorted core. */
id: string;
/** The short domain label (1–4 words), e.g. "Differential forms". */
label: string;
/** The member node ids the label was minted for, sorted — the survival core. */
core: string[];
createdAt: string;
}
/** One rendered cluster in the snapshot payload (vision §1). Members are
* node ids; the renderer draws the hull from member positions at draw
* time and gates the label inversely on the node-label zoom control. */
export interface Cluster {
/** The labeling record id when claimed; unlabeled clusters still
* appear (hull, no name) with a member-hash id. */
id: string;
/** null until the naming call has minted/claimed a labeling. */
label: string | null;
/** Node ids, sorted. Hull + centroid are renderer concerns. */
members: string[];
}
export interface GraphSnapshot {
nodes: GraphNode[];
edges: GraphEdge[];
@@ -286,6 +317,10 @@ export interface GraphSnapshot {
* shown next to the session list — the confidence history). Multiple
* notes for one (node, chat) collapse to the latest. */
nodeObservations: Record<string, Record<string, string>>;
/** Structural communities (Louvain over follows+supports), size ≥ 3,
* joined with persisted LLM labelings. Purely structural — never
* depends on layout. */
clusters: Cluster[];
}
/** Derive a short chat title from the learner's first message (pure).
+30 -2
View File
@@ -38,10 +38,11 @@
* adapters in.
*/
import { randomBytes } from "node:crypto";
import { createHash, randomBytes } from "node:crypto";
import { isToolUIPart, type UIMessage } from "ai";
import type {
AuditResult,
Cluster,
EmissionStats,
GraphEdge,
GraphNode,
@@ -50,6 +51,7 @@ import type {
LedgerEvent,
LedgerEventKind,
} from "./contracts.ts";
import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts";
import type { QuestionAnswerData } from "./question-core.ts";
import type { ChatStore, GraphStore } from "./ports.ts";
@@ -330,6 +332,16 @@ export function pendingQuizAnswers(chatId: string, deps: GraphDeps): number {
// ─── derivation (reads only — everything below is recomputable) ─────────────
/** The content identity of a community: "clu_" + sha256 of the sorted
* member ids (12 hex). Deterministic — the same member set always mints
* the same id — so re-naming races and idempotent re-runs dedupe at the
* store seam (appendClusterLabel). Lives here (server-only core), not in
* clusters.ts, because clusters.ts is barrel-exported and must stay
* browser-safe (no node:crypto). */
export function clusterContentId(members: readonly string[]): string {
return "clu_" + createHash("sha256").update([...members].sort().join("\n")).digest("hex").slice(0, 12);
}
export interface DerivedGraph {
nodes: GraphNode[];
edges: GraphEdge[];
@@ -485,7 +497,23 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
(nodeObservations[nodeId] ??= {})[chatId] = note;
}
return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations };
// Structural communities (vision §1): Louvain over the undirected
// follows+supports structure, joined with persisted labelings —
// core-survival matching, size-filtered at the presentation threshold.
// Detection is DETERMINISTIC (seeded), so no memo in v1: a snapshot
// read recomputes it (ms at current scale; revisit at vault scale).
const detection = detectCommunities(nodes, edges);
const claims = matchClusterLabelings(detection.communities, deps.graphStore.readClusterLabels());
const clusters: Cluster[] = detection.communities
.map((community, index) => ({ community, claim: claims[index] }))
.filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE)
.map(({ community, claim }) => ({
id: claim?.id ?? clusterContentId(community.members),
label: claim?.label ?? null,
members: community.members,
}));
return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations, clusters };
}
// Re-export: the contract owns the shape; legacy importers still reach it.
+6
View File
@@ -25,6 +25,7 @@ import type {
SessionCheckpoint,
TurnUsage,
NotesSource,
ClusterLabeling,
} from "./contracts.ts";
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
@@ -82,6 +83,11 @@ export interface GraphStore {
appendCheckpoint(checkpoint: SessionCheckpoint): void;
readCheckpoints(): SessionCheckpoint[];
/** Append one cluster labeling — idempotent by id (the id hashes the
* sorted core, so identical member sets dedupe). */
appendClusterLabel(labeling: ClusterLabeling): void;
readClusterLabels(): ClusterLabeling[];
/** Upsert one imported-notes source record (config.SOURCES_FILE — a
* JSON map, not a ledger; an upsert keeps the existing usage log). */
upsertSource(source: NotesSource): void;
+21
View File
@@ -31,6 +31,7 @@ import * as fs from "node:fs";
import * as path from "node:path";
import { LEDGER_FILES, SOURCES_FILE } from "../../config.ts";
import type {
ClusterLabeling,
GraphEdge,
GraphNode,
ExtractionRunUsage,
@@ -86,6 +87,14 @@ function readValid<T>(file: string, isValid: (v: unknown) => v is T): T[] {
const isProvenancePoint = (v: unknown): v is ProvenancePoint =>
isRecord(v) && isString(v.chatId) && isString(v.messageId);
const isClusterLabeling = (v: unknown): v is ClusterLabeling =>
isRecord(v) &&
isString(v.id) &&
isString(v.label) &&
Array.isArray(v.core) &&
v.core.every(isString) &&
isString(v.createdAt);
const isGraphNode = (v: unknown): v is GraphNode =>
isRecord(v) &&
isString(v.id) &&
@@ -164,6 +173,18 @@ export const graphStore: GraphStore = {
return readValid(LEDGER_FILES.sessions, isCheckpoint);
},
// ── cluster labelings (config.LEDGER_FILES.clusterLabels) ──
/** Idempotent by id: the id is a hash of the sorted core, so re-appending
* the same member set (re-naming races, idempotent re-runs) is a no-op. */
appendClusterLabel(labeling: ClusterLabeling): void {
if (this.readClusterLabels().some((l) => l.id === labeling.id)) return;
appendLine(LEDGER_FILES.clusterLabels, labeling);
},
readClusterLabels(): ClusterLabeling[] {
return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling);
},
// ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ──
/** Upsert: keeps the existing record's usage log when re-importing. */
+61 -4
View File
@@ -6,8 +6,9 @@
*/
import { strict as assert } from "node:assert";
import { describe, test } from "node:test";
import { detectCommunities } from "../src/core/clusters.ts";
import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "../src/core/clusters.ts";
import type { Community } from "../src/core/clusters.ts";
import type { ClusterLabeling } from "../src/core/contracts.ts";
const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`);
@@ -15,6 +16,62 @@ const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${pref
const clique = (nodeIds: string[]) =>
nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b })));
const labeling = (id: string, core: string[], createdAt = "2026-09-01T00:00:00.000Z"): ClusterLabeling => ({
id,
label: `label-${id}`,
core,
createdAt,
});
describe("matchClusterLabelings", () => {
test("claims a labeling while at least half its core survives in the community", () => {
const community: Community = { members: ["a", "b", "c", "d", "e"] };
const records = [labeling("clu_1", ["a", "b", "x", "y"])]; // 2 of 4 survive — exactly half
const [claim] = matchClusterLabelings([community], records);
assert.equal(claim?.id, "clu_1");
// One fewer survivor: under half → relabel.
const [dead] = matchClusterLabelings([{ members: ["a", "c", "d", "e", "f"] }], records);
assert.equal(dead, null);
});
test("single successor: a split core keeps its label in only one community", () => {
const records = [labeling("clu_1", ["a", "b", "c", "d"])];
const left: Community = { members: ["a", "b", "z"] }; // 2 of 4 — half
const right: Community = { members: ["c", "d", "w"] }; // 2 of 4 — half
// Tie: larger overlap wins; equal → both eligible, greedy takes first come.
const claims = matchClusterLabelings([left, right], records);
const claimed = claims.filter((c) => c !== null);
assert.equal(claimed.length, 1);
assert.equal(claimed[0]?.id, "clu_1");
});
test("greedy: the best-overlapping community wins the record", () => {
const records = [labeling("clu_1", ["a", "b", "c", "d"])];
const weak: Community = { members: ["a", "b", "q", "r", "s"] }; // 2/4
const strong: Community = { members: ["a", "b", "c", "x", "y"] }; // 3/4
const claims = matchClusterLabelings([weak, strong], records);
assert.equal(claims[0], null);
assert.equal(claims[1]?.id, "clu_1");
});
test("one record per community; newest record breaks ties", () => {
const oldR = labeling("clu_old", ["a", "b"], "2026-09-01T00:00:00.000Z");
const newR = labeling("clu_new", ["a", "b"], "2026-09-02T00:00:00.000Z");
const community: Community = { members: ["a", "b", "c"] };
const [claim] = matchClusterLabelings([community], [oldR, newR]);
assert.equal(claim?.id, "clu_new");
});
test("no labelings: every community unclaimed; empty communities array ok", () => {
assert.deepEqual(matchClusterLabelings([{ members: ["a", "b"] }], []), [null]);
assert.deepEqual(matchClusterLabelings([], [labeling("clu_1", ["a"])]), []);
});
test("MIN_CLUSTER_SIZE is 3 (the render/naming threshold)", () => {
assert.equal(MIN_CLUSTER_SIZE, 3);
});
});
describe("detectCommunities", () => {
test("two dense cliques joined by a single bridge split into two communities", () => {
const left = ids(4, "L");
@@ -57,7 +114,7 @@ describe("detectCommunities", () => {
{ from: "b", to: "c" },
]);
assert.equal(result.communities.length, 1);
assert.deepEqual(result.communities[0].members.sort(), ["a", "b", "c"]);
assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b", "c"]);
});
test("self-loops and unknown-endpoint edges are skipped defensively", () => {
@@ -69,7 +126,7 @@ describe("detectCommunities", () => {
{ from: "a", to: "b" }, // the one real edge
]);
assert.equal(result.communities.length, 1);
assert.deepEqual(result.communities[0].members.sort(), ["a", "b"]);
assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b"]);
});
test("members are sorted; communities ordered by size then id", () => {
@@ -84,7 +141,7 @@ describe("detectCommunities", () => {
assert.deepEqual(c.members, sorted);
}
// iso (isolated) is last: size 1 < 4.
const last = result.communities[result.communities.length - 1];
const last = result.communities[result.communities.length - 1]!;
assert.deepEqual(last.members, ["iso"]);
});
+83
View File
@@ -27,6 +27,7 @@ import type { UIMessage } from "ai";
import type { PiEngine } from "../src/agent/models.ts";
import type {
AuditResult,
ClusterLabeling,
ExtractionRunUsage,
GraphEdge,
GraphNode,
@@ -38,6 +39,7 @@ import type {
} from "../src/core/contracts.ts";
import type { GraphStore } from "../src/core/ports.ts";
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
import { chatStore } from "../src/store/chats.ts";
import { PATHS } from "../config.ts";
@@ -63,6 +65,7 @@ function memoryLedgers() {
const mappings: NodeMapping[] = [];
const checkpoints: SessionCheckpoint[] = [];
const sources = new Map<string, NotesSource>();
const clusterLabels: ClusterLabeling[] = [];
return {
appendNode: (n: GraphNode) => void nodes.push(n),
readNodes: () => nodes.map((n) => ({ ...n })),
@@ -87,6 +90,11 @@ function memoryLedgers() {
source.extractions = [...(source.extractions ?? []), entry];
}
},
// Cluster labelings — mirrors the real store: append dedupes by id.
appendClusterLabel: (l: ClusterLabeling) => {
if (!clusterLabels.some((x) => x.id === l.id)) clusterLabels.push(l);
},
readClusterLabels: () => clusterLabels.map((l) => ({ ...l, core: [...l.core] })),
} satisfies GraphStore;
}
@@ -306,6 +314,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
graphStore: ledgers,
extractor: submitModel("submit_findings", FINDINGS),
auditor: submitModel("submit_review", verdict),
labeler: submitModel("submit_labels", { labels: [] }),
});
const stats = await pipeline.runExtraction(chatId);
@@ -346,6 +355,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
graphStore: ledgers,
extractor: submitModel("submit_findings", FINDINGS),
auditor: silentModel(), // never submits the review
labeler: submitModel("submit_labels", { labels: [] }),
});
await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/);
assert.equal(ledgers.readNodes().length, 0);
@@ -356,6 +366,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
graphStore: ledgers,
extractor: silentModel(),
auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }),
labeler: submitModel("submit_labels", { labels: [] }),
});
await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/);
assert.equal(ledgers.readCheckpoints().length, 0);
@@ -388,6 +399,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
// registry must already carry this chat at the right phase.
extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())),
auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())),
labeler: submitModel("submit_labels", { labels: [] }),
});
await pipeline.runExtraction(chatId);
assert.deepEqual(seen, ["extracting", "auditing"]);
@@ -422,6 +434,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
auditor: submitModel("submit_review", verdict, () =>
usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens),
),
labeler: submitModel("submit_labels", { labels: [] }),
});
await pipeline.runExtraction(chatId);
@@ -458,6 +471,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
graphStore: ledgers,
extractor: hangingModel(), // hangs until aborted — no step finishes
auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }),
labeler: submitModel("submit_labels", { labels: [] }),
});
const runPromise = pipeline.runExtraction(chatId);
@@ -508,6 +522,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
graphStore: ledgers,
extractor: gatedExtractor,
auditor: silentModel(),
labeler: submitModel("submit_labels", { labels: [] }),
});
const first = pipeline.runExtraction(chatId);
@@ -525,6 +540,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
graphStore: ledgers,
extractor: silentModel(),
auditor: silentModel(),
labeler: submitModel("submit_labels", { labels: [] }),
});
await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
});
@@ -562,6 +578,70 @@ const NOTES_VERDICT = {
dropped: [],
} as const;
const CLUSTER_FINDINGS = {
propositions: [
{ id: "p1", name: "Modal editing", statement: "Vim edits text through modal command modes.", aliases: [], messages: [], evidence: ["Normal mode keys are commands."], understanding: "solid" },
{ id: "p2", name: "Verb-object grammar", statement: "Vim commands compose as verb and object.", aliases: [], messages: [], evidence: ["d then motion deletes."], understanding: "solid" },
{ id: "p3", name: "Operator-pending", statement: "Vim enters operator-pending mode between verb and object.", aliases: [], messages: [], evidence: ["After d, a motion is expected."], understanding: "solid" },
],
edges: [
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
],
} as const;
const CLUSTER_VERDICT = {
propositions: CLUSTER_FINDINGS.propositions,
edges: CLUSTER_FINDINGS.edges,
dropped: [],
} as const;
describe("cluster naming (the post-extraction garnish)", () => {
it("names a new community after a notes merge — one record, claimed on the next refresh", async () => {
const ledgers = memoryLedgers();
const namerCalls: number[] = [];
const labeler = submitModel("submit_labels", { labels: [{ key: "c0", label: "Vim modal grammar" }] }, () => namerCalls.push(1));
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
auditor: submitModel("submit_review", CLUSTER_VERDICT),
labeler,
});
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes, any format" });
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 });
// The refresh is fire-and-forget inside the pipeline — settle it.
await clusterNamingSettled();
assert.equal(namerCalls.length, 1, "ONE singular naming call");
const records = ledgers.readClusterLabels();
assert.equal(records.length, 1);
assert.equal(records[0]!.label, "Vim modal grammar");
assert.equal(records[0]!.core.length, 3);
assert.match(records[0]!.id, /^clu_/);
// Second refresh: core-survival claims it — nothing new to name.
const again = await refreshClusterLabels({ graphStore: ledgers, engine: labeler });
assert.equal(again.named, 0);
assert.equal(again.appended, 0);
assert.ok(again.claimed >= 1);
assert.equal(ledgers.readClusterLabels().length, 1, "no duplicate records");
});
it("naming failure never fails the run", async () => {
const ledgers = memoryLedgers();
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
auditor: submitModel("submit_review", CLUSTER_VERDICT),
labeler: silentModel(), // never submits — the namer throws
});
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes" });
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }, "the merge itself succeeded");
await clusterNamingSettled();
assert.equal(ledgers.readClusterLabels().length, 0, "nothing named, nothing appended");
});
});
describe("the notes front door (importNotes — format-agnostic)", () => {
it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
const ledgers = memoryLedgers();
@@ -575,6 +655,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
seen.push({ kind: run?.kind, label: run?.label });
}),
labeler: submitModel("submit_labels", { labels: [] }),
});
const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
@@ -618,6 +699,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
graphStore: ledgers,
extractor: submitModel("submit_findings", NOTES_FINDINGS),
auditor: submitModel("submit_review", NOTES_VERDICT),
labeler: submitModel("submit_labels", { labels: [] }),
});
const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
@@ -637,6 +719,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
graphStore: ledgers,
extractor: hangingModel(),
auditor: submitModel("submit_review", NOTES_VERDICT),
labeler: submitModel("submit_labels", { labels: [] }),
});
const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
+48 -1
View File
@@ -20,7 +20,7 @@ import { after, describe, it } from "node:test";
import type { UIMessage } from "ai";
import type { AuditResult } from "../src/core/contracts.ts";
import type { QuestionAnswerData } from "../src/core/question-core.ts";
import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
import { graphStore } from "../src/store/graph.ts";
import { chatStore } from "../src/store/chats.ts";
import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";
@@ -407,6 +407,53 @@ describe("the merger (mergeFindings)", () => {
assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
});
it("derives structural clusters in graphSnapshot, joined with persisted labelings", () => {
// A self-contained 3-clique: three propositions, all mutually linked.
const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const;
const merge = mergeFindings(
"src_clusterfixture",
{
propositions: tri.map((statement, i) => ({
id: `p${i + 1}`,
name: `Cluster node ${i + 1}`,
statement,
aliases: [],
messages: [],
evidence: ["fixture evidence"],
})),
edges: [
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
{ from: "p1", to: "p3", type: "follows", messages: [], evidence: ["e"] },
],
attribution: { quiz: {} },
dropped: [],
},
graphDeps,
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
);
assert.equal(merge.nodes, 3);
const snap = graphSnapshot(graphDeps);
// Presentation threshold: every cluster is >= 3 members.
assert.ok(snap.clusters.every((c) => c.members.length >= 3));
// Our trio is exactly one cluster, unlabeled until the namer runs.
const mine = snap.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap.nodes.find((n) => n.statement === s)!.id)));
assert.ok(mine, "the 3-clique is a cluster");
assert.equal(mine!.label, null);
assert.match(mine!.id, /^clu_/);
// Mint the labeling the namer would append — the next read joins it.
graphStore.appendClusterLabel({
id: clusterContentId(mine!.members),
label: "Test region",
core: mine!.members,
createdAt: "2026-09-13T00:00:00.000Z",
});
const snap2 = graphSnapshot(graphDeps);
const claimed = snap2.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap2.nodes.find((n) => n.statement === s)!.id)))!;
assert.equal(claimed.label, "Test region");
assert.equal(claimed.id, clusterContentId(mine!.members), "the claimed cluster carries the record's id");
});
it("derives sessions and per-session understanding observations in graphSnapshot", () => {
const snap = graphSnapshot(graphDeps);
const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));