Clusters phases B+C: labeling records, the naming call, snapshot payload

Core-survival labelings (vision section 1, DECIDED 2026-09-13):
ClusterLabeling records in a new append-only ledger
(data/graph/cluster-labels.jsonl), immutable, id = clu_ + sha256 of the
sorted core so re-naming races and re-runs dedupe at the store seam.
matchClusterLabelings (core/clusters.ts, pure): a community keeps a
record's label while at least half its core survives in it; both sides
single-successor (a split keeps one name; greedy by overlap, ties to
the newer record).

The namer (src/agent/subagents/cluster-namer.ts): ONE singular cheap LLM
call (new 'labeler' role, glm-flash/low) names every new/changed
community at once, given member names + statements (capped at 12
members per cluster). refreshClusterLabels runs POST-EXTRACTION
(user-decided) from both front doors, fire-and-forget — naming failure
never fails the run; refreshes serialize behind a chain so parallel
imports never mint duplicate labels. import-notes.ts awaits
clusterNamingSettled before exit.

Payload: GraphSnapshot.clusters (>= MIN_CLUSTER_SIZE 3) joins detection
with labelings at read time — claimed clusters carry the record's label
and id; unlabeled ones carry a member-hash id and null label. Detection
is deterministic so no memo in v1 (ms at current scale; revisit at
vault scale).

Tests (372/372): matcher half-rule/split/merge/greedy/tie cases; naming
end-to-end through importNotes (one call, one record, claim on
re-refresh), naming failure non-fatal; snapshot derivation with
labeling join; 11 test pipelines now pin an explicit labeler seam.
This commit is contained in:
amosblomqvist
2026-09-13 03:59:48 +03:00
parent 95a4477283
commit f0267a5fd9
12 changed files with 562 additions and 8 deletions
+200
View File
@@ -0,0 +1,200 @@
/**
* CLUSTER NAMING (vision §1, DECIDED 2026-09-13): the garnish layer over
* community detection. Detection is purely structural and deterministic
* (core/clusters.ts); naming is ONE singular cheap LLM call that labels
* every NEW/CHANGED community at once, given member names + statements.
*
* Core-survival: a community keeps an existing labeling while at least
* half its core survives in it (matchClusterLabelings); only communities
* with no surviving label AND at least MIN_CLUSTER_SIZE members are sent
* to the namer. Records are append-only and idempotent (content-hashed
* ids dedupe at the store seam), so a naming race or a re-run appends
* nothing new.
*
* Runs POST-EXTRACTION (user-decided 2026-09-13): both front doors call
* refreshClusterLabels after a successful merge. Failure is NON-FATAL —
* the graph and hulls render fine unlabeled; the next run retries.
*/
import { z } from "zod";
import { lhTool, toPiTools } from "../lh-tool.ts";
import type { PiEngine } from "../models.ts";
import { runSubagent } from "./pi-run.ts";
import type { GraphNode } from "../../core/contracts.ts";
import { getGraph, clusterContentId } from "../../core/graph.ts";
import {
detectCommunities,
matchClusterLabelings,
MIN_CLUSTER_SIZE,
type Community,
} from "../../core/clusters.ts";
import type { GraphStore } from "../../core/ports.ts";
/** How many member propositions the namer sees per cluster — enough to
* recognize the domain without paying for the whole region. */
const MAX_MEMBERS_PER_CLUSTER = 12;
const CLUSTER_NAMER_INSTRUCTIONS = `You name communities of propositions in a personal knowledge graph.
You are given clusters. Each cluster is a set of related propositions
(theory fragments the author is learning) plus, for context, a sketch of
every OTHER cluster already named. For each UNNAMED cluster, mint ONE
short DOMAIN LABEL: 1-4 words naming the region of the world model those
propositions belong to together — like a section heading in a textbook.
Use the members' own vocabulary ("Differential forms", "Sales
psychology", "Server architecture"), not generic words ("Topics",
"Various concepts"). If the propositions are unrelated, that is fine:
name the best common denominator anyway — a reader should recognize the
region at a glance.
Your entire deliverable is ONE submit_labels call. Include EVERY
top-level field even when empty (labels: [] when nothing to name).`;
const labelsSchema = z.object({
labels: z
.array(
z.object({
/** The cluster key from the prompt ("c1", "c2", ...). */
key: z.string(),
label: z.string(),
}),
)
.default([]),
});
export interface UnnamedCluster {
key: string;
members: string[];
nodes: Pick<GraphNode, "name" | "statement">[];
}
/** One single-submit namer run (the same shape as every extraction stage):
* the whole deliverable is ONE zod-validated submit call. */
async function nameClusters(
engine: PiEngine | undefined,
signal: AbortSignal | undefined,
unnamed: UnnamedCluster[],
): Promise<{ key: string; label: string }[]> {
let submitted: { key: string; label: string }[] | undefined;
await runSubagent(
toPiTools({
submit_labels: lhTool({
name: "submit_labels",
description: "Submit the cluster labels.",
schema: labelsSchema,
execute: async (labels) => {
submitted = labels.labels;
return labels;
},
}),
}),
{
role: "labeler",
engine,
systemPrompt: CLUSTER_NAMER_INSTRUCTIONS,
prompt: JSON.stringify(
{
unnamedClusters: unnamed.map((c) => ({
key: c.key,
members: c.nodes.map((n) => ({ name: n.name, statement: n.statement })),
})),
},
null,
2,
),
maxTurns: 4,
abortSignal: signal,
shouldStop: (_message, toolResults) =>
toolResults.some((r) => r.toolName === "submit_labels" && !r.isError),
},
);
if (!submitted) throw new Error("the cluster namer never submitted labels");
return submitted;
}
export interface ClusterRefreshStats {
/** Communities the current graph divides into (all sizes). */
detected: number;
/** Communities that kept a label via core-survival. */
claimed: number;
/** Communities sent to the namer (unnamed, size >= MIN_CLUSTER_SIZE). */
named: number;
/** Labeling records actually appended (idempotent dedupe may shrink this). */
appended: number;
}
/**
* The post-extraction garnish: detect communities on the CURRENT graph,
* join persisted labelings (core-survival), and name what is new — one
* singular LLM call. Appends ClusterLabeling records; the NEXT snapshot
* read joins them (the payload is derived, never stored).
*
* Serialized: concurrent merges (parallel notes imports) chain here, so
* two runs never mint duplicate labels for the same community.
*/
async function doRefresh(deps: {
graphStore: GraphStore;
/** Test seam: a scripted engine (production uses the labeler role). */
engine?: PiEngine;
signal?: AbortSignal;
}): Promise<ClusterRefreshStats> {
const { graphStore, engine, signal } = deps;
const { nodes, edges } = getGraph(graphStore);
const detection = detectCommunities(nodes, edges);
const labelings = graphStore.readClusterLabels();
const claims = matchClusterLabelings(detection.communities, labelings);
const byId = new Map(nodes.map((n) => [n.id, n]));
const claimed = claims.filter((c) => c !== null).length;
const unnamed = detection.communities
.filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE)
.map((c: Community, i) => {
const nodesFor = c.members
.map((id) => byId.get(id))
.filter((n): n is GraphNode => n !== undefined)
.slice(0, MAX_MEMBERS_PER_CLUSTER)
.map((n) => ({ name: n.name, statement: n.statement }));
return { key: `c${i}`, members: c.members, nodes: nodesFor };
});
if (unnamed.length === 0) return { detected: detection.communities.length, claimed, named: 0, appended: 0 };
const labels = await nameClusters(engine, signal, unnamed);
let appended = 0;
const seen = new Set<string>();
for (const { key, members } of unnamed) {
if (seen.has(key)) continue;
const label = labels.find((l) => l.key === key)?.label?.trim();
if (!label) continue;
seen.add(key);
graphStore.appendClusterLabel({
id: clusterContentId(members),
label,
core: members,
createdAt: new Date().toISOString(),
});
appended++;
}
return { detected: detection.communities.length, claimed, named: unnamed.length, appended };
}
let namingTail: Promise<unknown> = Promise.resolve();
/**
* Fire the refresh, serialized behind any refresh already running (a
* parallel import's merge waits its turn; each run reads fresh ledgers).
* Waivable: call sites may fire-and-forget; scripts await the return.
*/
export function refreshClusterLabels(deps: {
graphStore: GraphStore;
engine?: PiEngine;
signal?: AbortSignal;
}): Promise<ClusterRefreshStats> {
const run = namingTail.then(() => doRefresh(deps));
namingTail = run.catch(() => {});
return run;
}
/** Await every refresh fired so far (scripts: don't exit mid-naming). */
export async function clusterNamingSettled(): Promise<void> {
await namingTail;
}
+15
View File
@@ -16,6 +16,7 @@ import type {
SessionFindings,
} from "../../core/contracts.ts";
import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
import { refreshClusterLabels } from "./cluster-namer.ts";
/**
* The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the
@@ -548,6 +549,8 @@ export interface ExtractionModels {
/** Test seam: a scripted engine per stage (production uses the roles). */
extractor?: PiEngine;
auditor?: PiEngine;
/** Test seam for the cluster namer (post-extraction garnish). */
labeler?: PiEngine;
}
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
@@ -716,6 +719,13 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
`[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
`(${Math.round(performance.now() - t0)}ms total)`,
);
// Post-extraction garnish (DECIDED 2026-09-13): name new/changed
// clusters. Fire-and-forget — naming never fails the run.
refreshClusterLabels({
graphStore,
engine: deps.labeler,
signal: runningExtractions.get(chatId)?.controller.signal,
}).catch(() => {});
return stats;
} catch (err) {
persistRunUsage(chatId, Math.round(performance.now() - t0), undefined);
@@ -819,6 +829,11 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
`[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
`(${Math.round(performance.now() - t0)}ms total)`,
);
refreshClusterLabels({
graphStore,
engine: deps.labeler,
signal: runningExtractions.get(id)?.controller.signal,
}).catch(() => {});
return stats;
} catch (err) {
persistRunUsage(id, Math.round(performance.now() - t0), undefined);
+50
View File
@@ -26,6 +26,7 @@
*/
import Graph from "graphology";
import louvain from "graphology-communities-louvain";
import type { ClusterLabeling } from "./contracts.ts";
/** A detected community. Members are SORTED node ids — the member set IS
* the identity a labeling record can outlive a reshuffle with. */
@@ -33,6 +34,11 @@ export interface Community {
members: string[];
}
/** Communities below this size are not rendered, not named — presentation
* rule (user-decided 2026-09-13): a region of the world model needs at
* least three propositions to be worth a hull and a label. */
export const MIN_CLUSTER_SIZE = 3;
export interface CommunityDetection {
communities: Community[];
/** Modularity of the found partition — the one quality number. */
@@ -116,3 +122,47 @@ export function detectCommunities(
);
return { communities, modularity: details.modularity };
}
/**
* Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED
* 2026-09-13): a community keeps a record's label while AT LEAST HALF of
* the record's core survives in it (overlap * 2 >= core.length; exactly
* half survives). Both sides are single-successor: one community holds at
* most one label, one record is claimed by at most one community — on a
* split, only the branch carrying the majority of the core keeps the name;
* on a merge, the best-overlapping record wins.
*
* Greedy assignment by overlap size (largest first; ties broken by newer
* record). Returns a claim (or null) per community, in input order —
* unclaimed communities are the namer's input. Pure: same inputs, same
* claims.
*/
export function matchClusterLabelings(
communities: readonly Community[],
labelings: readonly ClusterLabeling[],
): (ClusterLabeling | null)[] {
interface Pair {
community: number;
labeling: ClusterLabeling;
overlap: number;
}
const pairs: Pair[] = [];
communities.forEach((community, ci) => {
const members = new Set(community.members);
for (const labeling of labelings) {
const overlap = labeling.core.reduce((n, id) => n + (members.has(id) ? 1 : 0), 0);
if (overlap * 2 >= labeling.core.length && overlap > 0) {
pairs.push({ community: ci, labeling, overlap });
}
}
});
pairs.sort((a, b) => b.overlap - a.overlap || (a.labeling.createdAt < b.labeling.createdAt ? 1 : -1));
const claims: (ClusterLabeling | null)[] = communities.map(() => null);
const takenRecords = new Set<string>();
for (const pair of pairs) {
if (claims[pair.community] || takenRecords.has(pair.labeling.id)) continue;
claims[pair.community] = pair.labeling;
takenRecords.add(pair.labeling.id);
}
return claims;
}
+35
View File
@@ -270,6 +270,37 @@ export interface AuditResult {
}
/** The GET /api/graph payload — everything the home view's PURE renderer needs. */
/** cluster-labels.jsonl line — one LLM-named community (vision §1,
* DECIDED 2026-09-13). Append-only history: records are immutable, never
* edited or deleted; dead labelings remain as history. IDENTITY is the
* sorted member set ("core"), hashed into the id, so re-naming the same
* community dedupes and a labeling survives graph edits as long as at
* least half its core survives in some single community (core-survival;
* under half → the community is renamed, splits can keep only one
* successor). */
export interface ClusterLabeling {
/** "clu_" + content hash of the sorted core. */
id: string;
/** The short domain label (1–4 words), e.g. "Differential forms". */
label: string;
/** The member node ids the label was minted for, sorted — the survival core. */
core: string[];
createdAt: string;
}
/** One rendered cluster in the snapshot payload (vision §1). Members are
* node ids; the renderer draws the hull from member positions at draw
* time and gates the label inversely on the node-label zoom control. */
export interface Cluster {
/** The labeling record id when claimed; unlabeled clusters still
* appear (hull, no name) with a member-hash id. */
id: string;
/** null until the naming call has minted/claimed a labeling. */
label: string | null;
/** Node ids, sorted. Hull + centroid are renderer concerns. */
members: string[];
}
export interface GraphSnapshot {
nodes: GraphNode[];
edges: GraphEdge[];
@@ -286,6 +317,10 @@ export interface GraphSnapshot {
* shown next to the session list — the confidence history). Multiple
* notes for one (node, chat) collapse to the latest. */
nodeObservations: Record<string, Record<string, string>>;
/** Structural communities (Louvain over follows+supports), size ≥ 3,
* joined with persisted LLM labelings. Purely structural — never
* depends on layout. */
clusters: Cluster[];
}
/** Derive a short chat title from the learner's first message (pure).
+30 -2
View File
@@ -38,10 +38,11 @@
* adapters in.
*/
import { randomBytes } from "node:crypto";
import { createHash, randomBytes } from "node:crypto";
import { isToolUIPart, type UIMessage } from "ai";
import type {
AuditResult,
Cluster,
EmissionStats,
GraphEdge,
GraphNode,
@@ -50,6 +51,7 @@ import type {
LedgerEvent,
LedgerEventKind,
} from "./contracts.ts";
import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts";
import type { QuestionAnswerData } from "./question-core.ts";
import type { ChatStore, GraphStore } from "./ports.ts";
@@ -330,6 +332,16 @@ export function pendingQuizAnswers(chatId: string, deps: GraphDeps): number {
// ─── derivation (reads only — everything below is recomputable) ─────────────
/** The content identity of a community: "clu_" + sha256 of the sorted
* member ids (12 hex). Deterministic — the same member set always mints
* the same id — so re-naming races and idempotent re-runs dedupe at the
* store seam (appendClusterLabel). Lives here (server-only core), not in
* clusters.ts, because clusters.ts is barrel-exported and must stay
* browser-safe (no node:crypto). */
export function clusterContentId(members: readonly string[]): string {
return "clu_" + createHash("sha256").update([...members].sort().join("\n")).digest("hex").slice(0, 12);
}
export interface DerivedGraph {
nodes: GraphNode[];
edges: GraphEdge[];
@@ -485,7 +497,23 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
(nodeObservations[nodeId] ??= {})[chatId] = note;
}
return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations };
// Structural communities (vision §1): Louvain over the undirected
// follows+supports structure, joined with persisted labelings —
// core-survival matching, size-filtered at the presentation threshold.
// Detection is DETERMINISTIC (seeded), so no memo in v1: a snapshot
// read recomputes it (ms at current scale; revisit at vault scale).
const detection = detectCommunities(nodes, edges);
const claims = matchClusterLabelings(detection.communities, deps.graphStore.readClusterLabels());
const clusters: Cluster[] = detection.communities
.map((community, index) => ({ community, claim: claims[index] }))
.filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE)
.map(({ community, claim }) => ({
id: claim?.id ?? clusterContentId(community.members),
label: claim?.label ?? null,
members: community.members,
}));
return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations, clusters };
}
// Re-export: the contract owns the shape; legacy importers still reach it.
+6
View File
@@ -25,6 +25,7 @@ import type {
SessionCheckpoint,
TurnUsage,
NotesSource,
ClusterLabeling,
} from "./contracts.ts";
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
@@ -82,6 +83,11 @@ export interface GraphStore {
appendCheckpoint(checkpoint: SessionCheckpoint): void;
readCheckpoints(): SessionCheckpoint[];
/** Append one cluster labeling — idempotent by id (the id hashes the
* sorted core, so identical member sets dedupe). */
appendClusterLabel(labeling: ClusterLabeling): void;
readClusterLabels(): ClusterLabeling[];
/** Upsert one imported-notes source record (config.SOURCES_FILE — a
* JSON map, not a ledger; an upsert keeps the existing usage log). */
upsertSource(source: NotesSource): void;
+21
View File
@@ -31,6 +31,7 @@ import * as fs from "node:fs";
import * as path from "node:path";
import { LEDGER_FILES, SOURCES_FILE } from "../../config.ts";
import type {
ClusterLabeling,
GraphEdge,
GraphNode,
ExtractionRunUsage,
@@ -86,6 +87,14 @@ function readValid<T>(file: string, isValid: (v: unknown) => v is T): T[] {
const isProvenancePoint = (v: unknown): v is ProvenancePoint =>
isRecord(v) && isString(v.chatId) && isString(v.messageId);
const isClusterLabeling = (v: unknown): v is ClusterLabeling =>
isRecord(v) &&
isString(v.id) &&
isString(v.label) &&
Array.isArray(v.core) &&
v.core.every(isString) &&
isString(v.createdAt);
const isGraphNode = (v: unknown): v is GraphNode =>
isRecord(v) &&
isString(v.id) &&
@@ -164,6 +173,18 @@ export const graphStore: GraphStore = {
return readValid(LEDGER_FILES.sessions, isCheckpoint);
},
// ── cluster labelings (config.LEDGER_FILES.clusterLabels) ──
/** Idempotent by id: the id is a hash of the sorted core, so re-appending
* the same member set (re-naming races, idempotent re-runs) is a no-op. */
appendClusterLabel(labeling: ClusterLabeling): void {
if (this.readClusterLabels().some((l) => l.id === labeling.id)) return;
appendLine(LEDGER_FILES.clusterLabels, labeling);
},
readClusterLabels(): ClusterLabeling[] {
return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling);
},
// ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ──
/** Upsert: keeps the existing record's usage log when re-importing. */