Clusters phases B+C: labeling records, the naming call, snapshot payload
Core-survival labelings (vision section 1, DECIDED 2026-09-13): ClusterLabeling records in a new append-only ledger (data/graph/cluster-labels.jsonl), immutable, id = clu_ + sha256 of the sorted core so re-naming races and re-runs dedupe at the store seam. matchClusterLabelings (core/clusters.ts, pure): a community keeps a record's label while at least half its core survives in it; both sides single-successor (a split keeps one name; greedy by overlap, ties to the newer record). The namer (src/agent/subagents/cluster-namer.ts): ONE singular cheap LLM call (new 'labeler' role, glm-flash/low) names every new/changed community at once, given member names + statements (capped at 12 members per cluster). refreshClusterLabels runs POST-EXTRACTION (user-decided) from both front doors, fire-and-forget — naming failure never fails the run; refreshes serialize behind a chain so parallel imports never mint duplicate labels. import-notes.ts awaits clusterNamingSettled before exit. Payload: GraphSnapshot.clusters (>= MIN_CLUSTER_SIZE 3) joins detection with labelings at read time — claimed clusters carry the record's label and id; unlabeled ones carry a member-hash id and null label. Detection is deterministic so no memo in v1 (ms at current scale; revisit at vault scale). Tests (372/372): matcher half-rule/split/merge/greedy/tie cases; naming end-to-end through importNotes (one call, one record, claim on re-refresh), naming failure non-fatal; snapshot derivation with labeling join; 11 test pipelines now pin an explicit labeler seam.
This commit is contained in:
@@ -27,6 +27,7 @@ import type { UIMessage } from "ai";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
import type {
|
||||
AuditResult,
|
||||
ClusterLabeling,
|
||||
ExtractionRunUsage,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
@@ -38,6 +39,7 @@ import type {
|
||||
} from "../src/core/contracts.ts";
|
||||
import type { GraphStore } from "../src/core/ports.ts";
|
||||
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
|
||||
import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
|
||||
@@ -63,6 +65,7 @@ function memoryLedgers() {
|
||||
const mappings: NodeMapping[] = [];
|
||||
const checkpoints: SessionCheckpoint[] = [];
|
||||
const sources = new Map<string, NotesSource>();
|
||||
const clusterLabels: ClusterLabeling[] = [];
|
||||
return {
|
||||
appendNode: (n: GraphNode) => void nodes.push(n),
|
||||
readNodes: () => nodes.map((n) => ({ ...n })),
|
||||
@@ -87,6 +90,11 @@ function memoryLedgers() {
|
||||
source.extractions = [...(source.extractions ?? []), entry];
|
||||
}
|
||||
},
|
||||
// Cluster labelings — mirrors the real store: append dedupes by id.
|
||||
appendClusterLabel: (l: ClusterLabeling) => {
|
||||
if (!clusterLabels.some((x) => x.id === l.id)) clusterLabels.push(l);
|
||||
},
|
||||
readClusterLabels: () => clusterLabels.map((l) => ({ ...l, core: [...l.core] })),
|
||||
} satisfies GraphStore;
|
||||
}
|
||||
|
||||
@@ -306,6 +314,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
graphStore: ledgers,
|
||||
extractor: submitModel("submit_findings", FINDINGS),
|
||||
auditor: submitModel("submit_review", verdict),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const stats = await pipeline.runExtraction(chatId);
|
||||
|
||||
@@ -346,6 +355,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
graphStore: ledgers,
|
||||
extractor: submitModel("submit_findings", FINDINGS),
|
||||
auditor: silentModel(), // never submits the review
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/);
|
||||
assert.equal(ledgers.readNodes().length, 0);
|
||||
@@ -356,6 +366,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
graphStore: ledgers,
|
||||
extractor: silentModel(),
|
||||
auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/);
|
||||
assert.equal(ledgers.readCheckpoints().length, 0);
|
||||
@@ -388,6 +399,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
// registry must already carry this chat at the right phase.
|
||||
extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())),
|
||||
auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await pipeline.runExtraction(chatId);
|
||||
assert.deepEqual(seen, ["extracting", "auditing"]);
|
||||
@@ -422,6 +434,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
auditor: submitModel("submit_review", verdict, () =>
|
||||
usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens),
|
||||
),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await pipeline.runExtraction(chatId);
|
||||
|
||||
@@ -458,6 +471,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
graphStore: ledgers,
|
||||
extractor: hangingModel(), // hangs until aborted — no step finishes
|
||||
auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const runPromise = pipeline.runExtraction(chatId);
|
||||
|
||||
@@ -508,6 +522,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
graphStore: ledgers,
|
||||
extractor: gatedExtractor,
|
||||
auditor: silentModel(),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
|
||||
const first = pipeline.runExtraction(chatId);
|
||||
@@ -525,6 +540,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
graphStore: ledgers,
|
||||
extractor: silentModel(),
|
||||
auditor: silentModel(),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
|
||||
});
|
||||
@@ -562,6 +578,70 @@ const NOTES_VERDICT = {
|
||||
dropped: [],
|
||||
} as const;
|
||||
|
||||
const CLUSTER_FINDINGS = {
|
||||
propositions: [
|
||||
{ id: "p1", name: "Modal editing", statement: "Vim edits text through modal command modes.", aliases: [], messages: [], evidence: ["Normal mode keys are commands."], understanding: "solid" },
|
||||
{ id: "p2", name: "Verb-object grammar", statement: "Vim commands compose as verb and object.", aliases: [], messages: [], evidence: ["d then motion deletes."], understanding: "solid" },
|
||||
{ id: "p3", name: "Operator-pending", statement: "Vim enters operator-pending mode between verb and object.", aliases: [], messages: [], evidence: ["After d, a motion is expected."], understanding: "solid" },
|
||||
],
|
||||
edges: [
|
||||
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
|
||||
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
|
||||
],
|
||||
} as const;
|
||||
|
||||
const CLUSTER_VERDICT = {
|
||||
propositions: CLUSTER_FINDINGS.propositions,
|
||||
edges: CLUSTER_FINDINGS.edges,
|
||||
dropped: [],
|
||||
} as const;
|
||||
|
||||
describe("cluster naming (the post-extraction garnish)", () => {
|
||||
it("names a new community after a notes merge — one record, claimed on the next refresh", async () => {
|
||||
const ledgers = memoryLedgers();
|
||||
const namerCalls: number[] = [];
|
||||
const labeler = submitModel("submit_labels", { labels: [{ key: "c0", label: "Vim modal grammar" }] }, () => namerCalls.push(1));
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
|
||||
auditor: submitModel("submit_review", CLUSTER_VERDICT),
|
||||
labeler,
|
||||
});
|
||||
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes, any format" });
|
||||
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 });
|
||||
// The refresh is fire-and-forget inside the pipeline — settle it.
|
||||
await clusterNamingSettled();
|
||||
assert.equal(namerCalls.length, 1, "ONE singular naming call");
|
||||
const records = ledgers.readClusterLabels();
|
||||
assert.equal(records.length, 1);
|
||||
assert.equal(records[0]!.label, "Vim modal grammar");
|
||||
assert.equal(records[0]!.core.length, 3);
|
||||
assert.match(records[0]!.id, /^clu_/);
|
||||
// Second refresh: core-survival claims it — nothing new to name.
|
||||
const again = await refreshClusterLabels({ graphStore: ledgers, engine: labeler });
|
||||
assert.equal(again.named, 0);
|
||||
assert.equal(again.appended, 0);
|
||||
assert.ok(again.claimed >= 1);
|
||||
assert.equal(ledgers.readClusterLabels().length, 1, "no duplicate records");
|
||||
});
|
||||
|
||||
it("naming failure never fails the run", async () => {
|
||||
const ledgers = memoryLedgers();
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
|
||||
auditor: submitModel("submit_review", CLUSTER_VERDICT),
|
||||
labeler: silentModel(), // never submits — the namer throws
|
||||
});
|
||||
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes" });
|
||||
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }, "the merge itself succeeded");
|
||||
await clusterNamingSettled();
|
||||
assert.equal(ledgers.readClusterLabels().length, 0, "nothing named, nothing appended");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the notes front door (importNotes — format-agnostic)", () => {
|
||||
it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
|
||||
const ledgers = memoryLedgers();
|
||||
@@ -575,6 +655,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
|
||||
const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
|
||||
seen.push({ kind: run?.kind, label: run?.label });
|
||||
}),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||
|
||||
@@ -618,6 +699,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
|
||||
graphStore: ledgers,
|
||||
extractor: submitModel("submit_findings", NOTES_FINDINGS),
|
||||
auditor: submitModel("submit_review", NOTES_VERDICT),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||
assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
|
||||
@@ -637,6 +719,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
|
||||
graphStore: ledgers,
|
||||
extractor: hangingModel(),
|
||||
auditor: submitModel("submit_review", NOTES_VERDICT),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||
|
||||
|
||||
Reference in New Issue
Block a user