Wave 7 close-out: the embedding ledger seam dies; notes birth via mapping lines

Part 1 — the ledger + remnants interlude (master-dispatched, owner ruling
2026-09-23: nothing configures the graph beyond chat models):

- The embedding ledger surface is gone: readEmbeddings/writeEmbeddings
  ports, the EmbeddingsLedger contract, EMBEDDINGS_FILE (config, store
  impls, graphRevision source list), and graph-deletion's vector-pruning
  step. On-disk data files stay untouched (inert history).
- The extraction embedder seam dies wholesale: ExtractionPipelineDeps
  drops the embedder dep (chat-handler + backfill/import-notes stubs go
  with it), the 'embedding' phase leaves ExtractionPhase, the usage
  capture/persist helpers die, and stages.embedding leaves
  ExtractionRunUsage (historical records keep their shape — reads are
  lenient). Extraction is two stages again, like the local-embedder era.
- Consumer-less remnants: latestQuizOutcomes + QUIZ_EVENT_KINDS,
  clusterContentId, matchClusterLabelings + Community + ClusterLabeling,
  the readClusterLabels/appendClusterLabel port seams + store impls +
  LEDGER_FILES.clusterLabels. NamingRunUsage/readNamingRuns STAY
  (standing ruling — the cost pill renders historical naming spend);
  clusterHue + MIN_CLUSTER_SIZE stay (live).
- The 'labeler' role dies: BACKGROUND_MODEL_ROLES, ROLE_SLOT, ModelRole,
  the catalog pin projections, and the UI roster strings. RolePins keys
  shrink with the union.

The notes birth fallback (owner-approved 2026-09-23, surfaced by the
interlude's live smoke: production notes imports create provenance-EMPTY
nodes (importNotes passes messages: null — the file's anchor is the
mapping line), so the provenance-only birth rule left every imported
file unclustered and invisible to the prober's session list, against the
locked file-birth ruling. A provenance-empty node now births under its
FIRST mapping line's chatId (ledger order); only notes-born nodes ever
hit the fallback (chat nodes always carry provenance; a re-encountered
node's link-append re-opens the provenance rule). Deletion-safe: mapping
lines of deleted chats are stripped alongside their provenance.

Live-verified on port 3004: a fresh notes import clusters under its
src_ id with the file path as the label, end to end with no embedder.

845/845 tests, typecheck/lint/build green.
EOF
)
This commit is contained in:
amosblomqvist
2026-09-23 15:37:34 +03:00
parent 1c3a5e121c
commit d1b46132b9
30 changed files with 181 additions and 595 deletions
+5 -20
View File
@@ -23,7 +23,7 @@ import type { QuestionAnswerData } from "../src/core/question-core.ts";
import { getGraph, graphSnapshot, mergeFindings, type GraphDeps } from "../src/core/graph.ts";
import { graphStore } from "../src/store/graph.ts";
import { chatStore } from "../src/store/chats.ts";
import { LEDGER_FILES, PATHS, SOURCES_FILE, EMBEDDINGS_FILE } from "../config.ts";
import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";
/** The wired fs adapters, exactly what src/server/stores.ts passes in production. */
const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps;
@@ -38,9 +38,7 @@ function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: strin
// ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ──────────────
// EMBEDDINGS_FILE too: the semantic-cluster source (read by graphSnapshot)
// — isolated like the rest so this suite never touches the real vectors.
const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE, EMBEDDINGS_FILE];
const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE];
const backups: Array<{ file: string; bak: string }> = [];
const createdChats: string[] = [];
@@ -681,21 +679,14 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
mergeFindings(dB.id, auditB, graphDeps, sessionSource(dB.id));
const zId = graphStore.readMappings().find((m) => m.chatId === dB.id && m.sourceId === "q2")!.graphNodeId;
// Coverage rows (both chats) + positions + embeddings garnish — the
// deletion prunes all three ledgers.
// Coverage rows (both chats) + positions garnish — the deletion
// prunes both ledgers.
for (const chat of [dA, dB]) {
for (let i = 0; i < 2; i++) {
graphStore.appendCoverage({ chatId: chat.id, fp: `${chat.id}-${i}`, tokens: 10, foldedAt: "2026-09-23T00:02:00.000Z" });
}
}
graphStore.writePositions({ ...graphStore.readPositions(), [xId]: { x: 1, y: 1 }, [yId]: { x: 2, y: 2 }, [zId]: { x: 3, y: 3 } });
const emb = graphStore.readEmbeddings();
graphStore.writeEmbeddings({
...emb,
vectors: { ...emb.vectors, [xId]: [1, 0], [yId]: [0, 1], [zId]: [1, 1] },
clusters: [...emb.clusters, { members: [xId, yId, zId] }],
updatedAt: "2026-09-23T00:03:00.000Z",
});
return { dA, dB, xId, yId, zId };
}
@@ -738,14 +729,10 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
// Mappings: dA's audit trail died; dB's stays.
const mappings = graphStore.readMappings().filter((m) => [dA.id, dB.id].includes(m.chatId));
assert.deepEqual(mappings, [{ chatId: dB.id, sourceId: "q1", graphNodeId: xId }, { chatId: dB.id, sourceId: "q2", graphNodeId: zId }]);
// Positions/embeddings: the orphan pruned everywhere.
// Positions: the orphan pruned.
const positions = graphStore.readPositions();
assert.equal(positions[yId], undefined);
assert.deepEqual(positions[xId], { x: 1, y: 1 });
const emb = graphStore.readEmbeddings();
assert.equal(emb.vectors[yId], undefined);
assert.ok(emb.vectors[xId] && emb.vectors[zId]);
assert.deepEqual(emb.clusters.at(-1), { members: [xId, zId] }, "dead members pruned from the hull");
// The snapshot resolves X to dB only — the deleted chat left no ghost.
const snapshot = graphSnapshot(graphDeps);
assert.deepEqual(snapshot.nodeSessions[xId], [dB.id]);
@@ -768,7 +755,6 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
events: graphStore.readEvents(),
mappings: graphStore.readMappings(),
positions: graphStore.readPositions(),
embeddings: graphStore.readEmbeddings(),
};
const stats = deleteChatTraces(graphStore, dA.id, "preserve-graph");
assert.deepEqual(stats, {
@@ -784,7 +770,6 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () =
assert.deepEqual(graphStore.readEvents(), before.events);
assert.deepEqual(graphStore.readMappings(), before.mappings);
assert.deepEqual(graphStore.readPositions(), before.positions);
assert.deepEqual(graphStore.readEmbeddings(), before.embeddings);
assert.equal(graphStore.readCoverage().some((row) => row.chatId === dA.id), false);
assert.equal(graphStore.readCoverage().some((row) => row.chatId === dB.id), true);
void xId; void yId; void zId;