Wave 7 close-out: the embedding ledger seam dies; notes birth via mapping lines
Part 1 — the ledger + remnants interlude (master-dispatched, owner ruling 2026-09-23: nothing configures the graph beyond chat models): - The embedding ledger surface is gone: readEmbeddings/writeEmbeddings ports, the EmbeddingsLedger contract, EMBEDDINGS_FILE (config, store impls, graphRevision source list), and graph-deletion's vector-pruning step. On-disk data files stay untouched (inert history). - The extraction embedder seam dies wholesale: ExtractionPipelineDeps drops the embedder dep (chat-handler + backfill/import-notes stubs go with it), the 'embedding' phase leaves ExtractionPhase, the usage capture/persist helpers die, and stages.embedding leaves ExtractionRunUsage (historical records keep their shape — reads are lenient). Extraction is two stages again, like the local-embedder era. - Consumer-less remnants: latestQuizOutcomes + QUIZ_EVENT_KINDS, clusterContentId, matchClusterLabelings + Community + ClusterLabeling, the readClusterLabels/appendClusterLabel port seams + store impls + LEDGER_FILES.clusterLabels. NamingRunUsage/readNamingRuns STAY (standing ruling — the cost pill renders historical naming spend); clusterHue + MIN_CLUSTER_SIZE stay (live). - The 'labeler' role dies: BACKGROUND_MODEL_ROLES, ROLE_SLOT, ModelRole, the catalog pin projections, and the UI roster strings. RolePins keys shrink with the union. The notes birth fallback (owner-approved 2026-09-23, surfaced by the interlude's live smoke: production notes imports create provenance-EMPTY nodes (importNotes passes messages: null — the file's anchor is the mapping line), so the provenance-only birth rule left every imported file unclustered and invisible to the prober's session list, against the locked file-birth ruling. A provenance-empty node now births under its FIRST mapping line's chatId (ledger order); only notes-born nodes ever hit the fallback (chat nodes always carry provenance; a re-encountered node's link-append re-opens the provenance rule). Deletion-safe: mapping lines of deleted chats are stripped alongside their provenance. Live-verified on port 3004: a fresh notes import clusters under its src_ id with the file path as the label, end to end with no embedder. 845/845 tests, typecheck/lint/build green. EOF )
This commit is contained in:
amosblomqvist
+1
-80
@@ -29,10 +29,8 @@
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { EMBEDDINGS_FILE, LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts";
|
||||
import { LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts";
|
||||
import type {
|
||||
ClusterLabeling,
|
||||
EmbeddingsLedger,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
ExtractionRunUsage,
|
||||
@@ -104,14 +102,6 @@ const isNamingRun = (v: unknown): v is NamingRunUsage =>
|
||||
Array.isArray(v.steps) &&
|
||||
isNamingStats(v.stats);
|
||||
|
||||
const isClusterLabeling = (v: unknown): v is ClusterLabeling =>
|
||||
isRecord(v) &&
|
||||
isString(v.id) &&
|
||||
isString(v.label) &&
|
||||
Array.isArray(v.core) &&
|
||||
v.core.every(isString) &&
|
||||
isString(v.createdAt);
|
||||
|
||||
const isGraphNode = (v: unknown): v is GraphNode =>
|
||||
isRecord(v) &&
|
||||
isString(v.id) &&
|
||||
@@ -200,18 +190,6 @@ export const graphStore: GraphStore = {
|
||||
return readValid(LEDGER_FILES.coverage, isCoverageEntry);
|
||||
},
|
||||
|
||||
// ── cluster labelings (config.LEDGER_FILES.clusterLabels) ──
|
||||
|
||||
/** Idempotent by id: the id is a hash of the sorted core, so re-appending
|
||||
* the same member set (re-naming races, idempotent re-runs) is a no-op. */
|
||||
appendClusterLabel(labeling: ClusterLabeling): void {
|
||||
if (this.readClusterLabels().some((l) => l.id === labeling.id)) return;
|
||||
appendLine(LEDGER_FILES.clusterLabels, labeling);
|
||||
},
|
||||
readClusterLabels(): ClusterLabeling[] {
|
||||
return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling);
|
||||
},
|
||||
|
||||
// ── cluster-naming run usage (config.LEDGER_FILES.namingRuns) ──
|
||||
|
||||
/** Idempotent per startedAt — the same discipline as every usage
|
||||
@@ -254,12 +232,6 @@ export const graphStore: GraphStore = {
|
||||
writePositions(positions: Record<string, Position>): void {
|
||||
writePositionMap(positions);
|
||||
},
|
||||
readEmbeddings(): EmbeddingsLedger {
|
||||
return readEmbeddingsLedger();
|
||||
},
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void {
|
||||
writeEmbeddingsLedger(ledger);
|
||||
},
|
||||
|
||||
/** THE DESTRUCTIVE SEAM (wave 6 session deletion): atomic full-content
|
||||
* replacement per provided ledger — write a uniquely-named temp file
|
||||
@@ -339,57 +311,6 @@ function writePositionMap(positions: Record<string, Position>): void {
|
||||
);
|
||||
}
|
||||
|
||||
/** embeddings.json — the whole EmbeddingsLedger at once. Missing/corrupt
|
||||
* file reads as an EMPTY ledger (model "") so consumers see "nothing
|
||||
* embedded yet" rather than an error; the embedder backfills. */
|
||||
function readEmbeddingsLedger(): EmbeddingsLedger {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(EMBEDDINGS_FILE, "utf-8");
|
||||
} catch {
|
||||
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!isRecord(parsed)) return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
const model = typeof parsed.model === "string" ? parsed.model : "";
|
||||
const dim = typeof parsed.dim === "number" && Number.isFinite(parsed.dim) ? parsed.dim : 0;
|
||||
const vectors: Record<string, number[]> = {};
|
||||
if (isRecord(parsed.vectors)) {
|
||||
for (const [id, value] of Object.entries(parsed.vectors)) {
|
||||
if (
|
||||
Array.isArray(value) &&
|
||||
value.length > 0 &&
|
||||
value.every((x) => typeof x === "number" && Number.isFinite(x))
|
||||
) {
|
||||
vectors[id] = value as number[];
|
||||
}
|
||||
}
|
||||
}
|
||||
const clusters: Array<{ members: string[] }> = [];
|
||||
if (Array.isArray(parsed.clusters)) {
|
||||
for (const c of parsed.clusters) {
|
||||
if (
|
||||
isRecord(c) &&
|
||||
Array.isArray(c.members) &&
|
||||
c.members.length > 0 &&
|
||||
c.members.every((m) => typeof m === "string")
|
||||
) {
|
||||
clusters.push({ members: [...(c.members as string[])] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { model, dim, vectors, clusters, updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "" };
|
||||
} catch {
|
||||
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
}
|
||||
}
|
||||
|
||||
function writeEmbeddingsLedger(ledger: EmbeddingsLedger): void {
|
||||
fs.mkdirSync(path.dirname(EMBEDDINGS_FILE), { recursive: true });
|
||||
fs.writeFileSync(EMBEDDINGS_FILE, JSON.stringify(ledger, null, "\t"), "utf-8");
|
||||
}
|
||||
|
||||
const isPosition = (v: unknown): v is Position =>
|
||||
isRecord(v) && typeof v.x === "number" && Number.isFinite(v.x) && typeof v.y === "number" && Number.isFinite(v.y);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user