From eda549101077b0ae5336123fb7e44a74ca93412e Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sat, 10 Oct 2026 13:54:17 +0300 Subject: [PATCH] schema stamps: lazy v1 for chats + the graph family, honest refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chat records gain version (CURRENT_CHAT_VERSION), stamped by a single writeChat choke point in the store; absence reads as v1 — existing files migrate on their next write, no mass rewrite. The graph family (four JSONL ledgers, sources.json, render-settings.json) shares one lazy sidecar, graph/format.json; positions stays unstamped (rebuildable) and jobs stay unstamped (transient). Downgrade scenario ruled honest-refusal: records stamped newer than the running build read as absent (chats, logged) or throw (graph family) — a read-modify-write store never rewrites what it doesn't understand. Golden fixtures stay the unstamped legacy shape on purpose (lazy v1 is current-valid by design); the stamped write path is pinned by tests/schema-stamps.test.ts. Reads never write — the golden byte-identical pin holds. --- config.ts | 7 ++ docs/development.md | 4 +- src/core/contracts/chat.ts | 9 +++ src/core/contracts/graph.ts | 4 + src/store/chats.ts | 37 ++++++--- src/store/graph-format.ts | 60 ++++++++++++++ src/store/graph.ts | 5 ++ src/store/render-settings.ts | 3 + tests/fixtures/golden-data/README.md | 9 +++ tests/schema-stamps.test.ts | 117 +++++++++++++++++++++++++++ 10 files changed, 245 insertions(+), 10 deletions(-) create mode 100644 src/store/graph-format.ts create mode 100644 tests/schema-stamps.test.ts diff --git a/config.ts b/config.ts index ac17747..860b571 100644 --- a/config.ts +++ b/config.ts @@ -86,6 +86,13 @@ export const POSITIONS_FILE = path.join(PATHS.graph, "positions.json"); */ export const RENDER_SETTINGS_FILE = path.join(PATHS.graph, "render-settings.json"); +/** The graph-family FORMAT STAMP sidecar (schema-versioning wave, + * 2026-10-10): one version for the whole family — the four JSONL ledgers, + * sources.json, render-settings.json. LAZY v1: absent = v1; the first + * graph-family write creates it. positions.json is REBUILDABLE and rides + * along unversioned by design (owner ruling). */ +export const GRAPH_FORMAT_FILE = path.join(PATHS.graph, "format.json"); + /** * THE USER SETTINGS RECORD (settings-storage wave, 2026-09-19): one * versioned, DELTA-shaped JSON blob at the data-dir seam holding every diff --git a/docs/development.md b/docs/development.md index bae0a61..a6c5026 100644 --- a/docs/development.md +++ b/docs/development.md @@ -178,4 +178,6 @@ The teaching methodology comes from an Obsidian-based POC (its `knowledge graph ## Data -Nothing under `data/` is ever tracked by git. Chats, graph ledgers, jobs, positions, and the settings records are plain files (JSON/JSONL) — inspect them freely; they are the canonical records. The desktop app relocates the whole directory via `LH_DATA_DIR`. \ No newline at end of file +Nothing under `data/` is ever tracked by git. Chats, graph ledgers, jobs, positions, and the settings records are plain files (JSON/JSONL) — inspect them freely; they are the canonical records. The desktop app relocates the whole directory via `LH_DATA_DIR`. + +**Schema stamps (2026-10-10):** chat records carry `version` (stamped at the store's write choke point; absence reads as v1 — lazy migration, no mass rewrite), and the graph family (ledgers, sources, render-settings) shares one sidecar `graph/format.json`, created on the family's first write. `positions.json` is deliberately unstamped (rebuildable). A record stamped NEWER than the running build is refused on read — honest refusal for the downgrade scenario: a read-modify-write store never rewrites what it doesn't understand. \ No newline at end of file diff --git a/src/core/contracts/chat.ts b/src/core/contracts/chat.ts index 850d370..2fb70e2 100644 --- a/src/core/contracts/chat.ts +++ b/src/core/contracts/chat.ts @@ -52,8 +52,17 @@ export interface ChatRecord { extractions?: ExtractionRunUsage[]; /** Per-chat model/thinking overrides for the conversation agents. */ settings?: ChatSettings; + /** SCHEMA STAMP (schema-versioning wave, owner 2026-10-10): LAZY v1 — + * existing records stay unstamped (absence reads as v1); every write + * stamps the current version. A record stamped NEWER than this build + * (the downgrade scenario) is REFUSED on read: a read-modify-write + * store must never rewrite what it doesn't understand. */ + version?: number; } +/** The chat record format this build writes and understands. */ +export const CURRENT_CHAT_VERSION = 1; + /** The conversation agents' runtime model choice (model-control wave, * 2026-09-12): a per-chat override of config.ts's pinned prober/teacher * spec. Absent fields fall back to config; written by the model picker diff --git a/src/core/contracts/graph.ts b/src/core/contracts/graph.ts index 5dccfb5..01b9eeb 100644 --- a/src/core/contracts/graph.ts +++ b/src/core/contracts/graph.ts @@ -120,6 +120,10 @@ export interface NamingRunStats { appended: number; } +/** The graph-family format version this build writes and understands + * (the format.json sidecar — src/store/graph-format.ts). */ +export const CURRENT_GRAPH_FORMAT_VERSION = 1; + export interface GraphNode { /** Opaque stable id, store-assigned: "n_" + nanoid-like. */ diff --git a/src/store/chats.ts b/src/store/chats.ts index b062f36..1d7a635 100644 --- a/src/store/chats.ts +++ b/src/store/chats.ts @@ -13,6 +13,7 @@ import { PATHS } from "../../config.ts"; import { emit } from "../core/events.ts"; import type { ChatStore } from "../core/ports.ts"; import type { ChatRecord, ChatSettings, ChatSummary, ExtractionRunUsage, PlanTree, TurnUsage } from "../core/contracts.ts"; +import { CURRENT_CHAT_VERSION } from "../core/contracts.ts"; import type { LhMessage } from "../core/native-messages.ts"; import { activeProblemOf, danglingQuestionOf } from "../core/native-messages.ts"; import { phaseAt3Lh } from "../core/phases.ts"; @@ -44,17 +45,35 @@ function createChat(title: string, kind?: "practice"): ChatRecord { ...(kind === "practice" ? { kind: "practice" as const } : {}), }; fs.mkdirSync(PATHS.chats, { recursive: true }); - fs.writeFileSync(chatFile(record.id), JSON.stringify(record, null, "\t"), "utf-8"); + writeChat(record); emit("chats-changed", {}); - return record; + return { ...record, version: CURRENT_CHAT_VERSION }; // the in-memory twin of what landed on disk +} + +/** THE WRITE CHOKE POINT (schema-versioning wave, owner 2026-10-10): every + * record persist goes through here and carries the CURRENT version stamp — + * lazy migration means existing files only get stamped on their next write. + * A future-version guard lives on the read side (getChat). */ +function writeChat(record: ChatRecord): void { + fs.writeFileSync(chatFile(record.id), JSON.stringify({ ...record, version: CURRENT_CHAT_VERSION }, null, "\t"), "utf-8"); } function getChat(id: string): ChatRecord | null { + let parsed: unknown; try { - return JSON.parse(fs.readFileSync(chatFile(id), "utf-8")) as ChatRecord; + parsed = JSON.parse(fs.readFileSync(chatFile(id), "utf-8")); } catch { return null; } + // Honest refusal (owner ruling 2026-10-10): a record stamped NEWER than + // this build (the downgrade scenario) reads as absent, never misparsed — + // the next read-modify-write must not rewrite what it doesn't understand. + const v = (parsed as { version?: unknown }).version; + if (typeof v === "number" && v > CURRENT_CHAT_VERSION) { + console.error(`[chat ${id}] record stamped with schema version ${v} > ${CURRENT_CHAT_VERSION} — this app build is older; refusing to read it`); + return null; + } + return parsed as ChatRecord; } function listChats(): ChatSummary[] { @@ -109,7 +128,7 @@ function saveMessages(id: string, messages: LhMessage[]): void { if (!record) throw new Error(`Chat not found: ${id}`); record.messages = messages; record.updatedAt = new Date().toISOString(); - fs.writeFileSync(chatFile(id), JSON.stringify(record, null, "\t"), "utf-8"); + writeChat(record); } /** Append one usage entry to the record's turn log — idempotent per @@ -122,7 +141,7 @@ function appendTurnUsage(id: string, entry: TurnUsage): void { record.turns = record.turns?.some((t) => t.id === entry.id) ? record.turns : [...(record.turns ?? []), entry]; - fs.writeFileSync(chatFile(id), JSON.stringify(record, null, "\t"), "utf-8"); + writeChat(record); } /** Append one extraction-run usage entry to the record's extractions @@ -134,7 +153,7 @@ function appendExtractionUsage(id: string, entry: ExtractionRunUsage): void { record.extractions = record.extractions?.some((r) => r.startedAt === entry.startedAt) ? record.extractions : [...(record.extractions ?? []), entry]; - fs.writeFileSync(chatFile(id), JSON.stringify(record, null, "\t"), "utf-8"); + writeChat(record); } /** Persist the per-chat model/thinking settings — replace-whole (the @@ -145,7 +164,7 @@ function saveSettings(id: string, settings: ChatSettings): void { // Like saveTree: a settings tweak is not conversation activity, so it // deliberately does NOT bump updatedAt (sidebar order must not churn). record.settings = settings; - fs.writeFileSync(chatFile(id), JSON.stringify(record, null, "\t"), "utf-8"); + writeChat(record); } /** Rename a session (the header's click-to-edit title, 2026-09-17). Like @@ -157,7 +176,7 @@ function renameChat(id: string, title: string): ChatRecord { const record = getChat(id); if (!record) throw new Error(`Chat not found: ${id}`); record.title = title.trim() || "Untitled session"; - fs.writeFileSync(chatFile(id), JSON.stringify(record, null, "\t"), "utf-8"); + writeChat(record); emit("chats-changed", {}); return record; } @@ -183,7 +202,7 @@ function saveTree(id: string, tree: PlanTree): void { record.tree = tree; // Deliberately does NOT bump updatedAt: tree mutations piggyback on the // turn's saveMessages, and sidebar ordering shouldn't churn on tree edits. - fs.writeFileSync(chatFile(id), JSON.stringify(record, null, "\t"), "utf-8"); + writeChat(record); } export const chatStore: ChatStore = { diff --git a/src/store/graph-format.ts b/src/store/graph-format.ts new file mode 100644 index 0000000..5b511ce --- /dev/null +++ b/src/store/graph-format.ts @@ -0,0 +1,60 @@ +/** + * The graph-family FORMAT STAMP (schema-versioning wave, owner 2026-10-10): + * data/graph/format.json — one sidecar versioning the whole graph dir + * family (the four JSONL ledgers, sources.json, render-settings.json). + * JSONL ledgers can't carry inline stamps without per-line bloat, and the + * JSON files share the family's fate, so the stamp lives beside them. + * positions.json is rebuildable (owner ruling) and needs no protection. + * + * The contract: + * - LAZY v1: NO sidecar = v1 (the pre-stamp world stays valid forever); + * the first graph-family WRITE creates it. Reads never write. + * - Honest refusal (owner: refuse-honestly ruling): a sidecar stamped + * NEWER than this build (the downgrade scenario — user reverted the + * app) makes graph-family reads THROW. Read-modify-write stores must + * never rewrite what they don't understand; a ledger store must never + * silently skip lines it can't interpret as current-format data. + * + * Version constants: contracts/graph.ts CURRENT_GRAPH_FORMAT_VERSION. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { GRAPH_FORMAT_FILE } from "../../config.ts"; +import { CURRENT_GRAPH_FORMAT_VERSION } from "../core/contracts/graph.ts"; + +function stampedVersion(): number | null { + let raw: string; + try { + raw = fs.readFileSync(GRAPH_FORMAT_FILE, "utf-8"); + } catch { + return null; // no sidecar — the lazy-v1 pre-stamp world + } + try { + const parsed = JSON.parse(raw) as { version?: unknown }; + return typeof parsed.version === "number" && Number.isInteger(parsed.version) ? parsed.version : null; + } catch { + return null; // corrupt sidecar reads as unstamped (v1), not fatal + } +} + +/** READ-SIDE GUARD: throws when the data dir was written by a NEWER build. + * Called by every graph-family read — never writes (the golden fixtures + * pin reads as byte-identical). */ +export function assertGraphFormat(): void { + const v = stampedVersion(); + if (v !== null && v > CURRENT_GRAPH_FORMAT_VERSION) { + throw new Error( + `Graph data is stamped with format version ${v}, but this build understands only ${CURRENT_GRAPH_FORMAT_VERSION}. ` + + "The app is older than its data — upgrade the app before touching the knowledge map.", + ); + } +} + +/** WRITE-SIDE STAMP: creates the sidecar on the first graph-family write + * (lazy); afterwards a no-op existence-wise. Callers keep their own mkdir. */ +export function ensureGraphFormat(): void { + if (stampedVersion() !== null) return; + fs.mkdirSync(path.dirname(GRAPH_FORMAT_FILE), { recursive: true }); + fs.writeFileSync(GRAPH_FORMAT_FILE, JSON.stringify({ version: CURRENT_GRAPH_FORMAT_VERSION }, null, "\t") + "\n", "utf-8"); +} diff --git a/src/store/graph.ts b/src/store/graph.ts index 7753654..63c7e31 100644 --- a/src/store/graph.ts +++ b/src/store/graph.ts @@ -35,6 +35,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts"; +import { assertGraphFormat, ensureGraphFormat } from "./graph-format.ts"; import type { GraphChain, GraphNode, @@ -51,6 +52,7 @@ import type { GraphStore } from "../core/ports.ts"; function appendLine(file: string, record: T): void { fs.mkdirSync(path.dirname(file), { recursive: true }); + ensureGraphFormat(); // the lazy format stamp (graph-format.ts) fs.appendFileSync(file, `${JSON.stringify(record)}\n`, "utf-8"); } @@ -83,6 +85,7 @@ const isString = (v: unknown): v is string => typeof v === "string"; /** Strict shape guards — anything not matching a W4 ledger shape is skipped. */ function readValid(file: string, isValid: (v: unknown) => v is T): T[] { + assertGraphFormat(); // honest refusal on a newer-stamped data dir return readRawLines(file).filter(isValid); } @@ -280,6 +283,7 @@ const isPosition = (v: unknown): v is Position => /** sources.json — { id: NotesSource }. Read-modify-write, NOT append-only: * the usage log lives inside the record (like a chat's JSON file). */ function readSourceMap(): Record { + assertGraphFormat(); // honest refusal on a newer-stamped data dir let raw: string; try { raw = fs.readFileSync(SOURCES_FILE, "utf-8"); @@ -301,6 +305,7 @@ function readSourceMap(): Record { function writeSourceMap(map: Record): void { fs.mkdirSync(path.dirname(SOURCES_FILE), { recursive: true }); + ensureGraphFormat(); // the lazy format stamp (graph-format.ts) fs.writeFileSync(SOURCES_FILE, JSON.stringify(map, null, "\t"), "utf-8"); } diff --git a/src/store/render-settings.ts b/src/store/render-settings.ts index 754135c..8a3d4f2 100644 --- a/src/store/render-settings.ts +++ b/src/store/render-settings.ts @@ -13,9 +13,11 @@ import fs from "node:fs"; import path from "node:path"; import { RENDER_SETTINGS_FILE } from "../../config.ts"; import { DEFAULT_RENDER_SETTINGS, sanitizeRenderSettings, type GraphRenderSettings } from "../core/render-settings.ts"; +import { assertGraphFormat, ensureGraphFormat } from "./graph-format.ts"; /** The persisted settings, or null when no file exists yet (first boot). */ export function readRenderSettings(): GraphRenderSettings | null { + assertGraphFormat(); // honest refusal on a newer-stamped data dir let raw: string; try { raw = fs.readFileSync(RENDER_SETTINGS_FILE, "utf-8"); @@ -33,6 +35,7 @@ export function readRenderSettings(): GraphRenderSettings | null { export function writeRenderSettings(raw: unknown): GraphRenderSettings { const settings = sanitizeRenderSettings(raw); fs.mkdirSync(path.dirname(RENDER_SETTINGS_FILE), { recursive: true }); + ensureGraphFormat(); // the lazy format stamp (graph-format.ts) fs.writeFileSync(RENDER_SETTINGS_FILE, JSON.stringify(settings, null, "\t"), "utf-8"); return settings; } diff --git a/tests/fixtures/golden-data/README.md b/tests/fixtures/golden-data/README.md index a4efa00..60dce53 100644 --- a/tests/fixtures/golden-data/README.md +++ b/tests/fixtures/golden-data/README.md @@ -36,3 +36,12 @@ core tool surface must appear in real data here. Run the app (`npm run dev`), exercise the feature against `data/`, then copy the produced record in verbatim. Do not hand-edit content; if a record needs pruning, re-run the flow instead. + +## Stamps + +These files are deliberately the UNSTAMPED legacy shape (schema-stamp wave, +2026-10-10): lazy v1 means absence reads as current-valid, so this set pins +the pre-stamp read path forever. Stamped writes (chat `version: 1`, the +`graph/format.json` sidecar) are pinned by `tests/schema-stamps.test.ts`. +New records copied in after their first post-stamp write will carry the +stamps — that shape is equally current-valid. diff --git a/tests/schema-stamps.test.ts b/tests/schema-stamps.test.ts new file mode 100644 index 0000000..bfb1cd3 --- /dev/null +++ b/tests/schema-stamps.test.ts @@ -0,0 +1,117 @@ +/** + * SCHEMA STAMPS (schema-versioning wave, owner 2026-10-10): LAZY v1 for + * chats (record.version, stamped at the write choke point) and the graph + * family (data/graph/format.json sidecar — ledgers, sources, + * render-settings; positions deliberately unversioned, rebuildable). + * Honest refusal: records stamped NEWER than this build are refused on + * read (the downgrade scenario) — never misparsed, never rewritten. + * + * Runs under an isolated LH_DATA_DIR (the chat-birth-stamp discipline: + * the seam is set before any config-loading import, so every app-module + * import is dynamic). + */ + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import test from "node:test"; + +// ── the seam, BEFORE any config-loading import ────────────────────────────── +const TMP_DATA = fs.mkdtempSync(path.join(os.tmpdir(), "lh-stamps-")); +process.env.LH_DATA_DIR = TMP_DATA; + +const { chatStore } = await import("../src/store/chats.ts"); +const { graphStore } = await import("../src/store/graph.ts"); +const { readRenderSettings, writeRenderSettings } = await import("../src/store/render-settings.ts"); +const { GRAPH_FORMAT_FILE } = await import("../config.ts"); + +test.after(() => { + fs.rmSync(TMP_DATA, { recursive: true, force: true }); +}); + +// ── chats: lazy stamp at the write choke point ────────────────────────────── + +test("chats: createChat stamps version 1 on disk", () => { + const r = chatStore.createChat("stamp test"); + assert.equal(r.version, 1, "the in-memory twin is stamped"); + const onDisk = JSON.parse(fs.readFileSync(path.join(TMP_DATA, "chats", `${r.id}.json`), "utf8")); + assert.equal(onDisk.version, 1, "the record carries the stamp"); +}); + +test("chats: an UNSTAMPED legacy record reads fine (absence = v1) and is stamped by its next write", () => { + const id = "chat_legacy000_stamp0"; + fs.mkdirSync(path.join(TMP_DATA, "chats"), { recursive: true }); + fs.writeFileSync( + path.join(TMP_DATA, "chats", `${id}.json`), + JSON.stringify({ id, title: "legacy", createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), messages: [] }), + "utf-8", + ); + const r = chatStore.getChat(id); + assert.ok(r, "unstamped record reads (lazy v1)"); + chatStore.saveMessages(id, []); + const onDisk = JSON.parse(fs.readFileSync(path.join(TMP_DATA, "chats", `${id}.json`), "utf8")); + assert.equal(onDisk.version, 1, "the write choke point stamped it"); +}); + +test("chats: a NEWER-stamped record is refused honestly (the downgrade scenario)", () => { + const id = "chat_future000_stamp0"; + fs.writeFileSync( + path.join(TMP_DATA, "chats", `${id}.json`), + JSON.stringify({ id, title: "from the future", createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), messages: [], version: 99 }), + "utf-8", + ); + assert.equal(chatStore.getChat(id), null, "getChat refuses: reads as absent, never misparsed"); + assert.ok(!chatStore.listChats().some((s) => s.id === id), "listChats skips it too"); + // and the refused record is NOT rewritten or downgraded by the visit: + const onDisk = JSON.parse(fs.readFileSync(path.join(TMP_DATA, "chats", `${id}.json`), "utf8")); + assert.equal(onDisk.version, 99, "the future record survives untouched"); +}); + +// ── the graph family: the lazy format.json sidecar ────────────────────────── + +test("graph: reads never write — no sidecar appears from reading alone", () => { + assert.ok(!fs.existsSync(GRAPH_FORMAT_FILE), "no sidecar before any write"); + assert.deepEqual(graphStore.readNodes(), [], "ledger reads fine without a sidecar"); + assert.equal(readRenderSettings(), null, "render-settings reads fine without a sidecar"); + assert.ok(!fs.existsSync(GRAPH_FORMAT_FILE), "reads did NOT create the sidecar (lazy = write-side only)"); +}); + +test("graph: the first ledger append creates the sidecar at version 1", () => { + graphStore.appendNode({ + id: "n_stamp0", + name: "Stamps", + statement: "The graph family carries one format sidecar.", + aliases: [], + createdAt: new Date().toISOString(), + provenance: [], + }); + assert.ok(fs.existsSync(GRAPH_FORMAT_FILE), "the first write stamped the family"); + assert.equal(JSON.parse(fs.readFileSync(GRAPH_FORMAT_FILE, "utf8")).version, 1); + assert.equal(graphStore.readNodes().length, 1, "the ledger reads back with the sidecar present"); +}); + +test("graph: a NEWER-stamped sidecar is refused honestly", () => { + fs.mkdirSync(path.dirname(GRAPH_FORMAT_FILE), { recursive: true }); + fs.writeFileSync(GRAPH_FORMAT_FILE, JSON.stringify({ version: 99 }), "utf-8"); + assert.throws(() => graphStore.readNodes(), /older than its data|format version 99/); + assert.throws(() => graphStore.readSources(), /older than its data|format version 99/); + assert.throws(() => readRenderSettings(), /older than its data|format version 99/); +}); + +test("render-settings: its write stamps the family sidecar too", () => { + fs.rmSync(path.dirname(GRAPH_FORMAT_FILE), { recursive: true, force: true }); + assert.ok(!fs.existsSync(GRAPH_FORMAT_FILE)); + const written = writeRenderSettings({ sizeUnits: "world", nodeSize: 3 }); + assert.equal(written.nodeSize, 3); + assert.equal(JSON.parse(fs.readFileSync(GRAPH_FORMAT_FILE, "utf8")).version, 1, "the render-settings write stamped the family"); +}); + +// ── the fixtures stay the legacy pin: golden data reads unstamped ─────────── + +test("golden fixtures are deliberately the unstamped legacy shape (lazy v1 = current-valid)", () => { + const here = path.dirname(new URL(import.meta.url).pathname); + const teach = JSON.parse(fs.readFileSync(path.join(here, "fixtures", "golden-data", "chats", "chat_mv27rrra_feff23b6.json"), "utf8")); + assert.equal(teach.version, undefined, "fixture records pin the unstamped read path"); + assert.ok(!fs.existsSync(path.join(here, "fixtures", "golden-data", "graph", "format.json")), "no sidecar in the fixture graph dir"); +});