Notes front door: format-agnostic import into the graph pipeline

The same extractor→auditor→merger pipeline now runs over raw personal notes in arbitrary formats (vision §1, DECIDED 2026-09-13): - NotesSource records (data/graph/sources.json, deterministic id from path hash → idempotent re-imports) play the chatId role in mappings, checkpoints, and observation events; usage lands on the source record (appendSourceUsage, idempotent per startedAt, kept across upserts). - mergeFindings is chat-store-free: callers pass MergeSource {messages, foldedAt} — sessions pass the chat record, notes pass messages:null (no quiz-event rebuild; a deterministic rebuild needs structured quiz tool parts). - Second extractor prompt (NOTES_EXTRACTOR_INSTRUCTIONS) for general- purpose extraction; understanding judges the AUTHOR's grasp. Auditor wording input-agnostic. extract/audit share one generic runSubmitAgent. - graphSnapshot.fileSources (srcId→filePath); notes sources render as static path entries in the detail card (FileTextIcon, never a link). - One shared pipeline instance (exported from chat-handler): same single-flight registry, live status chips (label = file path), usage accounting, and true abort for both front doors. - Transport: POST /api/graph/import (multipart, 3-wide worker pool) + Import notes… button in the graph view; scripts/import-notes.ts for batch runs (concurrency 4). - real-data-extraction/ gitignored (raw import sources never committed). 355/355 tests; typecheck, lint, build green.
2026-09-13 02:08:55 +03:00
parent b769d32bc6
commit 543cf33789
16 changed files with 961 additions and 114 deletions
@@ -23,14 +23,22 @@ import type { QuestionAnswerData } from "../src/core/question-core.ts";
 import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
 import { graphStore } from "../src/store/graph.ts";
 import { chatStore } from "../src/store/chats.ts";
-import { LEDGER_FILES, PATHS } from "../config.ts";
+import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";

 /** The wired fs adapters, exactly what src/server/stores.ts passes in production. */
 const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps;

+/** The session merge source for a chat — what the pipeline passes: the live
+ *  transcript + its updatedAt checkpoint stamp. (Notes imports pass
+ *  messages: null — see the notes-merger test below.) */
+function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: string } {
+	const record = chatStore.getChat(chatId)!;
+	return { messages: record.messages as UIMessage[], foldedAt: record.updatedAt };
+}
+
 // ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ──────────────

-const LEDGER_PATHS = Object.values(LEDGER_FILES);
+const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE];
 const backups: Array<{ file: string; bak: string }> = [];
 const createdChats: string[] = [];

@@ -233,7 +241,7 @@ function scopedGraph(chatIds: string[]) {

 describe("the merger (mergeFindings)", () => {
 	it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => {
-		const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps);
+		const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
 		assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 });

 		const graph = scopedGraph([chat1.id]);
@@ -302,7 +310,7 @@ describe("the merger (mergeFindings)", () => {
 	});

 	it("is idempotent — re-merging the same verdicts appends nothing", () => {
-		const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps);
+		const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
 		assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 });
 		const graph = scopedGraph([chat1.id]);
 		assert.equal(graph.nodes.length, 3);
@@ -346,6 +354,7 @@ describe("the merger (mergeFindings)", () => {
 				dropped: [],
 			},
 			graphDeps,
+			sessionSource(chat2.id),
 		);
 		assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 });

@@ -363,6 +372,41 @@ describe("the merger (mergeFindings)", () => {
 		assert.deepEqual(m, { chatId: chat2.id, sourceId: "q1", graphNodeId: alphaId });
 	});

+	it("resolves imported-notes ids to their file paths in the snapshot (static provenance)", () => {
+		// The notes front door in miniature: a source record + a merge under
+		// its src id — the snapshot must label it with the file path (a
+		// static entry), never as a chat session.
+		graphStore.upsertSource({ id: "src_snapfixture", filePath: "/vault/learn vim.md", importedAt: "2026-09-13T00:00:00.000Z" });
+		mergeFindings(
+			"src_snapfixture",
+			{
+				propositions: [
+					{
+						id: "p1",
+						name: "Vim registers",
+						statement: "Vim registers map single keys to editing commands.",
+						aliases: [],
+						messages: [],
+						evidence: ["fixture evidence"],
+						understanding: "solid",
+					},
+				],
+				edges: [],
+				attribution: { quiz: {} },
+				dropped: [],
+			},
+			graphDeps,
+			{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
+		);
+		const snap = graphSnapshot(graphDeps);
+		const node = snap.nodes.find((n) => n.statement === "Vim registers map single keys to editing commands.")!;
+		assert.equal(snap.fileSources["src_snapfixture"], "/vault/learn vim.md");
+		assert.deepEqual(snap.nodeSessions[node.id], ["src_snapfixture"], "the file entry joins the node's sessions list");
+		assert.equal(snap.sessions["src_snapfixture"], undefined, "a notes source is never a chat session");
+		// The understanding observation joins under the source id like any session's.
+		assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
+	});
+
 	it("derives sessions and per-session understanding observations in graphSnapshot", () => {
 		const snap = graphSnapshot(graphDeps);
 		const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));
@@ -408,17 +452,54 @@ describe("the merger (mergeFindings)", () => {
 				dropped: [],
 			},
 			graphDeps,
+			sessionSource(chat3.id),
 		);
 		assert.equal(stats.nodes, 1, "created — the merger does not match strings");
 		assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2);
 	});

-	it("returns zeros for unknown chats", () => {
-		assert.deepEqual(mergeFindings("chat_does_not_exist", fixtureAudit(), graphDeps), {
-			nodes: 0,
-			edges: 0,
-			events: 0,
+	it("merges the notes front door: source id as chatId, no quiz events, checkpoint at import time", () => {
+		// Raw personal notes carry no transcript — the merger gets no
+		// message parts, so NO quiz events are rebuilt; the understanding
+		// observation still lands, keyed to the source id.
+		const stats = mergeFindings(
+			"src_notesfixture",
+			{
+				propositions: [
+					{
+						id: "p1",
+						name: "Vim normal mode",
+						statement: "Vim starts in Normal mode, where keys are commands.",
+						aliases: [],
+						messages: [], // notes carry no message anchors — the file is the provenance
+						evidence: ["fixture evidence"],
+						understanding: "solid",
+					},
+				],
+				edges: [],
+				attribution: { quiz: {} }, // normalized by the notes pipeline — nothing to attribute
+				dropped: [],
+			},
+			graphDeps,
+			{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
+		);
+		assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
+
+		const node = getGraph(graphStore).nodes.find((n) => n.statement.startsWith("Vim starts"))!;
+		assert.deepEqual(node.provenance, [], "no message anchors — the mapping line carries the source");
+		assert.deepEqual(graphStore.readMappings().find((m) => m.chatId === "src_notesfixture"), {
+			chatId: "src_notesfixture",
+			sourceId: "p1",
+			graphNodeId: node.id,
 		});
+		// The observation event is file-scoped; no quiz events exist.
+		const events = graphStore.readEvents().filter((e) => e.chatId === "src_notesfixture");
+		assert.deepEqual(
+			events.map((e) => [e.kind, e.ref, e.note]),
+			[["observation", "obs:p1", "solid"]],
+		);
+		// The checkpoint stamps the import time.
+		assert.equal(graphStore.readCheckpoints().find((c) => c.chatId === "src_notesfixture")?.foldedAt, "2026-09-13T00:00:00.000Z");
 	});
 });

@@ -452,7 +533,7 @@ describe("the trigger helper (pendingQuizAnswers)", () => {
 		assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending");
 		assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0);

-		mergeFindings(chat.id, fixtureAudit(), graphDeps);
+		mergeFindings(chat.id, fixtureAudit(), graphDeps, sessionSource(chat.id));
 		// call-s is unattributed here (fixtureAudit has no call-s attribution)
 		// — but it still becomes an event, so nothing stays pending.
 		assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending");