Notes front door: format-agnostic import into the graph pipeline

The same extractor→auditor→merger pipeline now runs over raw personal
notes in arbitrary formats (vision §1, DECIDED 2026-09-13):

- NotesSource records (data/graph/sources.json, deterministic id from
  path hash → idempotent re-imports) play the chatId role in mappings,
  checkpoints, and observation events; usage lands on the source record
  (appendSourceUsage, idempotent per startedAt, kept across upserts).
- mergeFindings is chat-store-free: callers pass MergeSource
  {messages, foldedAt} — sessions pass the chat record, notes pass
  messages:null (no quiz-event rebuild; a deterministic rebuild needs
  structured quiz tool parts).
- Second extractor prompt (NOTES_EXTRACTOR_INSTRUCTIONS) for general-
  purpose extraction; understanding judges the AUTHOR's grasp. Auditor
  wording input-agnostic. extract/audit share one generic runSubmitAgent.
- graphSnapshot.fileSources (srcId→filePath); notes sources render as
  static path entries in the detail card (FileTextIcon, never a link).
- One shared pipeline instance (exported from chat-handler): same
  single-flight registry, live status chips (label = file path), usage
  accounting, and true abort for both front doors.
- Transport: POST /api/graph/import (multipart, 3-wide worker pool) +
  Import notes… button in the graph view; scripts/import-notes.ts for
  batch runs (concurrency 4).
- real-data-extraction/ gitignored (raw import sources never committed).

355/355 tests; typecheck, lint, build green.
This commit is contained in:
amosblomqvist
2026-09-13 02:08:55 +03:00
parent b769d32bc6
commit 543cf33789
16 changed files with 961 additions and 114 deletions
+60 -1
View File
@@ -15,6 +15,10 @@
* (chatId, sourceId) → graphNodeId
* sessions.jsonl SessionCheckpoint— emission checkpoints (foldedAt)
*
* Plus config.SOURCES_FILE (sources.json): the imported-notes bookkeeping
* (NOT a ledger) — one JSON map, src-id → NotesSource, read-modify-write
* like a chat file. The format-agnostic front door (2026-09-13).
*
* Implements core/ports.ts GraphStore. data/graph/* is exclusively this
* adapter's. Reads are strict against the contracts.ts shapes (the 2026-09-02
* graph redesign): corrupt lines are skipped, never guessed at — the ledgers
@@ -25,12 +29,14 @@
import * as fs from "node:fs";
import * as path from "node:path";
import { LEDGER_FILES } from "../../config.ts";
import { LEDGER_FILES, SOURCES_FILE } from "../../config.ts";
import type {
GraphEdge,
GraphNode,
ExtractionRunUsage,
LedgerEvent,
LedgerEventKind,
NotesSource,
NodeMapping,
ProvenancePoint,
SessionCheckpoint,
@@ -157,4 +163,57 @@ export const graphStore: GraphStore = {
readCheckpoints(): SessionCheckpoint[] {
return readValid(LEDGER_FILES.sessions, isCheckpoint);
},
// ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ──
/** Upsert: keeps the existing record's usage log when re-importing. */
upsertSource(source: NotesSource): void {
const all = readSourceMap();
const prev = all[source.id];
all[source.id] = prev ? { ...source, extractions: prev.extractions } : source;
writeSourceMap(all);
},
readSources(): NotesSource[] {
return Object.values(readSourceMap());
},
/** Idempotent per startedAt — same discipline as chats.appendExtractionUsage. */
appendSourceUsage(id: string, entry: ExtractionRunUsage): void {
const all = readSourceMap();
const source = all[id];
if (!source) return; // no such source — nothing to bill against
source.extractions = source.extractions?.some((r) => r.startedAt === entry.startedAt)
? source.extractions
: [...(source.extractions ?? []), entry];
writeSourceMap(all);
},
};
/** sources.json — { id: NotesSource }. Read-modify-write, NOT append-only:
* the usage log lives inside the record (like a chat's JSON file). */
function readSourceMap(): Record<string, NotesSource> {
let raw: string;
try {
raw = fs.readFileSync(SOURCES_FILE, "utf-8");
} catch {
return {};
}
try {
const parsed: unknown = JSON.parse(raw);
if (!isRecord(parsed)) return {};
const out: Record<string, NotesSource> = {};
for (const [id, value] of Object.entries(parsed)) {
if (isNotesSource(value)) out[id] = value;
}
return out;
} catch {
return {};
}
}
function writeSourceMap(map: Record<string, NotesSource>): void {
fs.mkdirSync(path.dirname(SOURCES_FILE), { recursive: true });
fs.writeFileSync(SOURCES_FILE, JSON.stringify(map, null, "\t"), "utf-8");
}
const isNotesSource = (v: unknown): v is NotesSource =>
isRecord(v) && isString(v.id) && isString(v.filePath) && isString(v.importedAt);