Notes front door: format-agnostic import into the graph pipeline
The same extractor→auditor→merger pipeline now runs over raw personal
notes in arbitrary formats (vision §1, DECIDED 2026-09-13):
- NotesSource records (data/graph/sources.json, deterministic id from
path hash → idempotent re-imports) play the chatId role in mappings,
checkpoints, and observation events; usage lands on the source record
(appendSourceUsage, idempotent per startedAt, kept across upserts).
- mergeFindings is chat-store-free: callers pass MergeSource
{messages, foldedAt} — sessions pass the chat record, notes pass
messages:null (no quiz-event rebuild; a deterministic rebuild needs
structured quiz tool parts).
- Second extractor prompt (NOTES_EXTRACTOR_INSTRUCTIONS) for general-
purpose extraction; understanding judges the AUTHOR's grasp. Auditor
wording input-agnostic. extract/audit share one generic runSubmitAgent.
- graphSnapshot.fileSources (srcId→filePath); notes sources render as
static path entries in the detail card (FileTextIcon, never a link).
- One shared pipeline instance (exported from chat-handler): same
single-flight registry, live status chips (label = file path), usage
accounting, and true abort for both front doors.
- Transport: POST /api/graph/import (multipart, 3-wide worker pool) +
Import notes… button in the graph view; scripts/import-notes.ts for
batch runs (concurrency 4).
- real-data-extraction/ gitignored (raw import sources never committed).
355/355 tests; typecheck, lint, build green.
This commit is contained in:
amosblomqvist
@@ -20,6 +20,8 @@
|
||||
/** Two relations only (DECIDED 2026-09-02): follows = b is derivable from a;
|
||||
* supports = a grounds b without entailing it. No scalars, no encoded
|
||||
* strength — strength is derived at read time from topology. */
|
||||
import type { ExtractionRunUsage } from "./chat.ts";
|
||||
|
||||
export type EdgeType = "follows" | "supports";
|
||||
|
||||
/** What one extraction run merged into the graph ledgers — the merger's
|
||||
@@ -37,10 +39,32 @@ export interface EmissionStats {
|
||||
* edges record every (chat, message) where they came up, deep-linkable to
|
||||
* the exact transcript point. */
|
||||
export interface ProvenancePoint {
|
||||
/** The chat this point anchors to — or a notes source id ("src_…"):
|
||||
* imports carry no message anchors (provenance is the file itself,
|
||||
* recorded through the mapping line + the source record). */
|
||||
chatId: string;
|
||||
messageId: string;
|
||||
}
|
||||
|
||||
/** One imported notes file — the format-agnostic front door's bookkeeping
|
||||
* (DECIDED 2026-09-13): personal notes in ARBITRARY formats run through the
|
||||
* same extractor → auditor → merger pipeline, with the file itself as the
|
||||
* provenance anchor. There is no chat: the detail card shows the file path
|
||||
* as a STATIC entry (not a link), and understanding observations still
|
||||
* apply (judged from the author's own voice in the notes). The run's usage
|
||||
* log lives here, mirroring ChatRecord.extractions. Ids are "src_" + hash
|
||||
* of the file path — deterministic, so re-importing the same file re-runs
|
||||
* extraction against the same source id (mappings/events dedupe by it). */
|
||||
export interface NotesSource {
|
||||
id: string;
|
||||
filePath: string;
|
||||
/** ISO — when this file was imported (the checkpoint's foldedAt). */
|
||||
importedAt: string;
|
||||
/** The run's usage log — same append-only discipline as
|
||||
* ChatRecord.extractions, idempotent per startedAt. */
|
||||
extractions?: ExtractionRunUsage[];
|
||||
}
|
||||
|
||||
export interface GraphNode {
|
||||
/** Opaque stable id, store-assigned: "n_" + nanoid-like. */
|
||||
id: string;
|
||||
@@ -107,6 +131,8 @@ export interface LedgerEvent {
|
||||
/** null = not yet attributed. */
|
||||
nodeId: string | null;
|
||||
kind: LedgerEventKind;
|
||||
/** The chat that produced this evidence — or a notes source id
|
||||
* ("src_…") for imported-notes observations. */
|
||||
chatId: string;
|
||||
at: string;
|
||||
/** Idempotency key: the quiz part's toolCallId, or `obs:<sourceId>` for
|
||||
@@ -249,6 +275,10 @@ export interface GraphSnapshot {
|
||||
edges: GraphEdge[];
|
||||
/** chatId → title, for every chat that touched the graph. */
|
||||
sessions: Record<string, string>;
|
||||
/** srcId → file path, for every IMPORTED NOTES file that touched the
|
||||
* graph (the format-agnostic front door): a static provenance entry —
|
||||
* the detail card shows the path as plain text, not a link. */
|
||||
fileSources: Record<string, string>;
|
||||
/** nodeId → chatIds that touched the node (click-through provenance). */
|
||||
nodeSessions: Record<string, string[]>;
|
||||
/** nodeId → chatId → that session's understanding observation (the
|
||||
|
||||
+38
-14
@@ -64,6 +64,17 @@ function newGraphNodeId(): string {
|
||||
return `n_${randomBytes(6).toString("hex")}`;
|
||||
}
|
||||
|
||||
/** What one merge run merges FROM (the caller supplies it — the merger never
|
||||
* reads the chat store): a session's transcript + its `updatedAt`
|
||||
* checkpoint, or — for the format-agnostic notes front door (2026-09-13) —
|
||||
* `messages: null`: raw notes carry no tool parts, so no quiz events are
|
||||
* rebuilt (quiz outcomes inside notes surface only through the extractor's
|
||||
* understanding observations), and the checkpoint stamps the import. */
|
||||
export interface MergeSource {
|
||||
messages: UIMessage[] | null;
|
||||
foldedAt: string;
|
||||
}
|
||||
|
||||
// ─── transcript scanning (shared by the merger + trigger) ─────────────────────
|
||||
|
||||
/** A flattened, order-preserving view of every tool part in a transcript. */
|
||||
@@ -126,11 +137,15 @@ const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean =>
|
||||
* parts, nodeId from the extractor's attribution through this run's
|
||||
* mapping) plus one observation event per proposition carrying an
|
||||
* understanding note (ref `obs:<sourceId>`). No taught events, ever.
|
||||
*
|
||||
* `chatId` is the run's source key: a real chat, or a notes source id
|
||||
* ("src_…") for the format-agnostic import path — same ledgers, same
|
||||
* idempotency, one graph. The `source` parameter carries the transcript
|
||||
* being merged (null messages = raw notes: no quiz events) and the
|
||||
* checkpoint stamp; the merger itself is chat-store-free.
|
||||
*/
|
||||
export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps): EmissionStats {
|
||||
const { graphStore: ledgers, chatStore } = deps;
|
||||
const record = chatStore.getChat(chatId);
|
||||
if (!record) return { nodes: 0, edges: 0, events: 0 };
|
||||
export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps, source: MergeSource): EmissionStats {
|
||||
const { graphStore: ledgers } = deps;
|
||||
const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 };
|
||||
|
||||
// 1. Resolution: this run's propositions → graph node ids. Existing
|
||||
@@ -214,7 +229,7 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
|
||||
for (const event of ledgers.readEvents()) {
|
||||
if (event.chatId === chatId && event.ref) seenRefs.add(event.ref);
|
||||
}
|
||||
for (const part of toolParts(record.messages)) {
|
||||
for (const part of source.messages ? toolParts(source.messages) : []) {
|
||||
if (part.name !== "quiz" || part.state !== "output-available") continue;
|
||||
const event = quizEvent(part, chatId, audit, mapping, now);
|
||||
if (event?.ref !== undefined && !seenRefs.has(event.ref)) {
|
||||
@@ -241,8 +256,9 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
|
||||
stats.events++;
|
||||
}
|
||||
|
||||
// 4. Checkpoint: this chat is extracted through its current updatedAt.
|
||||
ledgers.appendCheckpoint({ chatId, foldedAt: record.updatedAt });
|
||||
// 4. Checkpoint: this source is extracted through its foldedAt (the
|
||||
// chat's updatedAt, or the import time for notes).
|
||||
ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt });
|
||||
return stats;
|
||||
}
|
||||
|
||||
@@ -423,20 +439,28 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
|
||||
if (event.nodeId !== null) touch(event.nodeId, event.chatId);
|
||||
}
|
||||
|
||||
// Titles for every chat the graph can point back at.
|
||||
const chatIds = new Set<string>();
|
||||
// Labels for every source the graph can point back at: real chats
|
||||
// (title, deep-linkable) and imported notes files (path, static — the
|
||||
// format-agnostic front door's provenance).
|
||||
const sourceIds = new Set<string>();
|
||||
for (const chats of nodeChats.values()) {
|
||||
for (const id of chats) chatIds.add(id);
|
||||
for (const id of chats) sourceIds.add(id);
|
||||
}
|
||||
const sessions: Record<string, string> = {};
|
||||
for (const id of chatIds) {
|
||||
const fileSources: Record<string, string> = {};
|
||||
for (const id of sourceIds) {
|
||||
const chat = deps.chatStore.getChat(id);
|
||||
if (chat) sessions[id] = chat.title;
|
||||
if (chat) {
|
||||
sessions[id] = chat.title;
|
||||
continue;
|
||||
}
|
||||
const source = deps.graphStore.readSources().find((s) => s.id === id);
|
||||
if (source) fileSources[id] = source.filePath;
|
||||
}
|
||||
|
||||
const nodeSessions: Record<string, string[]> = {};
|
||||
for (const [nodeId, chats] of nodeChats) {
|
||||
const ids = [...chats].filter((id) => sessions[id] !== undefined);
|
||||
const ids = [...chats].filter((id) => sessions[id] !== undefined || fileSources[id] !== undefined);
|
||||
if (ids.length > 0) nodeSessions[nodeId] = ids;
|
||||
}
|
||||
|
||||
@@ -461,7 +485,7 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
|
||||
(nodeObservations[nodeId] ??= {})[chatId] = note;
|
||||
}
|
||||
|
||||
return { nodes, edges, sessions, nodeSessions, nodeObservations };
|
||||
return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations };
|
||||
}
|
||||
|
||||
// Re-export: the contract owns the shape; legacy importers still reach it.
|
||||
|
||||
@@ -24,6 +24,7 @@ import type {
|
||||
PlanTree,
|
||||
SessionCheckpoint,
|
||||
TurnUsage,
|
||||
NotesSource,
|
||||
} from "./contracts.ts";
|
||||
|
||||
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
|
||||
@@ -80,6 +81,15 @@ export interface GraphStore {
|
||||
/** Append one emission checkpoint. */
|
||||
appendCheckpoint(checkpoint: SessionCheckpoint): void;
|
||||
readCheckpoints(): SessionCheckpoint[];
|
||||
|
||||
/** Upsert one imported-notes source record (config.SOURCES_FILE — a
|
||||
* JSON map, not a ledger; an upsert keeps the existing usage log). */
|
||||
upsertSource(source: NotesSource): void;
|
||||
/** Every notes source record, by id order-insensitive. */
|
||||
readSources(): NotesSource[];
|
||||
/** Append one run's usage to a source record — idempotent per
|
||||
* startedAt, mirroring ChatRecord.extractions. */
|
||||
appendSourceUsage(id: string, entry: ExtractionRunUsage): void;
|
||||
}
|
||||
|
||||
// ─── JobStore — background sub-agent jobs (data/jobs) ─────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user