Notes front door: format-agnostic import into the graph pipeline

The same extractor→auditor→merger pipeline now runs over raw personal
notes in arbitrary formats (vision §1, DECIDED 2026-09-13):

- NotesSource records (data/graph/sources.json, deterministic id from
  path hash → idempotent re-imports) play the chatId role in mappings,
  checkpoints, and observation events; usage lands on the source record
  (appendSourceUsage, idempotent per startedAt, kept across upserts).
- mergeFindings is chat-store-free: callers pass MergeSource
  {messages, foldedAt} — sessions pass the chat record, notes pass
  messages:null (no quiz-event rebuild; a deterministic rebuild needs
  structured quiz tool parts).
- Second extractor prompt (NOTES_EXTRACTOR_INSTRUCTIONS) for general-
  purpose extraction; understanding judges the AUTHOR's grasp. Auditor
  wording input-agnostic. extract/audit share one generic runSubmitAgent.
- graphSnapshot.fileSources (srcId→filePath); notes sources render as
  static path entries in the detail card (FileTextIcon, never a link).
- One shared pipeline instance (exported from chat-handler): same
  single-flight registry, live status chips (label = file path), usage
  accounting, and true abort for both front doors.
- Transport: POST /api/graph/import (multipart, 3-wide worker pool) +
  Import notes… button in the graph view; scripts/import-notes.ts for
  batch runs (concurrency 4).
- real-data-extraction/ gitignored (raw import sources never committed).

355/355 tests; typecheck, lint, build green.
This commit is contained in:
amosblomqvist
2026-09-13 02:08:55 +03:00
parent b769d32bc6
commit 543cf33789
16 changed files with 961 additions and 114 deletions
+2
View File
@@ -7,3 +7,5 @@ data/
.env.local .env.local
.memory/ .memory/
SCOPE.md SCOPE.md
# raw real-data import sources (user directive 2026-09-13: never commit)
real-data-extraction/
+52
View File
@@ -0,0 +1,52 @@
/**
* POST /api/graph/import — the format-agnostic front door's transport
* (vision §1, 2026-09-13): raw personal-notes files in, one extraction run
* per file through the SAME pipeline the sessions use. Multipart form data
* (field "files", any text format — the extractor is format-agnostic); the
* pipeline mints a deterministic source id per file path, persists the
* source record (the file's provenance + the run's usage), and runs
* extractor → auditor → merger over the raw text.
*
* Small concurrency cap: the model calls are the slow part, and a burst of
* 25 simultaneous runs would trip provider rate limits — a 3-wide worker
* pool keeps the run registry (the dock's status chips) and the ledgers
* healthy. The dock's chips + graph refetch track progress live through
* /api/graph/status, as with session runs.
*/
import { extraction } from "@/src/server";
import type { EmissionStats } from "@/src/core";
export const dynamic = "force-dynamic";
interface ImportResult {
filePath: string;
ok: boolean;
stats?: EmissionStats;
error?: string;
}
export async function POST(request: Request) {
const form = await request.formData();
const files = form.getAll("files").filter((f): f is File => f instanceof File);
if (files.length === 0) {
return Response.json({ results: [] as ImportResult[], error: "no files" }, { status: 400 });
}
const results: ImportResult[] = [];
const queue = [...files];
const workers = Array.from({ length: Math.min(3, files.length) }, async () => {
for (;;) {
const file = queue.shift();
if (!file) return;
try {
const text = await file.text();
const stats = await extraction.importNotes({ filePath: file.name, text });
results.push({ filePath: file.name, ok: true, stats });
} catch (err) {
results.push({ filePath: file.name, ok: false, error: err instanceof Error ? err.message : String(err) });
}
}
});
await Promise.all(workers);
return Response.json({ results });
}
+3 -1
View File
@@ -12,7 +12,9 @@ export const dynamic = "force-dynamic";
export async function GET() { export async function GET() {
const runs = extractionStatuses().map((run) => ({ const runs = extractionStatuses().map((run) => ({
chatId: run.chatId, chatId: run.chatId,
title: chatStore.getChat(run.chatId)?.title ?? run.chatId, // The run's own label (the file path for notes imports) wins; chat
// runs fall back to the chat store's title.
title: run.label ?? chatStore.getChat(run.chatId)?.title ?? run.chatId,
phase: run.phase, phase: run.phase,
startedAt: run.startedAt, startedAt: run.startedAt,
// Live per-stage usage (the money channel): model + per-step CallUsage, // Live per-stage usage (the money channel): model + per-step CallUsage,
+72 -10
View File
@@ -56,7 +56,7 @@
import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
import Link from "next/link"; import Link from "next/link";
import { Loader2Icon, SearchIcon, WaypointsIcon } from "lucide-react"; import { FileTextIcon, Loader2Icon, SearchIcon, UploadIcon, WaypointsIcon } from "lucide-react";
import Graph from "graphology"; import Graph from "graphology";
import forceAtlas2 from "graphology-layout-forceatlas2"; import forceAtlas2 from "graphology-layout-forceatlas2";
import type { NodeHoverDrawingFunction, NodeLabelDrawingFunction } from "sigma/rendering"; import type { NodeHoverDrawingFunction, NodeLabelDrawingFunction } from "sigma/rendering";
@@ -322,7 +322,9 @@ function tokenHex(cssVar: string): string {
interface Selection { interface Selection {
node: GraphNode; node: GraphNode;
chatIds: string[]; /** The sources that touched this node: chat ids (deep-linkable sessions)
* and notes-source ids ("src_…" — imported files, static entries). */
sourceIds: string[];
/** Node position in container pixels (from graphToViewport at click time). */ /** Node position in container pixels (from graphToViewport at click time). */
anchor: { x: number; y: number }; anchor: { x: number; y: number };
} }
@@ -368,6 +370,26 @@ export function GraphView() {
}; };
}, [refreshTick]); }, [refreshTick]);
// Notes import (the format-agnostic front door): file picker → POST
// /api/graph/import; the runs surface in the status chips below, and the
// graph refetches when they finish (the runs-empty transition).
const fileInputRef = useRef<HTMLInputElement>(null);
const [importing, setImporting] = useState(false);
const onImportFiles = (files: FileList | null) => {
if (!files || files.length === 0) return;
setImporting(true);
const form = new FormData();
for (const file of Array.from(files)) form.append("files", file);
fetch("/api/graph/import", { method: "POST", body: form })
.then(async (res) => {
const body = (await res.json()) as { results: { filePath: string; ok: boolean }[] };
const failed = body.results.filter((r) => !r.ok);
if (failed.length > 0) console.error("notes import failed for:", failed);
})
.catch((err) => console.error("import failed:", err))
.finally(() => setImporting(false));
};
// Background pipeline status: poll /api/graph/status; when a run // Background pipeline status: poll /api/graph/status; when a run
// finishes (some -> none), refetch the graph. // finishes (some -> none), refetch the graph.
const [pipelineRuns, setPipelineRuns] = useState<{ chatId: string; title: string; phase: string; startedAt: number }[]>([]); const [pipelineRuns, setPipelineRuns] = useState<{ chatId: string; title: string; phase: string; startedAt: number }[]>([]);
@@ -490,7 +512,7 @@ export function GraphView() {
const anchor = renderer?.graphToViewport({ x: pos.x ?? 0, y: pos.y ?? 0 }); const anchor = renderer?.graphToViewport({ x: pos.x ?? 0, y: pos.y ?? 0 });
setSelected({ setSelected({
node: graphNode, node: graphNode,
chatIds: snapshot.nodeSessions[node] ?? [], sourceIds: snapshot.nodeSessions[node] ?? [],
anchor: anchor ?? { x: 0, y: 0 }, anchor: anchor ?? { x: 0, y: 0 },
}); });
}); });
@@ -640,6 +662,28 @@ export function GraphView() {
No matching nodes. No matching nodes.
</div> </div>
)} )}
<div className="mt-1.5 border-t pt-1.5">
<input
ref={fileInputRef}
type="file"
multiple
accept=".md,.markdown,.txt"
className="hidden"
onChange={(e) => {
onImportFiles(e.target.files);
e.target.value = ""; // allow re-picking the same file
}}
/>
<button
type="button"
disabled={importing}
onClick={() => fileInputRef.current?.click()}
className="flex h-7 w-full items-center justify-center gap-1.5 rounded-lg border text-xs text-muted-foreground transition-colors hover:bg-accent disabled:opacity-50"
>
<UploadIcon className="size-3" />
{importing ? "Importing…" : "Import notes…"}
</button>
</div>
</div> </div>
)} )}
@@ -670,20 +714,38 @@ export function GraphView() {
> >
<div className="text-sm font-medium leading-snug">{selected.node.name}</div> <div className="text-sm font-medium leading-snug">{selected.node.name}</div>
<div className="mt-1 text-xs leading-relaxed text-muted-foreground">{selected.node.statement}</div> <div className="mt-1 text-xs leading-relaxed text-muted-foreground">{selected.node.statement}</div>
{selected.chatIds.length > 0 ? ( {selected.sourceIds.length > 0 ? (
<div className="mt-3 flex flex-col gap-1"> <div className="mt-3 flex flex-col gap-1">
<div className="text-xs text-muted-foreground">Sessions</div> <div className="text-xs text-muted-foreground">Sessions</div>
{selected.chatIds.map((chatId) => { {selected.sourceIds.map((sourceId) => {
// That session's understanding observation — the // That source's understanding observation — the
// extractor's named-word note (confidence history). // extractor's named-word note (confidence history).
const note = snapshot?.nodeObservations?.[selected.node.id]?.[chatId]; const note = snapshot?.nodeObservations?.[selected.node.id]?.[sourceId];
// An imported-notes file: a STATIC entry — the path,
// plain text, never a link (no session to open).
if (snapshot?.fileSources?.[sourceId] !== undefined) {
return (
<div key={sourceId} className="flex items-center gap-1.5">
<FileTextIcon className="size-3.5 shrink-0 text-muted-foreground" />
<span className="min-w-0 flex-1 truncate px-1 py-1 text-sm text-muted-foreground">
{snapshot.fileSources[sourceId]}
</span>
{note && (
<span className="max-w-28 shrink-0 truncate whitespace-nowrap rounded-full border px-2 py-0.5 text-[10px] text-muted-foreground">
{note}
</span>
)}
</div>
);
}
// A real session: deep-linkable provenance.
return ( return (
<div key={chatId} className="flex items-center gap-1.5"> <div key={sourceId} className="flex items-center gap-1.5">
<Link <Link
href={`/chat/${chatId}`} href={`/chat/${sourceId}`}
className="min-w-0 flex-1 truncate rounded-md px-2 py-1 text-sm text-foreground transition-colors hover:bg-accent" className="min-w-0 flex-1 truncate rounded-md px-2 py-1 text-sm text-foreground transition-colors hover:bg-accent"
> >
{snapshot?.sessions[chatId] ?? chatId} {snapshot?.sessions[sourceId] ?? sourceId}
</Link> </Link>
{/* The note is a fixed-scale word (solid/shaky/...) — the {/* The note is a fixed-scale word (solid/shaky/...) — the
session name is the main thing; the chip is bounded so session name is the main thing; the chip is bounded so
+7
View File
@@ -72,3 +72,10 @@ export const LEDGER_FILES = {
mappings: path.join(PATHS.graph, "mappings.jsonl"), mappings: path.join(PATHS.graph, "mappings.jsonl"),
sessions: path.join(PATHS.graph, "sessions.jsonl"), sessions: path.join(PATHS.graph, "sessions.jsonl"),
} as const; } as const;
/**
* Notes-import bookkeeping (NOT a ledger): one JSON map, src-id → NotesSource
* record (contracts/graph.ts) — the file path + import time + the run's usage
* log. Read-modify-write like a chat file, not append-only like the ledgers.
*/
export const SOURCES_FILE = path.join(PATHS.graph, "sources.json");
+26 -1
View File
@@ -43,12 +43,37 @@ Every node records every **(chat, message) point** where it came up; every edge
Three stages, one pipeline — the sole graph writer: Three stages, one pipeline — the sole graph writer:
- **Extractor (LLM) — session-only scope.** Sees ONLY the session transcript (tree-blind, with message ids). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD. - **Extractor (LLM) — single-source scope.** Sees ONLY the source text — a session transcript (tree-blind, with message ids) or raw notes (the front door's second prompt, above). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD.
- **Auditor (LLM) — the global-graph scope.** Sees the extractor's output AND the existing graph. Two jobs: the faithfulness net (every candidate verified against verbatim evidence — unchanged), and the **identity judgment**: link each proposition to an existing node (recording the session's phrasing as an alias) or create it new. Conservative: duplicates over wrong merges. - **Auditor (LLM) — the global-graph scope.** Sees the extractor's output AND the existing graph. Two jobs: the faithfulness net (every candidate verified against verbatim evidence — unchanged), and the **identity judgment**: link each proposition to an existing node (recording the session's phrasing as an alias) or create it new. Conservative: duplicates over wrong merges.
- **Merger (deterministic).** Append-only ledger writes; quiz event kinds/refs rebuilt mechanically from quiz tool parts. Every judgment is an LLM's; everything mechanical stays code. - **Merger (deterministic).** Append-only ledger writes; quiz event kinds/refs rebuilt mechanically from quiz tool parts. Every judgment is an LLM's; everything mechanical stays code.
**Event ledger (amended):** quiz outcomes are rebuilt deterministically from quiz tool parts (correct / miss / don't-know, plus the learner's note). **There is no `taught` event** — no marking-things-done. The extractor's per-proposition understanding levels join the ledger as session-scoped observations (one closed-scale word per node per session). Identity is no longer exact-string matching — it is the auditor's judgment, with aliases as the phrasing record. **Event ledger (amended):** quiz outcomes are rebuilt deterministically from quiz tool parts (correct / miss / don't-know, plus the learner's note). **There is no `taught` event** — no marking-things-done. The extractor's per-proposition understanding levels join the ledger as session-scoped observations (one closed-scale word per node per session). Identity is no longer exact-string matching — it is the auditor's judgment, with aliases as the phrasing record.
### The format-agnostic front door — notes imports (DECIDED 2026-09-13)
The same three-stage pipeline (extractor → auditor → merger) runs over **raw
personal notes in arbitrary formats** — an Obsidian vault, plain markdown,
anything textual. The extractor is format-blind by construction (it always
saw rendered text, never ChatRecord JSON), so the notes path needs only a
second extractor prompt (general-purpose extraction; the understanding word
judges the **author's** grasp from their own commentary) and a source
abstraction: each imported file becomes a `NotesSource` record (deterministic
id from the path hash — re-imports are idempotent), playing the `chatId` role
in mappings, checkpoints, and observation events.
Decided edges:
- **No session shaping.** Notes are never bent into teacher/learner
transcripts; quiz outcomes inside notes surface only via the extractor's
understanding observations (the deterministic quiz-event rebuild is a
session-pipeline feature — it needs structured quiz tool parts).
- **Static provenance.** Imported nodes show the file path in the sessions
list as a static entry — never a link. Understanding-assessment levels
apply to personal notes as to sessions.
- **Same single-flight registry and observability** as session runs: live
status, usage accounting per stage, true abort. One pipeline instance,
two front doors (quiz-triggered sessions; POST /api/graph/import).
### Understanding is derived (amended 2026-09-02) ### Understanding is derived (amended 2026-09-02)
Known vs. understood, event ledger as source of truth, derived state recomputed on read — carried over from the design doc in spirit. The state model itself is now an explicit design discussion: **node states should be session-independent and computed from the depth and strength of dependencies** (the `follows`/`supports` topology) plus quiz events and understanding notes. That redesign is pending ideation; the transitional five-class derivation (taught / known / missed / don't-know / candidate) was **removed** (2026-09-08) — the graph view renders state-neutral nodes until the state model is designed. The anchored calibration rubric stays parked until real data. Known vs. understood, event ledger as source of truth, derived state recomputed on read — carried over from the design doc in spirit. The state model itself is now an explicit design discussion: **node states should be session-independent and computed from the depth and strength of dependencies** (the `follows`/`supports` topology) plus quiz events and understanding notes. That redesign is pending ideation; the transitional five-class derivation (taught / known / missed / don't-know / candidate) was **removed** (2026-09-08) — the graph view renders state-neutral nodes until the state model is designed. The anchored calibration rubric stays parked until real data.
+62
View File
@@ -0,0 +1,62 @@
/**
* Import raw notes files through the format-agnostic front door (vision §1,
* 2026-09-13) — every *.md / *.txt file in a directory becomes one
* extraction run (extractor → auditor → merger) over its raw text. Run
* from this worktree only — data never crosses trees:
*
* npx tsx --env-file=.env.local scripts/import-notes.ts <dir> [concurrency]
*
* Default dir: real-data-extraction/learn-sessions (the prototype
* learning-session exports — deliberately NOT imported as chats: they run
* through the notes path, raw). Runs in this process: the dev server's
* status chips do NOT see them (registry is per-process) — the dev server
* sees runs triggered through /api/graph/import.
*/
import * as fs from "node:fs";
import * as path from "node:path";
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
import { getGraph } from "../src/core/graph.ts";
import { stores } from "../src/server/stores.ts";
async function main() {
const dir = process.argv[2] ?? "real-data-extraction/learn-sessions";
const limit = Number(process.argv[3] ?? 4);
const files = fs
.readdirSync(dir)
.filter((f) => f.endsWith(".md") || f.endsWith(".markdown") || f.endsWith(".txt"))
.sort();
console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`);
const pipeline = createExtractionPipeline(stores);
const queue = [...files];
const totals = { nodes: 0, edges: 0, events: 0, failed: 0 };
const workers = Array.from({ length: Math.min(limit, files.length) }, async () => {
for (;;) {
const file = queue.shift();
if (!file) return;
const filePath = path.join(dir, file);
const t0 = Date.now();
try {
const stats = await pipeline.importNotes({ filePath, text: fs.readFileSync(filePath, "utf-8") });
totals.nodes += stats.nodes;
totals.edges += stats.edges;
totals.events += stats.events;
console.log(
`[import-notes] ${file}: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events in ${((Date.now() - t0) / 1000).toFixed(1)}s`,
);
} catch (err) {
totals.failed++;
console.error(`[import-notes] ${file} FAILED:`, err);
}
}
});
await Promise.all(workers);
const graph = getGraph(stores.graphStore);
console.log(
`[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` +
`graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events`,
);
}
main();
+329 -75
View File
@@ -1,7 +1,9 @@
import * as crypto from "node:crypto";
import type { UIMessage } from "ai"; import type { UIMessage } from "ai";
import type { Usage } from "@earendil-works/pi-ai"; import type { Usage } from "@earendil-works/pi-ai";
import { z } from "zod"; import { z } from "zod";
import { lhTool, toPiTools } from "../lh-tool.ts"; import { lhTool, toPiTools } from "../lh-tool.ts";
import type { ModelRole } from "../../../config.ts";
import type { PiEngine } from "../models.ts"; import type { PiEngine } from "../models.ts";
import { runSubagent } from "./pi-run.ts"; import { runSubagent } from "./pi-run.ts";
import type { import type {
@@ -42,6 +44,17 @@ import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../
* whatever the in-flight run missed). On failure nothing is checkpointed — * whatever the in-flight run missed). On failure nothing is checkpointed —
* the next trigger re-runs the whole thing. * the next trigger re-runs the whole thing.
* *
* The FORMAT-AGNOSTIC FRONT DOOR (2026-09-13): importNotes() runs the SAME
* three stages over raw personal notes (arbitrary format — no chat, no
* session, no message ids). A notes-mode extractor prompt replaces the
* session one; the auditor and merger are shared (identity + deterministic
* appends don't care where text came from). Provenance is the file itself:
* source ids ("src_" + path hash) play the chatId role in mappings, events
* and checkpoints, and the source record (graphStore sources seam) carries
* the file path + the run's usage. No quiz events are ever built from
* notes — quiz outcomes inside the text surface only through the
* extractor's understanding observations (user-decided 2026-09-13).
*
* Observability (§5 baseline): every stage logs one line per step, its * Observability (§5 baseline): every stage logs one line per step, its
* start, and its duration; the pipeline logs a failure line before * start, and its duration; the pipeline logs a failure line before
* rethrowing, so the server log reads start → stage timings → merged * rethrowing, so the server log reads start → stage timings → merged
@@ -166,6 +179,44 @@ const findingsSchema = z.object({
.describe("Quiz attribution — your only event judgment."), .describe("Quiz attribution — your only event judgment."),
}); });
const NOTES_EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE source —
the user's own NOTES, in an ARBITRARY format (personal notes, an exported conversation, a document — anything) — and
report what those notes show the author engaging with, as structured findings for their global knowledge graph.
There is no session and no chat structure here: the source is raw text, the author's own material. Read it as such.
Report, with verbatim evidence quotes for everything:
NODE THEORY — the graph is a set of TRUE propositions about the world, full stop. A node is never anything about the
author or the source. The author's relationship to a proposition is expressed ENTIRELY by your understanding note; it
never leaks into the node itself. So no "the author understands X", no "mentioned in these notes": the node is just X.
A misconception is NOT a node of its own wrong claim — state the TRUE correction as the proposition.
1. PROPOSITIONS — every distinct reasoning-step-sized proposition the notes actually engage with:
- Each has a "name" — a VERY short label, a few words, a compact handle like a glossary headword ("Will to power",
"Divine command theory"). Not a sentence; it becomes the node's label in the graph, so keep it glanceable.
- And a "statement" — the self-contained TRUE proposition at full precision.
- Notes may explain things at length or quote material the author is studying: extract the propositions themselves,
one reasoning step each — never a topic blob, never a sub-step.
- "messages": ALWAYS an empty list — provenance is the file itself.
- "aliases": verbatim phrasings the notes used for the same proposition.
- "understanding": ONE word from the fixed scale — "solid", "shaky", "accepted", "resisted", or "not yet engaged" —
judging the AUTHOR's grasp of the proposition as the notes show it: their own commentary, self-assessments,
recorded quiz outcomes, the way they write about it. Strictly one word, no numbers, no justification (a
wrong-shaped note is rejected by the schema). When the notes give no signal, omit the field.
2. EDGES between propositions:
- "follows": to is DERIVABLE from from (understanding from makes understanding to achievable).
- "supports": from GROUNDS to without entailing it (helpful context, not a prerequisite).
- Only relations the notes actually articulate — with evidence.
There is no quiz attribution for notes — the source has no quiz events, and the schema has no attribution field.
Evidence discipline: every quote must appear VERBATIM in the source. The auditor rejects anything unbacked. When in
doubt, report; the auditor decides.
Your entire deliverable is ONE submit_findings call.`;
const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session
transcript and report what was learned — as structured findings for the learner's global knowledge graph. transcript and report what was learned — as structured findings for the learner's global knowledge graph.
@@ -210,23 +261,30 @@ transcript). The auditor rejects anything unbacked. When in doubt, report; the a
Your entire deliverable is ONE submit_findings call.`; Your entire deliverable is ONE submit_findings call.`;
/** One extractor run: the whole deliverable is ONE zod-validated submit /** One single-submit agent run — the shared shape of EVERY LLM stage
* call (a pi agent purely so the model can retry a schema-invalid call). * here (session + notes): the whole deliverable is ONE zod-validated
* Returns the submitted findings — captured from the execute seam, which * submit call (a pi agent purely so the model can retry a schema-invalid
* call). The submitted value is captured from the execute seam, which
* only fires on a schema-valid call. */ * only fires on a schema-valid call. */
async function runExtractor( async function runSubmitAgent<T>(opts: {
engine: PiEngine | undefined, role: ModelRole;
prompt: string, toolName: string;
signal: AbortSignal | undefined, toolDescription: string;
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, schema: z.ZodType<T>;
): Promise<SessionFindings> { systemPrompt: string;
let submitted: SessionFindings | undefined; prompt: string;
engine: PiEngine | undefined;
signal: AbortSignal | undefined;
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void;
neverSubmitted: string;
}): Promise<T> {
let submitted: T | undefined;
const result = await runSubagent( const result = await runSubagent(
toPiTools({ toPiTools({
submit_findings: lhTool({ [opts.toolName]: lhTool({
name: "submit_findings", name: opts.toolName,
description: "Submit the session findings.", description: opts.toolDescription,
schema: findingsSchema, schema: opts.schema,
execute: async (findings) => { execute: async (findings) => {
submitted = findings; submitted = findings;
return findings; return findings;
@@ -234,28 +292,71 @@ async function runExtractor(
}), }),
}), }),
{ {
role: "extractor", role: opts.role,
engine, engine: opts.engine,
systemPrompt: EXTRACTOR_INSTRUCTIONS, systemPrompt: opts.systemPrompt,
prompt, prompt: opts.prompt,
maxTurns: 4, maxTurns: 4,
abortSignal: signal, abortSignal: opts.signal,
onAssistantMessage: (message, step) => onAssistantMessage: (message, step) =>
onStep( opts.onStep(
message.usage, message.usage,
message.model, message.model,
step, step,
message.stopReason, message.stopReason,
message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)", message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)",
), ),
shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_findings" && !r.isError), shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === opts.toolName && !r.isError),
}, },
); );
void result; void result;
if (!submitted) throw new Error("the extractor never submitted findings"); if (!submitted) throw new Error(opts.neverSubmitted);
return submitted; return submitted;
} }
/** The extractor over ONE session transcript (the chat front door). */
function runExtractor(
engine: PiEngine | undefined,
prompt: string,
signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<SessionFindings> {
return runSubmitAgent({
role: "extractor",
toolName: "submit_findings",
toolDescription: "Submit the session findings.",
schema: findingsSchema,
systemPrompt: EXTRACTOR_INSTRUCTIONS,
prompt,
engine,
signal,
onStep,
neverSubmitted: "the extractor never submitted findings",
});
}
/** The extractor over ONE source of raw personal notes (the format-agnostic
* front door): same single-submit shape, notes-mode prompt + schema. */
function runNotesExtractor(
engine: PiEngine | undefined,
prompt: string,
signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<NotesFindings> {
return runSubmitAgent({
role: "extractor",
toolName: "submit_findings",
toolDescription: "Submit the notes findings.",
schema: notesFindingsSchema,
systemPrompt: NOTES_EXTRACTOR_INSTRUCTIONS,
prompt,
engine,
signal,
onStep,
neverSubmitted: "the extractor never submitted findings",
});
}
// ─── the auditor ─────────────────────────────────────────────────────────── // ─── the auditor ───────────────────────────────────────────────────────────
const auditSchema = findingsSchema.extend({ const auditSchema = findingsSchema.extend({
@@ -279,11 +380,21 @@ const auditSchema = findingsSchema.extend({
.describe("Everything you dropped, with the reason — the audit trail."), .describe("Everything you dropped, with the reason — the audit trail."),
}); });
const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive // The notes front door's shapes: everything session findings have, minus
ONE session transcript, a set of proposed findings (propositions, edges, quiz attribution), and the CURRENT // quiz attribution — raw notes have no quiz tool parts, so there is
knowledge graph. Your jobs: the faithfulness net, and identity. // nothing to attribute (and no event to build: outcomes inside the text
// surface only through understanding observations, user-decided).
const notesFindingsSchema = findingsSchema.omit({ attribution: true });
const notesAuditSchema = auditSchema.omit({ attribution: true });
type NotesFindings = Omit<SessionFindings, "attribution">;
type NotesAuditResult = Omit<AuditResult, "attribution">;
FAITHFULNESS — verify every candidate against the transcript: const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive
ONE source — a session transcript, or the author's personal notes in an ARBITRARY format (the notes front door) — a
set of proposed findings (propositions, edges, and quiz attribution where the source has quiz lines), and the
CURRENT knowledge graph. Your jobs: the faithfulness net, and identity.
FAITHFULNESS — verify every candidate against the source text:
- PROPOSITIONS: every evidence quote must appear (near-)verbatim in the transcript view, and the statement - PROPOSITIONS: every evidence quote must appear (near-)verbatim in the transcript view, and the statement
must be a faithful reading of it. The node theory applies: a statement must be a TRUE, self-contained, must be a faithful reading of it. The node theory applies: a statement must be a TRUE, self-contained,
world-facing fact — reword learner- or session-relative statements ("the learner understands X" → "X"), state world-facing fact — reword learner- or session-relative statements ("the learner understands X" → "X"), state
@@ -293,8 +404,10 @@ FAITHFULNESS — verify every candidate against the transcript:
- EDGES: the evidence must actually articulate the relation, in the direction claimed. Drop - EDGES: the evidence must actually articulate the relation, in the direction claimed. Drop
direction-swapped or speculative edges. direction-swapped or speculative edges.
- ATTRIBUTION: the QUIZ line's question must genuinely test the attributed proposition. Drop wrong - ATTRIBUTION: the QUIZ line's question must genuinely test the attributed proposition. Drop wrong
attributions to null, or re-point them to the proposition the question actually tests. attributions to null, or re-point them to the proposition the question actually tests. (Session
- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence; each note must be sources only — notes findings carry no attribution; there is nothing to verify.)
- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence — or, for notes,
the author's own words; each note must be
exactly one word from the fixed scale ("solid", "shaky", "accepted", "resisted", "not yet engaged") — exactly one word from the fixed scale ("solid", "shaky", "accepted", "resisted", "not yet engaged") —
reword or drop anything else. No justifications. reword or drop anything else. No justifications.
@@ -324,47 +437,47 @@ function graphView(nodes: GraphNode[]): string {
return lines.join("\n"); return lines.join("\n");
} }
/** One auditor run — same single-submit shape as the extractor. */ /** The auditor over session findings (the chat front door). */
async function runAuditor( function runAuditor(
engine: PiEngine | undefined, engine: PiEngine | undefined,
prompt: string, prompt: string,
signal: AbortSignal | undefined, signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<AuditResult> { ): Promise<AuditResult> {
let submitted: AuditResult | undefined; return runSubmitAgent({
const result = await runSubagent( role: "auditor",
toPiTools({ toolName: "submit_review",
submit_review: lhTool({ toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.",
name: "submit_review", schema: auditSchema,
description: "Submit the verified findings subset (with identity verdicts) + the dropped list.", systemPrompt: AUDITOR_INSTRUCTIONS,
schema: auditSchema, prompt,
execute: async (review) => { engine,
submitted = review; signal,
return review; onStep,
}, neverSubmitted: "the auditor never submitted a review",
}), });
}), }
{
role: "auditor", /** The auditor over notes findings (the format-agnostic front door): same
engine, * faithfulness + identity job, notes-mode schema (no attribution). */
systemPrompt: AUDITOR_INSTRUCTIONS, function runNotesAuditor(
prompt, engine: PiEngine | undefined,
maxTurns: 4, prompt: string,
abortSignal: signal, signal: AbortSignal | undefined,
onAssistantMessage: (message, step) => onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
onStep( ): Promise<NotesAuditResult> {
message.usage, return runSubmitAgent({
message.model, role: "auditor",
step, toolName: "submit_review",
message.stopReason, toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.",
message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)", schema: notesAuditSchema,
), systemPrompt: AUDITOR_INSTRUCTIONS,
shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_review" && !r.isError), prompt,
}, engine,
); signal,
void result; onStep,
if (!submitted) throw new Error("the auditor never submitted a review"); neverSubmitted: "the auditor never submitted a review",
return submitted; });
} }
// ─── the pipeline status registry ───────────────────────────────────────────── // ─── the pipeline status registry ─────────────────────────────────────────────
@@ -376,6 +489,13 @@ export type ExtractionPhase = "extracting" | "auditing" | "merging";
* Exported for the status payload's consumer typing (the dock). */ * Exported for the status payload's consumer typing (the dock). */
export interface RunningExtraction { export interface RunningExtraction {
chatId: string; chatId: string;
/** Which front door the run came through — label + usage-persistence
* depend on it (chats bill their chat record; notes bill the source
* record). */
kind: "chat" | "notes";
/** Display label (the file path for notes imports); the status route
* falls back to the chat store for chat runs. */
label?: string;
phase: ExtractionPhase; phase: ExtractionPhase;
startedAt: number; startedAt: number;
/** Live per-stage usage, accumulated as steps bill (the dock's cost / /** Live per-stage usage, accumulated as steps bill (the dock's cost /
@@ -388,6 +508,10 @@ export interface RunningExtraction {
const runningExtractions = new Map<string, RunningExtraction>(); const runningExtractions = new Map<string, RunningExtraction>();
/** Last minted startedAt — kept monotonic so two runs starting in the same
* millisecond never collide on the usage idempotency key. */
let lastStartedAt = 0;
/** In-flight pipeline runs (oldest first) — the /api/graph/status payload. */ /** In-flight pipeline runs (oldest first) — the /api/graph/status payload. */
export function extractionStatuses(): RunningExtraction[] { export function extractionStatuses(): RunningExtraction[] {
return [...runningExtractions.values()].sort((a, b) => a.startedAt - b.startedAt); return [...runningExtractions.values()].sort((a, b) => a.startedAt - b.startedAt);
@@ -415,22 +539,38 @@ export interface ExtractionModels {
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {} export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
export interface NotesImportInput {
/** The source's file path — its identity (deterministic src id) AND its
* provenance label in the detail card. */
filePath: string;
/** The file's raw content, VERBATIM — any format. */
text: string;
}
export interface ExtractionPipeline { export interface ExtractionPipeline {
/** Extract + audit + merge ONE chat. Single-flight per chat; resolves to /** Extract + audit + merge ONE chat. Single-flight per chat; resolves to
* the merger's append stats (or throws — no checkpoint on failure). */ * the merger's append stats (or throws — no checkpoint on failure). */
runExtraction(chatId: string): Promise<EmissionStats>; runExtraction(chatId: string): Promise<EmissionStats>;
/** The format-agnostic front door (2026-09-13): run the same pipeline
* over ONE raw notes file. The source id is deterministic per file
* path, so re-importing the same file re-runs extraction against the
* same source identity (mappings + events dedupe by it). */
importNotes(input: NotesImportInput): Promise<EmissionStats>;
} }
export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline { export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline {
const { chatStore, graphStore } = deps; const { chatStore, graphStore } = deps;
const inFlight = new Set<string>(); const inFlight = new Set<string>();
function setPhase(chatId: string, phase: ExtractionPhase): void { function setPhase(chatId: string, phase: ExtractionPhase, kind: "chat" | "notes", label?: string): void {
const prev = runningExtractions.get(chatId); const prev = runningExtractions.get(chatId);
if (!prev) lastStartedAt = Math.max(Date.now(), lastStartedAt + 1);
runningExtractions.set(chatId, { runningExtractions.set(chatId, {
chatId, chatId,
kind,
label: prev?.label ?? label, // phase changes never lose the label
phase, phase,
startedAt: prev?.startedAt ?? Date.now(), startedAt: prev?.startedAt ?? lastStartedAt,
usage: prev?.usage ?? {}, // phase changes never lose usage usage: prev?.usage ?? {}, // phase changes never lose usage
controller: prev?.controller ?? new AbortController(), controller: prev?.controller ?? new AbortController(),
}); });
@@ -525,7 +665,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
function persistRunUsage(chatId: string, durationMs: number, stats: EmissionStats | undefined): void { function persistRunUsage(chatId: string, durationMs: number, stats: EmissionStats | undefined): void {
const run = runningExtractions.get(chatId); const run = runningExtractions.get(chatId);
if (!run) return; if (!run) return;
chatStore.appendExtractionUsage(chatId, { const entry: ExtractionRunUsage = {
startedAt: run.startedAt, startedAt: run.startedAt,
at: new Date().toISOString(), at: new Date().toISOString(),
durationMs, durationMs,
@@ -534,7 +674,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
extractor: run.usage.extractor ?? { model: "unknown", steps: [] }, extractor: run.usage.extractor ?? { model: "unknown", steps: [] },
auditor: run.usage.auditor ?? { model: "unknown", steps: [] }, auditor: run.usage.auditor ?? { model: "unknown", steps: [] },
}, },
}); };
if (run.kind === "notes") graphStore.appendSourceUsage(chatId, entry);
else chatStore.appendExtractionUsage(chatId, entry);
} }
async function runExtraction(chatId: string): Promise<EmissionStats> { async function runExtraction(chatId: string): Promise<EmissionStats> {
@@ -543,14 +685,19 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
const t0 = performance.now(); const t0 = performance.now();
try { try {
console.info(`[extraction ${chatId}] pipeline start`); console.info(`[extraction ${chatId}] pipeline start`);
setPhase(chatId, "extracting"); setPhase(chatId, "extracting", "chat");
const findings = await extract(chatId); const findings = await extract(chatId);
setPhase(chatId, "auditing"); setPhase(chatId, "auditing", "chat");
const auditResult = await audit(chatId, findings); const auditResult = await audit(chatId, findings);
setPhase(chatId, "merging"); setPhase(chatId, "merging", "chat");
// The merger re-reads the chat record itself — events are rebuilt // The merger merges from the CURRENT record — events are rebuilt
// from the transcript, never from stale copies. // from the live transcript, never from stale copies.
const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }); const record = chatStore.getChat(chatId);
if (!record) throw new Error(`Chat not found: ${chatId}`);
const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }, {
messages: record.messages as UIMessage[],
foldedAt: record.updatedAt,
});
persistRunUsage(chatId, Math.round(performance.now() - t0), stats); persistRunUsage(chatId, Math.round(performance.now() - t0), stats);
console.info( console.info(
`[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + `[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
@@ -577,6 +724,113 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
} }
} }
return { runExtraction }; // ── the notes front door (format-agnostic: raw text in, same graph out) ──
/** The raw source view the notes stages see: the file path + the text
* VERBATIM. No message ids — provenance is the file itself. */
function notesView(input: { filePath: string; text: string }): string {
return `SOURCE (the author's personal notes): ${input.filePath}\n\n${input.text}`;
}
async function notesExtract(input: { id: string; filePath: string; text: string }): Promise<NotesFindings> {
const signal = runningExtractions.get(input.id)?.controller.signal;
signal?.throwIfAborted(); // an abort during a sync gap ends the run here
const view = notesView(input);
console.info(`[extraction ${input.id}] notes extractor starting (${view.split("\n").length} source lines)`);
const t0 = performance.now();
const findings = await runNotesExtractor(
deps.extractor,
view,
signal,
(usage, model, step, finishReason, tools) => {
recordStep(input.id, "extractor", callUsage(usage), model);
console.info(`[extraction] extractor step ${step}: ${finishReason}, tools: ${tools}`);
},
);
console.info(`[extraction ${input.id}] notes extractor done in ${Math.round(performance.now() - t0)}ms`);
return findings;
}
async function notesAudit(input: { id: string; filePath: string; text: string }, findings: NotesFindings): Promise<NotesAuditResult> {
const signal = runningExtractions.get(input.id)?.controller.signal;
signal?.throwIfAborted(); // an abort during the extract→audit gap ends here
const graph = getGraph(graphStore);
console.info(
`[extraction ${input.id}] notes auditor starting (${findings.propositions.length} propositions, ` +
`${findings.edges.length} edges, ${graph.nodes.length} graph nodes)`,
);
const t0 = performance.now();
const auditResult = await runNotesAuditor(
deps.auditor,
`Source:\n\n${notesView(input)}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}\n\n` +
`Current knowledge graph (${graph.nodes.length} nodes):\n\n${graphView(graph.nodes)}`,
signal,
(usage, model, step, finishReason, tools) => {
recordStep(input.id, "auditor", callUsage(usage), model);
console.info(`[extraction] auditor step ${step}: ${finishReason}, tools: ${tools}`);
},
);
if (auditResult.dropped.length > 0) {
console.info(
`[extraction ${input.id}] audit dropped ${auditResult.dropped.length}: ` +
auditResult.dropped.map((d) => `${d.ref} (${d.reason})`).join("; "),
);
}
console.info(
`[extraction ${input.id}] audit kept ${auditResult.propositions.length}/${findings.propositions.length} propositions, ` +
`${auditResult.propositions.filter((p) => p.linkTo !== undefined).length} linked to existing nodes`,
);
console.info(`[extraction ${input.id}] notes auditor done in ${Math.round(performance.now() - t0)}ms`);
return auditResult;
}
async function runNotesExtraction(input: { id: string; filePath: string; text: string; importedAt: string }): Promise<EmissionStats> {
const { id } = input;
if (inFlight.has(id)) return { nodes: 0, edges: 0, events: 0 };
inFlight.add(id);
const t0 = performance.now();
try {
console.info(`[extraction ${id}] notes pipeline start (${input.filePath})`);
setPhase(id, "extracting", "notes", input.filePath);
const findings = await notesExtract(input);
setPhase(id, "auditing", "notes", input.filePath);
const notesResult = await notesAudit(input, findings);
setPhase(id, "merging", "notes", input.filePath);
// Normalize to the merger's findings shape: notes carry no quiz
// attribution, and the merger scans no quiz parts for notes
// (messages: null) — nothing to attribute, nothing to build.
const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } };
const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt });
persistRunUsage(id, Math.round(performance.now() - t0), stats);
console.info(
`[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
`(${Math.round(performance.now() - t0)}ms total)`,
);
return stats;
} catch (err) {
persistRunUsage(id, Math.round(performance.now() - t0), undefined);
if (err instanceof Error && err.name === "AbortError") {
console.info(`[extraction ${id}] aborted (${Math.round(performance.now() - t0)}ms) — nothing merged, nothing checkpointed`);
return { nodes: 0, edges: 0, events: 0 };
}
console.error(`[extraction ${id}] notes pipeline failed (${Math.round(performance.now() - t0)}ms):`, err);
throw err;
} finally {
inFlight.delete(id);
runningExtractions.delete(id); // done or failed — no longer running
}
}
/** Import ONE raw notes file: register the source (deterministic id per
* file path — re-import keeps the same identity and its usage history)
* and run the pipeline over it. */
async function importNotes(input: NotesImportInput): Promise<EmissionStats> {
const id = `src_${crypto.createHash("sha256").update(input.filePath).digest("hex").slice(0, 12)}`;
const importedAt = new Date().toISOString();
graphStore.upsertSource({ id, filePath: input.filePath, importedAt });
return runNotesExtraction({ id, ...input, importedAt });
}
return { runExtraction, importNotes };
} }
+30
View File
@@ -20,6 +20,8 @@
/** Two relations only (DECIDED 2026-09-02): follows = b is derivable from a; /** Two relations only (DECIDED 2026-09-02): follows = b is derivable from a;
* supports = a grounds b without entailing it. No scalars, no encoded * supports = a grounds b without entailing it. No scalars, no encoded
* strength — strength is derived at read time from topology. */ * strength — strength is derived at read time from topology. */
import type { ExtractionRunUsage } from "./chat.ts";
export type EdgeType = "follows" | "supports"; export type EdgeType = "follows" | "supports";
/** What one extraction run merged into the graph ledgers — the merger's /** What one extraction run merged into the graph ledgers — the merger's
@@ -37,10 +39,32 @@ export interface EmissionStats {
* edges record every (chat, message) where they came up, deep-linkable to * edges record every (chat, message) where they came up, deep-linkable to
* the exact transcript point. */ * the exact transcript point. */
export interface ProvenancePoint { export interface ProvenancePoint {
/** The chat this point anchors to — or a notes source id ("src_…"):
* imports carry no message anchors (provenance is the file itself,
* recorded through the mapping line + the source record). */
chatId: string; chatId: string;
messageId: string; messageId: string;
} }
/** One imported notes file — the format-agnostic front door's bookkeeping
* (DECIDED 2026-09-13): personal notes in ARBITRARY formats run through the
* same extractor → auditor → merger pipeline, with the file itself as the
* provenance anchor. There is no chat: the detail card shows the file path
* as a STATIC entry (not a link), and understanding observations still
* apply (judged from the author's own voice in the notes). The run's usage
* log lives here, mirroring ChatRecord.extractions. Ids are "src_" + hash
* of the file path — deterministic, so re-importing the same file re-runs
* extraction against the same source id (mappings/events dedupe by it). */
export interface NotesSource {
id: string;
filePath: string;
/** ISO — when this file was imported (the checkpoint's foldedAt). */
importedAt: string;
/** The run's usage log — same append-only discipline as
* ChatRecord.extractions, idempotent per startedAt. */
extractions?: ExtractionRunUsage[];
}
export interface GraphNode { export interface GraphNode {
/** Opaque stable id, store-assigned: "n_" + nanoid-like. */ /** Opaque stable id, store-assigned: "n_" + nanoid-like. */
id: string; id: string;
@@ -107,6 +131,8 @@ export interface LedgerEvent {
/** null = not yet attributed. */ /** null = not yet attributed. */
nodeId: string | null; nodeId: string | null;
kind: LedgerEventKind; kind: LedgerEventKind;
/** The chat that produced this evidence — or a notes source id
* ("src_…") for imported-notes observations. */
chatId: string; chatId: string;
at: string; at: string;
/** Idempotency key: the quiz part's toolCallId, or `obs:<sourceId>` for /** Idempotency key: the quiz part's toolCallId, or `obs:<sourceId>` for
@@ -249,6 +275,10 @@ export interface GraphSnapshot {
edges: GraphEdge[]; edges: GraphEdge[];
/** chatId → title, for every chat that touched the graph. */ /** chatId → title, for every chat that touched the graph. */
sessions: Record<string, string>; sessions: Record<string, string>;
/** srcId → file path, for every IMPORTED NOTES file that touched the
* graph (the format-agnostic front door): a static provenance entry —
* the detail card shows the path as plain text, not a link. */
fileSources: Record<string, string>;
/** nodeId → chatIds that touched the node (click-through provenance). */ /** nodeId → chatIds that touched the node (click-through provenance). */
nodeSessions: Record<string, string[]>; nodeSessions: Record<string, string[]>;
/** nodeId → chatId → that session's understanding observation (the /** nodeId → chatId → that session's understanding observation (the
+38 -14
View File
@@ -64,6 +64,17 @@ function newGraphNodeId(): string {
return `n_${randomBytes(6).toString("hex")}`; return `n_${randomBytes(6).toString("hex")}`;
} }
/** What one merge run merges FROM (the caller supplies it — the merger never
* reads the chat store): a session's transcript + its `updatedAt`
* checkpoint, or — for the format-agnostic notes front door (2026-09-13) —
* `messages: null`: raw notes carry no tool parts, so no quiz events are
* rebuilt (quiz outcomes inside notes surface only through the extractor's
* understanding observations), and the checkpoint stamps the import. */
export interface MergeSource {
messages: UIMessage[] | null;
foldedAt: string;
}
// ─── transcript scanning (shared by the merger + trigger) ───────────────────── // ─── transcript scanning (shared by the merger + trigger) ─────────────────────
/** A flattened, order-preserving view of every tool part in a transcript. */ /** A flattened, order-preserving view of every tool part in a transcript. */
@@ -126,11 +137,15 @@ const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean =>
* parts, nodeId from the extractor's attribution through this run's * parts, nodeId from the extractor's attribution through this run's
* mapping) plus one observation event per proposition carrying an * mapping) plus one observation event per proposition carrying an
* understanding note (ref `obs:<sourceId>`). No taught events, ever. * understanding note (ref `obs:<sourceId>`). No taught events, ever.
*
* `chatId` is the run's source key: a real chat, or a notes source id
* ("src_…") for the format-agnostic import path — same ledgers, same
* idempotency, one graph. The `source` parameter carries the transcript
* being merged (null messages = raw notes: no quiz events) and the
* checkpoint stamp; the merger itself is chat-store-free.
*/ */
export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps): EmissionStats { export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps, source: MergeSource): EmissionStats {
const { graphStore: ledgers, chatStore } = deps; const { graphStore: ledgers } = deps;
const record = chatStore.getChat(chatId);
if (!record) return { nodes: 0, edges: 0, events: 0 };
const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 }; const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 };
// 1. Resolution: this run's propositions → graph node ids. Existing // 1. Resolution: this run's propositions → graph node ids. Existing
@@ -214,7 +229,7 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
for (const event of ledgers.readEvents()) { for (const event of ledgers.readEvents()) {
if (event.chatId === chatId && event.ref) seenRefs.add(event.ref); if (event.chatId === chatId && event.ref) seenRefs.add(event.ref);
} }
for (const part of toolParts(record.messages)) { for (const part of source.messages ? toolParts(source.messages) : []) {
if (part.name !== "quiz" || part.state !== "output-available") continue; if (part.name !== "quiz" || part.state !== "output-available") continue;
const event = quizEvent(part, chatId, audit, mapping, now); const event = quizEvent(part, chatId, audit, mapping, now);
if (event?.ref !== undefined && !seenRefs.has(event.ref)) { if (event?.ref !== undefined && !seenRefs.has(event.ref)) {
@@ -241,8 +256,9 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
stats.events++; stats.events++;
} }
// 4. Checkpoint: this chat is extracted through its current updatedAt. // 4. Checkpoint: this source is extracted through its foldedAt (the
ledgers.appendCheckpoint({ chatId, foldedAt: record.updatedAt }); // chat's updatedAt, or the import time for notes).
ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt });
return stats; return stats;
} }
@@ -423,20 +439,28 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
if (event.nodeId !== null) touch(event.nodeId, event.chatId); if (event.nodeId !== null) touch(event.nodeId, event.chatId);
} }
// Titles for every chat the graph can point back at. // Labels for every source the graph can point back at: real chats
const chatIds = new Set<string>(); // (title, deep-linkable) and imported notes files (path, static — the
// format-agnostic front door's provenance).
const sourceIds = new Set<string>();
for (const chats of nodeChats.values()) { for (const chats of nodeChats.values()) {
for (const id of chats) chatIds.add(id); for (const id of chats) sourceIds.add(id);
} }
const sessions: Record<string, string> = {}; const sessions: Record<string, string> = {};
for (const id of chatIds) { const fileSources: Record<string, string> = {};
for (const id of sourceIds) {
const chat = deps.chatStore.getChat(id); const chat = deps.chatStore.getChat(id);
if (chat) sessions[id] = chat.title; if (chat) {
sessions[id] = chat.title;
continue;
}
const source = deps.graphStore.readSources().find((s) => s.id === id);
if (source) fileSources[id] = source.filePath;
} }
const nodeSessions: Record<string, string[]> = {}; const nodeSessions: Record<string, string[]> = {};
for (const [nodeId, chats] of nodeChats) { for (const [nodeId, chats] of nodeChats) {
const ids = [...chats].filter((id) => sessions[id] !== undefined); const ids = [...chats].filter((id) => sessions[id] !== undefined || fileSources[id] !== undefined);
if (ids.length > 0) nodeSessions[nodeId] = ids; if (ids.length > 0) nodeSessions[nodeId] = ids;
} }
@@ -461,7 +485,7 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
(nodeObservations[nodeId] ??= {})[chatId] = note; (nodeObservations[nodeId] ??= {})[chatId] = note;
} }
return { nodes, edges, sessions, nodeSessions, nodeObservations }; return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations };
} }
// Re-export: the contract owns the shape; legacy importers still reach it. // Re-export: the contract owns the shape; legacy importers still reach it.
+10
View File
@@ -24,6 +24,7 @@ import type {
PlanTree, PlanTree,
SessionCheckpoint, SessionCheckpoint,
TurnUsage, TurnUsage,
NotesSource,
} from "./contracts.ts"; } from "./contracts.ts";
// ─── ChatStore — the session record's home (data/chats) ───────────────────── // ─── ChatStore — the session record's home (data/chats) ─────────────────────
@@ -80,6 +81,15 @@ export interface GraphStore {
/** Append one emission checkpoint. */ /** Append one emission checkpoint. */
appendCheckpoint(checkpoint: SessionCheckpoint): void; appendCheckpoint(checkpoint: SessionCheckpoint): void;
readCheckpoints(): SessionCheckpoint[]; readCheckpoints(): SessionCheckpoint[];
/** Upsert one imported-notes source record (config.SOURCES_FILE — a
* JSON map, not a ledger; an upsert keeps the existing usage log). */
upsertSource(source: NotesSource): void;
/** Every notes source record, by id order-insensitive. */
readSources(): NotesSource[];
/** Append one run's usage to a source record — idempotent per
* startedAt, mirroring ChatRecord.extractions. */
appendSourceUsage(id: string, entry: ExtractionRunUsage): void;
} }
// ─── JobStore — background sub-agent jobs (data/jobs) ───────────────────────── // ─── JobStore — background sub-agent jobs (data/jobs) ─────────────────────────
+4 -1
View File
@@ -37,7 +37,10 @@ const jobRunner = createJobRunner({
jobStore: stores.jobStore, jobStore: stores.jobStore,
deliver: deliverJobResult, deliver: deliverJobResult,
}); });
const extraction = createExtractionPipeline(stores); /** The app's ONE extraction pipeline instance — both front doors share it:
* quiz-triggered session runs (turn-service) and notes imports
* (/api/graph/import) hit the same single-flight registry. */
export const extraction = createExtractionPipeline(stores);
// Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline // Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline
// fires its server-side teacher turns through THIS composition root, and // fires its server-side teacher turns through THIS composition root, and
+60 -1
View File
@@ -15,6 +15,10 @@
* (chatId, sourceId) → graphNodeId * (chatId, sourceId) → graphNodeId
* sessions.jsonl SessionCheckpoint— emission checkpoints (foldedAt) * sessions.jsonl SessionCheckpoint— emission checkpoints (foldedAt)
* *
* Plus config.SOURCES_FILE (sources.json): the imported-notes bookkeeping
* (NOT a ledger) — one JSON map, src-id → NotesSource, read-modify-write
* like a chat file. The format-agnostic front door (2026-09-13).
*
* Implements core/ports.ts GraphStore. data/graph/* is exclusively this * Implements core/ports.ts GraphStore. data/graph/* is exclusively this
* adapter's. Reads are strict against the contracts.ts shapes (the 2026-09-02 * adapter's. Reads are strict against the contracts.ts shapes (the 2026-09-02
* graph redesign): corrupt lines are skipped, never guessed at — the ledgers * graph redesign): corrupt lines are skipped, never guessed at — the ledgers
@@ -25,12 +29,14 @@
import * as fs from "node:fs"; import * as fs from "node:fs";
import * as path from "node:path"; import * as path from "node:path";
import { LEDGER_FILES } from "../../config.ts"; import { LEDGER_FILES, SOURCES_FILE } from "../../config.ts";
import type { import type {
GraphEdge, GraphEdge,
GraphNode, GraphNode,
ExtractionRunUsage,
LedgerEvent, LedgerEvent,
LedgerEventKind, LedgerEventKind,
NotesSource,
NodeMapping, NodeMapping,
ProvenancePoint, ProvenancePoint,
SessionCheckpoint, SessionCheckpoint,
@@ -157,4 +163,57 @@ export const graphStore: GraphStore = {
readCheckpoints(): SessionCheckpoint[] { readCheckpoints(): SessionCheckpoint[] {
return readValid(LEDGER_FILES.sessions, isCheckpoint); return readValid(LEDGER_FILES.sessions, isCheckpoint);
}, },
// ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ──
/** Upsert: keeps the existing record's usage log when re-importing. */
upsertSource(source: NotesSource): void {
const all = readSourceMap();
const prev = all[source.id];
all[source.id] = prev ? { ...source, extractions: prev.extractions } : source;
writeSourceMap(all);
},
readSources(): NotesSource[] {
return Object.values(readSourceMap());
},
/** Idempotent per startedAt — same discipline as chats.appendExtractionUsage. */
appendSourceUsage(id: string, entry: ExtractionRunUsage): void {
const all = readSourceMap();
const source = all[id];
if (!source) return; // no such source — nothing to bill against
source.extractions = source.extractions?.some((r) => r.startedAt === entry.startedAt)
? source.extractions
: [...(source.extractions ?? []), entry];
writeSourceMap(all);
},
}; };
/** sources.json — { id: NotesSource }. Read-modify-write, NOT append-only:
* the usage log lives inside the record (like a chat's JSON file). */
function readSourceMap(): Record<string, NotesSource> {
let raw: string;
try {
raw = fs.readFileSync(SOURCES_FILE, "utf-8");
} catch {
return {};
}
try {
const parsed: unknown = JSON.parse(raw);
if (!isRecord(parsed)) return {};
const out: Record<string, NotesSource> = {};
for (const [id, value] of Object.entries(parsed)) {
if (isNotesSource(value)) out[id] = value;
}
return out;
} catch {
return {};
}
}
function writeSourceMap(map: Record<string, NotesSource>): void {
fs.mkdirSync(path.dirname(SOURCES_FILE), { recursive: true });
fs.writeFileSync(SOURCES_FILE, JSON.stringify(map, null, "\t"), "utf-8");
}
const isNotesSource = (v: unknown): v is NotesSource =>
isRecord(v) && isString(v.id) && isString(v.filePath) && isString(v.importedAt);
+136
View File
@@ -27,10 +27,12 @@ import type { UIMessage } from "ai";
import type { PiEngine } from "../src/agent/models.ts"; import type { PiEngine } from "../src/agent/models.ts";
import type { import type {
AuditResult, AuditResult,
ExtractionRunUsage,
GraphEdge, GraphEdge,
GraphNode, GraphNode,
LedgerEvent, LedgerEvent,
NodeMapping, NodeMapping,
NotesSource,
SessionCheckpoint, SessionCheckpoint,
SessionFindings, SessionFindings,
} from "../src/core/contracts.ts"; } from "../src/core/contracts.ts";
@@ -60,6 +62,7 @@ function memoryLedgers() {
const events: LedgerEvent[] = []; const events: LedgerEvent[] = [];
const mappings: NodeMapping[] = []; const mappings: NodeMapping[] = [];
const checkpoints: SessionCheckpoint[] = []; const checkpoints: SessionCheckpoint[] = [];
const sources = new Map<string, NotesSource>();
return { return {
appendNode: (n: GraphNode) => void nodes.push(n), appendNode: (n: GraphNode) => void nodes.push(n),
readNodes: () => nodes.map((n) => ({ ...n })), readNodes: () => nodes.map((n) => ({ ...n })),
@@ -71,6 +74,19 @@ function memoryLedgers() {
readMappings: () => mappings.map((m) => ({ ...m })), readMappings: () => mappings.map((m) => ({ ...m })),
appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c), appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c),
readCheckpoints: () => checkpoints.map((c) => ({ ...c })), readCheckpoints: () => checkpoints.map((c) => ({ ...c })),
upsertSource: (s: NotesSource) => {
// Mirrors the real store: an upsert KEEPS the existing usage log.
const prev = sources.get(s.id);
sources.set(s.id, { ...s, extractions: [...(prev?.extractions ?? [])] });
},
readSources: () => [...sources.values()].map((s) => ({ ...s, extractions: [...(s.extractions ?? [])] })),
appendSourceUsage: (id: string, entry: ExtractionRunUsage) => {
const source = sources.get(id);
if (!source) return;
if (!source.extractions?.some((r) => r.startedAt === entry.startedAt)) {
source.extractions = [...(source.extractions ?? []), entry];
}
},
} satisfies GraphStore; } satisfies GraphStore;
} }
@@ -513,3 +529,123 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/); await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
}); });
}); });
// ── the notes front door (format-agnostic import) ────────────────────────────
/** Raw personal notes — arbitrary format, no session, no message ids. */
const NOTES_TEXT = `> [!quote] YOU
> teach me vim
> [!abstract] PI
> Vim always starts in Normal mode — keys are commands, not text.`;
/** What the notes-mode extractor submits: same shapes minus attribution,
* and messages always empty (the file is the provenance). */
const NOTES_FINDINGS = {
propositions: [
{
id: "p1",
name: "Vim normal mode",
statement: "Vim starts in Normal mode, where keys are commands.",
aliases: [],
messages: [],
evidence: ["Vim always starts in Normal mode — keys are commands, not text."],
understanding: "solid",
},
],
edges: [],
} as const;
const NOTES_VERDICT = {
propositions: [NOTES_FINDINGS.propositions[0]],
edges: [],
dropped: [],
} as const;
describe("the notes front door (importNotes — format-agnostic)", () => {
it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
const ledgers = memoryLedgers();
const seen: { kind?: string; label?: string }[] = [];
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
extractor: submitModel("submit_findings", NOTES_FINDINGS),
// During the auditor's call the registry must show the notes run.
auditor: submitModel("submit_review", NOTES_VERDICT, () => {
const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
seen.push({ kind: run?.kind, label: run?.label });
}),
});
const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
// The run surfaced as a NOTES run labeled with the file path.
assert.deepEqual(seen, [{ kind: "notes", label: "/vault/learn vim.md" }]);
assert.equal(extractionStatuses().length, 0, "the registry clears when the run ends");
// The source record: deterministic src id, the file path, the usage log.
const source = ledgers.readSources()[0]!;
assert.match(source.id, /^src_/);
assert.equal(source.filePath, "/vault/learn vim.md");
assert.ok(Date.parse(source.importedAt));
assert.equal(source.extractions?.length, 1, "the run's usage persisted into the source record");
assert.equal(source.extractions![0]!.stats?.nodes, 1);
assert.ok(source.extractions![0]!.stages.extractor.steps.length > 0, "faux usage captured");
// File-scoped provenance: the mapping line carries the source id;
// the node carries NO message anchors.
const node = ledgers.readNodes()[0]!;
assert.equal(node.statement, "Vim starts in Normal mode, where keys are commands.");
assert.deepEqual(node.provenance, []);
assert.deepEqual(ledgers.readMappings(), [{ chatId: source.id, sourceId: "p1", graphNodeId: node.id }]);
// One observation event, zero quiz events — outcomes inside notes
// surface only through understanding observations (user-decided).
const events = ledgers.readEvents();
assert.deepEqual(
events.map((e) => [e.kind, e.chatId, e.ref, e.note]),
[["observation", source.id, "obs:p1", "solid"]],
);
// The checkpoint stamps the import.
assert.equal(ledgers.readCheckpoints()[0]!.chatId, source.id);
});
it("re-importing the same file is idempotent: same source identity, refs dedupe, usage history grows", async () => {
const ledgers = memoryLedgers();
const mk = () =>
createExtractionPipeline({
chatStore,
graphStore: ledgers,
extractor: submitModel("submit_findings", NOTES_FINDINGS),
auditor: submitModel("submit_review", NOTES_VERDICT),
});
const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
const second = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
assert.deepEqual(second, { nodes: 0, edges: 0, events: 0 }, "the same source id dedupes everything");
assert.equal(ledgers.readSources().length, 1, "one source identity — the path hashes to the same id");
assert.equal(ledgers.readSources()[0]!.extractions?.length, 2, "both runs billed their usage");
assert.equal(ledgers.readNodes().length, 1, "no duplicate node");
});
it("aborts like a chat run: true cancellation, nothing merged, partial usage in the source record", async () => {
const ledgers = memoryLedgers();
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
extractor: hangingModel(),
auditor: submitModel("submit_review", NOTES_VERDICT),
});
const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
const srcId = extractionStatuses().find((r) => r.chatId.startsWith("src_"))!.chatId;
assert.equal(abortExtraction(srcId), true);
assert.deepEqual(await runPromise, { nodes: 0, edges: 0, events: 0 });
assert.equal(ledgers.readNodes().length, 0, "nothing merged");
assert.equal(ledgers.readCheckpoints().length, 0, "nothing checkpointed");
assert.equal(ledgers.readSources()[0]!.extractions?.length, 1, "partial usage still persisted");
assert.equal(ledgers.readSources()[0]!.extractions![0]!.stats, undefined, "never merged → no stats");
});
});
+91 -10
View File
@@ -23,14 +23,22 @@ import type { QuestionAnswerData } from "../src/core/question-core.ts";
import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts"; import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
import { graphStore } from "../src/store/graph.ts"; import { graphStore } from "../src/store/graph.ts";
import { chatStore } from "../src/store/chats.ts"; import { chatStore } from "../src/store/chats.ts";
import { LEDGER_FILES, PATHS } from "../config.ts"; import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";
/** The wired fs adapters, exactly what src/server/stores.ts passes in production. */ /** The wired fs adapters, exactly what src/server/stores.ts passes in production. */
const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps; const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps;
/** The session merge source for a chat — what the pipeline passes: the live
* transcript + its updatedAt checkpoint stamp. (Notes imports pass
* messages: null — see the notes-merger test below.) */
function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: string } {
const record = chatStore.getChat(chatId)!;
return { messages: record.messages as UIMessage[], foldedAt: record.updatedAt };
}
// ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ────────────── // ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ──────────────
const LEDGER_PATHS = Object.values(LEDGER_FILES); const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE];
const backups: Array<{ file: string; bak: string }> = []; const backups: Array<{ file: string; bak: string }> = [];
const createdChats: string[] = []; const createdChats: string[] = [];
@@ -233,7 +241,7 @@ function scopedGraph(chatIds: string[]) {
describe("the merger (mergeFindings)", () => { describe("the merger (mergeFindings)", () => {
it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => { it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => {
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps); const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 }); assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 });
const graph = scopedGraph([chat1.id]); const graph = scopedGraph([chat1.id]);
@@ -302,7 +310,7 @@ describe("the merger (mergeFindings)", () => {
}); });
it("is idempotent — re-merging the same verdicts appends nothing", () => { it("is idempotent — re-merging the same verdicts appends nothing", () => {
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps); const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 }); assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 });
const graph = scopedGraph([chat1.id]); const graph = scopedGraph([chat1.id]);
assert.equal(graph.nodes.length, 3); assert.equal(graph.nodes.length, 3);
@@ -346,6 +354,7 @@ describe("the merger (mergeFindings)", () => {
dropped: [], dropped: [],
}, },
graphDeps, graphDeps,
sessionSource(chat2.id),
); );
assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 }); assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 });
@@ -363,6 +372,41 @@ describe("the merger (mergeFindings)", () => {
assert.deepEqual(m, { chatId: chat2.id, sourceId: "q1", graphNodeId: alphaId }); assert.deepEqual(m, { chatId: chat2.id, sourceId: "q1", graphNodeId: alphaId });
}); });
it("resolves imported-notes ids to their file paths in the snapshot (static provenance)", () => {
// The notes front door in miniature: a source record + a merge under
// its src id — the snapshot must label it with the file path (a
// static entry), never as a chat session.
graphStore.upsertSource({ id: "src_snapfixture", filePath: "/vault/learn vim.md", importedAt: "2026-09-13T00:00:00.000Z" });
mergeFindings(
"src_snapfixture",
{
propositions: [
{
id: "p1",
name: "Vim registers",
statement: "Vim registers map single keys to editing commands.",
aliases: [],
messages: [],
evidence: ["fixture evidence"],
understanding: "solid",
},
],
edges: [],
attribution: { quiz: {} },
dropped: [],
},
graphDeps,
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
);
const snap = graphSnapshot(graphDeps);
const node = snap.nodes.find((n) => n.statement === "Vim registers map single keys to editing commands.")!;
assert.equal(snap.fileSources["src_snapfixture"], "/vault/learn vim.md");
assert.deepEqual(snap.nodeSessions[node.id], ["src_snapfixture"], "the file entry joins the node's sessions list");
assert.equal(snap.sessions["src_snapfixture"], undefined, "a notes source is never a chat session");
// The understanding observation joins under the source id like any session's.
assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
});
it("derives sessions and per-session understanding observations in graphSnapshot", () => { it("derives sessions and per-session understanding observations in graphSnapshot", () => {
const snap = graphSnapshot(graphDeps); const snap = graphSnapshot(graphDeps);
const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id])); const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));
@@ -408,17 +452,54 @@ describe("the merger (mergeFindings)", () => {
dropped: [], dropped: [],
}, },
graphDeps, graphDeps,
sessionSource(chat3.id),
); );
assert.equal(stats.nodes, 1, "created — the merger does not match strings"); assert.equal(stats.nodes, 1, "created — the merger does not match strings");
assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2); assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2);
}); });
it("returns zeros for unknown chats", () => { it("merges the notes front door: source id as chatId, no quiz events, checkpoint at import time", () => {
assert.deepEqual(mergeFindings("chat_does_not_exist", fixtureAudit(), graphDeps), { // Raw personal notes carry no transcript — the merger gets no
nodes: 0, // message parts, so NO quiz events are rebuilt; the understanding
edges: 0, // observation still lands, keyed to the source id.
events: 0, const stats = mergeFindings(
"src_notesfixture",
{
propositions: [
{
id: "p1",
name: "Vim normal mode",
statement: "Vim starts in Normal mode, where keys are commands.",
aliases: [],
messages: [], // notes carry no message anchors — the file is the provenance
evidence: ["fixture evidence"],
understanding: "solid",
},
],
edges: [],
attribution: { quiz: {} }, // normalized by the notes pipeline — nothing to attribute
dropped: [],
},
graphDeps,
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
);
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
const node = getGraph(graphStore).nodes.find((n) => n.statement.startsWith("Vim starts"))!;
assert.deepEqual(node.provenance, [], "no message anchors — the mapping line carries the source");
assert.deepEqual(graphStore.readMappings().find((m) => m.chatId === "src_notesfixture"), {
chatId: "src_notesfixture",
sourceId: "p1",
graphNodeId: node.id,
}); });
// The observation event is file-scoped; no quiz events exist.
const events = graphStore.readEvents().filter((e) => e.chatId === "src_notesfixture");
assert.deepEqual(
events.map((e) => [e.kind, e.ref, e.note]),
[["observation", "obs:p1", "solid"]],
);
// The checkpoint stamps the import time.
assert.equal(graphStore.readCheckpoints().find((c) => c.chatId === "src_notesfixture")?.foldedAt, "2026-09-13T00:00:00.000Z");
}); });
}); });
@@ -452,7 +533,7 @@ describe("the trigger helper (pendingQuizAnswers)", () => {
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending"); assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending");
assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0); assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0);
mergeFindings(chat.id, fixtureAudit(), graphDeps); mergeFindings(chat.id, fixtureAudit(), graphDeps, sessionSource(chat.id));
// call-s is unattributed here (fixtureAudit has no call-s attribution) // call-s is unattributed here (fixtureAudit has no call-s attribution)
// — but it still becomes an event, so nothing stays pending. // — but it still becomes an event, so nothing stays pending.
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending"); assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending");
+39 -1
View File
@@ -10,7 +10,8 @@ import test from "node:test";
import * as fs from "node:fs"; import * as fs from "node:fs";
import { chatStore } from "../src/store/chats.ts"; import { chatStore } from "../src/store/chats.ts";
import { PATHS } from "../config.ts"; import { graphStore } from "../src/store/graph.ts";
import { PATHS, SOURCES_FILE } from "../config.ts";
import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts"; import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts";
import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts"; import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts";
@@ -170,3 +171,40 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`); fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
}); });
test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => {
// The real fs graphStore — sources.json backed up and restored around
// this test (same discipline as the chat fixture files).
fs.mkdirSync(PATHS.graph, { recursive: true });
let bak: string | undefined;
if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`;
if (bak) fs.renameSync(SOURCES_FILE, bak);
try {
const runEntry = (startedAt: number): ExtractionRunUsage => ({
startedAt,
at: "2026-09-13T00:00:00Z",
durationMs: 42_000,
stats: { nodes: 1, edges: 2, events: 3 },
stages: {
extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
},
});
const id = "src_usagetest01";
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" });
graphStore.appendSourceUsage(id, runEntry(1));
graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op
graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
// An upsert (re-import) KEEPS the usage history.
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" });
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
// Unknown ids are a silent no-op (nothing to bill against).
graphStore.appendSourceUsage("src_nope", runEntry(1));
} finally {
fs.rmSync(SOURCES_FILE, { force: true });
if (bak) fs.renameSync(bak, SOURCES_FILE);
}
});