Notes front door: format-agnostic import into the graph pipeline
The same extractor→auditor→merger pipeline now runs over raw personal
notes in arbitrary formats (vision §1, DECIDED 2026-09-13):
- NotesSource records (data/graph/sources.json, deterministic id from
path hash → idempotent re-imports) play the chatId role in mappings,
checkpoints, and observation events; usage lands on the source record
(appendSourceUsage, idempotent per startedAt, kept across upserts).
- mergeFindings is chat-store-free: callers pass MergeSource
{messages, foldedAt} — sessions pass the chat record, notes pass
messages:null (no quiz-event rebuild; a deterministic rebuild needs
structured quiz tool parts).
- Second extractor prompt (NOTES_EXTRACTOR_INSTRUCTIONS) for general-
purpose extraction; understanding judges the AUTHOR's grasp. Auditor
wording input-agnostic. extract/audit share one generic runSubmitAgent.
- graphSnapshot.fileSources (srcId→filePath); notes sources render as
static path entries in the detail card (FileTextIcon, never a link).
- One shared pipeline instance (exported from chat-handler): same
single-flight registry, live status chips (label = file path), usage
accounting, and true abort for both front doors.
- Transport: POST /api/graph/import (multipart, 3-wide worker pool) +
Import notes… button in the graph view; scripts/import-notes.ts for
batch runs (concurrency 4).
- real-data-extraction/ gitignored (raw import sources never committed).
355/355 tests; typecheck, lint, build green.
This commit is contained in:
+39
-1
@@ -10,7 +10,8 @@ import test from "node:test";
|
||||
import * as fs from "node:fs";
|
||||
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
import { PATHS, SOURCES_FILE } from "../config.ts";
|
||||
import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts";
|
||||
import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts";
|
||||
|
||||
@@ -170,3 +171,40 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl
|
||||
|
||||
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
|
||||
});
|
||||
|
||||
test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => {
|
||||
// The real fs graphStore — sources.json backed up and restored around
|
||||
// this test (same discipline as the chat fixture files).
|
||||
fs.mkdirSync(PATHS.graph, { recursive: true });
|
||||
let bak: string | undefined;
|
||||
if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`;
|
||||
if (bak) fs.renameSync(SOURCES_FILE, bak);
|
||||
try {
|
||||
const runEntry = (startedAt: number): ExtractionRunUsage => ({
|
||||
startedAt,
|
||||
at: "2026-09-13T00:00:00Z",
|
||||
durationMs: 42_000,
|
||||
stats: { nodes: 1, edges: 2, events: 3 },
|
||||
stages: {
|
||||
extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
},
|
||||
});
|
||||
const id = "src_usagetest01";
|
||||
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" });
|
||||
graphStore.appendSourceUsage(id, runEntry(1));
|
||||
graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op
|
||||
graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend
|
||||
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
|
||||
|
||||
// An upsert (re-import) KEEPS the usage history.
|
||||
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" });
|
||||
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
|
||||
|
||||
// Unknown ids are a silent no-op (nothing to bill against).
|
||||
graphStore.appendSourceUsage("src_nope", runEntry(1));
|
||||
} finally {
|
||||
fs.rmSync(SOURCES_FILE, { force: true });
|
||||
if (bak) fs.renameSync(bak, SOURCES_FILE);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user