From 1b500135510b99c9a88810a7fd2a5e95de0b09ca Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Mon, 28 Sep 2026 16:54:44 +0300 Subject: [PATCH] extraction cleanup: events ledger dies, markdown source view, auditor-as-linker (2026-09-27 rulings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The graph becomes raw reasoning chains only. Quiz outcomes and understanding observations are no longer formalized; assessment moves to the future practice module. Events ledger removed end-to-end: LedgerEvent/LedgerEventKind/ UnderstandingLevel/ExtractionAttribution/AuditResult.dropped and snapshot nodeObservations deleted from contracts; quizEvent builder and the events step of mergeFindings gone; appendEvent/readEvents dropped from the port and store; LEDGER_FILES loses events.jsonl with a one-time purge of a stale file next to the nodes ledger; graph-deletion and the delete dialog lose event purge + quiz-answer impact counts; GraphSessionProposition becomes a pure birth list. Whole-session markdown source view: new core/session-markdown.ts is the shared formatting engine (id-anchored speaker blocks, quiz/ recall/question blocks, tool one-liners) — the extractor's prompt view and the future user-facing export render from one engine. Coverage fingerprints deliberately stay on the old line format (content-hash semantics; re-formatting never re-opens covered history). The whole session goes to the extractor — no phase cut. Auditor refocused as the identity linker: never sees the source; explores the graph via new auditor-specific tools in agent/tools/graph.ts (find — glob over name/statement/aliases; list_sessions; list_propositions with full statement text; no timestamps) instead of an injected full-graph dump; equivalence bias — paraphrases merge, only genuinely distinct propositions stay apart; no drop channel, the extractor is fully trusted; may draw bridging edges (typed follows/supports) between existing nodes and the batch, resolved by the merger with chat-scoped message-less provenance. Agentic loop, maxTurns 16. UI: the node detail card gains an edges section above the sessions list (follows-from / supported-by / what-follows / what-it-supports, rendered only when present, click-to-select neighbors); the understanding-note chips are gone. Prober list_propositions reworded to what a session covered — names only, no levels. docs/architecture.md + README updated to the as-built pipeline. Gates: typecheck, 1059/1059, lint, build. --- README.md | 2 +- components/graph-view.tsx | 60 ++-- config.ts | 1 - docs/architecture.md | 20 +- scripts/backfill-graph.ts | 4 +- scripts/import-notes.ts | 9 +- src/agent/subagents/extraction.ts | 447 ++++++++++-------------------- src/agent/tools/graph.ts | 101 ++++++- src/core/contracts/chat.ts | 2 +- src/core/contracts/graph.ts | 141 +++------- src/core/graph-deletion.ts | 43 +-- src/core/graph.ts | 318 +++++---------------- src/core/ports.ts | 4 - src/core/session-markdown.ts | 126 +++++++++ src/server/graph.ts | 1 - src/store/graph.ts | 40 ++- tests/extraction.test.ts | 180 +++++------- tests/graph-tools.test.ts | 117 +++++--- tests/graph.test.ts | 175 +++--------- tests/session-clusters.test.ts | 40 +-- tests/session-markdown.test.ts | 138 +++++++++ tests/usage.test.ts | 4 +- 22 files changed, 909 insertions(+), 1064 deletions(-) create mode 100644 src/core/session-markdown.ts create mode 100644 tests/session-markdown.test.ts diff --git a/README.md b/README.md index abe03be..a860dc9 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ data/ chats/ · graph/ (six ledgers + positions/source **The async sub-agent protocol (ack-and-steer).** `research` / `demo` / `visual` are server-executed tools: the call resolves immediately mid-turn with a fire-and-forget ack and the teacher keeps teaching; a background runner works in parallel (the client polls `/api/activity` and `/api/jobs`). On completion the runner builds a **steer message** — a user-role `UIMessage` whose text part is the model-facing result and whose `data-subagent-artifact` part carries the full result for the client — and the **wake pipeline** delivers it: a per-chat serialized queue waits for the turn to end and no question to be pending, then fires a server-side teacher turn that weaves the result into the lesson. Delivery is exactly-once (deterministic steer ids, delivered-flag checks at every claim point); stalled jobs are recovered on read; aborted jobs deliver an abort steer. The client adopts the wake turn through the same `turnInFlight` machinery as returning mid-stream (busy shimmer, composer gate) and renders steers as artifacts — never a bubble. Sub-agents **verify, never one-shot**: the demo-builder headlessly executes its HTML in system Chrome before delivery; the visual-maker renders each SVG and looks at it (vision-capable model — the `vision` slot in settings). -**The knowledge graph (two channels, one writer).** The write channel: the extraction pipeline — extractor (LLM, tree-blind: sees only the transcript view — propositions as true statements, `follows`/`supports` edges, retroactive quiz attribution, one-word understanding observations) → auditor (LLM: sees findings + the current graph; drops anything without verbatim evidence, stamps each survivor with an identity verdict — link to an existing node or create) → merger (`mergeFindings`, deterministic: applies the verdicts, link-appends phrasings/provenance, rebuilds quiz events from the transcript's tool parts, dedupes, checkpoints coverage) → layout settle. Triggered when a chat's **uncovered tokens** (message-view fingerprints, `coverage.jsonl`) cross `graph.extractionChunkTokens` (a settings knob, default 4096, floor 500 — read fresh per turn); failure checkpoints nothing, the next trigger re-runs. The same pipeline serves the **notes front door** (`POST /api/graph/import`: any `*.md`/`*.txt` becomes a source; re-imports are idempotent by path hash). The read channel: `GET /api/graph` derives the snapshot from the ledgers — a pure read; the home view is a hand-rolled Canvas 2D renderer (one rAF loop, dirty-flag idle, d3-zoom camera, zoom-gated labels, cluster hulls) over a d3-force layout, with server-persisted render settings and drag-committed positions. Clusters are **birth-session groupings**: a node joins the cluster of the session that birthed it (a chat, or an imported notes file as a file-kind session), derived at read — no embedding pass, no naming pass. The teacher reaches the graph on its own terms through the gated prober tools (`list_sessions` / `list_propositions`, the `agent.graphTools` setting) — there is no automatic context injection. +**The knowledge graph (two channels, one writer).** The write channel: the extraction pipeline — extractor (LLM: sees only the session, rendered as readable markdown by the shared `core/session-markdown.ts` engine — propositions as true statements, `follows`/`supports` edges with verbatim evidence; fully trusted, graph-blind) → auditor (LLM, an agentic loop that never sees the source: explores the existing graph through `find` / `list_sessions` / `list_propositions` tools, stamps each proposition with an identity verdict — link to an existing node (equivalence bias: paraphrases merge) or create — and may draw bridging edges between existing nodes and the new batch) → merger (`mergeFindings`, deterministic: applies the verdicts, link-appends phrasings/provenance, resolves and dedupes edges, checkpoints coverage) → layout settle. No quiz outcomes or understanding assessments live in the graph (2026-09-27 cleanup); assessment becomes practice-module territory. Triggered when a chat's **uncovered tokens** (message-view fingerprints, `coverage.jsonl`) cross `graph.extractionChunkTokens` (a settings knob, default 4096, floor 500 — read fresh per turn); failure checkpoints nothing, the next trigger re-runs. The same pipeline serves the **notes front door** (`POST /api/graph/import`: any `*.md`/`*.txt` becomes a source; re-imports are idempotent by path hash). The read channel: `GET /api/graph` derives the snapshot from the ledgers — a pure read; the home view is a hand-rolled Canvas 2D renderer (one rAF loop, dirty-flag idle, d3-zoom camera, zoom-gated labels, cluster hulls) over a d3-force layout, with server-persisted render settings and drag-committed positions. Clusters are **birth-session groupings**: a node joins the cluster of the session that birthed it (a chat, or an imported notes file as a file-kind session), derived at read — no embedding pass, no naming pass. The teacher reaches the graph on its own terms through the gated prober tools (`list_sessions` / `list_propositions`, the `agent.graphTools` setting) — there is no automatic context injection. **Dictation.** Optional speech-to-text: the server mints short-lived Deepgram tokens; audio streams browser→Deepgram over WebSocket, never through the server. diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 22a12ea..c5a2f87 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -1563,13 +1563,54 @@ export function GraphView() { >
{selected.node.name}
{selected.node.statement}
+ {(() => { + // EDGES (owner 2026-09-27): edge classification was entirely + // hidden in the UI — the card now shows every edge this node + // participates in, grouped by direction + type, above the + // sessions list. Only present groups render. + const id = selected.node.id; + const nodeById = new Map((snapshot?.nodes ?? []).map((n) => [n.id, n])); + const label = (nid: string) => nodeById.get(nid)?.name ?? nid; + const groups: { title: string; names: string[] }[] = [ + { + title: "Follows from", + names: (snapshot?.edges ?? []).filter((e) => e.from === id && e.type === "follows").map((e) => label(e.to)), + }, + { + title: "Supported by", + names: (snapshot?.edges ?? []).filter((e) => e.from === id && e.type === "supports").map((e) => label(e.to)), + }, + { + title: "What follows from this", + names: (snapshot?.edges ?? []).filter((e) => e.to === id && e.type === "follows").map((e) => label(e.from)), + }, + { + title: "What this supports", + names: (snapshot?.edges ?? []).filter((e) => e.to === id && e.type === "supports").map((e) => label(e.from)), + }, + ].filter((g) => g.names.length > 0); + if (groups.length === 0) return null; + return ( +
+ {groups.map((g) => ( +
+
{g.title}
+
+ {g.names.map((name, i) => ( + + {name} + + ))} +
+
+ ))} +
+ ); + })()} {selected.sourceIds.length > 0 ? (
Sessions
{selected.sourceIds.map((sourceId) => { - // That source's understanding observation — the - // extractor's named-word note (confidence history). - const note = snapshot?.nodeObservations?.[selected.node.id]?.[sourceId]; // An imported-notes file: a STATIC entry — the path, // plain text, never a link (no session to open). if (snapshot?.fileSources?.[sourceId] !== undefined) { @@ -1579,11 +1620,6 @@ export function GraphView() { {snapshot.fileSources[sourceId]} - {note && ( - - {note} - - )}
); } @@ -1596,14 +1632,6 @@ export function GraphView() { > {snapshot?.sessions[sourceId] ?? sourceId} - {/* The note is a fixed-scale word (solid/shaky/...) — the - session name is the main thing; the chip is bounded so - it can never push or overflow the row. */} - {note && ( - - {note} - - )} ); })} diff --git a/config.ts b/config.ts index a06c8d0..3cf9c5b 100644 --- a/config.ts +++ b/config.ts @@ -58,7 +58,6 @@ export type ModelRole = export const LEDGER_FILES = { nodes: path.join(PATHS.graph, "nodes.jsonl"), edges: path.join(PATHS.graph, "edges.jsonl"), - events: path.join(PATHS.graph, "events.jsonl"), mappings: path.join(PATHS.graph, "mappings.jsonl"), sessions: path.join(PATHS.graph, "sessions.jsonl"), /** Per-message extraction coverage (contracts/graph.ts CoverageEntry; diff --git a/docs/architecture.md b/docs/architecture.md index 25a3a68..3238e7a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -331,24 +331,24 @@ flowchart TB TRIG["trigger seam (turn-service): coverageOf(record, coverageLedger)
— uncovered tokens ≥ graph.extractionChunkTokens (settings)?"] -->|"yes, single-flight per source"| RUN["extraction.runExtraction(chatId)"] IMPORT["POST /api/graph/import (notes front door)
deterministic src id = sha256(path) — idempotent re-imports"] -->|"importNotes: shared pipeline, second extractor prompt"| RUN RUN --> BATCH["batchRuns — greedy token-budgeted batches of oldest uncovered messages"] - BATCH --> EXT["EXTRACTOR (LLM) — sees ONLY the transcript view
(tree-blind; the same messageViewLines the fingerprint hashes)"] - EXT --> AUD["AUDITOR (LLM) — sees findings + current graph;
verifies evidence verbatim, judges identity (link-or-create), drops + logs unbacked"] - AUD --> MERGE["MERGER (deterministic, core/graph.ts) — the ONLY ledger writer:
identity via validated linkTo · dedupe edges · rebuild quiz events from tool parts · coverage checkpoint"] + BATCH --> EXT["EXTRACTOR (LLM) — sees ONLY the session markdown view
(tree-blind, graph-blind; shared engine: core/session-markdown.ts)"] + EXT --> AUD["AUDITOR (LLM, agentic loop) — sees findings ONLY, never the source;
explores the graph via find / list_sessions / list_propositions tools,
judges identity (link-or-create, equivalence bias) + may draw bridging edges"] + AUD --> MERGE["MERGER (deterministic, core/graph.ts) — the ONLY ledger writer:
identity via validated linkTo · bridging-edge resolution · dedupe edges · coverage checkpoint"] MERGE --> SETTLE["settleGraphLayout — resting positions ledger"] MERGE -->|"failure at ANY stage"| NOTHING["checkpoints nothing — the next trigger re-runs"] ``` -**Tree-blind and post-hoc** (vision §1): the extractor reads ONLY the transcript — never the session's plan tree — and proposes free-form propositions (true statements; a misconception is stated as its true correction), `follows`/`supports` edges, per-proposition understanding observations (a closed one-word scale), and message-granular provenance. Every node is an objective, world-facing fact; the learner's relationship to it lives entirely in the event ledger. +**Raw reasoning chains only** (2026-09-27 cleanup): the extractor reads ONLY the session — the whole thing, rendered as readable markdown by the shared engine (`core/session-markdown.ts`; the same renderer will serve future user-facing session export) — and proposes free-form propositions (true statements; a misconception is stated as its true correction) and `follows`/`supports` edges with verbatim evidence. Every node is an objective, world-facing fact. No quiz outcomes, no understanding assessments — the events ledger and `UnderstandingLevel` are deleted; the learner's relationship to a proposition becomes practice-module territory (assessment records, later). -**Division of trust** (the pipeline's core rule): the LLM stages supply only judgment — *what was learned*, *what follows/supports what*, *which proposition each quiz tested*, and the auditor's *identity* verdict (whether a proposition IS an existing node). Everything the model could silently mis-copy — event kinds (`correct`/`miss`/`dont-know`), timestamps, idempotency refs, notes, provenance points — is rebuilt deterministically by the merger from the transcript's tool parts. The extractor files verbatim evidence quotes; the auditor drops anything unbacked and logs every drop. Conservative throughout: duplicates over wrong merges, drop over invent. +**Division of trust** (the pipeline's core rule): the extractor is fully trusted and is the only stage that ever sees the source. The auditor never sees the transcript — it receives the findings JSON alone and explores the existing graph through read-only tools (`find`, `list_sessions`, `list_propositions` — statement text included, no timestamps) to reach identity verdicts: `linkTo` an existing node or let the merger create one. Equivalence bias (owner ruling 2026-09-27): paraphrases, rewordings, and compressions of the same claim MERGE; only genuinely distinct propositions stay apart. The auditor may also draw bridging edges — typed `follows`/`supports` connections between existing graph nodes and the batch's propositions (the auditor is the asserter: chat-scoped provenance, no message anchor). The deterministic merger validates and appends: linkTo resolution, dedupe by `(from, to, type)`, coverage checkpoints. There is no drop channel — nothing rejects the extractor's output. -**Coverage, not quiz-counting** (supersedes the v1 every-N-quiz-answers trigger): each message's *view* (the same lines the extractor sees) is hashed into a per-chat fingerprint ledger (`coverage.jsonl`); "covered" means exactly "what the model saw." Uncovered = fingerprint diff, never dual-tracked; runs are greedy token-budgeted batches (`graph.extractionChunkTokens` — a settings knob, default 4096, floor 500, read fresh each turn) of the oldest uncovered messages — quiz-less sessions extract too. A coarse `sessions.jsonl` checkpoint still marks each merged run. The trigger seam (`turn-service.ts`) is trigger-agnostic from the pipeline's side: when a better mechanism lands (session end, idle, explicit flush), only the seam changes. Single-flight per source; the run registry powers the coverage UI and true abort. +**Coverage, not quiz-counting** (supersedes the v1 every-N-quiz-answers trigger): each message's *view* (the compact line format `core/coverage.ts` has always hashed) is hashed into a per-chat fingerprint ledger (`coverage.jsonl`); "covered" means exactly "a run committed on this content." The prompt view is the richer markdown document — a deliberate 2026-09-27 split: re-formatting presentation never re-opens covered history. Uncovered = fingerprint diff, never dual-tracked; runs are greedy token-budgeted batches (`graph.extractionChunkTokens` — a settings knob, default 4096, floor 500, read fresh each turn) of the oldest uncovered messages — quiz-less sessions extract too. A coarse `sessions.jsonl` checkpoint still marks each merged run. The trigger seam (`turn-service.ts`) is trigger-agnostic from the pipeline's side: when a better mechanism lands (session end, idle, explicit flush), only the seam changes. Single-flight per source; the run registry powers the coverage UI and true abort. **Two front doors, one pipeline**: chat transcripts (quiz-triggered) and raw personal notes (`POST /api/graph/import` — each file becomes a `NotesSource` playing the `chatId` role, deterministic id from the path hash). -**The read channel is pure derivation.** `GET /api/graph` reads the ledgers (`graphSnapshot`: merged nodes/edges, folded observations, birth-session clusters, positions) — the home view never triggers a write. Node states are always recomputed from the ledger, never stored. +**The read channel is pure derivation.** `GET /api/graph` reads the ledgers (`graphSnapshot`: merged nodes/edges, birth-session clusters, positions) — the home view never triggers a write. Node detail cards show a node's edges (follows-from / supported-by / what-follows / what-it-supports) and its birth sessions. -**The teacher probes, nothing is injected** (supersedes the semantic-injection experiment, killed 2026-09-23): the teacher reaches the graph on its own terms through two read-only prober tools — `list_sessions` (birth sessions, the learner's files) and `list_propositions` (a session's extracted statements) — gated by the `agent.graphTools` setting (composing with `graph.enabled`). No topic embedding, no top-K selection, no session-start steer; the agent decides what to look at, and the graph never writes into a lesson unbidden. +**The teacher probes, nothing is injected** (supersedes the semantic-injection experiment, killed 2026-09-23): the teacher reaches the graph on its own terms through two read-only prober tools — `list_sessions` (birth sessions, the learner's files) and `list_propositions` (the names of what a session covered) — gated by the `agent.graphTools` setting (composing with `graph.enabled`). No topic embedding, no top-K selection, no session-start steer; the agent decides what to look at, and the graph never writes into a lesson unbidden. **Rendering**: a hand-rolled Canvas 2D renderer (`components/graph-canvas.ts` — one rAF loop, dirty-flag idle, d3-zoom camera, zoom-gated label fade, cluster hulls computed at draw time) over a d3-force layout in core (`layout.ts`, warm-started `incrementalLayout` + `settleGraphLayout`). Render settings and drag-committed positions persist server-side; the camera persists per-device in localStorage. @@ -406,9 +406,9 @@ The Electron main is a launcher + security guard, never a host for app logic: si | **One composition root per concern** — `stores.ts` (storage), `chat-handler.ts` (a turn) | | **One writer per ledger** — the merger is the sole graph writer; chat records written only by the turn pipeline; job records only by the runner; positions PUT is the only sanctioned graph-adjacent exception | | **The store never decides** — adapters persist and shape-guard; merge/dedup/derivation semantics are reader-side, in core | -| **Derived, never dual-tracked** — append-only evidence; every state recomputed at read (snapshot, plan statuses, phases, understanding, costs, coverage) | +| **Derived, never dual-tracked** — append-only evidence; every state recomputed at read (snapshot, plan statuses, phases, costs, coverage) | | **Idempotency via deterministic keys** — steer ids, ledger refs, coverage fingerprints, mapping `(chatId, sourceId)`, cluster content hashes; re-running anything converges | -| **LLMs judge; code mechanizes** — grading, merging, event kinds, fingerprints, layout are deterministic; a failed LLM stage checkpoints nothing | +| **LLMs judge; code mechanizes** — grading, merging, fingerprints, layout are deterministic; a failed LLM stage checkpoints nothing | | **The persisted record is the only source of truth** — agents are stateless per-turn rebuilds; no unanswered tool call reaches a provider | | **Failure containment** — side channels (context, extraction, usage, naming) are fire-and-forget behind try/catch seams; a side-channel failure never touches the response or the persist; the turn registry clears on every path | | **Absence is not zero** — unreported tokens, absent prices, absent levels render as absent, never fabricated | diff --git a/scripts/backfill-graph.ts b/scripts/backfill-graph.ts index ba1dd6c..e708b8f 100644 --- a/scripts/backfill-graph.ts +++ b/scripts/backfill-graph.ts @@ -27,7 +27,7 @@ async function main() { const stats = await pipeline.runExtraction(chat.id); console.log( `[backfill] ${chat.id} (${chat.title.slice(0, 40)}): ` + - `${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events in ${((Date.now() - t0) / 1000).toFixed(1)}s`, + `${stats.nodes} nodes, ${stats.edges} edges in ${((Date.now() - t0) / 1000).toFixed(1)}s`, ); } catch (err) { console.error(`[backfill] ${chat.id} FAILED:`, err); @@ -35,7 +35,7 @@ async function main() { } const graph = getGraph(graphStore); console.log( - `[backfill] done: ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events`, + `[backfill] done: ${graph.nodes.length} nodes, ${graph.edges.length} edges`, ); } diff --git a/scripts/import-notes.ts b/scripts/import-notes.ts index 0ded548..2d91932 100644 --- a/scripts/import-notes.ts +++ b/scripts/import-notes.ts @@ -33,7 +33,7 @@ async function main() { getChunkTokens: () => effectiveSettings().graph.extractionChunkTokens, }); const queue = [...files]; - const totals = { nodes: 0, edges: 0, events: 0, failed: 0 }; + const totals = { nodes: 0, edges: 0, failed: 0 }; const workers = Array.from({ length: Math.min(limit, files.length) }, async () => { for (;;) { const file = queue.shift(); @@ -44,9 +44,8 @@ async function main() { const stats = await pipeline.importNotes({ filePath, text: fs.readFileSync(filePath, "utf-8") }); totals.nodes += stats.nodes; totals.edges += stats.edges; - totals.events += stats.events; console.log( - `[import-notes] ${file}: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events in ${((Date.now() - t0) / 1000).toFixed(1)}s`, + `[import-notes] ${file}: ${stats.nodes} nodes, ${stats.edges} edges in ${((Date.now() - t0) / 1000).toFixed(1)}s`, ); } catch (err) { totals.failed++; @@ -58,8 +57,8 @@ async function main() { const graph = getGraph(stores.graphStore); console.log( - `[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` + - `graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events`, + `[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, ${totals.failed} failed — ` + + `graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges`, ); } diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index 3294da4..b41f257 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -20,33 +20,43 @@ import type { ChatRecord, ExtractionRunUsage, ExtractionStageUsage, - GraphNode, SessionFindings, } from "../../core/contracts.ts"; import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts"; import type { GraphStore } from "../../core/ports.ts"; import { settleGraphLayout } from "../../core/layout-settle.ts"; -import { batchRuns, batchTag, coverageOf, messageViewLines } from "../../core/coverage.ts"; +import { batchRuns, coverageOf, messageViewLines } from "../../core/coverage.ts"; +import { sessionMarkdown } from "../../core/session-markdown.ts"; +import { auditorGraphTools } from "../tools/graph.ts"; /** - * The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the - * sole graph writer. Three stages, one orchestrator: + * The extraction pipeline (vision §1 construction, 2026-09-02 redesign; + * relinked 2026-09-27) — the sole graph writer. Three stages, one + * orchestrator: * - * extractor (LLM) transcript → SessionFindings (free-form propositions, - * follows/supports edges, quiz attribution, understanding - * notes — each with verbatim evidence). TREE-BLIND: it - * sees only the transcript, never the plan tree. - * auditor (LLM) findings + THE CURRENT GRAPH → AuditResult: the - * verified subset with an identity verdict per - * proposition (linkTo an existing node, or create), plus - * the drops for the log. + * extractor (LLM) the ONE stage that sees the raw source (a session + * rendered as markdown by the shared session-markdown + * engine, or a notes file verbatim) → SessionFindings: + * propositions + follows/supports edges, each with + * verbatim evidence. TREE-BLIND, graph-blind, FULLY + * TRUSTED — nothing downstream re-checks it. + * auditor (LLM) never sees the source. Its job is IDENTITY: for each + * proposition, linkTo an existing graph node or create — + * decided by EXPLORING the graph through tools (find / + * list_sessions / list_propositions), not by an injected + * graph dump (context bloat, owner 2026-09-27). It may + * also draw BRIDGING edges from existing graph nodes to + * the batch's propositions. * merger (core) mergeFindings() → deterministic ledger appends * - * The LLM stages supply only the judgment calls; the deterministic rebuild - * in the merger owns everything that must never be mis-copied (event kinds, - * timestamps, refs, provenance points). Both agents are single-tool-call - * workers: their whole deliverable is one TypeBox-validated submit call — a - * ToolLoopAgent purely so the model can retry a schema-invalid call. + * The events ledger is GONE (2026-09-27): quiz outcomes and understanding + * observations are no longer formalized in the graph — the graph is purely + * extracted reasoning chains (nodes + edges). Understanding-level + * assessment is practice-module territory. + * + * The extractor is a single-tool-call worker (one TypeBox-validated submit + * call — a pi agent purely so the model can retry a schema-invalid call). + * The auditor is a small AGENTIC loop: exploration tools plus one submit. * * Created via createExtractionPipeline(deps) — the composition root * (chat-handler) injects the storage ports + models; this module never @@ -60,11 +70,9 @@ import { batchRuns, batchTag, coverageOf, messageViewLines } from "../../core/co * session, no message ids). A notes-mode extractor prompt replaces the * session one; the auditor and merger are shared (identity + deterministic * appends don't care where text came from). Provenance is the file itself: - * source ids ("src_" + path hash) play the chatId role in mappings, events - * and checkpoints, and the source record (graphStore sources seam) carries - * the file path + the run's usage. No quiz events are ever built from - * notes — quiz outcomes inside the text surface only through the - * extractor's understanding observations (user-decided 2026-09-13). + * source ids ("src_" + path hash) play the chatId role in mappings and + * checkpoints, and the source record (graphStore sources seam) carries + * the file path + the run's usage. * * Observability (§5 baseline): every stage logs one line per step, its * start, and its duration; the pipeline logs a failure line before @@ -72,32 +80,23 @@ import { batchRuns, batchTag, coverageOf, messageViewLines } from "../../core/co * stats (or failed) for every run. */ -// ─── the transcript view (what the extractor + auditor see) ───────────────── +// ─── the source view (what the extractor sees) ────────────────────────────── /** - * A quiz tool part's outcome, one line: the attribution evidence. - * (question text + what the learner did — kind/correct are rebuilt by the - * merger; the extractor only needs enough to attribute.) - */ -/** - * The compact transcript both LLM stages see: user/teacher text VERBATIM, - * each tagged with its message id (provenance anchors — the extractor - * reports them back per proposition/edge), and quiz outcome lines. - * TREE-BLIND by design (2026-09-02): no plan tree, no mark_done — - * extraction is free-form, decoupled from the session's plan. Job tool - * parts (research/demo/visual) are noise for extraction — their outcomes - * are summarized as one line each. + * A session rendered as readable markdown — the EXTRACTOR's view (owner + * ruling 2026-09-27): the whole session goes in, formatted by the ONE + * shared session-markdown engine (core/session-markdown.ts — the same + * engine the user-facing session export will use), with the message-id + * anchors preserved (the extractor reports them back per + * proposition/edge — they become provenance). TREE-BLIND by design: + * no plan tree, no mark_done. + * + * Presentation-only: coverage fingerprints hash core/coverage.ts's line + * format, which is SEPARATE and unchanged — re-formatting the view never + * re-opens covered history (the deliberate 2026-09-27 split). */ export function transcriptView(record: ChatRecord): string { - // The shared seam (core/coverage.ts messageViewLines): the LLM's prompt - // view and the coverage fingerprint use ONE line builder, so "covered" - // means — exactly — what the extractor saw. Wave D: the record is native - // history — the view reads the native messages directly. - const lines = [`SESSION: ${record.title}`]; - record.messages.forEach((message, index) => { - lines.push(...messageViewLines(message, (message as { id?: string }).id || `m${index}`)); - }); - return lines.join("\n"); + return sessionMarkdown(record.messages, { title: record.title }); } // ─── the extractor ────────────────────────────────────────────────────────── @@ -111,24 +110,6 @@ const messagesSchema = Type.Array(Type.String(), { description: "The message ids (from the [id] tags in the transcript) where this came up — provenance.", }); -const understandingSchema = Type.Optional( - Type.Union( - [ - Type.Literal("solid"), - Type.Literal("shaky"), - Type.Literal("accepted"), - Type.Literal("resisted"), - Type.Literal("not yet engaged"), - ], - { - description: - "ONE word — the learner's grasp of this proposition in THIS session, from the fixed scale " + - '"solid" | "shaky" | "accepted" | "resisted" | "not yet engaged". Nothing else: no numbers, ' + - "no justification, no explanation.", - }, - ), -); - const propositionProps = { id: Type.String({ description: 'Local id: "p1", "p2", … — referenced by edges and attribution.' }), name: Type.String({ @@ -142,14 +123,13 @@ const propositionProps = { "The proposition as one reasoning step — a TRUE, self-contained, world-facing statement. " + 'Never reference the learner, the teacher, or the session: not "the learner understands X" but ' + '"X". A misconception is stated as the TRUE correction ("Nietzsche did not endorse ' + - 'might-makes-right social Darwinism") — the wrongness lives in the quiz outcomes, never here.', + 'might-makes-right social Darwinism") — wrongness is never recorded in the graph.', }), aliases: Type.Array(Type.String(), { description: "Verbatim phrasings this session used for the same proposition, if any.", }), messages: messagesSchema, evidence: evidenceSchema, - understanding: understandingSchema, }; const edgeSchema = Type.Object({ @@ -177,20 +157,6 @@ const findingsProps = { "Relations between propositions: follows (derivable) / supports (grounds without entailing). Pass [] when there are none — never omit the field.", }), ), - attribution: Type.Optional( - Type.Object( - { - quiz: Type.Record(Type.String(), Type.Union([Type.String(), Type.Null()]), { - description: - "toolCallId of EVERY quiz line → the proposition id it was really testing. Attribute RETROACTIVELY — the whole transcript is visible; map each quiz to the proposition it tests (often articulated elsewhere in the session). null when genuinely unattributable.", - }), - }, - { - description: - "Quiz attribution — your only event judgment. Pass { quiz: {} } when the session has no quiz lines — never omit the field.", - }, - ), - ), }; const findingsSchema = Type.Object(findingsProps); @@ -203,7 +169,7 @@ const auditProps = { linkTo: Type.Optional( Type.String({ description: - "The id of the EXISTING graph node this proposition IS — your identity verdict. Absent = create a new node. When unsure, omit: duplicates over wrong merges.", + "The id of the EXISTING graph node this proposition IS — your identity verdict. Absent = create a new node. EQUIVALENCE BIAS (2026-09-27): paraphrases, rewordings, and compressions of the same claim MERGE; only genuinely distinct propositions stay apart.", }), ), }), @@ -211,34 +177,15 @@ const auditProps = { description: "Every distinct proposition the session touched — true statements, misconception-free.", }, ), - dropped: Type.Optional( - Type.Array( - Type.Object({ - ref: Type.String({ - description: 'Proposition id, "from→to" for an edge, or attribution toolCallId.', - }), - reason: Type.String(), - }), - { - description: - "Everything you dropped, with the reason — the audit trail. Pass [] when you dropped nothing — never omit the field.", - }, - ), - ), }; const auditSchema = Type.Object(auditProps); -// The notes front door's shapes: everything session findings have, minus -// quiz attribution — raw notes have no quiz tool parts, so there is -// nothing to attribute (and no event to build: outcomes inside the text -// surface only through understanding observations, user-decided). -const { attribution: _omitted, ...notesFindingsProps } = findingsProps; -const notesFindingsSchema = Type.Object(notesFindingsProps); -const { attribution: _omitted2, ...notesAuditProps } = auditProps; -const notesAuditSchema = Type.Object(notesAuditProps); -type NotesFindings = Omit; -type NotesAuditResult = Omit; +// The notes front door's shapes are IDENTICAL to the session shapes now +// (the only difference was quiz attribution, which no longer exists): +// one schema, one findings type, both front doors. +type NotesFindings = SessionFindings; +type NotesAuditResult = AuditResult; // ── the weak-model ??-fills (the old zod defaults, applied at submit) ── @@ -246,39 +193,26 @@ function fillFindings(raw: Static): SessionFindings { return { propositions: raw.propositions, edges: raw.edges ?? [], - attribution: raw.attribution ?? { quiz: {} }, }; } -function fillNotesFindings(raw: Static): NotesFindings { - return { propositions: raw.propositions, edges: raw.edges ?? [] }; -} - function fillAudit(raw: Static): AuditResult { return { propositions: raw.propositions, edges: raw.edges ?? [], - attribution: raw.attribution ?? { quiz: {} }, - dropped: raw.dropped ?? [], }; } -function fillNotesAudit(raw: Static): NotesAuditResult { - return { propositions: raw.propositions, edges: raw.edges ?? [], dropped: raw.dropped ?? [] }; -} - const NOTES_EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE source — the user's own NOTES, in an ARBITRARY format (personal notes, an exported conversation, a document — anything) — and report what those notes show the author engaging with, as structured findings for their global knowledge graph. There is no session and no chat structure here: the source is raw text, the author's own material. Read it as such. +You are the ONLY stage that sees this source — you are fully trusted; report carefully. -Report, with verbatim evidence quotes for everything: - -NODE THEORY — the graph is a set of TRUE propositions about the world, full stop. A node is never anything about the -author or the source. The author's relationship to a proposition is expressed ENTIRELY by your understanding note; it -never leaks into the node itself. So no "the author understands X", no "mentioned in these notes": the node is just X. -A misconception is NOT a node of its own wrong claim — state the TRUE correction as the proposition. +The graph is a set of TRUE propositions about the world, full stop. A node is never anything about the author or the +source — no "the author understands X", no "mentioned in these notes": the node is just X. A misconception is NOT a +node of its own wrong claim — state the TRUE correction as the proposition. 1. PROPOSITIONS — every distinct reasoning-step-sized proposition the notes actually engage with: - Each has a "name" — a VERY short label, a few words, a compact handle like a glossary headword ("Will to power", @@ -288,20 +222,14 @@ A misconception is NOT a node of its own wrong claim — state the TRUE correcti one reasoning step each — never a topic blob, never a sub-step. - "messages": ALWAYS an empty list — provenance is the file itself. - "aliases": verbatim phrasings the notes used for the same proposition. - - "understanding": ONE word from the fixed scale — "solid", "shaky", "accepted", "resisted", or "not yet engaged" — - judging the AUTHOR's grasp of the proposition as the notes show it: their own commentary, self-assessments, - recorded quiz outcomes, the way they write about it. Strictly one word, no numbers, no justification (a - wrong-shaped note is rejected by the schema). When the notes give no signal, omit the field. 2. EDGES between propositions: - "follows": to is DERIVABLE from from (understanding from makes understanding to achievable). - "supports": from GROUNDS to without entailing it (helpful context, not a prerequisite). - Only relations the notes actually articulate — with evidence. -There is no quiz attribution for notes — the source has no quiz events, and the schema has no attribution field. - -Evidence discipline: every quote must appear VERBATIM in the source. The auditor rejects anything unbacked. When in -doubt, report; the auditor decides. +Evidence discipline: every quote must appear VERBATIM in the source — this is your self-grounding, and the proof the +statements come from the material rather than invention. Include EVERY top-level field of the call even when empty — "edges": [] when there are no relations. NEVER omit a field because it is empty. @@ -309,19 +237,18 @@ a field because it is empty. Your entire deliverable is ONE submit_findings call.`; const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session -transcript and report what was learned — as structured findings for the learner's global knowledge graph. +rendering and report what that session teaches — as structured findings for the learner's global knowledge graph. -The transcript view shows: the conversation verbatim (LEARNER / TEACHER lines, each tagged with its message id) -and QUIZ outcome lines. +The session is rendered as readable markdown: Teacher and Learner passages in order (each header carries its message +id — the [id] anchor), Quiz / Recall-check / Question blocks for the embedded checks, and one-line notes where other +tools ran. Read ALL of it as the session's teaching content. -Report, with verbatim evidence quotes for everything: +You are the ONLY stage that sees this source, and you are FULLY TRUSTED: nothing downstream re-checks your work. +Extract carefully — the graph will hold exactly what you report. -NODE THEORY — the graph is a set of TRUE propositions about the world, full stop. A node is never anything -about the learner or the session. The learner's relationship to a proposition — whether they know it, missed -it, resisted it — is expressed ENTIRELY by the quiz outcomes and your understanding note; it never leaks into -the node itself. So no "the learner understands X", no "discussed in this session": the node is just X. A -misconception is NOT a node of its own wrong claim — state the TRUE correction as the proposition, and let the -quiz outcomes carry that the learner believed the wrong version. +The graph is a set of TRUE propositions about the world, full stop. A node is never anything about the learner or the +session — no "the learner understands X", no "discussed in this session": the node is just X. A misconception is NOT +a node of its own wrong claim — state the TRUE correction as the proposition. 1. PROPOSITIONS — every distinct reasoning-step-sized proposition the session touched: - Each has a "name" — a VERY short label, a few words, a compact handle like a glossary headword @@ -332,26 +259,19 @@ quiz outcomes carry that the learner believed the wrong version. in the graph only as this one statement). - Include corrections of misconceptions the teacher articulated, and true facts explained in prose. - One reasoning step per proposition: never a topic blob, never a sub-step. - - "messages": the [id] tags of every message where the proposition came up — provenance, report it. + - "messages": the id from the header anchor of every passage where the proposition came up — provenance, report it. - "aliases": verbatim phrasings this session used for the same proposition. - - "understanding": ONE word from the fixed scale — "solid", "shaky", "accepted", "resisted", or - "not yet engaged" — judging the learner's grasp of the proposition in THIS session. Strictly one - word, no numbers, no justification, no explanation (a wrong-shaped note is rejected by the schema). 2. EDGES between propositions: - "follows": to is DERIVABLE from from (understanding from makes understanding to achievable). - "supports": from GROUNDS to without entailing it (helpful context, not a prerequisite). - - Only relations the transcript actually articulates — with evidence. + - Only relations the session actually articulates — with evidence. -3. ATTRIBUTION — every QUIZ line gets a mapping: toolCallId → the proposition id it was really testing. - Attribute RETROACTIVELY — a quiz often tests a proposition the session only articulates later; the - question text is your guide. If you truly cannot tell, use null. +Evidence discipline: every quote must appear VERBATIM in the source markdown (a Quiz block's question counts as the +source). This is your self-grounding: the proof that statements come from the material rather than invention. -Evidence discipline: every quote must appear VERBATIM in the transcript view (a QUIZ line counts as the -transcript). The auditor rejects anything unbacked. When in doubt, report; the auditor decides. - -Include EVERY top-level field of the call even when empty — "edges": [], "attribution": { quiz: {} } when there -are no quizzes. NEVER omit a field because it is empty. +Include EVERY top-level field of the call even when empty — "edges": [] when there are no relations. NEVER omit a +field because it is empty. Your entire deliverable is ONE submit_findings call.`; @@ -375,10 +295,17 @@ async function runSubmitAgent(opts: { signal: AbortSignal | undefined; onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void; neverSubmitted: string; + /** Extra tools beside the submit call (the auditor's graph-exploration + * set, 2026-09-27): a small agentic loop instead of a bare retry. */ + extraTools?: Record>; + /** Turn budget for the loop (default 4 = retry-only; the auditor runs + * an exploration loop instead — 16). */ + maxTurns?: number; }): Promise { let submitted: T | undefined; const result = await runSubagent( [ + ...(opts.extraTools ? Object.values(opts.extraTools) : []), { name: opts.toolName, label: opts.toolName, @@ -395,7 +322,7 @@ async function runSubmitAgent(opts: { engine: opts.engine, systemPrompt: opts.systemPrompt, prompt: opts.prompt, - maxTurns: 4, + maxTurns: opts.maxTurns ?? 4, abortSignal: opts.signal, onAssistantMessage: (message, step) => opts.onStep( @@ -450,8 +377,8 @@ function runNotesExtractor( role: "extractor", toolName: "submit_findings", toolDescription: "Submit the notes findings.", - parameters: notesFindingsSchema, - fill: fillNotesFindings, + parameters: findingsSchema, + fill: fillFindings, systemPrompt: NOTES_EXTRACTOR_INSTRUCTIONS, prompt, engine, @@ -464,68 +391,52 @@ function runNotesExtractor( // ─── the auditor ─────────────────────────────────────────────────────────── -const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive -ONE source — a session transcript, or the author's personal notes in an ARBITRARY format (the notes front door) — a -set of proposed findings (propositions, edges, and quiz attribution where the source has quiz lines), and the -CURRENT knowledge graph. Your jobs: the faithfulness net, and identity. +const AUDITOR_INSTRUCTIONS = `You are the linking stage of a learning application's extraction pipeline. An extractor +has read a source (a teaching session, or the author's personal notes) and reported findings: propositions and +follows/supports edges. You NEVER see the source itself — the extractor is fully trusted, and you do not re-check +its work. Your job is IDENTITY: connecting those findings to the CURRENT knowledge graph, preparing the merge. -FAITHFULNESS — verify every candidate against the source text: -- PROPOSITIONS: every evidence quote must appear (near-)verbatim in the transcript view, and the statement - must be a faithful reading of it. The node theory applies: a statement must be a TRUE, self-contained, - world-facing fact — reword learner- or session-relative statements ("the learner understands X" → "X"), state - misconceptions as the true correction, or drop them when no objective fact is backed. Drop invented or - bloated statements. Conservative: when two proposed propositions are really one, keep the better-stated one - and drop the other. -- EDGES: the evidence must actually articulate the relation, in the direction claimed. Drop - direction-swapped or speculative edges. -- ATTRIBUTION: the QUIZ line's question must genuinely test the attributed proposition. Drop wrong - attributions to null, or re-point them to the proposition the question actually tests. (Session - sources only — notes findings carry no attribution; there is nothing to verify.) -- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence — or, for notes, - the author's own words; each note must be - exactly one word from the fixed scale ("solid", "shaky", "accepted", "resisted", "not yet engaged") — - reword or drop anything else. No justifications. +You explore the graph through your tools: find (glob over names + statements + aliases), list_sessions, and +list_propositions (full statement text, per session). The graph may be large — search, don't assume. IDENTITY — link-or-create, per proposition: -- You see the current graph (every node: id, name, statement, aliases). For each verified proposition, - decide: does an EXISTING node already state this same proposition? -- If yes: set "linkTo" to that node's id. The session's phrasings will join its aliases and its provenance; - the node keeps its existing name (labels stay stable across sessions — ignore your name for linked nodes). -- If no: omit "linkTo" and a new node is created, with the extractor's name as its label. Reword only a - clearly unglanceable name (an unwieldy sentence of a name); keep it short. -- When unsure, omit — duplicates are recoverable later; wrong merges are not. +- For each proposition, search the graph (its key words, its synonyms, its phrasings). Does an EXISTING node already + state this same proposition? +- If yes: set "linkTo" to that node's id. The session's phrasings join its aliases and its provenance; the node + keeps its existing name (labels stay stable — ignore your name for linked nodes). +- If no: omit "linkTo" and a new node is created with the extractor's name as its label. +- EQUIVALENCE BIAS (owner ruling): paraphrases, rewordings, and compressions of the same claim MERGE. When two + statements are the same claim at different precision or from different angles, LINK them. Only genuinely distinct + propositions — different claims about the world — stay apart. You are NOT guarding against wrong merges; you are + hunting for right ones. -Also drop propositions not referenced by anything else ONLY if they have no other backing — an unreferenced but -well-evidenced proposition is still knowledge. +BRIDGING EDGES — the one place you create edges: +- You may add edges whose endpoints are an EXISTING graph node id and one of the batch's propositions (use the + existing node's real id as "from" or "to"). This is how the new material gets welded into the old graph: when the + existing graph's content makes a proposition follow, or grounds it, draw the edge. +- Keep edges typed like the extractor's: "follows" = derivable; "supports" = grounds without entailing. +- NEVER drop or rewrite the extractor's own propositions or edges — pass them through unchanged (adding linkTo is + the only edit). Never invent propositions. -Never invent new candidates. Your entire deliverable is ONE submit_review call containing the verified findings -(the same shapes you received, with linkTo added) plus the dropped list. +Reproduce every proposition and edge you received, with their evidence and messages verbatim. Your entire deliverable +is ONE submit_review call. -Include EVERY top-level field of the call even when empty — "edges": [], "dropped": []. NEVER omit a field -because it is empty.`; +Include EVERY top-level field of the call even when empty — "edges": []. NEVER omit a field because it is empty.`; -/** The current graph, compact — the auditor's identity context. */ -function graphView(nodes: GraphNode[]): string { - if (nodes.length === 0) return "(the graph is currently empty — every proposition is a new node)"; - const lines = nodes.map( - (n) => - `- ${n.id} :: ${n.name}\n statement: ${n.statement}` + - (n.aliases.length > 0 ? `\n aliases: ${n.aliases.join(" | ")}` : ""), - ); - return lines.join("\n"); -} - -/** The auditor over session findings (the chat front door). */ +/** The auditor — one shape for both front doors now (the notes schema was + * identical once attribution died). An agentic loop: graph-exploration + * tools plus the one submit call. */ function runAuditor( engine: PiEngine | undefined, prompt: string, signal: AbortSignal | undefined, onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, + tools: Record>, ): Promise { return runSubmitAgent({ role: "auditor", toolName: "submit_review", - toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.", + toolDescription: "Submit the findings with identity verdicts (linkTo) and any bridging edges — for the merge.", parameters: auditSchema, fill: fillAudit, systemPrompt: AUDITOR_INSTRUCTIONS, @@ -534,29 +445,8 @@ function runAuditor( signal, onStep, neverSubmitted: "the auditor never submitted a review", - }); -} - -/** The auditor over notes findings (the format-agnostic front door): same - * faithfulness + identity job, notes-mode schema (no attribution). */ -function runNotesAuditor( - engine: PiEngine | undefined, - prompt: string, - signal: AbortSignal | undefined, - onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, -): Promise { - return runSubmitAgent({ - role: "auditor", - toolName: "submit_review", - toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.", - parameters: notesAuditSchema, - fill: fillNotesAudit, - systemPrompt: AUDITOR_INSTRUCTIONS, - prompt, - engine, - signal, - onStep, - neverSubmitted: "the auditor never submitted a review", + extraTools: tools, + maxTurns: 16, }); } @@ -683,7 +573,7 @@ export interface ExtractionPipeline { /** The format-agnostic front door (2026-09-13): run the same pipeline * over ONE raw notes file. The source id is deterministic per file * path, so re-importing the same file re-runs extraction against the - * same source identity (mappings + events dedupe by it). */ + * same source identity (mappings dedupe by it). */ importNotes(input: NotesImportInput): Promise; } @@ -770,12 +660,12 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti async function extract(chatId: string, view: string): Promise { const signal = runningExtractions.get(chatId)?.controller.signal; signal?.throwIfAborted(); // an abort during a sync gap ends the run here - console.info(`[extraction ${chatId}] extractor starting (${view.split("\n").length} transcript lines)`); + console.info(`[extraction ${chatId}] extractor starting (${view.split("\n").length} markdown lines)`); const t0 = performance.now(); const findings = await runExtractor( stageEngine("extractor", deps.extractor), - `Transcript:\n\n${view}`, + view, signal, (usage, model, step, finishReason, tools) => { recordStep(chatId, "extractor", callUsage(usage), model); @@ -786,36 +676,36 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti return findings; } - async function audit(chatId: string, findings: SessionFindings, view: string): Promise { + async function audit(chatId: string, findings: SessionFindings): Promise { const signal = runningExtractions.get(chatId)?.controller.signal; signal?.throwIfAborted(); // an abort during the extract→audit gap ends here const graph = getGraph(graphStore); console.info( `[extraction ${chatId}] auditor starting (${findings.propositions.length} propositions, ${findings.edges.length} edges, ` + - `${Object.keys(findings.attribution.quiz).length} attributions, ${graph.nodes.length} graph nodes)`, + `${graph.nodes.length} graph nodes)`, ); const t0 = performance.now(); + // The auditor NEVER sees the source (owner 2026-09-27): findings + // JSON + identity instructions, and the graph reached through + // exploration tools instead of an injected dump. const auditResult = await runAuditor( stageEngine("auditor", deps.auditor), - `Transcript:\n\n${view}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}\n\n` + - `Current knowledge graph (${graph.nodes.length} nodes):\n\n${graphView(graph.nodes)}`, + `Proposed findings from the extractor (the source itself is not available to you — the extractor is fully trusted):\n\n` + + `${JSON.stringify(findings, null, 2)}\n\n` + + `The current knowledge graph has ${graph.nodes.length} node(s). Explore it with your tools and link what links.`, signal, (usage, model, step, finishReason, tools) => { recordStep(chatId, "auditor", callUsage(usage), model); console.info(`[extraction] auditor step ${step}: ${finishReason}, tools: ${tools}`); }, + auditorGraphTools({ chatStore, graphStore }), ); - if (auditResult.dropped.length > 0) { - console.info( - `[extraction ${chatId}] audit dropped ${auditResult.dropped.length}: ` + - auditResult.dropped.map((d) => `${d.ref} (${d.reason})`).join("; "), - ); - } + const bridged = auditResult.edges.filter((e) => !e.from.startsWith("p") || !e.to.startsWith("p")).length; console.info( - `[extraction ${chatId}] audit kept ${auditResult.propositions.length}/${findings.propositions.length} propositions, ` + - `${auditResult.edges.length}/${findings.edges.length} edges, ` + - `${auditResult.propositions.filter((p) => p.linkTo !== undefined).length} linked to existing nodes`, + `[extraction ${chatId}] audit: ${auditResult.propositions.length} propositions through, ` + + `${auditResult.propositions.filter((p) => p.linkTo !== undefined).length} linked to existing nodes, ` + + `${auditResult.edges.length} edges (${bridged} bridging)`, ); console.info(`[extraction ${chatId}] auditor done in ${Math.round(performance.now() - t0)}ms`); return auditResult; @@ -848,7 +738,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti } async function runExtraction(chatId: string): Promise { - if (inFlight.has(chatId)) return { nodes: 0, edges: 0, events: 0 }; + if (inFlight.has(chatId)) return { nodes: 0, edges: 0 }; inFlight.add(chatId); const t0 = performance.now(); // COVERAGE (vision-extraction-coverage.md, 2026-09-15): extract ONLY @@ -859,7 +749,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti // error below loses at most the in-flight batch — every earlier // batch stays covered, the rest stays visibly uncovered. let batchesDone = 0; - const total: EmissionStats = { nodes: 0, edges: 0, events: 0 }; + const total: EmissionStats = { nodes: 0, edges: 0 }; try { const record = chatStore.getChat(chatId); if (!record) throw new Error(`Chat not found: ${chatId}`); @@ -871,7 +761,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti const covered = coverageOf(record.messages, graphStore.readCoverage(), chatId); if (covered.uncovered.length === 0) { console.info(`[extraction ${chatId}] nothing uncovered — no-op`); - return { nodes: 0, edges: 0, events: 0 }; + return { nodes: 0, edges: 0 }; } const chunkTokens = deps.getChunkTokens?.() ?? FACTORY_SETTINGS.graph.extractionChunkTokens; // The batcher's one rule (owner, Wave D): a tool-call message and @@ -883,17 +773,19 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti `${covered.uncoveredCount} messages, ${batches.length} batch(es) of ≤${chunkTokens} tok`, ); for (const batch of batches) { - // The batch's prompt view: SESSION header + the batch's lines, - // same provenance [id] tags the fingerprints used (coverage - // and prompt share ONE line builder — core/coverage.ts). - const view = [ - `SESSION: ${record.title}`, - ...batch.flatMap((slot) => messageViewLines(slot.message, slot.message.id)), - ].join("\n"); + // The batch's source view: the ONE shared markdown engine + // (owner 2026-09-27) over the batch's messages — the same + // rendering the user-facing session export will use, id + // anchors preserved. The fingerprint's line format is + // separate and unchanged (no mass re-extraction). + const view = sessionMarkdown( + batch.map((slot) => slot.message), + { title: record.title }, + ); setPhase(chatId, "extracting", "chat"); const findings = await extract(chatId, view); setPhase(chatId, "auditing", "chat"); - const auditResult = await audit(chatId, findings, view); + const auditResult = await audit(chatId, findings); setPhase(chatId, "merging", "chat"); // SESSION-DELETION GUARD (wave 6): a chat deleted mid-run must never // merge. The deletion route aborts this run first and runs its purge @@ -911,20 +803,17 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti // one and it re-opens: honest by construction. const foldedAt = new Date().toISOString(); const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }, { - messages: batch.map((slot) => slot.message), foldedAt, coverage: batch .filter((slot) => messageViewLines(slot.message, slot.message.id).length > 0) .map((slot) => ({ chatId, fp: slot.fp, tokens: slot.tokens, foldedAt })), - batchTag: batchTag(batch), }); total.nodes += stats.nodes; total.edges += stats.edges; - total.events += stats.events; batchesDone++; console.info( `[extraction ${chatId}] batch ${batchesDone}/${batches.length} merged: ` + - `${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events`, + `${stats.nodes} nodes, ${stats.edges} edges`, ); } const stats = total; @@ -933,7 +822,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti settleGraphLayout(graphStore); persistRunUsage(chatId, Math.round(performance.now() - t0), stats); console.info( - `[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + + `[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges ` + `(${Math.round(performance.now() - t0)}ms total)`, ); return stats; @@ -989,42 +878,12 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti return findings; } - async function notesAudit(input: { id: string; filePath: string; text: string }, findings: NotesFindings): Promise { - const signal = runningExtractions.get(input.id)?.controller.signal; - signal?.throwIfAborted(); // an abort during the extract→audit gap ends here - const graph = getGraph(graphStore); - console.info( - `[extraction ${input.id}] notes auditor starting (${findings.propositions.length} propositions, ` + - `${findings.edges.length} edges, ${graph.nodes.length} graph nodes)`, - ); - const t0 = performance.now(); - const auditResult = await runNotesAuditor( - stageEngine("auditor", deps.auditor), - `Source:\n\n${notesView(input)}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}\n\n` + - `Current knowledge graph (${graph.nodes.length} nodes):\n\n${graphView(graph.nodes)}`, - signal, - (usage, model, step, finishReason, tools) => { - recordStep(input.id, "auditor", callUsage(usage), model); - console.info(`[extraction] auditor step ${step}: ${finishReason}, tools: ${tools}`); - }, - ); - if (auditResult.dropped.length > 0) { - console.info( - `[extraction ${input.id}] audit dropped ${auditResult.dropped.length}: ` + - auditResult.dropped.map((d) => `${d.ref} (${d.reason})`).join("; "), - ); - } - console.info( - `[extraction ${input.id}] audit kept ${auditResult.propositions.length}/${findings.propositions.length} propositions, ` + - `${auditResult.propositions.filter((p) => p.linkTo !== undefined).length} linked to existing nodes`, - ); - console.info(`[extraction ${input.id}] notes auditor done in ${Math.round(performance.now() - t0)}ms`); - return auditResult; - } + // The notes auditor is the SAME identity-linker as the chat one (one + // schema now that attribution is gone) — audit() above, verbatim. async function runNotesExtraction(input: { id: string; filePath: string; text: string; importedAt: string }): Promise { const { id } = input; - if (inFlight.has(id)) return { nodes: 0, edges: 0, events: 0 }; + if (inFlight.has(id)) return { nodes: 0, edges: 0 }; inFlight.add(id); const t0 = performance.now(); try { @@ -1036,18 +895,14 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti setPhase(id, "extracting", "notes", input.filePath); const findings = await notesExtract(input); setPhase(id, "auditing", "notes", input.filePath); - const notesResult = await notesAudit(input, findings); + const auditResult = await audit(id, findings); setPhase(id, "merging", "notes", input.filePath); - // Normalize to the merger's findings shape: notes carry no quiz - // attribution, and the merger scans no quiz parts for notes - // (messages: null) — nothing to attribute, nothing to build. - const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } }; - const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt }); + const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { foldedAt: input.importedAt }); // Same post-merge settle as the chat path. settleGraphLayout(graphStore); persistRunUsage(id, Math.round(performance.now() - t0), stats); console.info( - `[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + + `[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges ` + `(${Math.round(performance.now() - t0)}ms total)`, ); return stats; @@ -1055,7 +910,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti persistRunUsage(id, Math.round(performance.now() - t0), undefined); if (err instanceof Error && err.name === "AbortError") { console.info(`[extraction ${id}] aborted (${Math.round(performance.now() - t0)}ms) — nothing merged, nothing checkpointed`); - return { nodes: 0, edges: 0, events: 0 }; + return { nodes: 0, edges: 0 }; } console.error(`[extraction ${id}] notes pipeline failed (${Math.round(performance.now() - t0)}ms):`, err); throw err; diff --git a/src/agent/tools/graph.ts b/src/agent/tools/graph.ts index b9130aa..fcf703c 100644 --- a/src/agent/tools/graph.ts +++ b/src/agent/tools/graph.ts @@ -1,7 +1,7 @@ import { Type } from "typebox"; import type { AgentTool } from "@earendil-works/pi-agent-core"; import { serverExecute } from "../tool-execute.ts"; -import { graphSessionIndex, graphSessionPropositions } from "../../core/graph.ts"; +import { getGraph, graphSessionIndex, graphSessionPropositions } from "../../core/graph.ts"; import type { ChatStore, GraphStore } from "../../core/ports.ts"; /** @@ -63,18 +63,107 @@ export function proberGraphTools(ctx: GraphToolContext): Record => { const propositions = graphSessionPropositions(deps, session); if (propositions.length === 0) { return `No propositions born in session "${session}". Call list_sessions for the current ids.`; } - const lines = propositions.map( - (p) => `- ${p.name} — ${p.understanding ?? "never evaluated"}${p.evaluatedAt ? ` (as of ${p.evaluatedAt})` : ""}`, + const lines = propositions.map((p) => `- ${p.name}`); + return `${propositions.length} proposition(s) covered in this session:\n${lines.join("\n")}`; + }), + }, + }; +} + +// ─── the auditor's graph tools (2026-09-27 relink) ─────────────────────────── + +/** + * THE AUDITOR'S TOOLSET — the extraction auditor explores the graph to + * place new propositions instead of receiving it injected (owner ruling + * 2026-09-27: a full-graph dump bloats context as the graph grows; the + * auditor looks up identity candidates itself). Auditor-specific + * versions, deliberately different from the prober's: + * + * find glob over node names + statements + aliases — the + * identity workhorse; returns id + name + statement + * (statement text is ESSENTIAL for identity work, + * unlike the prober's names-only rule). + * list_sessions the birth-session index, no timestamps. + * list_propositions full proposition text (same reasoning as find). + * + * No timestamps anywhere: identity work needs content, not chronology. + * Read-only, pure core seams, same ports the snapshot reads. + */ + +/** Glob match — `*` spans anything, case-insensitive. `*needle*` is the + * expected shape; a bare term gets implicit surrounding stars. */ +function globMatch(pattern: string, text: string): boolean { + const raw = pattern.includes("*") ? pattern : `*${pattern}*`; + const re = new RegExp( + `^${raw.replace(/[.*+?^${}()|[\]\\]/g, (ch) => (ch === "*" ? "[\\s\\S]*" : `\\${ch}`))}$`, + "i", + ); + return re.test(text); +} + +export function auditorGraphTools(ctx: GraphToolContext): Record> { + const deps = { chatStore: ctx.chatStore, graphStore: ctx.graphStore }; + const findInput = Type.Object({ + query: Type.String({ + minLength: 1, + description: "Glob pattern over node names, statements, and aliases — e.g. *will to power*. A bare term gets implicit surrounding *.", + }), + }); + return { + find: { + name: "find", + label: "find", + description: + "Search the knowledge graph for existing propositions — glob over names, statements, and aliases. " + + "Returns each match's id, name, and full statement. This is how you decide whether a new proposition IS an existing one: search its key words, its synonyms, its phrasings.", + parameters: findInput, + execute: serverExecute("find", findInput, async ({ query }): Promise => { + const nodes = getGraph(deps.graphStore).nodes; + const hits = nodes.filter( + (n) => globMatch(query, n.name) || globMatch(query, n.statement) || n.aliases.some((a) => globMatch(query, a)), ); + if (hits.length === 0) return `No existing proposition matches "${query}".`; + const lines = hits.map((n) => `- ${n.id} :: ${n.name}\n statement: ${n.statement}`); + return `${hits.length} match(es):\n${lines.join("\n")}`; + }), + }, + list_sessions: { + name: "list_sessions", + label: "list_sessions", + description: + "List the knowledge graph's birth sessions — every chat or imported-notes file that ever contributed a proposition. " + + "Each line: name, kind, proposition count, id. Use the id with list_propositions to see what a session covered.", + parameters: Type.Object({}), + execute: serverExecute("list_sessions", Type.Object({}), async (): Promise => { + const sessions = graphSessionIndex(deps); + if (sessions.length === 0) return "The knowledge graph is empty — no session has contributed propositions yet."; + const lines = sessions.map((s, i) => `${i + 1}. ${s.name} (${s.kind}, ${s.propositions} proposition${s.propositions === 1 ? "" : "s"}) — id: ${s.id}`); + return `${sessions.length} session(s):\n${lines.join("\n")}`; + }), + }, + list_propositions: { + name: "list_propositions", + label: "list_propositions", + description: + "List the propositions BORN in one session, with FULL statement text — the content-level view for identity work " + + "(deciding whether a new proposition is really one of these).", + parameters: listPropositionsInput, + execute: serverExecute("list_propositions", listPropositionsInput, async ({ session }): Promise => { + const propositions = graphSessionPropositions(deps, session); + if (propositions.length === 0) return `No propositions born in session "${session}". Call list_sessions for the current ids.`; + const nodes = new Map(getGraph(deps.graphStore).nodes.map((n) => [n.id, n])); + const lines = propositions.map((p) => { + const node = nodes.get(p.nodeId); + return `- ${p.nodeId} :: ${p.name}\n statement: ${node?.statement ?? "(unknown)"}`; + }); return `${propositions.length} proposition(s) born in this session:\n${lines.join("\n")}`; }), }, diff --git a/src/core/contracts/chat.ts b/src/core/contracts/chat.ts index 6efd56f..dc5a3fb 100644 --- a/src/core/contracts/chat.ts +++ b/src/core/contracts/chat.ts @@ -9,7 +9,7 @@ import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; import type { LhMessage } from "../native-messages.ts"; import type { Phase3 } from "../phases.ts"; import type { PlanNodeStatus, PlanTree } from "./plan.ts"; -import type { EmissionStats, UnderstandingLevel } from "./graph.ts"; +import type { EmissionStats } from "./graph.ts"; // ─── Chat records (the session store's on-disk format) ─────────────────────── diff --git a/src/core/contracts/graph.ts b/src/core/contracts/graph.ts index 6c7b945..a5e8703 100644 --- a/src/core/contracts/graph.ts +++ b/src/core/contracts/graph.ts @@ -5,10 +5,11 @@ * 2026-09-02 redesign): every node is a TRUE proposition; edges are follows * (derivable) and supports (grounds without entailing); provenance is * message-granular; extraction is free-form and post-hoc, fully decoupled - * from the session's plan tree; the event ledger records quiz outcomes plus - * the extractor's per-proposition understanding observations. Anything not - * yet decided (the state model, strength derivation, understanding - * treatment, selection strategies) is deliberately NOT here — transitional + * from the session's plan tree. The graph is an extraction of the reasoning + * chains from all teaching sessions (and imported notes) — NOTHING else: + * no quiz outcomes, no understanding assessments (2026-09-27 cleanup — + * understanding becomes practice-module territory). Anything not + * yet decided (the state model, strength derivation, selection strategies) is deliberately NOT here — transitional * stubs carry the renderer until those land. * * Part of the domain-split contracts (src/core/contracts/): each domain file @@ -31,13 +32,13 @@ export interface EmissionStats { nodes: number; /** New edge assertions appended. */ edges: number; - /** Ledger events rebuilt (quiz outcomes + observations). */ - events: number; } /** One provenance point — message-granular (DECIDED 2026-09-02): nodes and * edges record every (chat, message) where they came up, deep-linkable to - * the exact transcript point. */ + * the exact transcript point. Auditor-drawn bridging edges carry the chat + * but NO message anchor (nothing in the transcript asserts them — the + * auditor is the asserter); messageId is "" there. */ export interface ProvenancePoint { /** The chat this point anchors to — or a notes source id ("src_…"): * imports carry no message anchors (provenance is the file itself, @@ -50,11 +51,10 @@ export interface ProvenancePoint { * (DECIDED 2026-09-13): personal notes in ARBITRARY formats run through the * same extractor → auditor → merger pipeline, with the file itself as the * provenance anchor. There is no chat: the detail card shows the file path - * as a STATIC entry (not a link), and understanding observations still - * apply (judged from the author's own voice in the notes). The run's usage - * log lives here, mirroring ChatRecord.extractions. Ids are "src_" + hash - * of the file path — deterministic, so re-importing the same file re-runs - * extraction against the same source id (mappings/events dedupe by it). */ + * as a STATIC entry (not a link). The run's usage log lives here, + * mirroring ChatRecord.extractions. Ids are "src_" + hash of the file + * path — deterministic, so re-importing the same file re-runs extraction + * against the same source id (mappings dedupe by it). */ export interface NotesSource { id: string; filePath: string; @@ -149,39 +149,9 @@ export interface CoverageEntry { foldedAt: string; } -/** - * Event kinds (DECIDED 2026-09-02): quiz outcomes only — correct / miss / - * dont-know — plus `observation`, the extractor's per-proposition - * understanding note joining the ledger as a session-scoped observation - * (named-word scale; treatment pending the state-model ideation). No - * `taught` events at all; no misconception kind — a misconception is a - * TRUE correction in the graph, and the wrongness lives here, in the - * ledger, until the state model exists. - */ -export type LedgerEventKind = "correct" | "miss" | "dont-know" | "observation"; - -export interface LedgerEvent { - /** null = not yet attributed. */ - nodeId: string | null; - kind: LedgerEventKind; - /** The chat that produced this evidence — or a notes source id - * ("src_…") for imported-notes observations. */ - chatId: string; - at: string; - /** Idempotency key: the quiz part's toolCallId, or `obs:` for - * understanding observations — re-emission of the same chat must never - * double-record an event. */ - ref?: string; - /** The learner's quiz note (diagnostic gold), or the understanding - * LEVEL word for observations — a session-relative judgment. */ - note?: string; - /** The quiz question text, when nodeId is null (later attribution input). */ - question?: string; -} - /** * Store format (src/store/graph.ts): JSONL ledgers at config.LEDGER_FILES - * (nodes/edges/events/mappings/sessions), all append-only. + * (nodes/edges/mappings/sessions), all append-only. * * nodes.jsonl one line per node — the FIRST line for an id defines * name/statement/createdAt; later lines for the @@ -198,27 +168,23 @@ export interface LedgerEvent { * (never extract the same turns twice). * * The EXTRACTION PIPELINE is the sole graph writer (extractor → auditor → - * merger), tree-blind and post-hoc: the extractor reads only the transcript - * and proposes propositions/edges/quiz-attribution with verbatim evidence; - * the auditor sees the extractor's output PLUS the current graph, owns - * faithfulness (evidence check) and identity (link-or-create: point each - * proposition at the existing node it is, or let it be created — recording - * the session's phrasings as aliases); the merger (core/graph.ts, - * deterministic) applies the verdicts and appends. + * merger), tree-blind and post-hoc (2026-09-27 cleanup — clear separation + * of work): the extractor is the ONLY stage that sees the raw source + * (whole session as readable markdown, or raw notes) and extracts the + * reasoning chains — propositions (lonely relevant ones included) plus + * follows/supports edges, with verbatim evidence; the auditor NEVER sees + * the source — its job is preparing the merge: identity verdicts + * (link-or-create, merging equivalent forms of the same proposition) via + * graph-exploration tools (find / list_sessions / list_propositions), and + * it may draw BRIDGING edges from existing graph nodes to the new + * propositions; the merger (core/graph.ts, deterministic) applies the + * verdicts and appends. Quiz events and understanding observations are + * GONE — the pipeline is the only session→graph bridge, and it carries + * propositions + edges, nothing else. */ // ─── Extraction findings (extractor output → audit → merger input) ─────────── -/** - * The extractor's standardized per-session understanding scale (DECIDED - * 2026-09-08): a CLOSED named-word list — no numbers, no prose, no - * justifications (a justification, if the agent wants one, is dropped; - * it is not stored). The level is session- and extraction-run-relative - * judgment about the learner, shown next to its session — never a node - * property. - */ -export type UnderstandingLevel = "solid" | "shaky" | "accepted" | "resisted" | "not yet engaged"; - /** One proposed proposition — a node candidate (or a re-encounter of one). */ export interface ExtractedProposition { /** Extractor-local id ("p1", …) referenced by edges + attribution. */ @@ -231,7 +197,8 @@ export interface ExtractedProposition { /** The minimal proposition — a true, self-contained, world-facing * statement, at full precision. No learner / teacher / session * references: misconceptions are stated as the true correction, and - * wrongness lives in the event ledger, never in the node. */ + * wrongness is never recorded in the graph (2026-09-27: the event + * ledger is gone). */ statement: string; /** Verbatim phrasings the session used. */ aliases: string[]; @@ -239,17 +206,17 @@ export interface ExtractedProposition { messages: string[]; /** Verbatim transcript quotes backing the proposition. */ evidence: string[]; - /** The extractor's understanding level for this proposition IN THIS - * session — one word from the UnderstandingLevel scale. Joins the - * ledger as an observation event (note = the level word); treatment - * pending the state-model ideation. */ - understanding?: UnderstandingLevel; } -/** One proposed relation between two propositions. */ +/** One proposed relation. Endpoints are extractor-local proposition ids + * ("p1", …); the AUDITOR may also emit bridging edges whose from/to is an + * EXISTING graph node id (connecting the new content to what the graph + * already holds) — the merger resolves both. */ export interface ExtractedEdge { - /** Proposition id that is followed / supported. */ + /** Proposition id (or, for auditor-drawn bridges, a graph node id) + * that is followed / supported. */ from: string; + /** Proposition id (or graph node id) that follows / is supported. */ to: string; type: EdgeType; /** The message ids where this relation was asserted — provenance. */ @@ -258,26 +225,10 @@ export interface ExtractedEdge { evidence: string[]; } -/** - * The extractor's quiz attribution — its ONLY say about quiz events. Event - * kinds, timestamps, idempotency refs, notes and question texts are all - * rebuilt deterministically by the merger from the transcript's quiz tool - * parts; the LLM supplies exactly the judgment call: WHICH proposition each - * quiz tested. Tree-blind, so EVERY quiz attributes here (there is no tree - * mapping to lean on) — retroactively against the whole transcript. - */ -export interface ExtractionAttribution { - /** quiz toolCallId → proposition id; null / missing = unattributed (the - * event lands with nodeId null + the question text, a later- - * attribution input). */ - quiz: Record; -} - -/** The extractor's output shape. */ +/** The extractor's output shape: the raw reasoning chains. */ export interface SessionFindings { propositions: ExtractedProposition[]; edges: ExtractedEdge[]; - attribution: ExtractionAttribution; } /** The auditor's verdict on one proposition: the findings, plus `linkTo` — @@ -286,20 +237,19 @@ export interface AuditedProposition extends ExtractedProposition { /** The existing GraphNode id this proposition IS — the auditor's * identity verdict. On link, the session's phrasings (statement, * aliases) join the node's aliases and the provenance unions. - * Conservative: when unsure, leave it out — duplicates over wrong - * merges. */ + * EQUIVALENCE BIAS (2026-09-27): paraphrases, rewordings, and + * compressions of the same claim MERGE — do not create a duplicate + * because the wording differs. Only genuinely distinct propositions + * stay apart. */ linkTo?: string; } -/** The auditor's output: the verified findings (with identity verdicts), - * plus the drops (the audit trail of what was rejected and why). */ +/** The auditor's output: the findings with identity verdicts, plus any + * bridging edges drawn to existing graph nodes. No drop channel — the + * auditor never rejects; the extractor is fully trusted (2026-09-27). */ export interface AuditResult { propositions: AuditedProposition[]; edges: ExtractedEdge[]; - attribution: ExtractionAttribution; - /** Dropped candidates: proposition id, edge "from→to", or attribution - * toolCallId — with the reason. */ - dropped: { ref: string; reason: string }[]; } /** The GET /api/graph payload — everything the home view's PURE renderer needs. */ @@ -328,11 +278,6 @@ export interface GraphSnapshot { fileSources: Record; /** nodeId → chatIds that touched the node (click-through provenance). */ nodeSessions: Record; - /** nodeId → chatId → that session's understanding observation (the - * extractor's per-session named-word note; session-relative data, - * shown next to the session list — the confidence history). Multiple - * notes for one (node, chat) collapse to the latest. */ - nodeObservations: Record>; /** Structural communities (Louvain over follows+supports), size ≥ 3, * joined with persisted LLM labelings. Purely structural — never * depends on layout. */ diff --git a/src/core/graph-deletion.ts b/src/core/graph-deletion.ts index 4a85518..f9e9ee5 100644 --- a/src/core/graph-deletion.ts +++ b/src/core/graph-deletion.ts @@ -13,17 +13,16 @@ * * preserve-graph the chat's extraction bookkeeping dies — its coverage * rows and its legacy checkpoint lines — and NOTHING else: - * nodes, edges, events, mappings stay byte-identical. + * nodes, edges, mappings stay byte-identical. * The graph keeps the session's contributions; the * snapshot simply no longer resolves the dead chatId to - * a title (nodeSessions/nodeObservations filter unknown - * ids at read time — no UI ghosting). + * a title (nodeSessions filters unknown ids at read + * time — no UI ghosting). * * delete-nodes "this session never happened": every trace of the * chat leaves the ledgers — its provenance points on * every node and edge line, its link lines (their - * phrasings die with them), its quiz events and - * observations, its mapping lines (the audit trail), its + * phrasings die with them), its mapping lines (the audit trail), its * coverage/checkpoint rows — and orphans die: a node * whose references all came from this chat disappears * entirely, pruned from the positions and embedding @@ -58,7 +57,6 @@ import type { CoverageEntry, GraphEdge, GraphNode, - LedgerEvent, NodeMapping, SessionCheckpoint, } from "./contracts.ts"; @@ -76,8 +74,6 @@ export interface ChatDeletionStats { nodesRemoved: number; /** Edge assertion LINES removed (empty-provenance or dead endpoint). */ edgesRemoved: number; - /** Events removed (quiz outcomes + observations). */ - eventsRemoved: number; /** Mapping lines removed (the chat's audit trail). */ mappingsRemoved: number; /** Coverage rows removed. */ @@ -89,7 +85,6 @@ export interface ChatDeletionStats { const EMPTY_STATS: ChatDeletionStats = { nodesRemoved: 0, edgesRemoved: 0, - eventsRemoved: 0, mappingsRemoved: 0, coverageRemoved: 0, checkpointsRemoved: 0, @@ -126,7 +121,6 @@ export function deleteChatTraces(graphStore: GraphStore, chatId: string, mode: C // ── delete-nodes: the full purge ────────────────────────────────────── const nodeLines = graphStore.readNodes(); const edgeLines = graphStore.readEdges(); - const events = graphStore.readEvents(); const mappings = graphStore.readMappings(); // Does the chat touch the graph at all? (A never-extracted chat still @@ -136,7 +130,6 @@ export function deleteChatTraces(graphStore: GraphStore, chatId: string, mode: C const touched = nodeLines.some((line) => line.provenance.some((pt) => pt.chatId === chatId)) || edgeLines.some((line) => line.provenance.some((pt) => pt.chatId === chatId)) || - events.some((event) => event.chatId === chatId) || mappings.some((m) => m.chatId === chatId); if (!touched) { if (stats.coverageRemoved > 0 || stats.checkpointsRemoved > 0) { @@ -149,7 +142,7 @@ export function deleteChatTraces(graphStore: GraphStore, chatId: string, mode: C // remains — another chat/src's provenance point on any of its lines, // a surviving mapping line (notes-front-door nodes carry EMPTY // provenance: the file IS the anchor, recorded through the mapping - // line), or a surviving event naming it. (Edge endpoints are deliberately + // line). (Edge endpoints are deliberately // NOT survival references: an edge's asserting session always left // provenance on its endpoints, so endpoint-only liveness cannot // occur in a consistent ledger — and counting it would keep nodes @@ -163,9 +156,6 @@ export function deleteChatTraces(graphStore: GraphStore, chatId: string, mode: C for (const mapping of mappings) { if (mapping.chatId !== chatId) live.add(mapping.graphNodeId); } - for (const event of events) { - if (event.chatId !== chatId && event.nodeId !== null) live.add(event.nodeId); - } // 2. Node lines. Per-line (each line is ONE extraction run's append, so // per-session alias provenance exists at line granularity): @@ -219,19 +209,16 @@ export function deleteChatTraces(graphStore: GraphStore, chatId: string, mode: C outEdges.push(points.length === line.provenance.length ? line : { ...line, provenance: points }); } - // 4. Events, mappings: the chat's rows die outright. - const outEvents = events.filter((event) => event.chatId !== chatId); + // 4. Mappings: the chat's rows die outright. const outMappings = mappings.filter((m) => m.chatId !== chatId); stats.nodesRemoved = [...new Set(nodeLines.map((line) => line.id))].filter((id) => !live.has(id)).length; stats.edgesRemoved = edgeLines.length - outEdges.length; - stats.eventsRemoved = events.length - outEvents.length; stats.mappingsRemoved = mappings.length - outMappings.length; graphStore.rewriteLedgers({ nodes: outNodes, edges: outEdges, - events: outEvents, mappings: outMappings, coverage, checkpoints, @@ -264,8 +251,6 @@ export function deleteChatTraces(graphStore: GraphStore, chatId: string, mode: C * * - every line of the node (create + link lines) * - every EDGE touching it (from or to), whatever session asserted it - * - every EVENT naming it (quiz outcomes + observations), from every - * session — the evidence history dies with the thing it was about * - every mapping line that anchored it (the audit trail; for a * notes-born node the file IS the anchor — its mapping line is the * only one) @@ -287,20 +272,16 @@ export function deleteChatTraces(graphStore: GraphStore, chatId: string, mode: C export interface NodeDeletionStats { /** Edge assertion LINES removed (every edge touching the node). */ edgesRemoved: number; - /** Events removed (quiz outcomes + observations naming the node). */ - eventsRemoved: number; /** Mapping lines removed (the node's audit trail). */ mappingsRemoved: number; } /** The confirm dialog's pre-flight facts (the GET on the same node id): * what full death would take with it. sessions is the deduped set of - * chats/notes sources that reference the node — provenance points, - * mapping lines, and events all count. */ + * chats/notes sources that reference the node — provenance points and + * mapping lines count. */ export interface NodeDeletionImpact { edges: number; - quizEvents: number; - observations: number; sessions: string[]; } @@ -313,7 +294,6 @@ function nodeLinesOf(graphStore: GraphStore, nodeId: string): GraphNode[] { export function nodeDeletionImpact(graphStore: GraphStore, nodeId: string): NodeDeletionImpact | null { if (nodeLinesOf(graphStore, nodeId).length === 0) return null; const edges = graphStore.readEdges().filter((line) => line.from === nodeId || line.to === nodeId); - const events = graphStore.readEvents().filter((event) => event.nodeId === nodeId); const sessions = new Set(); for (const line of nodeLinesOf(graphStore, nodeId)) { for (const pt of line.provenance) sessions.add(pt.chatId); @@ -321,11 +301,8 @@ export function nodeDeletionImpact(graphStore: GraphStore, nodeId: string): Node for (const mapping of graphStore.readMappings()) { if (mapping.graphNodeId === nodeId) sessions.add(mapping.chatId); } - for (const event of events) sessions.add(event.chatId); return { edges: edges.length, - quizEvents: events.filter((event) => event.kind !== "observation").length, - observations: events.filter((event) => event.kind === "observation").length, sessions: [...sessions].sort(), }; } @@ -341,11 +318,9 @@ export function deleteNodeTraces(graphStore: GraphStore, nodeId: string): NodeDe if (nodeLines.length === 0) return null; const edgeLines = graphStore.readEdges(); - const events = graphStore.readEvents(); const mappings = graphStore.readMappings(); const outEdges = edgeLines.filter((line) => line.from !== nodeId && line.to !== nodeId); - const outEvents = events.filter((event) => event.nodeId !== nodeId); const outMappings = mappings.filter((m) => m.graphNodeId !== nodeId); // Only the touched ledgers: the node's own lines always die (its file @@ -354,7 +329,6 @@ export function deleteNodeTraces(graphStore: GraphStore, nodeId: string): NodeDe graphStore.rewriteLedgers({ nodes: outNodes, ...(outEdges.length !== edgeLines.length ? { edges: outEdges } : {}), - ...(outEvents.length !== events.length ? { events: outEvents } : {}), ...(outMappings.length !== mappings.length ? { mappings: outMappings } : {}), }); @@ -367,7 +341,6 @@ export function deleteNodeTraces(graphStore: GraphStore, nodeId: string): NodeDe return { edgesRemoved: edgeLines.length - outEdges.length, - eventsRemoved: events.length - outEvents.length, mappingsRemoved: mappings.length - outMappings.length, }; } diff --git a/src/core/graph.ts b/src/core/graph.ts index 8810f93..d749aa2 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -2,45 +2,45 @@ * The knowledge graph: merger (the deterministic final stage of the * extraction pipeline — vision §1 construction) + derivation (reads). * - * The extraction pipeline is the SOLE graph writer, tree-blind and post-hoc: + * The extraction pipeline is the SOLE graph writer, tree-blind and post-hoc + * (2026-09-27 cleanup — clear separation of work): * - * extractor (LLM) → SessionFindings: propositions, edges, quiz - * attribution, per-proposition understanding notes — - * each with verbatim evidence, from the transcript ALONE - * auditor (LLM) → AuditResult: the verified subset, with the IDENTITY - * VERDICT per proposition (linkTo an existing node, or - * create), and the drops for the log + * extractor (LLM) → SessionFindings: the raw reasoning chains — + * propositions (lonely relevant ones included) and + * follows/supports edges, each with verbatim evidence, + * from the source ALONE (whole session as markdown, + * or raw notes). Fully trusted: no stage re-checks it. + * auditor (LLM) → AuditResult: identity verdicts (linkTo an existing + * node, or create) via graph-exploration tools, plus + * any BRIDGING edges it draws between existing graph + * nodes and the new propositions. Never sees the + * source; never drops. * merger (HERE) → mergeFindings(): deterministic ledger appends * * The merger is deliberately the ONLY stage that touches the ledgers, and it * is pure code: the LLM supplies exactly the judgment calls (what the - * propositions are, what follows/supports what, which proposition a quiz - * tested, whether a proposition IS an existing node), while event - * kinds/timestamps/refs/provenance — anything the model could silently - * mis-copy — are rebuilt deterministically from the transcript's tool parts - * and the message ids in the findings. Idempotent + incremental: refs are - * chat-scoped idempotency keys, edges dedupe by (from,to,type)-already- - * asserted-by-this-chat, mapping lines dedupe by (chat, sourceId), and - * sessions.jsonl checkpoints which chats have been extracted (never the - * same turns twice). + * propositions are, what follows/supports what, whether a proposition IS an + * existing node), while timestamps/provenance — anything the model could + * silently mis-copy — are rebuilt deterministically. Idempotent + + * incremental: edges dedupe by (from,to,type)-already-asserted-by-this-chat, + * mapping lines dedupe by (chat, sourceId), and coverage entries / + * sessions.jsonl checkpoints which messages and chats have been extracted + * (never the same turns twice). No events: the events ledger is GONE + * (2026-09-27) — the graph carries propositions + edges, nothing else. * - * Identity across re-encounters is the AUDITOR's judgment (link-or-create), - * not string matching: the conservative v1 exact-statement merge was - * replaced by the 2026-09-02 redesign. A re-extracted session's - * propositions link to the existing nodes; the link appends a node line - * whose phrasings join `aliases` and whose provenance unions (getGraph - * merges lines by id). + * Identity across re-encounters is the AUDITOR's judgment (link-or-create + * with an equivalence bias — paraphrases merge), not string matching. + * A re-extracted session's propositions link to the existing nodes; the + * link appends a node line whose phrasings join `aliases` and whose + * provenance unions (getGraph merges lines by id). * - * Derivation below (getGraph / deriveNodeStates / graphSnapshot) is reads - * only — everything is recomputed from the ledgers. Every function takes - * the storage ports (core/ports.ts) as explicit parameters — pure core, no - * adapter imports; the composition root (src/server/stores.ts) wires the fs - * adapters in. + * Derivation below (getGraph / graphSnapshot) is reads only — everything + * is recomputed from the ledgers. Every function takes the storage ports + * (core/ports.ts) as explicit parameters — pure core, no adapter imports; + * the composition root (src/server/stores.ts) wires the fs adapters in. */ import { randomBytes } from "node:crypto"; -import type { LhMessage } from "./native-messages.ts"; -import type { ToolResultMessage } from "@earendil-works/pi-ai"; import type { CoverageEntry } from "./contracts/graph.ts"; import type { AuditResult, @@ -50,13 +50,9 @@ import type { GraphNode, ProvenancePoint, GraphSnapshot, - LedgerEvent, - LedgerEventKind, - UnderstandingLevel, } from "./contracts.ts"; import { MIN_CLUSTER_SIZE } from "./clusters.ts"; import { birthMappingIndex, birthSessionOf, birthSessionPartition } from "./session-clusters.ts"; -import type { QuestionAnswerData } from "./question-core.ts"; import type { ChatStore, GraphStore } from "./ports.ts"; /** The storage ports every graph operation needs, passed explicitly. */ @@ -71,71 +67,22 @@ function newGraphNodeId(): string { } /** What one merge run merges FROM (the caller supplies it — the merger never - * reads the chat store): a session's transcript + its coverage checkpoint, - * or — for the format-agnostic notes front door (2026-09-13) — `messages: - * null`: raw notes carry no tool parts, so no quiz events are rebuilt (quiz - * outcomes inside notes surface only through the extractor's understanding - * observations), and the whole-file checkpoint stamps the import. + * reads the chat store): the coverage checkpoint for a session batch, or — + * for the format-agnostic notes front door (2026-09-13) — no coverage at + * all: the whole-file checkpoint stamps the import. * * Coverage (vision-extraction-coverage.md, settled 2026-09-15): a chat merge * passes the BATCH it extracted — the caller computed the fingerprints - * before the run, from the same snapshot the merger sees — and step 4 - * appends one CoverageEntry per message on success ("covered" means - * exactly this merge happened). `batchTag` prefixes the observation refs so - * re-runs of the SAME batch dedupe while a re-opened (mutated) message's - * fresh understanding observations land as new events; the notes path - * passes neither and keeps the whole-file checkpoint. */ + * before the run, from the same snapshot the merger sees — and the final + * step appends one CoverageEntry per message on success ("covered" means + * exactly this merge happened). The notes path passes neither and keeps + * the whole-file checkpoint. */ export interface MergeSource { - messages: LhMessage[] | null; foldedAt: string; - /** The batch being merged (chat path): one entry per message in - * `messages`, same order — undefined (notes path / coverage-less - * callers) → the legacy whole-chat checkpoint is written instead. */ + /** The batch being merged (chat path): one entry per message in the + * extracted batch — undefined (notes path / coverage-less callers) → + * the legacy whole-chat checkpoint is written instead. */ coverage?: CoverageEntry[]; - /** The batch's deterministic tag (core/coverage.ts batchTag) — obs refs. */ - batchTag?: string; -} - -// ─── transcript scanning (shared by the merger + trigger) ───────────────────── - -/** A flattened, order-preserving view of every tool call (+ its result - * state) in a native transcript. Wave D: the pairing is native — calls - * come from assistant content blocks; a result map built from the - * toolResult messages carries the outcome (a dangling call is - * input-available, an error result is output-error). */ -interface ToolPartView { - name: string; - toolCallId: string; - state: string; - input: unknown; - output: unknown; -} - -function toolParts(messages: LhMessage[]): ToolPartView[] { - const results = new Map(); - for (const m of messages) { - if ((m as { role?: string }).role === "toolResult") { - results.set((m as ToolResultMessage).toolCallId, m as ToolResultMessage); - } - } - const out: ToolPartView[] = []; - for (const m of messages) { - if ((m as { role?: string }).role !== "assistant") continue; - const content = (m as { content?: unknown }).content; - if (!Array.isArray(content)) continue; - for (const block of content as Array<{ type?: string; id?: string; name?: string; arguments?: unknown }>) { - if (block.type !== "toolCall" || typeof block.id !== "string") continue; - const result = results.get(block.id); - out.push({ - name: block.name ?? "", - toolCallId: block.id, - state: result ? (result.isError ? "output-error" : "output-available") : "input-available", - input: block.arguments, - output: result && !result.isError ? result.details : undefined, - }); - } - } - return out; } /** (chatId, messageIds) → provenance points; order-preserving, deduped. */ @@ -167,21 +114,14 @@ const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean => * Mapping lines record each verdict: one per (chat, sourceId). Idempotent: * a proposition id already mapped for this chat reuses its resolution. * - * Events are rebuilt deterministically from the transcript: quiz outcomes - * (correct / miss / dont-know; kinds + timestamps + refs from the tool - * parts, nodeId from the extractor's attribution through this run's - * mapping) plus one observation event per proposition carrying an - * understanding note (ref `obs:`). No taught events, ever. - * * `chatId` is the run's source key: a real chat, or a notes source id * ("src_…") for the format-agnostic import path — same ledgers, same - * idempotency, one graph. The `source` parameter carries the transcript - * being merged (null messages = raw notes: no quiz events) and the - * checkpoint stamp; the merger itself is chat-store-free. + * idempotency, one graph. The `source` parameter carries the checkpoint + * stamp; the merger itself is chat-store-free. */ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps, source: MergeSource): EmissionStats { const { graphStore: ledgers } = deps; - const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 }; + const stats: EmissionStats = { nodes: 0, edges: 0 }; // 1. Resolution: this run's propositions → graph node ids. Existing // mapping lines for THIS chat are loaded first — the audit trail doubles @@ -238,63 +178,41 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep } } - // 2. Edges. Resolve endpoints through the mapping; dedupe by - // (from,to,type) already asserted by THIS chat (an existing assertion - // by another chat stays untouched — provenance unions at read time). + // 2. Edges. Resolve endpoints through the mapping — with BRIDGING + // (2026-09-27): an endpoint the extractor never proposed but that + // names an EXISTING graph node is the auditor drawing a connection + // between the graph it explored and this batch's propositions. Such + // edges carry chat-scoped provenance with NO message anchor (nothing + // in the transcript asserts them — the auditor is the asserter); + // transcript-articulated edges keep message-granular provenance. + // Dedupe by (from,to,type) already asserted by THIS chat (an existing + // assertion by another chat stays untouched — provenance unions at + // read time). const assertedByChat = new Set(); for (const line of ledgers.readEdges()) { const key = `${line.from}\u0000${line.to}\u0000${line.type}`; if (line.provenance.some((pt) => pt.chatId === chatId)) assertedByChat.add(key); } for (const edge of audit.edges) { - const from = mapping.get(edge.from); - const to = mapping.get(edge.to); - if (!from || !to || from === to) continue; // unmapped endpoint — skip + const resolve = (endpoint: string): string | undefined => + mapping.get(endpoint) ?? (existingIds.has(endpoint) ? endpoint : undefined); + const from = resolve(edge.from); + const to = resolve(edge.to); + if (!from || !to || from === to) continue; // unresolvable endpoint — skip + const bridged = !mapping.has(edge.from) || !mapping.has(edge.to); const key = `${from}\u0000${to}\u0000${edge.type}`; if (assertedByChat.has(key)) continue; - ledgers.appendEdge({ from, to, type: edge.type, provenance: pointsOf(chatId, edge.messages) }); + ledgers.appendEdge({ + from, + to, + type: edge.type, + provenance: bridged ? [{ chatId, messageId: "" }] : pointsOf(chatId, edge.messages), + }); assertedByChat.add(key); stats.edges++; } - // 3. Events, rebuilt deterministically in transcript order. Refs are - // chat-scoped idempotency keys (quiz toolCallId, obs:) — - // a ref already in the ledger means this evidence was extracted before. - const seenRefs = new Set(); - for (const event of ledgers.readEvents()) { - if (event.chatId === chatId && event.ref) seenRefs.add(event.ref); - } - for (const part of source.messages ? toolParts(source.messages) : []) { - if (part.name !== "quiz" || part.state !== "output-available") continue; - const event = quizEvent(part, chatId, audit, mapping, now); - if (event?.ref !== undefined && !seenRefs.has(event.ref)) { - ledgers.appendEvent(event); - seenRefs.add(event.ref); - stats.events++; - } - } - // Understanding observations: one per proposition carrying a note, - // resolved to the node the verdict mapped it to. Refs carry the batch - // tag when there is one: run-local source ids ("p1"…) shift position - // between runs, so a bare obs:p1 could never both dedupe a re-run of - // the same batch AND let a re-opened message's fresh level through. - for (const p of audit.propositions) { - if (p.understanding === undefined || p.understanding.length === 0) continue; - const ref = source.batchTag !== undefined ? `obs:${source.batchTag}:${p.id}` : `obs:${p.id}`; - if (seenRefs.has(ref)) continue; - ledgers.appendEvent({ - nodeId: mapping.get(p.id) ?? null, - kind: "observation", - chatId, - at: now, - ref, - note: p.understanding, - }); - seenRefs.add(ref); - stats.events++; - } - - // 4. Checkpoint — THE COVERAGE DISCIPLINE (vision-extraction-coverage.md): + // 3. Checkpoint — THE COVERAGE DISCIPLINE (vision-extraction-coverage.md): // only now, after every append succeeded, is the batch covered. The // chat path appends one CoverageEntry per message (per-batch atomic — // an abort or failure mid-run leaves earlier batches covered, this @@ -308,50 +226,11 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep return stats; } -/** Build the ledger event for one answered quiz tool part (or null to skip). - * Deterministic in everything except the attribution-supplied nodeId. */ -function quizEvent( - part: ToolPartView, - chatId: string, - audit: AuditResult, - mapping: Map, - now: string, -): LedgerEvent | null { - const out = part.output; - if (typeof out !== "object" || out === null) return null; - const answer = out as Partial; - if (answer.kind !== "quiz") return null; // ask questions are not graded - if (answer.dismissed === true) return null; // no answer — no evidence - - let kind: LedgerEventKind; - if (answer.dontKnow === true) kind = "dont-know"; - else if (answer.correct === true) kind = "correct"; - else kind = "miss"; - - // Tree-blind: EVERY quiz attributes through the extractor's judgment — - // the vision's retroactive attribution, with no tree mapping to lean on. - const attributedId = audit.attribution.quiz[part.toolCallId]; - const nodeId = attributedId ? (mapping.get(attributedId) ?? null) : null; - - const event: LedgerEvent = { - nodeId, - kind, - chatId, - at: typeof answer.answeredAt === "string" ? answer.answeredAt : now, - ref: part.toolCallId, - }; - if (typeof answer.note === "string" && answer.note.length > 0) event.note = answer.note; - // The question text is the attribution input for unattributed quizzes. - if (nodeId === null && typeof answer.questionText === "string") event.question = answer.questionText; - return event; -} - // ─── derivation (reads only — everything below is recomputable) ───────────── export interface DerivedGraph { nodes: GraphNode[]; edges: GraphEdge[]; - events: LedgerEvent[]; } /** @@ -396,14 +275,14 @@ export function getGraph(graphStore: GraphStore): DerivedGraph { merged.push(edge); } } - return { nodes: [...byId.values()], edges: merged, events: graphStore.readEvents() }; + return { nodes: [...byId.values()], edges: merged }; } // ─── the /api/graph payload ────────────────────────────────────────────────── /** Everything the home graph view needs, in one fetch (shape: contracts). */ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { - const { nodes, edges, events } = getGraph(deps.graphStore); + const { nodes, edges } = getGraph(deps.graphStore); const nodeChats = new Map>(); const touch = (nodeId: string, chatId: string): void => { @@ -424,9 +303,6 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { touch(edge.to, pt.chatId); } } - for (const event of events) { - if (event.nodeId !== null) touch(event.nodeId, event.chatId); - } // Labels for every source the graph can point back at: real chats // (title, deep-linkable) and imported notes files (path, static — the @@ -453,27 +329,6 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { if (ids.length > 0) nodeSessions[nodeId] = ids; } - // Understanding observations per (node, chat) — the extractor's - // session-relative note (latest wins within a chat). Read-time join for - // the detail card: the confidence history next to the session list. - const nodeObservations: Record> = {}; - const latestObs = new Map(); - events.forEach((event, index) => { - if (event.kind !== "observation" || event.nodeId === null) return; - if (!event.note) return; - const key = `${event.nodeId}\u0000${event.chatId}`; - const current = latestObs.get(key); - if (!current || event.at > current.at || (event.at === current.at && index > current.index)) { - latestObs.set(key, { at: event.at, note: event.note, index }); - } - }); - for (const [key, { note }] of latestObs) { - const sep = key.indexOf("\u0000"); - const nodeId = key.slice(0, sep); - const chatId = key.slice(sep + 1); - (nodeObservations[nodeId] ??= {})[chatId] = note; - } - // BIRTH-SESSION CLUSTERS (wave 7, owner-locked 2026-09-23): embeddings // are dead — a cluster is the SESSION THAT CREATED ITS PROPOSITIONS, // derived on read from provenance (core/session-clusters.ts). The label @@ -502,7 +357,6 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { sessions, fileSources, nodeSessions, - nodeObservations, clusters, ...(Object.keys(positions).length > 0 ? { positions } : {}), }; @@ -577,48 +431,22 @@ export function graphSessionIndex(deps: GraphDeps): GraphSessionEntry[] { ); } -/** One birth proposition of one session, for the prober's per-session - * view. `understanding` is the extractor's latest observation for - * (node, session) — the named-word scale (solid/shaky/accepted/resisted/ - * not yet engaged) — and `evaluatedAt` is THAT event's own timestamp - * (owner ruling 2026-09-23: quiz outcomes are evidence in the ledger, - * but they never move the evaluation date — the level's event does). */ +/** One birth proposition of one session — WHAT WAS COVERED there (2026-09-27 + * cleanup: no understanding levels; assessment is practice-module + * territory and nothing writes it until that module exists). */ export interface GraphSessionProposition { nodeId: string; /** The compact handle — the same name the canvas shows. */ name: string; - understanding: UnderstandingLevel | null; - /** The `at` of the event that minted `understanding`; null when the - * session never evaluated this proposition. */ - evaluatedAt: string | null; } -/** The propositions BORN in one session, with the session's latest - * understanding of each. Latest observation wins by `at`, then by - * ledger order (the same tie rule the snapshot's nodeObservations - * uses). Members sorted by node id — deterministic. */ +/** The propositions BORN in one session — what that session first + * articulated. Members sorted by node id — deterministic. */ export function graphSessionPropositions(deps: GraphDeps, sessionId: string): GraphSessionProposition[] { const mappingFallback = birthMappingIndex(deps.graphStore.readMappings()); const born = getGraph(deps.graphStore).nodes.filter((node) => birthSessionOf(node, mappingFallback) === sessionId); - const latest = new Map(); - deps.graphStore.readEvents().forEach((event, index) => { - if (event.kind !== "observation" || event.nodeId === null || event.chatId !== sessionId) return; - if (!event.note) return; - const current = latest.get(event.nodeId); - if (!current || event.at > current.at || (event.at === current.at && index > current.index)) { - latest.set(event.nodeId, { at: event.at, note: event.note, index }); - } - }); return born - .map((node) => { - const observation = latest.get(node.id); - return { - nodeId: node.id, - name: node.name, - understanding: (observation?.note as UnderstandingLevel | undefined) ?? null, - evaluatedAt: observation?.at ?? null, - }; - }) + .map((node) => ({ nodeId: node.id, name: node.name })) .sort((a, b) => (a.nodeId < b.nodeId ? -1 : 1)); } diff --git a/src/core/ports.ts b/src/core/ports.ts index a4932ac..94436d4 100644 --- a/src/core/ports.ts +++ b/src/core/ports.ts @@ -18,7 +18,6 @@ import type { GraphNode, JobKind, JobRecord, - LedgerEvent, ExtractionRunUsage, NodeMapping, PlanTree, @@ -84,8 +83,6 @@ export interface GraphStore { readEdges(): GraphEdge[]; /** Append one quiz outcome / understanding observation. */ - appendEvent(event: LedgerEvent): void; - readEvents(): LedgerEvent[]; /** Append one identity-map line (chat's tree node → graph node). */ appendMapping(mapping: NodeMapping): void; @@ -130,7 +127,6 @@ export interface GraphStore { rewriteLedgers(next: { nodes?: GraphNode[]; edges?: GraphEdge[]; - events?: LedgerEvent[]; mappings?: NodeMapping[]; coverage?: CoverageEntry[]; checkpoints?: SessionCheckpoint[]; diff --git a/src/core/session-markdown.ts b/src/core/session-markdown.ts new file mode 100644 index 0000000..0a911b0 --- /dev/null +++ b/src/core/session-markdown.ts @@ -0,0 +1,126 @@ +/** + * Session → markdown — the ONE formatting engine (owner ruling 2026-09-27). + * + * The extractor's view of a teaching session and the future user-facing + * session export share this engine: a session is rendered as clean, + * readable markdown — speaker headers, quiz/check-in blocks, tool notes — + * with the message-id anchors preserved (extraction provenance depends on + * the extractor reporting the `[id]` tags it saw; `anchors: false` mints + * the same document without them, for the user export). + * + * Pure core: native messages in (LhMessage), string out. Content parity + * with the coverage line builder (core/coverage.ts messageViewLines) is + * deliberate — every message that produces view lines produces a markdown + * block, in the same order — but the formats are SEPARATE by design + * (2026-09-27): coverage fingerprints hash the line format and stay stable + * (no mass re-extraction when presentation changes); markdown is + * presentation. A failed tool call renders nothing (retry context, not + * learning evidence); custom markers are invisible, same as the view. + */ + +import type { LhMessage } from "./native-messages.ts"; +import type { ToolResultMessage } from "@earendil-works/pi-ai"; + +/** The user-facing text blocks of a native message (string or text parts). */ +function textOf(content: unknown): string[] { + if (typeof content === "string") return content.trim() ? [content] : []; + if (!Array.isArray(content)) return []; + return content + .filter((c): c is { type: "text"; text: string } => (c as { type?: string }).type === "text") + .map((c) => c.text) + .filter((t) => t.trim().length > 0); +} + +/** One tool result's learner-visible outcome, rendered as body lines. */ +function toolResultBody(result: ToolResultMessage): { heading: string; body: string[] } { + const out = result.details as + | Partial<{ + questionText: string; + correct: boolean; + dontKnow: boolean; + note: string; + text: string; + answers: string[]; + other: string; + dismissed: boolean; + }> + | undefined; + + if (result.toolName === "quiz" || result.toolName === "explain") { + const heading = result.toolName === "quiz" ? "Quiz" : "Recall check"; + const question = typeof out?.questionText === "string" && out.questionText ? out.questionText : "(no question text)"; + const body = [`**Q:** ${question}`]; + if (out?.dismissed === true) body.push("The learner skipped this check."); + else if (out?.dontKnow === true) body.push("The learner did not know."); + else if (out?.correct === true) body.push("The learner answered correctly."); + else if (out?.correct === false) body.push("The learner answered incorrectly."); + else if (typeof out?.text === "string" && out.text.trim()) body.push(`The learner answered in his own words: ${out.text.trim()}`); + if (typeof out?.note === "string" && out.note) body.push(`*(his thinking: ${out.note})*`); + return { heading, body }; + } + + if (result.toolName === "ask_user_question") { + const question = typeof out?.questionText === "string" && out.questionText ? out.questionText : "(no question text)"; + const body = [`**Q:** ${question}`]; + const chosen = [ + ...(Array.isArray(out?.answers) ? out!.answers! : []), + ...(typeof out?.other === "string" && out.other.trim() ? [out.other.trim()] : []), + ]; + if (typeof out?.text === "string" && out.text.trim()) body.push(`The learner answered: ${out.text.trim()}`); + else if (chosen.length > 0) body.push(`The learner chose: ${chosen.join(", ")}`); + else if (out?.dismissed === true) body.push("The learner skipped this question."); + if (typeof out?.note === "string" && out.note) body.push(`*(his thinking: ${out.note})*`); + return { heading: "Question", body }; + } + + // Parity with the line builder: other clean tool results are one line. + return { heading: "Tool", body: [`*${result.toolName} result delivered*`] }; +} + +/** + * ONE message as a markdown block — `null` when the message renders + * nothing (a failed tool call, a bare marker, an empty text part set). + * `mid` is the provenance anchor shown when `anchors` is on. + */ +export function messageMarkdown(message: LhMessage, mid: string, anchors = true): string | null { + const role = (message as { role?: string }).role; + const anchor = anchors ? ` · ${mid}` : ""; + const parts: string[] = []; + + if (role === "user") { + for (const text of textOf((message as { content: unknown }).content)) { + parts.push(`### Learner${anchor}\n\n${text.trim()}`); + } + } else if (role === "assistant") { + for (const text of textOf((message as { content: unknown }).content)) { + parts.push(`### Teacher${anchor}\n\n${text.trim()}`); + } + } else if (role === "toolResult") { + const result = message as ToolResultMessage; + if (result.isError) return null; // retry context, not learning evidence + const { heading, body } = toolResultBody(result); + parts.push(`### ${heading}${anchor}\n\n${body.join("\n\n")}`); + } + + // custom markers are invisible, same as the extraction view. + return parts.length > 0 ? parts.join("\n\n") : null; +} + +/** + * A whole session (or one extraction BATCH — same engine, subset of + * messages) as one markdown document. Blocks render in message order; + * messages that render nothing are skipped silently. The optional title + * mints the document header (`# `). + */ +export function sessionMarkdown( + messages: ReadonlyArray<LhMessage>, + opts: { title?: string; anchors?: boolean } = {}, +): string { + const blocks: string[] = []; + if (opts.title !== undefined && opts.title.length > 0) blocks.push(`# ${opts.title}`); + for (const message of messages) { + const block = messageMarkdown(message, (message as { id?: string }).id ?? "", opts.anchors ?? true); + if (block !== null) blocks.push(block); + } + return blocks.join("\n\n"); +} diff --git a/src/server/graph.ts b/src/server/graph.ts index 0ce4783..6b2d8a0 100644 --- a/src/server/graph.ts +++ b/src/server/graph.ts @@ -57,7 +57,6 @@ export function graphRevision(): number { for (const file of [ LEDGER_FILES.nodes, LEDGER_FILES.edges, - LEDGER_FILES.events, LEDGER_FILES.mappings, LEDGER_FILES.sessions, LEDGER_FILES.coverage, diff --git a/src/store/graph.ts b/src/store/graph.ts index 362c57f..cd8922a 100644 --- a/src/store/graph.ts +++ b/src/store/graph.ts @@ -1,5 +1,5 @@ /** - * GraphStore — the fs JSONL adapter: five append-only ledgers at + * GraphStore — the fs JSONL adapter: four append-only ledgers at * config.LEDGER_FILES (contracts.ts "v1 emission"). * nodes.jsonl GraphNode — one line per node write: the FIRST * line for an id defines it; a later @@ -10,7 +10,6 @@ * edges.jsonl GraphEdge — one line per edge ASSERTION (a second * session asserting the same (from,to,type) * appends another line; reads merge) - * events.jsonl LedgerEvent — quiz outcomes + understanding observations * mappings.jsonl NodeMapping — the session→graph identity map: * (chatId, sourceId) → graphNodeId * sessions.jsonl SessionCheckpoint— emission checkpoints (foldedAt) @@ -34,8 +33,6 @@ import type { GraphEdge, GraphNode, ExtractionRunUsage, - LedgerEvent, - LedgerEventKind, NotesSource, NodeMapping, Position, @@ -110,15 +107,6 @@ const isGraphEdge = (v: unknown): v is GraphEdge => Array.isArray(v.provenance) && v.provenance.every(isProvenancePoint); -const EVENT_KINDS: readonly LedgerEventKind[] = ["correct", "miss", "dont-know", "observation"]; - -const isLedgerEvent = (v: unknown): v is LedgerEvent => - isRecord(v) && - (v.nodeId === null || isString(v.nodeId)) && - (EVENT_KINDS as readonly string[]).includes(v.kind as string) && - isString(v.chatId) && - isString(v.at); - const isNodeMapping = (v: unknown): v is NodeMapping => isRecord(v) && isString(v.chatId) && isString(v.sourceId) && isString(v.graphNodeId); @@ -149,13 +137,6 @@ export const graphStore: GraphStore = { return readValid(LEDGER_FILES.edges, isGraphEdge); }, - appendEvent(event: LedgerEvent): void { - appendLine(LEDGER_FILES.events, event); - }, - readEvents(): LedgerEvent[] { - return readValid(LEDGER_FILES.events, isLedgerEvent); - }, - appendMapping(mapping: NodeMapping): void { appendLine(LEDGER_FILES.mappings, mapping); }, @@ -217,7 +198,6 @@ export const graphStore: GraphStore = { rewriteLedgers(next: { nodes?: GraphNode[]; edges?: GraphEdge[]; - events?: LedgerEvent[]; mappings?: NodeMapping[]; coverage?: CoverageEntry[]; checkpoints?: SessionCheckpoint[]; @@ -225,7 +205,6 @@ export const graphStore: GraphStore = { const files = { nodes: LEDGER_FILES.nodes, edges: LEDGER_FILES.edges, - events: LEDGER_FILES.events, mappings: LEDGER_FILES.mappings, coverage: LEDGER_FILES.coverage, checkpoints: LEDGER_FILES.sessions, @@ -326,5 +305,22 @@ function writeSourceMap(map: Record<string, NotesSource>): void { fs.writeFileSync(SOURCES_FILE, JSON.stringify(map, null, "\t"), "utf-8"); } +// ─── one-way migration: the events ledger is gone (2026-09-27) ──────────────── +// Quiz outcomes and understanding observations are no longer formalized in +// the graph — the pipeline writes nodes + edges only. A stale events.jsonl +// left by an earlier build is deleted once, here, at adapter load: dead +// ledgers are exactly the confusion the cleanup removes (owner ruling). +{ + const legacyEvents = path.join(path.dirname(LEDGER_FILES.nodes), "events.jsonl"); + try { + if (fs.existsSync(legacyEvents)) { + fs.rmSync(legacyEvents); + console.info(`[graph-store] purged legacy events ledger: ${legacyEvents}`); + } + } catch (err) { + console.warn(`[graph-store] could not purge legacy events ledger:`, err); + } +} + const isNotesSource = (v: unknown): v is NotesSource => isRecord(v) && isString(v.id) && isString(v.filePath) && isString(v.importedAt); diff --git a/tests/extraction.test.ts b/tests/extraction.test.ts index 7a3d7f7..9fde09b 100644 --- a/tests/extraction.test.ts +++ b/tests/extraction.test.ts @@ -1,18 +1,20 @@ /** - * The extraction pipeline tests (vision §1 construction, 2026-09-02 redesign): - * extractor → auditor → merger, the sole graph writer — tree-blind, - * free-form, with the auditor owning faithfulness and identity. + * The extraction pipeline tests (vision §1 construction, 2026-09-02 redesign; + * relinked 2026-09-27): extractor → auditor → merger, the sole graph writer — + * the extractor alone sees the source (whole-session markdown), the auditor + * never does (identity linking through graph tools), the merger is + * deterministic. * - * transcriptView — the compact view both LLM stages see (pure): - * message-id-tagged verbatim text + quiz lines, NO tree + * transcriptView — the markdown rendering the extractor sees (pure): + * id-anchored Teacher/Learner blocks + check blocks * pipeline — scripted faux engines drive both agents: findings out, * audit verdict (with linkTo identity) in, deterministic * merge applied; failure paths checkpoint nothing; * single-flight per chat * - * The merger itself (identity verdicts, event rebuild, idempotency) is - * tested in tests/graph.test.ts directly against mergeFindings — here it is - * the integration: what one full pipeline run does to the ledgers. + * The merger itself (identity verdicts, idempotency) is tested in + * tests/graph.test.ts directly against mergeFindings — here it is the + * integration: what one full pipeline run does to the ledgers. * * Ledger: in-memory (the GraphStore port) — this suite never touches * data/graph. Chats: real ChatRecords in the real chat store (created and @@ -33,7 +35,6 @@ import type { ExtractionRunUsage, GraphEdge, GraphNode, - LedgerEvent, NodeMapping, NotesSource, SessionCheckpoint, @@ -42,7 +43,7 @@ import type { import type { GraphStore } from "../src/core/ports.ts"; import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts"; import { ModelsNotConfiguredError, ProviderKeyMissingError, providerKeyMissing, type ModelSlotName } from "../src/core/settings.ts"; -import { batchTag, coverageOf, coverageSlots, messageViewLines } from "../src/core/coverage.ts"; +import { coverageOf, messageViewLines } from "../src/core/coverage.ts"; import { chatStore } from "../src/store/chats.ts"; import { PATHS } from "../config.ts"; @@ -64,7 +65,6 @@ function fauxEngine(steps: FauxResponseStep[]): PiEngine { function memoryLedgers() { const nodes: GraphNode[] = []; const edges: GraphEdge[] = []; - const events: LedgerEvent[] = []; const mappings: NodeMapping[] = []; const checkpoints: SessionCheckpoint[] = []; const coverage: CoverageEntry[] = []; @@ -75,8 +75,6 @@ function memoryLedgers() { readNodes: () => nodes.map((n) => ({ ...n })), appendEdge: (e: GraphEdge) => void edges.push(e), readEdges: () => edges.map((e) => ({ ...e })), - appendEvent: (e: LedgerEvent) => void events.push(e), - readEvents: () => events.map((e) => ({ ...e })), appendMapping: (m: NodeMapping) => void mappings.push(m), readMappings: () => mappings.map((m) => ({ ...m })), appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c), @@ -110,7 +108,6 @@ function memoryLedgers() { rewriteLedgers: (next) => { if (next.nodes) nodes.splice(0, nodes.length, ...next.nodes.map((n) => ({ ...n, provenance: [...n.provenance], aliases: [...n.aliases] }))); if (next.edges) edges.splice(0, edges.length, ...next.edges.map((e) => ({ ...e, provenance: [...e.provenance] }))); - if (next.events) events.splice(0, events.length, ...next.events.map((e) => ({ ...e }))); if (next.mappings) mappings.splice(0, mappings.length, ...next.mappings.map((m) => ({ ...m }))); if (next.coverage) coverage.splice(0, coverage.length, ...next.coverage.map((c) => ({ ...c }))); if (next.checkpoints) checkpoints.splice(0, checkpoints.length, ...next.checkpoints.map((c) => ({ ...c }))); @@ -224,7 +221,7 @@ after(() => { }); /** What the extractor would produce for the fixture session — tree-blind, - * free-form, with message-granular provenance + an understanding note. */ + * free-form, with message-granular provenance. */ const FINDINGS: SessionFindings = { propositions: [ { @@ -233,8 +230,7 @@ const FINDINGS: SessionFindings = { statement: "The derivative measures how fast a function changes", aliases: [], messages: ["a2"], - evidence: ["TEACHER [a2]: The derivative measures how fast a function changes."], - understanding: "solid", + evidence: ["The derivative measures how fast a function changes."], }, { id: "p2", @@ -242,7 +238,7 @@ const FINDINGS: SessionFindings = { statement: "The chain rule differentiates composed functions by multiplying local rates.", aliases: [], messages: ["a2"], - evidence: ["LEARNER [u1]: teach me the chain rule"], + evidence: ["teach me the chain rule"], }, ], edges: [ @@ -251,10 +247,9 @@ const FINDINGS: SessionFindings = { to: "p2", type: "follows", messages: ["a2"], - evidence: ["LEARNER [u1]: teach me the chain rule"], + evidence: ["teach me the chain rule"], }, ], - attribution: { quiz: { "probe-1": "p1", "check-1": "p2" } }, }; /** An engine whose ONE response is the `name` tool call with the given @@ -301,27 +296,30 @@ function silentModel(): PiEngine { // ─── transcriptView (pure) ─────────────────────────────────────────────────── -describe("transcriptView (pure)", () => { - it("shows message-id-tagged verbatim conversation + quiz lines — tree-blind, plan ops folded", () => { +describe("transcriptView (pure — the shared markdown engine, owner 2026-09-27)", () => { + it("renders id-anchored speaker blocks + check blocks — tree-blind, plan ops folded", () => { const { record } = fixtureChat(); const view = transcriptView(record); - assert.match(view, /SESSION: Extraction fixture/); + // The document header is the session title. + assert.match(view, /^# Extraction fixture$/m); - // Conversation verbatim, each line tagged with its message id — - // the provenance anchors the extractor reports back. - assert.match(view, /LEARNER \[u1\] teach me the chain rule/); - assert.match(view, /TEACHER \[a2\] The derivative measures how fast a function changes\./); + // Conversation verbatim, each block headed by its speaker + the + // message-id anchor — the provenance the extractor reports back. + assert.match(view, /### Learner · u1\n\nteach me the chain rule/); + assert.match(view, /### Teacher · a2\n\nThe derivative measures how fast a function changes\./); - // Structured events, attribution-ready. - assert.match(view, /QUIZ \[probe-1\] What is a derivative, roughly\? — the learner answered incorrectly/); - assert.match(view, /QUIZ \[check-1\] Check — the learner answered correctly/); + // Checks render as readable blocks (question + outcome). + assert.match(view, /### Quiz · r_probe-1/); + assert.match(view, /\*\*Q:\*\* What is a derivative, roughly\?/); + assert.match(view, /The learner answered incorrectly\./); + assert.match(view, /The learner answered correctly\./); // TREE-BLIND: no plan tree, no tree edges, no mark_done lines — plan // ops fold to the generic one-liner like every other non-quiz tool. assert.doesNotMatch(view, /FINAL PLAN TREE/); assert.doesNotMatch(view, /TREE EDGES/); - assert.match(view, /\(plan_init tool result delivered\)/); + assert.match(view, /\*plan_init result delivered\*/); assert.doesNotMatch(view, /output-available/); }); @@ -341,9 +339,9 @@ describe("transcriptView (pure)", () => { ]), ].flat()); const view = transcriptView(chatStore.getChat(chat.id)!); - assert.match(view, /\(research tool result delivered\)/); + assert.match(view, /\*research result delivered\*/); assert.doesNotMatch(view, /long brief/); - assert.doesNotMatch(view, /TEACHER: {2}/); + assert.doesNotMatch(view, /### Teacher/); }); }); @@ -355,7 +353,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const ledgers = memoryLedgers(); // The auditor passes p1 through and LINKS it to a pre-existing node - // (identity verdict), drops p2 (unbacked) + the edge with it. + // (identity verdict); p2 stays a verdict-free create. const existingId = "n_existing"; ledgers.appendNode({ id: existingId, @@ -370,8 +368,6 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { { ...FINDINGS.propositions[0]!, linkTo: existingId }, ], edges: [], - attribution: { quiz: { "probe-1": "p1" } }, - dropped: [{ ref: "p2", reason: "evidence not found" }, { ref: "p1→p2", reason: "endpoint dropped" }], }; const pipeline = createExtractionPipeline({ @@ -382,38 +378,18 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { }); const stats = await pipeline.runExtraction(chatId); - // The linked proposition created NO new node; the understanding note - // became an observation; both quizzes became events. - assert.deepEqual(stats, { nodes: 0, edges: 0, events: 3 }); + // The linked proposition created NO new node. + assert.deepEqual(stats, { nodes: 0, edges: 0 }); // Link-append: the phrasing joined the existing node's aliases and // the provenance unioned; no second node line defines a new node. assert.equal(ledgers.readNodes().length, 2); // the pre-seeded line + the link line assert.deepEqual(ledgers.readMappings(), [{ chatId, sourceId: "p1", graphNodeId: existingId }]); - // Events rebuilt deterministically: quizzes in transcript order, then - // the observation. check-1's attribution pointed at the DROPPED p2 → - // unresolvable → null (the question text is kept for later attribution). - // The observation ref is BATCH-TAGGED (obs:<tag>:p1) so a re-run of - // the same batch dedupes while a re-opened message's update lands as - // a new event (vision-extraction-coverage.md). - const events = ledgers.readEvents(); - const record = chatStore.getChat(chatId)!; - const expectedTag = batchTag(coverageSlots(record.messages)); - assert.deepEqual( - events.map((e) => [e.kind, e.ref, e.nodeId]), - [ - ["miss", "probe-1", existingId], - ["correct", "check-1", null], - ["observation", `obs:${expectedTag}:p1`, existingId], - ], - ); - assert.equal(events[0]!.nodeId, existingId, "probe attributed via the verdict's mapping"); - assert.equal(events[2]!.note, "solid", "the understanding note joins the ledger"); - // COVERAGE checkpointed per message (not a whole-chat checkpoint): // one entry per transcript message, fingerprinted content each — // the same turns are never extracted twice. + const record = chatStore.getChat(chatId)!; const covered = ledgers.readCoverage(); // Wave D: the coverage unit is the VISIBLE native message — bonded // riders with no view lines (a textless call message) are context, @@ -498,12 +474,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { it("the LLM cluster-namer is DEAD (wave 7): a merge mints no naming runs — labels are session names now", async () => { const { chatId } = fixtureChat(); const ledgers = memoryLedgers(); - // Identity-pass verdict: p1 becomes a new node, p2 dropped (no links). + // Identity-pass verdict: p1 becomes a new node, p2 unlinked. const verdict: AuditResult = { propositions: [{ ...FINDINGS.propositions[0]! }], edges: [], - attribution: { quiz: { "probe-1": "p1" } }, - dropped: [{ ref: "p2", reason: "evidence not found" }], }; const pipeline = createExtractionPipeline({ chatStore, @@ -512,7 +486,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { auditor: submitModel("submit_review", verdict), }); const stats = await pipeline.runExtraction(chatId); - assert.deepEqual(stats, { nodes: 1, edges: 0, events: 3 }); + assert.deepEqual(stats, { nodes: 1, edges: 0 }); // Wave 7 (950ddfd): birth-session clusters carry session-name labels, // so the post-merge naming garnish is gone — no model spend, no // naming-run records, no label writes. @@ -525,9 +499,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const ledgers = memoryLedgers(); const verdict: AuditResult = { propositions: FINDINGS.propositions, - edges: [], // the edge is dropped with it - attribution: FINDINGS.attribution, - dropped: [{ ref: "p1→p2", reason: "endpoint dropped" }], + edges: [], }; const consulted: string[] = []; const pipeline = createExtractionPipeline({ @@ -558,8 +530,6 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const verdict: AuditResult = { propositions: FINDINGS.propositions, edges: [], - attribution: FINDINGS.attribution, - dropped: [{ ref: "p1→p2", reason: "endpoint dropped" }], }; const pipeline = createExtractionPipeline({ chatStore, @@ -593,7 +563,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { chatStore, graphStore: ledgers, extractor: silentModel(), - auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }), + auditor: submitModel("submit_review", FINDINGS), }); await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/); assert.equal(ledgers.readCheckpoints().length, 0); @@ -616,8 +586,6 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const verdict: AuditResult = { propositions: [{ ...FINDINGS.propositions[0]!, linkTo: existingId }], edges: [], - attribution: { quiz: {} }, - dropped: [{ ref: "p2", reason: "evidence not found" }], }; const pipeline = createExtractionPipeline({ chatStore, @@ -648,8 +616,6 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const verdict: AuditResult = { propositions: [{ ...FINDINGS.propositions[0]!, linkTo: existingId }], edges: [], - attribution: { quiz: {} }, - dropped: [{ ref: "p2", reason: "evidence not found" }], }; const pipeline = createExtractionPipeline({ chatStore, @@ -675,7 +641,6 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { assert.equal(record.extractions?.length, 1); const run = record.extractions![0]!; assert.equal(run.stats?.nodes, 0); // linked, not created - assert.equal(run.stats?.events, 3); // the scripted engine's model, provider-qualified (model slots wave) assert.equal(run.stages.extractor.model, "faux:faux-1"); assert.equal(run.stages.extractor.steps.length, 1); // the submit call — the run STOPS after a clean submit @@ -696,7 +661,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { chatStore, graphStore: ledgers, extractor: hangingModel(), // hangs until aborted — no step finishes - auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }), + auditor: submitModel("submit_review", { propositions: [], edges: [] }), }); const runPromise = pipeline.runExtraction(chatId); @@ -706,7 +671,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const stats = await runPromise; // A deliberate stop, not a failure: zero emissions, no throw. - assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 }); + assert.deepEqual(stats, { nodes: 0, edges: 0 }); assert.equal(extractionStatuses().length, 0, "the registry clears"); assert.equal(ledgers.readNodes().length, 0, "nothing merged"); assert.equal(ledgers.readCheckpoints().length, 0, "nothing checkpointed — the next trigger re-runs"); @@ -751,7 +716,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const first = pipeline.runExtraction(chatId); const second = await pipeline.runExtraction(chatId); // in-flight → zeros - assert.deepEqual(second, { nodes: 0, edges: 0, events: 0 }); + assert.deepEqual(second, { nodes: 0, edges: 0 }); release(); await assert.rejects(first, /the extractor never submitted findings/); @@ -800,7 +765,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { ); const g = globalThis as { __lhExtractionRuns?: { running: Map<string, unknown> } }; assert.ok(g.__lhExtractionRuns?.running.has(chatId), "the registry lives on globalThis — no module-instance split can hide runs"); - assert.deepEqual(await observer.runExtraction(chatId), { nodes: 0, edges: 0, events: 0 }, "single-flight spans pipeline instances"); + assert.deepEqual(await observer.runExtraction(chatId), { nodes: 0, edges: 0 }, "single-flight spans pipeline instances"); release(); await assert.rejects(first, /the extractor never submitted findings/); @@ -829,8 +794,8 @@ const NOTES_TEXT = `> [!quote] YOU > [!abstract] PI > Vim always starts in Normal mode — keys are commands, not text.`; -/** What the notes-mode extractor submits: same shapes minus attribution, - * and messages always empty (the file is the provenance). */ +/** What the notes-mode extractor submits: same shapes as session findings, + * messages always empty (the file is the provenance). */ const NOTES_FINDINGS = { propositions: [ { @@ -840,7 +805,6 @@ const NOTES_FINDINGS = { aliases: [], messages: [], evidence: ["Vim always starts in Normal mode — keys are commands, not text."], - understanding: "solid", }, ], edges: [], @@ -849,11 +813,10 @@ const NOTES_FINDINGS = { const NOTES_VERDICT = { propositions: [NOTES_FINDINGS.propositions[0]], edges: [], - dropped: [], } as const; describe("the notes front door (importNotes — format-agnostic)", () => { - it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => { + it("imports raw notes end to end: source record, file-scoped provenance", async () => { const ledgers = memoryLedgers(); const seen: { kind?: string; label?: string }[] = []; const pipeline = createExtractionPipeline({ @@ -868,7 +831,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => { }); const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); - assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 }); + assert.deepEqual(stats, { nodes: 1, edges: 0 }); // The run surfaced as a NOTES run labeled with the file path. assert.deepEqual(seen, [{ kind: "notes", label: "/vault/learn vim.md" }]); assert.equal(extractionStatuses().length, 0, "the registry clears when the run ends"); @@ -888,14 +851,6 @@ describe("the notes front door (importNotes — format-agnostic)", () => { assert.equal(node.statement, "Vim starts in Normal mode, where keys are commands."); assert.deepEqual(node.provenance, []); assert.deepEqual(ledgers.readMappings(), [{ chatId: source.id, sourceId: "p1", graphNodeId: node.id }]); - - // One observation event, zero quiz events — outcomes inside notes - // surface only through understanding observations (user-decided). - const events = ledgers.readEvents(); - assert.deepEqual( - events.map((e) => [e.kind, e.chatId, e.ref, e.note]), - [["observation", source.id, "obs:p1", "solid"]], - ); // The checkpoint stamps the import. assert.equal(ledgers.readCheckpoints()[0]!.chatId, source.id); }); @@ -910,10 +865,10 @@ describe("the notes front door (importNotes — format-agnostic)", () => { auditor: submitModel("submit_review", NOTES_VERDICT), }); const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); - assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 }); + assert.deepEqual(first, { nodes: 1, edges: 0 }); const second = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); - assert.deepEqual(second, { nodes: 0, edges: 0, events: 0 }, "the same source id dedupes everything"); + assert.deepEqual(second, { nodes: 0, edges: 0 }, "the same source id dedupes everything"); assert.equal(ledgers.readSources().length, 1, "one source identity — the path hashes to the same id"); assert.equal(ledgers.readSources()[0]!.extractions?.length, 2, "both runs billed their usage"); @@ -932,7 +887,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => { const srcId = extractionStatuses().find((r) => r.chatId.startsWith("src_"))!.chatId; assert.equal(abortExtraction(srcId), true); - assert.deepEqual(await runPromise, { nodes: 0, edges: 0, events: 0 }); + assert.deepEqual(await runPromise, { nodes: 0, edges: 0 }); assert.equal(ledgers.readNodes().length, 0, "nothing merged"); assert.equal(ledgers.readCheckpoints().length, 0, "nothing checkpointed"); assert.equal(ledgers.readSources()[0]!.extractions?.length, 1, "partial usage still persisted"); @@ -943,8 +898,8 @@ describe("the notes front door (importNotes — format-agnostic)", () => { // ── the chunked pipeline (extraction coverage — vision-extraction-coverage.md) ── /** Pull the pipeline's prompt view out of a model call: the pipeline sends - * `Transcript:\n\n<view>` as the user turn, so the LAST user message's - * text IS what the extractor saw. */ + * the session's markdown document itself as the user turn, so the LAST + * user message's text IS what the extractor saw. */ function capturedView(context: unknown): string { const messages = (context as { messages?: { role: string; content: string }[] }).messages ?? []; const last = [...messages].reverse().find((m) => m.role === "user"); @@ -972,8 +927,6 @@ function passThroughAuditStep(): FauxResponseStep { { propositions: [{ ...FINDINGS.propositions[0]! }], edges: [], - attribution: { quiz: {} }, - dropped: [{ ref: "p2", reason: "evidence not found" }], } as unknown as ToolCall["arguments"], { id: "review-call" }, ), @@ -1014,15 +967,16 @@ describe("the chunked pipeline (extraction coverage)", () => { // Two batches: the first held messages 1-3, the second 4-5 — a // message is never split, batches are consecutive oldest-first. assert.equal(views.length, 2, "one extractor call per batch"); - assert.ok(views[0]!.includes("LEARNER [u1]") && views[0]!.includes("LEARNER [u3]")); - assert.ok(!views[0]!.includes("LEARNER [u4]"), "the batch stops at the token budget"); - assert.ok(views[1]!.includes("LEARNER [u4]") && views[1]!.includes("LEARNER [u5]")); - assert.ok(!views[1]!.includes("LEARNER [u3]")); + assert.ok(views[0]!.includes("### Learner · u1") && views[0]!.includes("### Learner · u3")); + assert.ok(!views[0]!.includes("### Learner · u4"), "the batch stops at the token budget"); + assert.ok(views[1]!.includes("### Learner · u4") && views[1]!.includes("### Learner · u5")); + assert.ok(!views[1]!.includes("### Learner · u3")); + // The document header is the session title. + assert.ok(views[0]!.startsWith("# Chunky coverage fixture")); // p1 was a fresh node in batch 1; batch 2's p1 mapped to the same - // node (chat-scoped mapping) — but its understanding note is a NEW - // batch-tagged observation. - assert.deepEqual(stats, { nodes: 1, edges: 0, events: 2 }); + // node (chat-scoped mapping). + assert.deepEqual(stats, { nodes: 1, edges: 0 }); // Per-batch coverage: every message checkpointed, nothing legacy. const coverage = ledgers.readCoverage(); @@ -1034,7 +988,7 @@ describe("the chunked pipeline (extraction coverage)", () => { // new usage entry (no money spent on covered ground). const before = views.length; const again = await pipeline.runExtraction(chatId); - assert.deepEqual(again, { nodes: 0, edges: 0, events: 0 }); + assert.deepEqual(again, { nodes: 0, edges: 0 }); assert.equal(views.length, before); assert.equal(chatStore.getChat(chatId)!.extractions?.length, 1, "the no-op run persists no usage"); }); @@ -1079,7 +1033,7 @@ describe("the chunked pipeline (extraction coverage)", () => { const stats = await runPromise; // Batch 1's emissions are kept and its messages stay covered. - assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 }); + assert.deepEqual(stats, { nodes: 1, edges: 0 }); assert.equal(ledgers.readCoverage().length, 3, "batch 1 checkpointed its three messages"); const cov = coverageOf(record.messages, ledgers.readCoverage(), chatId); assert.equal(cov.uncoveredCount, 2, "the in-flight batch's messages stay uncovered"); @@ -1095,8 +1049,8 @@ describe("the chunked pipeline (extraction coverage)", () => { }); await finisher.runExtraction(chatId); assert.equal(reRunViews.length, 1); - assert.ok(reRunViews[0]!.includes("LEARNER [u4]") && reRunViews[0]!.includes("LEARNER [u5]")); - assert.ok(!reRunViews[0]!.includes("LEARNER [u1]"), "the covered remainder never re-enters the prompt"); + assert.ok(reRunViews[0]!.includes("### Learner · u4") && reRunViews[0]!.includes("### Learner · u5")); + assert.ok(!reRunViews[0]!.includes("### Learner · u1"), "the covered remainder never re-enters the prompt"); assert.equal(ledgers.readCoverage().length, 5, "the re-run closed the gap"); }); @@ -1152,7 +1106,7 @@ describe("the chunked pipeline (extraction coverage)", () => { }); await reOpener.runExtraction(chatId); assert.equal(reRunViews.length, 1, "one batch: the single re-opened message"); - assert.ok(reRunViews[0]!.includes("LEARNER [s1]") && !reRunViews[0]!.includes("LEARNER [s2]")); + assert.ok(reRunViews[0]!.includes("### Learner · s1") && !reRunViews[0]!.includes("### Learner · s2")); assert.equal(coverageOf(messages(), ledgers.readCoverage(), chatId).uncoveredCount, 0, "re-covered"); }); }); @@ -1170,7 +1124,7 @@ describe("the session-deletion guard (chat deleted mid-run)", () => { chatStore, graphStore: ledgers, extractor: submitModel("submit_findings", FINDINGS), - auditor: submitModel("submit_review", { propositions: FINDINGS.propositions, edges: [], attribution: { quiz: {} }, dropped: [] }, () => chatStore.deleteChat(chatId)), + auditor: submitModel("submit_review", { propositions: FINDINGS.propositions, edges: [] }, () => chatStore.deleteChat(chatId)), }); const stats = await pipeline.runExtraction(chatId); @@ -1179,13 +1133,11 @@ describe("the session-deletion guard (chat deleted mid-run)", () => { assert.ok(stats); assert.equal(stats.nodes, 0); assert.equal(stats.edges, 0); - assert.equal(stats.events, 0); // Nothing merged, nothing checkpointed — the deleted chat leaves no // trace for a later snapshot to resurrect. assert.equal(ledgers.readNodes().length, 0); assert.equal(ledgers.readEdges().length, 0); - assert.equal(ledgers.readEvents().length, 0); assert.equal(ledgers.readMappings().length, 0); assert.equal(ledgers.readCoverage().length, 0); assert.equal(ledgers.readCheckpoints().length, 0); diff --git a/tests/graph-tools.test.ts b/tests/graph-tools.test.ts index 13f10f8..fc2a8e9 100644 --- a/tests/graph-tools.test.ts +++ b/tests/graph-tools.test.ts @@ -1,19 +1,21 @@ -// THE PROBER'S GRAPH TOOLS (wave 7) — pinned behaviors: +// THE PROBER'S GRAPH TOOLS (wave 7; understanding levels removed +// 2026-09-27) — pinned behaviors: // 1. REGISTRATION: the factory registers list_sessions/list_propositions // on the prober ONLY when the composition root passes a graphStore // (the graphToolsEnabled gate lives server-side; absent store = closed // gate — the factory must not reach for settings itself). -// 2. OUTPUT: names + levels + dating only — the owner ruling bars -// statement text and session contents from the prober's view of the map. +// 2. OUTPUT: names only — the owner ruling bars statement text and session +// contents from the prober's view of the map. // 3. HONESTY: empty graph, unknown session id — the tools answer in // their own voice, never throw, never guess. import assert from "node:assert/strict"; import { describe, it } from "node:test"; import { createProberTools } from "../src/agent/tools.ts"; +import { auditorGraphTools } from "../src/agent/tools/graph.ts"; import type { ChatStore, GraphStore, JobStore } from "../src/core/ports.ts"; import type { ChatSummary } from "../src/core/contracts/chat.ts"; -import type { GraphEdge, GraphNode, LedgerEvent } from "../src/core/contracts/graph.ts"; +import type { GraphEdge, GraphNode } from "../src/core/contracts/graph.ts"; // ─── fixtures ──────────────────────────────────────────────────────────────── @@ -31,27 +33,18 @@ const node = ( provenance, }); -const observation = ( - nodeId: string, - chatId: string, - note: string, - at: string, -): LedgerEvent => - ({ kind: "observation", nodeId, chatId, note, at }) as unknown as LedgerEvent; - function fakeStores( nodes: GraphNode[], over: { - events?: LedgerEvent[]; chats?: Array<Pick<ChatSummary, "id" | "title" | "createdAt" | "updatedAt">>; + mappings?: Array<{ chatId: string; sourceId: string; graphNodeId: string }>; } = {}, ): { chatStore: ChatStore; graphStore: GraphStore } { const edges: GraphEdge[] = []; const graphStore: GraphStore = { readNodes: () => nodes, readEdges: () => edges, - readEvents: () => over.events ?? [], - readMappings: () => [], + readMappings: () => over.mappings ?? [], readCheckpoints: () => [], readCoverage: () => [], readSources: () => [], @@ -134,7 +127,6 @@ describe("list_sessions", () => { updatedAt: "2026-09-21T10:00:00.000Z", }, ], - events: [], }, ); const tools = createProberTools({ ...baseCtx, chatStore, graphStore }); @@ -157,38 +149,28 @@ describe("list_sessions", () => { }); describe("list_propositions", () => { - it("names + latest understanding + that evaluation's date — the latest event wins", async () => { - const { chatStore, graphStore } = fakeStores( - [node("n1", "Water cycle", [pt("c1")]), node("n2", "Evaporation", [pt("c1")])], - { - events: [ - // The birth session's OWN evaluations only — a later - // session's observation must not leak in here. - observation("n1", "c1", "shaky", "2026-09-20T12:00:00.000Z"), - observation("n1", "c1", "solid", "2026-09-22T12:00:00.000Z"), - observation("n1", "c2", "resisted", "2026-09-23T12:00:00.000Z"), - observation("n2", "c1", "not yet engaged", "2026-09-21T12:00:00.000Z"), - ], - }, - ); + it("names only — what the session covered (2026-09-27: no levels, no dating)", async () => { + const { chatStore, graphStore } = fakeStores([ + node("n1", "Water cycle", [pt("c1")]), + node("n2", "Evaporation", [pt("c1")]), + ]); const tools = createProberTools({ ...baseCtx, chatStore, graphStore }); const out = (await tools.list_propositions!.execute!("tc", { session: "c1" })).details as string; - assert.match(out, /2 proposition\(s\) born in this session:/); - assert.match(out, /- Water cycle — solid \(as of 2026-09-22T12:00:00.000Z\)/); - assert.match(out, /- Evaporation — not yet engaged \(as of 2026-09-21T12:00:00.000Z\)/); - // Never the other session's word, never the statement text. - assert.ok(!out.includes("resisted")); + assert.match(out, /2 proposition\(s\) covered in this session:/); + assert.match(out, /- Water cycle/); + assert.match(out, /- Evaporation/); + // The owner ruling: no levels, no dating, never the statement text. + assert.ok(!out.includes("as of")); + assert.ok(!out.includes("solid")); assert.ok(!out.includes("statement of")); }); - it("a proposition with no observation reads 'never evaluated', undated", async () => { - const { chatStore, graphStore } = fakeStores([node("n1", "Water cycle", [pt("c1")])], { - events: [], - }); + it("no evaluation machinery at all — the covered list is the whole output", async () => { + const { chatStore, graphStore } = fakeStores([node("n1", "Water cycle", [pt("c1")])]); const tools = createProberTools({ ...baseCtx, chatStore, graphStore }); const out = (await tools.list_propositions!.execute!("tc", { session: "c1" })).details as string; - assert.match(out, /- Water cycle — never evaluated/); - assert.ok(!out.includes("(as of")); + assert.match(out, /- Water cycle/); + assert.ok(!out.includes("never evaluated")); }); it("an unknown session id — honest redirect, not a throw", async () => { @@ -198,3 +180,56 @@ describe("list_propositions", () => { assert.equal(out, 'No propositions born in session "no-such". Call list_sessions for the current ids.'); }); }); + +// ─── the auditor's toolset (2026-09-27 relink) ─────────────────────────────── + +describe("auditor graph tools", () => { + const nodes = [ + { ...node("n1", "Will to power", [pt("c1")]), statement: "The will to power is Nietzsche's central drive.", aliases: ["power drive"] }, + { ...node("n2", "Eternal recurrence", [pt("c1")]), statement: "Eternal recurrence is the thought experiment of living again.", aliases: [] }, + ]; + const { chatStore, graphStore } = fakeStores(nodes); + + it("find: glob over names + statements + aliases, returning id + name + statement", async () => { + const tools = auditorGraphTools({ chatStore, graphStore }); + const out = (await tools.find!.execute!("tc", { query: "*will to power*" })).details as string; + assert.match(out, /1 match\(es\):/); + assert.match(out, /- n1 :: Will to power/); + assert.match(out, /statement: The will to power is Nietzsche's central drive\./); + // Statements and aliases hit too — and a bare term gets implicit stars. + const byStatement = (await tools.find!.execute!("tc", { query: "living again" })).details as string; + assert.match(byStatement, /n2 :: Eternal recurrence/); + const byAlias = (await tools.find!.execute!("tc", { query: "power drive" })).details as string; + assert.match(byAlias, /n1/); + const miss = (await tools.find!.execute!("tc", { query: "*nothing matches this*" })).details as string; + assert.equal(miss, 'No existing proposition matches "*nothing matches this*".'); + }); + + it("list_sessions: names, kinds, counts, ids — NO timestamps (identity work needs content, not chronology)", async () => { + const { chatStore: cs, graphStore: gs } = fakeStores(nodes, { + chats: [{ id: "c1", title: "Nietzsche seminar", createdAt: "2026-09-20T10:00:00.000Z", updatedAt: "2026-09-21T10:00:00.000Z" }], + }); + const tools = auditorGraphTools({ chatStore: cs, graphStore: gs }); + const out = (await tools.list_sessions!.execute!("tc", {})).details as string; + assert.match(out, /1 session\(s\):/); + assert.match(out, /Nietzsche seminar \(chat, 2 propositions\) — id: c1/); + assert.ok(!out.includes("2026-09-20"), "no dates in the auditor's view"); + }); + + it("list_propositions: FULL statement text (identity work), no dates", async () => { + const { chatStore: cs, graphStore: gs } = fakeStores(nodes, { + mappings: [ + { chatId: "c1", sourceId: "p1", graphNodeId: "n1" }, + { chatId: "c1", sourceId: "p2", graphNodeId: "n2" }, + ], + }); + const tools = auditorGraphTools({ chatStore: cs, graphStore: gs }); + const out = (await tools.list_propositions!.execute!("tc", { session: "c1" })).details as string; + assert.match(out, /2 proposition\(s\) born in this session:/); + assert.match(out, /- n1 :: Will to power/); + assert.match(out, /statement: The will to power is Nietzsche's central drive\./); + assert.ok(!out.includes("2026-09"), "no dates in the auditor's view"); + const miss = (await tools.list_propositions!.execute!("tc", { session: "no-such" })).details as string; + assert.equal(miss, 'No propositions born in session "no-such". Call list_sessions for the current ids.'); + }); +}); diff --git a/tests/graph.test.ts b/tests/graph.test.ts index 05cd4fb..5208474 100644 --- a/tests/graph.test.ts +++ b/tests/graph.test.ts @@ -1,9 +1,11 @@ /** - * Graph merger + trigger + derivation tests (vision §1, 2026-09-02 redesign). + * Graph merger + trigger + derivation tests (vision §1, 2026-09-02 redesign; + * events cleanup 2026-09-27). * The merger (core/graph.ts mergeFindings) is the pipeline's deterministic * apply stage: AUDITED VERDICTS in, ledger appends out — link-or-create - * identity (the auditor's linkTo, never string matching), deterministic quiz - * events, understanding observations, and the idempotency guarantees. + * identity (the auditor's linkTo, never string matching) and the + * idempotency guarantees. No events ledger: quiz outcomes and + * understanding observations are no longer formalized in the graph. * * Ledger isolation: PATHS is fixed at import time, so the five data/graph * ledgers are backed up and cleared before the suite and restored after @@ -30,15 +32,12 @@ import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts"; /** The wired fs adapters, exactly what src/server/stores.ts passes in production. */ const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps; -/** The session merge source for a chat — what the pipeline passes: the live - * transcript + its updatedAt checkpoint stamp. (Notes imports pass - * messages: null — see the notes-merger test below.) */ -function sessionSource(chatId: string): { messages: LhMessage[]; foldedAt: string } { - // Wave B: the record is native history — the merger's transcript view - // reads through the one-way projection (same shape the extraction - // pipeline passes: its batch slots are projected messages). +/** The session merge source for a chat — what the pipeline passes: the + * fold stamp only (the merger reads no transcript; quiz-event rebuilding + * died with the events ledger, 2026-09-27). */ +function sessionSource(chatId: string): { foldedAt: string } { const record = chatStore.getChat(chatId)!; - return { messages: record.messages, foldedAt: record.updatedAt }; + return { foldedAt: record.updatedAt }; } // ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ────────────── @@ -215,9 +214,8 @@ function fixtureMessages(): LhMessage[] { ].flat(); } -/** The auditor's verdict for the fixture chat: three verified propositions - * (no linkTo — the graph starts empty), one follows edge, all quizzes - * attributed, one understanding note. */ +/** The auditor's verdict for the fixture chat: three propositions + * (no linkTo — the graph starts empty), one follows edge. */ function fixtureAudit(): AuditResult { return { propositions: [ @@ -228,7 +226,6 @@ function fixtureAudit(): AuditResult { aliases: ["alpha feel"], messages: ["a1", "a3"], evidence: ["fixture evidence"], - understanding: "solid", }, { id: "p2", @@ -256,8 +253,6 @@ function fixtureAudit(): AuditResult { evidence: ["fixture evidence"], }, ], - attribution: { quiz: { "call-1": "p1", "call-2": null, "call-3": "p1", "call-4": "p2", "call-5": "p3" } }, - dropped: [], }; } @@ -283,14 +278,13 @@ function scopedGraph(chatIds: string[]) { return { nodes: graph.nodes.filter((n) => nodeIds.has(n.id)), edges: graph.edges.filter((e) => e.provenance.some((pt) => chatIds.includes(pt.chatId))), - events: graph.events.filter((e) => chatIds.includes(e.chatId)), }; } describe("the merger (mergeFindings)", () => { - it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => { + it("creates nodes from the verdicts with message-granular provenance", () => { const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id)); - assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 }); + assert.deepEqual(stats, { nodes: 3, edges: 1 }); const graph = scopedGraph([chat1.id]); assert.equal(graph.nodes.length, 3); @@ -321,49 +315,14 @@ describe("the merger (mergeFindings)", () => { graphStore.readMappings().map((m) => m.sourceId), ["p1", "p2", "p3"], ); - - // Events — kinds, refs, notes, and the attribution paths. - assert.equal(graph.events.length, 6); - const byRef = new Map(graph.events.map((e) => [e.ref, e])); - - const miss = byRef.get("call-1")!; - assert.equal(miss.kind, "miss"); - assert.equal(miss.nodeId, idByStatement.get("Alpha intuition"), "attributed quiz"); - - const dontKnow = byRef.get("call-2")!; - assert.equal(dontKnow.kind, "dont-know"); - assert.equal(dontKnow.nodeId, null, "unattributable → null"); - assert.equal(dontKnow.question, "Q2 text"); - - const correct = byRef.get("call-3")!; - assert.equal(correct.kind, "correct"); - assert.equal(correct.nodeId, idByStatement.get("Alpha intuition"), "attribution wins over the ignored input.nodeId"); - assert.equal(correct.note, "recalled the diagram"); - - const miss2 = byRef.get("call-4")!; - assert.equal(miss2.kind, "miss"); - assert.equal(miss2.nodeId, idByStatement.get("Beta rule")); - - const dontKnow2 = byRef.get("call-5")!; - assert.equal(dontKnow2.kind, "dont-know"); - assert.equal(dontKnow2.nodeId, idByStatement.get("Gamma application")); - - const observation = byRef.get("obs:p1")!; - assert.equal(observation.kind, "observation"); - assert.equal(observation.nodeId, idByStatement.get("Alpha intuition")); - assert.equal(observation.note, "solid"); - - // The dismissed quiz and the ask question became nothing. - assert.equal([...byRef.keys()].filter((r) => r === "call-6" || r === "call-7").length, 0); }); it("is idempotent — re-merging the same verdicts appends nothing", () => { const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id)); - assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 }); + assert.deepEqual(stats, { nodes: 0, edges: 0 }); const graph = scopedGraph([chat1.id]); assert.equal(graph.nodes.length, 3); assert.equal(graph.edges.length, 1); - assert.equal(graph.events.length, 6); // exactly one checkpoint line per merge, though — checkpoints are a log assert.equal(graphStore.readCheckpoints().filter((c) => c.chatId === chat1.id).length, 2); }); @@ -398,13 +357,11 @@ describe("the merger (mergeFindings)", () => { }, ], edges: [], - attribution: { quiz: { "call-9": "q1" } }, - dropped: [], }, graphDeps, sessionSource(chat2.id), ); - assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 }); + assert.deepEqual(stats, { nodes: 0, edges: 0 }); const after = scopedGraph([chat1.id, chat2.id]); assert.equal(after.nodes.length, 3, "no duplicate node"); @@ -436,23 +393,18 @@ describe("the merger (mergeFindings)", () => { aliases: [], messages: [], evidence: ["fixture evidence"], - understanding: "solid", }, ], edges: [], - attribution: { quiz: {} }, - dropped: [], }, graphDeps, - { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + { foldedAt: "2026-09-13T00:00:00.000Z" }, ); const snap = graphSnapshot(graphDeps); const node = snap.nodes.find((n) => n.statement === "Vim registers map single keys to editing commands.")!; assert.equal(snap.fileSources["src_snapfixture"], "/vault/learn vim.md"); assert.deepEqual(snap.nodeSessions[node.id], ["src_snapfixture"], "the file entry joins the node's sessions list"); assert.equal(snap.sessions["src_snapfixture"], undefined, "a notes source is never a chat session"); - // The understanding observation joins under the source id like any session's. - assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid"); }); it("derives BIRTH-SESSION clusters in graphSnapshot, labeled with the session's name", () => { @@ -478,11 +430,9 @@ describe("the merger (mergeFindings)", () => { { from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] }, { from: "p1", to: "p3", type: "follows", messages: [], evidence: ["e"] }, ], - attribution: { quiz: {} }, - dropped: [], }, graphDeps, - { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + { foldedAt: "2026-09-13T00:00:00.000Z" }, ); assert.equal(merge.nodes, 3); const snap = graphSnapshot(graphDeps); @@ -496,12 +446,14 @@ describe("the merger (mergeFindings)", () => { assert.equal(mine!.label, "/vault/cluster notes.md", "a notes file is a birth-session — the label is its path"); }); - it("derives sessions and per-session understanding observations in graphSnapshot", () => { + it("derives the sessions index in graphSnapshot", () => { const snap = graphSnapshot(graphDeps); const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id])); - // No node-state classes anymore (2026-09-08): the snapshot carries - // only the objective graph + sessions; quiz outcomes stay ledger data. + // No node-state classes (2026-09-08) and no understanding + // observations (2026-09-27 cleanup): the snapshot carries only the + // objective graph + sessions. assert.equal("nodeStates" in snap, false); + assert.equal("nodeObservations" in snap, false); assert.equal(snap.sessions[chat1.id], "Graph fixture"); assert.equal(snap.sessions[chat2Id], "Graph fixture 2"); @@ -511,12 +463,6 @@ describe("the merger (mergeFindings)", () => { ); // Gamma was only ever touched by chat1. assert.deepEqual(snap.nodeSessions[idByStatement.get("Gamma application")!], [chat1.id]); - - // Understanding observations: (node → chat → the extractor's note). - // Alpha carries chat1's "solid" level; chat2's q1 has no level. - const alphaObs = snap.nodeObservations[idByStatement.get("Alpha intuition")!] ?? {}; - assert.equal(alphaObs[chat1.id], "solid"); - assert.equal(alphaObs[chat2Id], undefined); }); it("never string-matches without a verdict — an identical statement without linkTo is a NEW node", () => { @@ -537,8 +483,6 @@ describe("the merger (mergeFindings)", () => { }, ], edges: [], - attribution: { quiz: {} }, - dropped: [], }, graphDeps, sessionSource(chat3.id), @@ -547,10 +491,9 @@ describe("the merger (mergeFindings)", () => { assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2); }); - it("merges the notes front door: source id as chatId, no quiz events, checkpoint at import time", () => { - // Raw personal notes carry no transcript — the merger gets no - // message parts, so NO quiz events are rebuilt; the understanding - // observation still lands, keyed to the source id. + it("merges the notes front door: source id as chatId, checkpoint at import time", () => { + // Raw personal notes carry no transcript — no message anchors; the + // file is the provenance (and no events exist to build, 2026-09-27). const stats = mergeFindings( "src_notesfixture", { @@ -562,17 +505,14 @@ describe("the merger (mergeFindings)", () => { aliases: [], messages: [], // notes carry no message anchors — the file is the provenance evidence: ["fixture evidence"], - understanding: "solid", }, ], edges: [], - attribution: { quiz: {} }, // normalized by the notes pipeline — nothing to attribute - dropped: [], }, graphDeps, - { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + { foldedAt: "2026-09-13T00:00:00.000Z" }, ); - assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 }); + assert.deepEqual(stats, { nodes: 1, edges: 0 }); const node = getGraph(graphStore).nodes.find((n) => n.statement.startsWith("Vim starts"))!; assert.deepEqual(node.provenance, [], "no message anchors — the mapping line carries the source"); @@ -581,12 +521,6 @@ describe("the merger (mergeFindings)", () => { sourceId: "p1", graphNodeId: node.id, }); - // The observation event is file-scoped; no quiz events exist. - const events = graphStore.readEvents().filter((e) => e.chatId === "src_notesfixture"); - assert.deepEqual( - events.map((e) => [e.kind, e.ref, e.note]), - [["observation", "obs:p1", "solid"]], - ); // The checkpoint stamps the import time. assert.equal(graphStore.readCheckpoints().find((c) => c.chatId === "src_notesfixture")?.foldedAt, "2026-09-13T00:00:00.000Z"); }); @@ -673,12 +607,10 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () = ].flat()); const auditA: AuditResult = { propositions: [ - { id: "p1", name: "Shared core", statement: "Shared core statement", aliases: ["shared feel"], messages: ["du1"], evidence: ["e"], understanding: "solid" }, + { id: "p1", name: "Shared core", statement: "Shared core statement", aliases: ["shared feel"], messages: ["du1"], evidence: ["e"] }, { id: "p2", name: "Solo fact", statement: "Solo fact statement", aliases: ["solo feel"], messages: ["du1"], evidence: ["e"] }, ], edges: [{ from: "p1", to: "p2", type: "follows", messages: ["du1"], evidence: ["e"] }], - attribution: { quiz: { "d-call-1": "p1" } }, - dropped: [], }; mergeFindings(dA.id, auditA, graphDeps, sessionSource(dA.id)); // Resolve through THIS session's mappings — the shared ledger holds @@ -701,12 +633,10 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () = ].flat()); const auditB: AuditResult = { propositions: [ - { id: "q1", name: "Shared again", statement: "Shared core, rephrased", aliases: ["B phrasing"], messages: ["eu1"], evidence: ["e"], linkTo: xId, understanding: "shaky" }, + { id: "q1", name: "Shared again", statement: "Shared core, rephrased", aliases: ["B phrasing"], messages: ["eu1"], evidence: ["e"], linkTo: xId }, { id: "q2", name: "B extra", statement: "B extra statement", aliases: ["b extra feel"], messages: ["eu1"], evidence: ["e"] }, ], edges: [{ from: "q1", to: "q2", type: "follows", messages: ["eu1"], evidence: ["e"] }], - attribution: { quiz: { "e-call-1": "q1" } }, - dropped: [], }; mergeFindings(dB.id, auditB, graphDeps, sessionSource(dB.id)); const zId = graphStore.readMappings().find((m) => m.chatId === dB.id && m.sourceId === "q2")!.graphNodeId; @@ -730,7 +660,6 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () = assert.deepEqual(stats, { nodesRemoved: 1, edgesRemoved: 1, - eventsRemoved: 2, mappingsRemoved: 2, coverageRemoved: 2, checkpointsRemoved: 1, @@ -753,11 +682,6 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () = // Edges: dA's assertion X→Y died; dB's X→Z survived. const scoped = scopedGraph([dA.id, dB.id]); assert.deepEqual(scoped.edges, [{ from: xId, to: zId, type: "follows", provenance: [{ chatId: dB.id, messageId: "eu1" }] }]); - // Events: only dB's quiz + observation remain. - assert.deepEqual(scoped.events.map((e) => [e.kind, e.chatId === dB.id]), [ - ["miss", true], - ["observation", true], - ]); // Mappings: dA's audit trail died; dB's stays. const mappings = graphStore.readMappings().filter((m) => [dA.id, dB.id].includes(m.chatId)); assert.deepEqual(mappings, [{ chatId: dB.id, sourceId: "q1", graphNodeId: xId }, { chatId: dB.id, sourceId: "q2", graphNodeId: zId }]); @@ -768,15 +692,16 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () = // The snapshot resolves X to dB only — the deleted chat left no ghost. const snapshot = graphSnapshot(graphDeps); assert.deepEqual(snapshot.nodeSessions[xId], [dB.id]); - assert.deepEqual(snapshot.nodeObservations[xId], { [dB.id]: "shaky" }); assert.ok(snapshot.sessions[dB.id] && !snapshot.sessions[dA.id]); // Atomic rewrites left no temp files, and appends still work. assert.equal(fs.readdirSync(PATHS.graph).filter((f) => f.includes(".tmp-")).length, 0); - const eventsBefore = graphStore.readEvents().length; - graphStore.appendEvent({ nodeId: null, kind: "correct", chatId: "chat_del_append_probe", at: "2026-09-23T00:04:00.000Z" }); - assert.equal(graphStore.readEvents().length, eventsBefore + 1); + const nodesBefore = graphStore.readNodes().length; + // Foreign provenance: the probe is another chat's data, so later + // deletions must leave it untouched (the foreign-data doctrine). + graphStore.appendNode({ id: "n_del_append_probe", name: "Probe", statement: "Probe statement", aliases: [], createdAt: "2026-09-23T00:04:00.000Z", provenance: [{ chatId: "chat_foreign_probe", messageId: "" }] }); + assert.equal(graphStore.readNodes().length, nodesBefore + 1); // Idempotent: a second run finds nothing. - assert.deepEqual(deleteChatTraces(graphStore, dA.id, "delete-nodes"), { ...stats, nodesRemoved: 0, edgesRemoved: 0, eventsRemoved: 0, mappingsRemoved: 0, coverageRemoved: 0, checkpointsRemoved: 0 } satisfies ChatDeletionStats); + assert.deepEqual(deleteChatTraces(graphStore, dA.id, "delete-nodes"), { ...stats, nodesRemoved: 0, edgesRemoved: 0, mappingsRemoved: 0, coverageRemoved: 0, checkpointsRemoved: 0 } satisfies ChatDeletionStats); }); it("preserve-graph: only the extraction bookkeeping dies — the graph ledgers are byte-identical", () => { @@ -784,7 +709,6 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () = const before = { nodes: graphStore.readNodes(), edges: graphStore.readEdges(), - events: graphStore.readEvents(), mappings: graphStore.readMappings(), positions: graphStore.readPositions(), }; @@ -792,14 +716,12 @@ describe("session deletion (deleteChatTraces + deleteChatSession, wave 6)", () = assert.deepEqual(stats, { nodesRemoved: 0, edgesRemoved: 0, - eventsRemoved: 0, mappingsRemoved: 0, coverageRemoved: 2, checkpointsRemoved: 1, } satisfies ChatDeletionStats); assert.deepEqual(graphStore.readNodes(), before.nodes); assert.deepEqual(graphStore.readEdges(), before.edges); - assert.deepEqual(graphStore.readEvents(), before.events); assert.deepEqual(graphStore.readMappings(), before.mappings); assert.deepEqual(graphStore.readPositions(), before.positions); assert.equal(graphStore.readCoverage().some((row) => row.chatId === dA.id), false); @@ -853,7 +775,7 @@ describe("single-node deletion (deleteNodeTraces + deleteGraphNode, node interlu // Clean slate per test: each test in this describe builds its own // scenario from empty ledgers (the suite's isolation covers the // whole file; this covers describe-internal stacking). - graphStore.rewriteLedgers({ nodes: [], edges: [], events: [], mappings: [] }); + graphStore.rewriteLedgers({ nodes: [], edges: [], mappings: [] }); graphStore.writePositions({}); graphStore.appendNode({ id: "n_dead", @@ -884,10 +806,6 @@ describe("single-node deletion (deleteNodeTraces + deleteGraphNode, node interlu graphStore.appendEdge({ from: "n_dead", to: "n_keep", type: "follows", provenance: [{ chatId: "chat_node_a", messageId: "m1" }] }); graphStore.appendEdge({ from: "n_keep", to: "n_dead", type: "supports", provenance: [{ chatId: "chat_node_b", messageId: "m3" }] }); graphStore.appendEdge({ from: "n_keep", to: "n_other", type: "follows", provenance: [{ chatId: "chat_node_b", messageId: "m3" }] }); - // Events: two sessions' evidence on X, one on the neighbor. - graphStore.appendEvent({ nodeId: "n_dead", kind: "correct", chatId: "chat_node_a", at: "2026-09-23T00:01:00.000Z", ref: "q1" }); - graphStore.appendEvent({ nodeId: "n_dead", kind: "observation", chatId: "chat_node_b", at: "2026-09-23T00:02:00.000Z", note: "shaky", ref: "obs:p1" }); - graphStore.appendEvent({ nodeId: "n_keep", kind: "miss", chatId: "chat_node_a", at: "2026-09-23T00:03:00.000Z", ref: "q2" }); // Mappings: the notes anchor + the chat's audit trail. graphStore.appendMapping({ chatId: "src_nodefile", sourceId: "p1", graphNodeId: "n_dead" }); graphStore.appendMapping({ chatId: "chat_node_a", sourceId: "p1", graphNodeId: "n_dead" }); @@ -900,11 +818,9 @@ describe("single-node deletion (deleteNodeTraces + deleteGraphNode, node interlu buildNodeScenario(); assert.deepEqual(nodeDeletionImpact(graphStore, "n_dead"), { edges: 2, - quizEvents: 1, - observations: 1, - // The file anchor, the re-encounter, and the second session's - // observation — every referencing session, deduped and sorted. - sessions: ["chat_node_a", "chat_node_b", "src_nodefile"], + // The file anchor + the re-encounter — every referencing + // session, deduped and sorted. + sessions: ["chat_node_a", "src_nodefile"], }); // Unknown node: null (the route's 404). assert.equal(nodeDeletionImpact(graphStore, "n_missing"), null); @@ -914,7 +830,7 @@ describe("single-node deletion (deleteNodeTraces + deleteGraphNode, node interlu buildNodeScenario(); const keepLineBefore = JSON.stringify(graphStore.readNodes().find((n) => n.id === "n_keep")); const stats = deleteNodeTraces(graphStore, "n_dead"); - assert.deepEqual(stats, { edgesRemoved: 2, eventsRemoved: 2, mappingsRemoved: 2 } satisfies NodeDeletionStats); + assert.deepEqual(stats, { edgesRemoved: 2, mappingsRemoved: 2 } satisfies NodeDeletionStats); // The node's EVERY line is gone; the neighbor is byte-identical // (full death is node-scoped — other nodes' sessions keep their @@ -930,11 +846,6 @@ describe("single-node deletion (deleteNodeTraces + deleteGraphNode, node interlu graphStore.readEdges().map((e) => [e.from, e.to]), [["n_keep", "n_other"]], ); - // The evidence died with the thing it was about; the neighbor's stays. - assert.deepEqual( - graphStore.readEvents().map((e) => [e.nodeId, e.kind]), - [["n_keep", "miss"]], - ); // The audit trail: X's mappings gone, the neighbor's stay. assert.deepEqual(graphStore.readMappings().map((m) => m.graphNodeId), ["n_keep"]); // The resting position died with the node. @@ -957,7 +868,7 @@ describe("single-node deletion (deleteNodeTraces + deleteGraphNode, node interlu provenance: [{ chatId: "chat_node_solo", messageId: "m1" }], }); const stats = deleteNodeTraces(graphStore, "n_solo"); - assert.deepEqual(stats, { edgesRemoved: 0, eventsRemoved: 0, mappingsRemoved: 0 } satisfies NodeDeletionStats); + assert.deepEqual(stats, { edgesRemoved: 0, mappingsRemoved: 0 } satisfies NodeDeletionStats); assert.equal(graphStore.readNodes().some((n) => n.id === "n_solo"), false); }); @@ -1007,7 +918,7 @@ describe("single-node deletion (deleteNodeTraces + deleteGraphNode, node interlu const revisionBefore = graphRevision(); const outcome = deleteGraphNode("n_orch"); assert.ok(outcome.ok); - assert.deepEqual(outcome.stats, { edgesRemoved: 0, eventsRemoved: 0, mappingsRemoved: 0 }); + assert.deepEqual(outcome.stats, { edgesRemoved: 0, mappingsRemoved: 0 }); assert.ok(outcome.graphRevision > 0); assert.ok(outcome.graphRevision >= revisionBefore, "the version stamp moved"); assert.equal(graphStore.readNodes().some((n) => n.id === "n_orch"), false); diff --git a/tests/session-clusters.test.ts b/tests/session-clusters.test.ts index 50e952b..77ee960 100644 --- a/tests/session-clusters.test.ts +++ b/tests/session-clusters.test.ts @@ -11,7 +11,7 @@ import assert from "node:assert/strict"; import { describe, it } from "node:test"; -import type { GraphEdge, GraphNode, LedgerEvent, NotesSource, ChatSummary } from "../src/core/contracts.ts"; +import type { GraphEdge, GraphNode, NotesSource, ChatSummary } from "../src/core/contracts.ts"; import type { ChatStore, GraphStore } from "../src/core/ports.ts"; import { graphSessionIndex, @@ -36,7 +36,6 @@ const node = (id: string, name: string, provenance: Array<{ chatId: string; mess function fakeStores( nodes: GraphNode[], over: { - events?: LedgerEvent[]; chats?: Array<Pick<ChatSummary, "id" | "title" | "createdAt" | "updatedAt">>; sources?: Array<Pick<NotesSource, "id" | "filePath" | "importedAt">>; mappings?: Array<{ chatId: string; sourceId: string; graphNodeId: string }>; @@ -46,7 +45,6 @@ function fakeStores( const graphStore: GraphStore = { readNodes: () => nodes, readEdges: () => edges, - readEvents: () => over.events ?? [], readMappings: () => over.mappings ?? [], readCheckpoints: () => [], readCoverage: () => [], @@ -160,12 +158,8 @@ describe("the notes mapping-line fallback", () => { ], "the file births under its src id; the childless chat has no entry (birth sessions only)"); }); - it("graphSessionPropositions lists the file's births under the src id", () => { - const events: LedgerEvent[] = [ - { nodeId: "a", kind: "observation", chatId: "src_abc", at: "2026-09-22T00:00:00.000Z", note: "solid", ref: "obs:p1" }, - ]; + it("graphSessionPropositions lists the file's births under the src id (a pure birth list, 2026-09-27)", () => { const deps = fakeStores([node("a", "Alpha", []), node("b", "Beta", [])], { - events, mappings: [ { chatId: "src_abc", sourceId: "p1", graphNodeId: "a" }, { chatId: "src_abc", sourceId: "p2", graphNodeId: "b" }, @@ -173,8 +167,8 @@ describe("the notes mapping-line fallback", () => { sources: [{ id: "src_abc", filePath: "/vault/learn vim.md", importedAt: "2026-09-22T00:00:00.000Z" }], }); assert.deepEqual(graphSessionPropositions(deps, "src_abc"), [ - { nodeId: "a", name: "Alpha", understanding: "solid", evaluatedAt: "2026-09-22T00:00:00.000Z" }, - { nodeId: "b", name: "Beta", understanding: null, evaluatedAt: null }, + { nodeId: "a", name: "Alpha" }, + { nodeId: "b", name: "Beta" }, ]); }); }); @@ -272,34 +266,16 @@ describe("graphSessionIndex (the prober's session list)", () => { }); describe("graphSessionPropositions (the per-session view)", () => { - it("birth propositions with the latest extractor evaluation AND that event's own timestamp", () => { + it("birth propositions — a pure list, no evaluations (2026-09-27 cleanup)", () => { const nodes = [ node("a", "Alpha", [pt("chat1")]), node("b", "Beta", [pt("chat1")]), node("c", "Gamma", [pt("chat2")]), // born elsewhere — never listed under chat1 ]; - const events: LedgerEvent[] = [ - { nodeId: "a", kind: "observation", chatId: "chat1", at: "2026-09-20T00:00:00.000Z", note: "shaky", ref: "obs:p1" }, - // A LATER evaluation supersedes the earlier one for (a, chat1). - { nodeId: "a", kind: "observation", chatId: "chat1", at: "2026-09-21T00:00:00.000Z", note: "solid", ref: "obs:p2" }, - // Quiz evidence (owner ruling 2026-09-23): recorded in the ledger - // but it NEVER moves the evaluation date. - { nodeId: "a", kind: "correct", chatId: "chat1", at: "2026-09-22T00:00:00.000Z", note: "got it" }, - // Another session's evaluation of the same node is not ours. - { nodeId: "a", kind: "observation", chatId: "chat2", at: "2026-09-23T00:00:00.000Z", note: "resisted" }, - // Same-at events: ledger order breaks the tie (the snapshot's rule). - { nodeId: "b", kind: "observation", chatId: "chat1", at: "2026-09-20T00:00:00.000Z", note: "accepted", ref: "obs:p3" }, - { nodeId: "b", kind: "observation", chatId: "chat1", at: "2026-09-20T00:00:00.000Z", note: "not yet engaged", ref: "obs:p4" }, - ]; - const deps = fakeStores(nodes, { events }); + const deps = fakeStores(nodes); assert.deepEqual(graphSessionPropositions(deps, "chat1"), [ - { nodeId: "a", name: "Alpha", understanding: "solid", evaluatedAt: "2026-09-21T00:00:00.000Z" }, - { nodeId: "b", name: "Beta", understanding: "not yet engaged", evaluatedAt: "2026-09-20T00:00:00.000Z" }, - ]); - // A node the session never evaluated shows nulls, never a guess. - const fresh = fakeStores([node("x", "X", [pt("chat1")])], { events }); - assert.deepEqual(graphSessionPropositions(fresh, "chat1"), [ - { nodeId: "x", name: "X", understanding: null, evaluatedAt: null }, + { nodeId: "a", name: "Alpha" }, + { nodeId: "b", name: "Beta" }, ]); }); }); \ No newline at end of file diff --git a/tests/session-markdown.test.ts b/tests/session-markdown.test.ts new file mode 100644 index 0000000..d3eeee0 --- /dev/null +++ b/tests/session-markdown.test.ts @@ -0,0 +1,138 @@ +/** + * The shared session-markdown engine (core/session-markdown.ts, owner ruling + * 2026-09-27): ONE renderer for the extractor's source view and the future + * user-facing session export — id-anchored speaker blocks, readable + * quiz/recall/question blocks, one-line notes for other tools. Pinned here: + * block shapes, anchor handling (on/off), title header, empty-message + * skipping, and content parity with the coverage line builder. + */ + +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import type { LhMessage } from "../src/core/native-messages.ts"; +import { messageMarkdown, sessionMarkdown } from "../src/core/session-markdown.ts"; +import { messageViewLines } from "../src/core/coverage.ts"; +import { coverageSlots } from "../src/core/coverage.ts"; + +function user(id: string, text: string): LhMessage { + return { id, role: "user", content: text, timestamp: Date.now() } as never; +} + +function teacher(id: string, text: string): LhMessage { + return { + id, + role: "assistant", + content: [{ type: "text", text }], + api: "openai-completions", + provider: "openrouter", + model: "test", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + stopReason: "stop", + timestamp: Date.now(), + } as never; +} + +function result( + id: string, + toolCallId: string, + toolName: string, + details: unknown, + isError = false, +): LhMessage { + return { + id, + role: "toolResult", + toolCallId, + toolName, + content: [{ type: "text", text: "ok" }], + details, + isError, + timestamp: Date.now(), + } as never; +} + +describe("messageMarkdown", () => { + it("renders speaker blocks with the id anchor", () => { + const md = messageMarkdown(user("u1", "teach me the chain rule"), "u1"); + assert.equal(md, "### Learner · u1\n\nteach me the chain rule"); + const md2 = messageMarkdown(teacher("a1", "The derivative measures change."), "a1"); + assert.equal(md2, "### Teacher · a1\n\nThe derivative measures change."); + }); + + it("anchors off — the same document without the ids (the user export)", () => { + const md = messageMarkdown(user("u1", "hello"), "u1", false); + assert.equal(md, "### Learner\n\nhello"); + }); + + it("quiz blocks: question + outcome + the learner's note", () => { + const md = messageMarkdown( + result("r1", "q1", "quiz", { questionText: "What is a derivative?", correct: false, dontKnow: false, note: "mixed up signs" }), + "r1", + )!; + assert.match(md, /^### Quiz · r1$/m); + assert.match(md, /\*\*Q:\*\* What is a derivative\?/); + assert.match(md, /The learner answered incorrectly\./); + assert.match(md, /\*\(his thinking: mixed up signs\)\*/); + }); + + it("explain results: the learner's own words are shown", () => { + const md = messageMarkdown( + result("r1", "e1", "explain", { questionText: "Why does Mercury stay near the Sun?", text: "it never strays far" }), + "r1", + )!; + assert.match(md, /^### Recall check · r1$/m); + assert.match(md, /The learner answered in his own words: it never strays far/); + }); + + it("ask blocks: question + chosen options + free text", () => { + const chosen = messageMarkdown( + result("r1", "a1", "ask_user_question", { questionText: "Which way?", answers: ["second"], other: "also the third" }), + "r1", + )!; + assert.match(chosen, /^### Question · r1$/m); + assert.match(chosen, /The learner chose: second, also the third/); + const free = messageMarkdown( + result("r2", "a2", "ask_user_question", { questionText: "Why?", text: "because" }), + "r2", + )!; + assert.match(free, /The learner answered: because/); + }); + + it("other clean tool results: one line; failed calls and markers: nothing", () => { + assert.equal(messageMarkdown(result("r1", "j1", "research", { brief: "long" }), "r1"), "### Tool · r1\n\n*research result delivered*"); + assert.equal(messageMarkdown(result("r2", "j2", "research", {}, true), "r2"), null, "a failed call is retry context"); + assert.equal(messageMarkdown(user("u3", " "), "u3"), null, "empty text renders nothing"); + }); +}); + +describe("sessionMarkdown", () => { + it("assembles blocks in order under the title header, skipping nothings", () => { + const md = sessionMarkdown([user("u1", "go"), teacher("a1", "First."), user("u2", " ")], { title: "My session" }); + assert.equal(md, "# My session\n\n### Learner · u1\n\ngo\n\n### Teacher · a1\n\nFirst."); + }); + + it("no title → no header; anchors default on", () => { + const md = sessionMarkdown([user("u1", "go")]); + assert.equal(md, "### Learner · u1\n\ngo"); + }); + + it("content parity with the coverage line builder (2026-09-27 split)", () => { + // Every message that produces view lines produces a markdown block, + // and every message that renders nothing renders nothing in either — + // same messages, two formats, one seam decision. The FINGERPRINT + // stays on the line format (stable — no mass re-extraction when + // presentation changes); markdown is presentation only. + const messages: LhMessage[] = [ + user("u1", "teach me"), + teacher("a1", "The chain rule multiplies local rates."), + result("r1", "q1", "quiz", { questionText: "Q?", correct: true, dontKnow: false }), + result("r2", "j1", "research", { brief: "b" }), + result("r3", "j2", "research", {}, true), + ]; + for (const message of messages) { + const hasLines = messageViewLines(message, message.id).length > 0; + const hasMarkdown = messageMarkdown(message, message.id) !== null; + assert.equal(hasMarkdown, hasLines, `parity for ${message.id}`); + } + }); +}); diff --git a/tests/usage.test.ts b/tests/usage.test.ts index f520548..279d912 100644 --- a/tests/usage.test.ts +++ b/tests/usage.test.ts @@ -243,7 +243,7 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl startedAt, at: "2026-09-09T00:00:00Z", durationMs: 42_000, - stats: { nodes: 1, edges: 2, events: 3 }, + stats: { nodes: 1, edges: 2 }, stages: { extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] }, auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] }, @@ -280,7 +280,7 @@ test("appendSourceUsage: appends to the source record, idempotent per startedAt, startedAt, at: "2026-09-13T00:00:00Z", durationMs: 42_000, - stats: { nodes: 1, edges: 2, events: 3 }, + stats: { nodes: 1, edges: 2 }, stages: { extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] }, auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },