Merge feat/graph-rendering: wave 5 — semantic clusters, regions renderer, persisted render tuning
This commit is contained in:
@@ -7,3 +7,5 @@ data/
|
||||
.env.local
|
||||
.memory/
|
||||
SCOPE.md
|
||||
# raw real-data import sources (user directive 2026-09-13: never commit)
|
||||
real-data-extraction/
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* POST /api/graph/import — the format-agnostic front door's transport
|
||||
* (vision §1, 2026-09-13): raw personal-notes files in, one extraction run
|
||||
* per file through the SAME pipeline the sessions use. Multipart form data
|
||||
* (field "files", any text format — the extractor is format-agnostic); the
|
||||
* pipeline mints a deterministic source id per file path, persists the
|
||||
* source record (the file's provenance + the run's usage), and runs
|
||||
* extractor → auditor → merger over the raw text.
|
||||
*
|
||||
* Small concurrency cap: the model calls are the slow part, and a burst of
|
||||
* 25 simultaneous runs would trip provider rate limits — a 3-wide worker
|
||||
* pool keeps the run registry (the dock's status chips) and the ledgers
|
||||
* healthy. The dock's chips + graph refetch track progress live through
|
||||
* /api/graph/status, as with session runs.
|
||||
*/
|
||||
import { extraction } from "@/src/server";
|
||||
import type { EmissionStats } from "@/src/core";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
interface ImportResult {
|
||||
filePath: string;
|
||||
ok: boolean;
|
||||
stats?: EmissionStats;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const form = await request.formData();
|
||||
const files = form.getAll("files").filter((f): f is File => f instanceof File);
|
||||
if (files.length === 0) {
|
||||
return Response.json({ results: [] as ImportResult[], error: "no files" }, { status: 400 });
|
||||
}
|
||||
|
||||
const results: ImportResult[] = [];
|
||||
const queue = [...files];
|
||||
const workers = Array.from({ length: Math.min(3, files.length) }, async () => {
|
||||
for (;;) {
|
||||
const file = queue.shift();
|
||||
if (!file) return;
|
||||
try {
|
||||
const text = await file.text();
|
||||
const stats = await extraction.importNotes({ filePath: file.name, text });
|
||||
results.push({ filePath: file.name, ok: true, stats });
|
||||
} catch (err) {
|
||||
results.push({ filePath: file.name, ok: false, error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
return Response.json({ results });
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* PUT /api/graph/positions — the live-physics drag commit. The client
|
||||
* anneals a drag to rest and sends the new positions; the server
|
||||
* sanitizes (finite coords, known ids only), writes the ledger, and if
|
||||
* the graph grew mid-drag runs one incremental settle to absorb the new
|
||||
* nodes with the client's arrangement as anchors. See
|
||||
* src/server/commit-positions.ts.
|
||||
*
|
||||
* PUT → 200 {positions: {...}} with whatever survived sanitizing
|
||||
* (empty object = nothing written).
|
||||
*/
|
||||
import { commitPositions } from "@/src/server";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function PUT(request: Request) {
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
return Response.json({ error: "invalid JSON body" }, { status: 400 });
|
||||
}
|
||||
return Response.json({ positions: commitPositions(body) });
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* GET/PUT /api/graph/render-settings — server persistence for the graph
|
||||
* view's rendering controls (node/edge/arrow/label sizes, the label-zoom
|
||||
* threshold, the cluster-labels toggle). The file lives at
|
||||
* data/render-settings.json (config.RENDER_SETTINGS_FILE).
|
||||
*
|
||||
* GET → 200 {settings} once saved, 200 null on first boot (client keeps
|
||||
* in-memory defaults until the user changes something).
|
||||
* PUT → 200 {settings} (the sanitized copy actually persisted — the
|
||||
* client may echo it to stay honest with clamped values).
|
||||
*
|
||||
* Every body is passed through sanitizeRenderSettings (src/core) before
|
||||
* touching disk: garbage clamps or falls back to defaults, never throws.
|
||||
*/
|
||||
import { readRenderSettings, writeRenderSettings } from "@/src/server";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET() {
|
||||
return Response.json(readRenderSettings());
|
||||
}
|
||||
|
||||
export async function PUT(request: Request) {
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
return Response.json({ error: "invalid JSON body" }, { status: 400 });
|
||||
}
|
||||
return Response.json(writeRenderSettings(body));
|
||||
}
|
||||
@@ -12,7 +12,9 @@ export const dynamic = "force-dynamic";
|
||||
export async function GET() {
|
||||
const runs = extractionStatuses().map((run) => ({
|
||||
chatId: run.chatId,
|
||||
title: chatStore.getChat(run.chatId)?.title ?? run.chatId,
|
||||
// The run's own label (the file path for notes imports) wins; chat
|
||||
// runs fall back to the chat store's title.
|
||||
title: run.label ?? chatStore.getChat(run.chatId)?.title ?? run.chatId,
|
||||
phase: run.phase,
|
||||
startedAt: run.startedAt,
|
||||
// Live per-stage usage (the money channel): model + per-step CallUsage,
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
/**
|
||||
* Cluster color identity — pure presentation math for the knowledge-graph
|
||||
* regions. (The hull geometry that used to live here — convexHull,
|
||||
* outwardNormal, offsetPolygon, traceRoundedPath — was RETIRED with the
|
||||
* convex-hull renderer: regions are now a low-res world-space splat
|
||||
* bitmap in graph-canvas.ts, which renders concave cluster shapes
|
||||
* correctly and costs one blit per frame instead of 32 clusters × 4
|
||||
* offset polygons. The file keeps the name for the barrel imports.)
|
||||
*/
|
||||
|
||||
export type Pt = { x: number; y: number };
|
||||
|
||||
/** Stable hue per cluster id (content-hash): the region keeps its color
|
||||
* across reshuffles because the id only changes when the members do. */
|
||||
export function clusterHue(id: string): number {
|
||||
let h = 0;
|
||||
for (const ch of id) h = (h * 31 + ch.charCodeAt(0)) % 360;
|
||||
return h;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+706
-472
File diff suppressed because it is too large
Load Diff
@@ -47,6 +47,9 @@ export const MODELS = {
|
||||
extractor: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
||||
/** The auditor verifies evidence exists; keep it cheap. */
|
||||
auditor: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
||||
/** Names detected communities — one singular call, given member names
|
||||
* + statements. Purely a labeling flourish; keep it cheap. */
|
||||
labeler: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
||||
} satisfies Record<string, ModelSpec>;
|
||||
|
||||
export type ModelRole = keyof typeof MODELS;
|
||||
@@ -71,4 +74,29 @@ export const LEDGER_FILES = {
|
||||
events: path.join(PATHS.graph, "events.jsonl"),
|
||||
mappings: path.join(PATHS.graph, "mappings.jsonl"),
|
||||
sessions: path.join(PATHS.graph, "sessions.jsonl"),
|
||||
/** LLM-named community records (contracts/graph.ts ClusterLabeling). */
|
||||
clusterLabels: path.join(PATHS.graph, "cluster-labels.jsonl"),
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Notes-import bookkeeping (NOT a ledger): one JSON map, src-id → NotesSource
|
||||
* record (contracts/graph.ts) — the file path + import time + the run's usage
|
||||
* log. Read-modify-write like a chat file, not append-only like the ledgers.
|
||||
*/
|
||||
export const SOURCES_FILE = path.join(PATHS.graph, "sources.json");
|
||||
/** The layout ledger (read-modify-write like sources.json): nodeId →
|
||||
* {x,y} resting positions from the incremental settle (core/layout.ts).
|
||||
* Carries an updatedAt stamp for future live-physics conflict handling. */
|
||||
export const POSITIONS_FILE = path.join(PATHS.graph, "positions.json");
|
||||
|
||||
/** The embedding ledger: one L2-normalized vector per node id, all from
|
||||
* ONE model (swapping models re-embeds everything — vectors from
|
||||
* different models live in incompatible spaces). Read-modify-write
|
||||
* JSON like positions/sources. */
|
||||
export const EMBEDDINGS_FILE = path.join(PATHS.graph, "embeddings.json");
|
||||
|
||||
/**
|
||||
* Graph render settings (the settings-panel controls): one small JSON blob,
|
||||
* read-modify-write. Shared shape/sanitizer: src/core/render-settings.ts.
|
||||
*/
|
||||
export const RENDER_SETTINGS_FILE = path.join(PATHS.graph, "render-settings.json");
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
# Graph rendering & cluster-first layout — research findings
|
||||
|
||||
*Compiled 2026-09-13 from four research passes (library fact-check ×2, prior-art ×2), all
|
||||
claims source-verified by subagents against shipped package code (`npm pack` tarballs of
|
||||
`sigma@3.0.3`, `@sigma/layer-webgl@3.0.0`), the sigma.js GitHub storybook source, and
|
||||
primary docs. Speculation is marked where it appears. This document is the decision
|
||||
basis for the rendering rebuild; nothing has been implemented yet.*
|
||||
|
||||
---
|
||||
|
||||
## 1. Requirements (as of this research)
|
||||
|
||||
Current symptoms to eliminate:
|
||||
- Hull layers lag behind nodes during pan (desync).
|
||||
- Settings changes shift hulls up-left until the next interaction (canvas-sizing race).
|
||||
- GPU ~50–100% during pan/zoom (redundant full-canvas 2D repaints per camera event +
|
||||
WebGL re-render per frame; the earlier `ctx.filter: blur` per frame caused coil whine).
|
||||
- Stale/garbage boundary shapes at extreme zoom-out.
|
||||
|
||||
Feature set:
|
||||
- Circles + directed arrows; precomputed layout (no runtime physics).
|
||||
- Cluster regions: hull boundary around each community, dashed stroke, soft translucent
|
||||
fill, stable color per region, region name label.
|
||||
- Inverse zoom gates: node labels when zoomed IN, region labels when zoomed OUT,
|
||||
cross-fading over a shared span.
|
||||
- Pan, zoom-to-cursor, hover glow, click → detail card, edge-type filtering, camera
|
||||
survives UI changes. 60fps, minimal GPU, ordinary laptop.
|
||||
|
||||
New direction (user, 2026-09-13):
|
||||
- **Cluster-first placement**: layout depends on clustering — clusters placed as units,
|
||||
nodes within a cluster close together (compact, natural hull shapes as a consequence).
|
||||
- **Freeform experimentation**: a dev-only playground to try layouts/shapes/styles
|
||||
against real and synthetic data; renderer must be dumb, data-driven, swappable.
|
||||
|
||||
Scale: 338 nodes / 211 edges today; low thousands expected (notes imports).
|
||||
|
||||
## 2. Verified facts about the current stack (sigma.js v3)
|
||||
|
||||
- **v3.0.3 ships a PUBLIC custom-layer API** — `createLayer` / `createCanvas` /
|
||||
`createCanvasContext` / `killLayer` (typed in shipped `.d.ts`). Our current
|
||||
`firstChild` DOM injection was improvising a worse version of a supported API.
|
||||
- **Sigma sizes and DPR-scales ALL registered canvases** in its own `resize()` —
|
||||
using the public API eliminates the canvas-sizing race by construction.
|
||||
- **`afterRender` fires once per rendered frame** (rAF-debounced `scheduleRender()`).
|
||||
Camera motion → full re-render each rAF. **Hover is cheap**: it only redraws the
|
||||
`hoverNodes` WebGL layer + `edgeLabels` canvas via a separate rAF; no `afterRender`,
|
||||
no full re-render. Sigma is strictly event-driven: no continuous loop, no rendering
|
||||
when idle (verified: zero `setInterval`, all rAF sites are schedulers).
|
||||
- **`renderer.graphToViewport()` is public and typed** — the sanctioned graph→pixel
|
||||
path. `renderer.matrix` is **private**: building a `ctx.setTransform()` from it would
|
||||
be guessing at internals; per-vertex `graphToViewport()` is the supported route.
|
||||
- **The official cluster-label storybook example is exactly our pattern**:
|
||||
custom DOM layer + `afterRender` + per-cluster `graphToViewport()` re-projection,
|
||||
maintained in the sigma repo. Our minimal-change fix path is a demonstrated usage.
|
||||
- Real v3 layer order (verified in source, NOT in docs):
|
||||
`edges` (WebGL) → `edgeLabels` → `nodes` (WebGL) → `labels` → `hovers` →
|
||||
`hoverNodes` (WebGL) → `mouse`.
|
||||
|
||||
### `@sigma/layer-webgl@3.0.0` (official background-layer package)
|
||||
- `bindWebGLLayer` inserts a background WebGL layer (`beforeLayer: "edges"`), redrawn
|
||||
on `afterRender` with the same camera matrix — fully in-loop, no desync.
|
||||
- **`createContoursProgram` renders METABALL-style implicit fields, not hull polygons**
|
||||
(verified from the fragment shader: fullscreen quad, summed radial falloffs,
|
||||
analytic contour lines via `dFdx/dFdy`). Translucent fill: yes (arbitrary CSS color
|
||||
with alpha). **Dashed stroke: NO** — solid anti-aliased line only; no dash parameter
|
||||
exists. Per-community color: yes, but one program + one WebGL context per community,
|
||||
and each runs a fullscreen fragment pass per frame (browsers cap ~16 WebGL contexts —
|
||||
a problem for dozens of regions).
|
||||
- **Decisive fact**: our exact visual (hull polygon + translucent fill + dashed
|
||||
stroke) is **NOT achievable with shipped layer-webgl programs**. It would require a
|
||||
custom `WebGLLayerProgram` (supported extension point, moderate effort, dashes DIY in
|
||||
GLSL) — whereas Canvas 2D gives it for free with `ctx.setLineDash()`.
|
||||
- v4-era programs (heatmap, density-splat, dashed/fill layer factories) are NOT in the
|
||||
v3 package.
|
||||
|
||||
### sigma v4
|
||||
- Real but **beta** (4.0.0-beta.5; docs site self-describes as alpha). Single `stage`
|
||||
canvas + public `createLayer`, GPU picking with downsizing, SDF labels, depth
|
||||
layers. Breaking migration. Not a production bet today.
|
||||
|
||||
### Pan GPU cost at our scale
|
||||
- Structural facts verified (event-driven idle; full re-render per camera frame;
|
||||
synchronous `gl.readPixels` picking per mousemove pre-#1546, since fixed).
|
||||
- **Absolute GPU cost of panning 338–3k nodes in v3: UNVERIFIED** — no public
|
||||
benchmark exists; nobody has measured it. If it matters, we measure it ourselves.
|
||||
|
||||
## 3. Verified facts about the candidates
|
||||
|
||||
| Candidate | Renderer | True hull fill+dash | Picking | Camera | Maintenance | License | Fit |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| sigma v3 + public 2D canvas layer | WebGL (+our 2D) | yes (our canvas, free) | GPU color-pick | built-in | active (3.0.3, 2026-04) | MIT | 7/10 |
|
||||
| sigma v3 + layer-webgl contours | WebGL | **no** (metaballs, solid border) | — | built-in | active | MIT | 6/10 |
|
||||
| sigma v4 + layers | WebGL, SDF | via custom program (beta risk) | GPU picking | built-in | **beta** | MIT | 6 (risky) |
|
||||
| d3-zoom + Canvas 2D (hand-rolled loop) | Canvas 2D | yes, trivially | distance check | d3-zoom (verified math) | d3 very active | ISC | **8/10** |
|
||||
| PixiJS v8 + pixi-viewport | WebGL/WebGPU/Canvas | yes (Graphics; dashes via shim `pixi-dashed-line`) | built-in events | pixi-viewport 6.0.3 | very active | MIT | 7/10 |
|
||||
| cytoscape.js 3.34 | Canvas 2D | via bubblesets/layers extensions | built-in | built-in | active | MIT | 5/10 |
|
||||
| vis-network 10.1 | Canvas 2D | **no** — clustering = supernodes (collapse, opposite of our regions) | built-in | built-in | sporadic | MIT | 2/10 |
|
||||
| @cosmos.gl/graph (Cosmograph engine) | WebGL (luma.gl) | no (cluster = layout force only) | GPU | built-in | active | MIT | 3/10 |
|
||||
|
||||
Fact nuances we must not over-trust:
|
||||
- "Canvas 2D is GPU-rasterized": **half-true**. Skia may raster on CPU or GPU depending
|
||||
on browser/OS config. Either way the cost driver is **repaint count × pixel area
|
||||
(DPR²)**, not "Skia = GPU".
|
||||
- Canvas 2D at 60fps for hundreds–low-thousands of elements: **supported by practitioner
|
||||
evidence** (cytoscape is fully canvas-based into the thousands; 10k animated canvas
|
||||
elements ≈ 55fps in one 2025/26 benchmark). Pixel area matters more than node count
|
||||
at our scale.
|
||||
- Obsidian's graph view renderer: **UNVERIFIED** — no primary source. Do not cite.
|
||||
- PixiJS exact bundle size: unmeasured. yFiles: commercial, unverified details.
|
||||
|
||||
## 4. Prior art for our specific features
|
||||
|
||||
### Region/hull visuals
|
||||
- Nobody in the graph-tool world ships our region aesthetic: Gephi desktop, Gephi Lite
|
||||
(the flagship sigma app — zero hull code in its tree), Obsidian — all partition by
|
||||
**node coloring only**. The region feature is a differentiator, not a copied pattern.
|
||||
- The aesthetic reference: ggforce `geom_mark_hull` (expand + round the hull, low-alpha
|
||||
fill, integrated label placement).
|
||||
- **Bostock's thick-stroke hull trick**: one path, wide stroke (e.g. 32px) +
|
||||
`linejoin: round` = padded, rounded hull with zero polygon-offset math. Replaces our
|
||||
offset-normal code outright.
|
||||
- Named-region UX validation: Excalidraw frames / tldraw regions (dashed titled
|
||||
container around content).
|
||||
- Closest feature twin: **Cosmograph** — documents "cluster labels when zoomed out,
|
||||
point labels when zoomed in" as a first-class use case (our exact inverse gate).
|
||||
|
||||
### Zoom-gated labels
|
||||
- Consensus pattern = map-style per-layer zoom ranges (Mapbox/MapLibre
|
||||
`minzoom`/`maxzoom` on symbol layers). State labels at far zoom, city labels at near
|
||||
zoom is the standard cartographic solution; our inverse gates are the same mechanism
|
||||
with opposite ranges. Sigma's own `labelRenderedSizeThreshold` is the zoom-in gate.
|
||||
- Cross-fade + hysteresis over a shared span: no single canonical writeup (medium
|
||||
confidence); nearest established practice is map label collision systems.
|
||||
|
||||
### The sync pattern (fixes our desync class structurally)
|
||||
- Canonical and repeatedly documented: **one rAF loop with a dirty flag; input/DOM
|
||||
events only mutate state.** All layers drawn in the same frame, under the same
|
||||
camera transform — either same canvas (`setTransform` once, draw back-to-front) or
|
||||
stacked canvases all redrawn by the same loop.
|
||||
Sources: browser.engineering, MDN "Optimizing canvas", web.dev canvas performance.
|
||||
|
||||
### Picking
|
||||
- Canvas 2D at our scale: nearest-node distance check per pointer event is the norm,
|
||||
trivially fast (µs for 3k nodes; spatial grid only past ~10k).
|
||||
|
||||
## 5. Cluster-first layout (the new direction)
|
||||
|
||||
- **Two-level pattern is well-trodden**: place cluster anchors first (quotient graph —
|
||||
one super-node per community, Gephi's "Groups by Partition" plugin is the desktop
|
||||
prior art), layout nodes within each cluster, compose by translating local layouts
|
||||
to cluster anchors. Bostock's Clustered Force Layout lineage; cosmos.gl's cluster
|
||||
force verified from source (per-frame center-of-mass pull, or user-pinned positions —
|
||||
cluster anchors as data independent of member layout).
|
||||
- **graphology FA2 has NO cluster-aware settings** (full settings list read; gravity is
|
||||
global). **LinLog mode** (`linLogMode: true`) is the documented "communities separate
|
||||
tightly" switch — a one-afternoon A/B candidate before any custom pipeline.
|
||||
- **No turnkey JS package does the composed two-level pipeline** — assemble from
|
||||
graphology primitives (~150 lines of glue: quotient build, per-cluster FA2 with
|
||||
circular init, compose transform, `noverlap` cleanup). The combination is our
|
||||
synthesis from verified primitives.
|
||||
- Deterministic given fixed init positions (no RNG dependency in FA2).
|
||||
- **Server-side at extraction time is the better fit** for our architecture: positions
|
||||
become data, rendering stays dumb. Browser worker fallback for live re-layout.
|
||||
- Harness precedents: yFiles layout playground (polished), Observable/bl.ocks lineage
|
||||
(notebook = graph + sliders + canvas), react-sigma layouts example. No canonical OSS
|
||||
playground for our stack — copy the config-driven pattern (algorithm dropdown +
|
||||
parameter sliders + seed input + live canvas).
|
||||
|
||||
## 6. Options (the decision)
|
||||
|
||||
**Option A — Stay on sigma v3; mount the hull canvas via the public
|
||||
`createCanvas` API; redraw on `afterRender` via `graphToViewport()`.**
|
||||
Supported, minimal-change, demonstrated by sigma's own cluster-label storybook. Kills
|
||||
the desync, the sizing race, and the multi-event redraws (once per frame, sigma-sized).
|
||||
Our exact visuals stay (dashes free in 2D). Keeps sigma's camera/labels/picking
|
||||
machinery. Residual: full-canvas 2D repaint per interaction frame remains (raster cost,
|
||||
mitigable with move-gating like sigma's own `hideLabelsOnMove`); WebGL + 2D raster
|
||||
still both run per frame. Est: ~1 day.
|
||||
|
||||
**Option A′ — sigma + custom `WebGLLayerProgram` for hulls.** Supported extension
|
||||
point, but dashes are DIY GLSL and we'd rebuild in shaders what Canvas 2D gives free.
|
||||
No reason given A exists. Est: several days, for a worse iteration loop.
|
||||
|
||||
**Option B — d3-zoom + hand-rolled single Canvas 2D renderer.**
|
||||
One canvas, one rAF + dirty flag, world-space geometry + one transform, all layers
|
||||
back-to-front in the same frame. Lowest GPU/CPU, total control, perfectly matches the
|
||||
new requirements: renderer is a dumb draw function over data (layout experiments can't
|
||||
fight it), playground is trivial, cluster-first layout slots in as pure data. Costs
|
||||
~600 lines + camera feel + picking + label gating owned by us. Est: ~1 focused day.
|
||||
|
||||
**Option C — PixiJS v8 + pixi-viewport.** Cleanest scene-graph answer (one ticker,
|
||||
sync by construction), WebGPU headroom. But: dashes need a third-party shim, no graph
|
||||
semantics, bundle, more machinery than Canvas 2D needs at our scale. Est: 1–2 days.
|
||||
|
||||
**Recommendation**: **Option B as the target** — it is the only option that structurally
|
||||
matches all three requirement sets (bugs, cluster-first placement, freeform
|
||||
experimentation), scored highest (8/10), and removes the vendor-internals class of
|
||||
problems permanently. **Option A is a legitimate ~1-day stabilization** if we want the
|
||||
current bugs dead immediately while deferring the rebuild; it reuses all our existing
|
||||
visual code and is the officially sanctioned pattern, but it keeps us on sigma's
|
||||
opinionated machinery and the experimentation direction will keep pushing against it.
|
||||
|
||||
**Layout work rides along independent of renderer choice**: first A/B
|
||||
LinLog-only FA2 (one afternoon), then the two-level pipeline
|
||||
(quotient → per-cluster FA2 → compose → noverlap) if needed, developed in the playground.
|
||||
|
||||
## 7. Facts we should NOT rely on
|
||||
|
||||
- "Obsidian's graph view uses renderer X" — unverified.
|
||||
- "Sigma's documented canvas DOM order" — source-only, implementation detail, changed
|
||||
in v4.
|
||||
- "Canvas 2D is always GPU-rasterized" — false as stated.
|
||||
- "Sigma v4 is ready" — beta.
|
||||
- "Sigma pan cost at our scale is X%" — nobody has measured it publicly.
|
||||
- "@cosmograph/cosmos is the maintained Cosmograph engine" — stale npm channel (Dec
|
||||
2024); current engine is `@cosmos.gl/graph` v3.
|
||||
- Any PixiJS bundle-size number — unmeasured.
|
||||
|
||||
## 8. Open items for the implementation phase
|
||||
|
||||
- Measure pan GPU/CPU ourselves (Chrome profiler) before/after whichever path is chosen.
|
||||
- Playground design: `/playground` dev-only route, config-driven (layout algorithm
|
||||
dropdown, parameter sliders, seed input, synthetic + real datasets), same renderer.
|
||||
- Hull styling decision inside Option B: Bostock thick-stroke trick vs offset polygons
|
||||
(geometry already unit-tested in `components/cluster-geometry.ts`).
|
||||
- Label-gate hysteresis: small dead-band in log-zoom to avoid threshold flicker.
|
||||
- Whether layout runs server-side at extraction (persisted as data) or in-browser.
|
||||
+46
-2
@@ -43,18 +43,60 @@ Every node records every **(chat, message) point** where it came up; every edge
|
||||
|
||||
Three stages, one pipeline — the sole graph writer:
|
||||
|
||||
- **Extractor (LLM) — session-only scope.** Sees ONLY the session transcript (tree-blind, with message ids). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD.
|
||||
- **Extractor (LLM) — single-source scope.** Sees ONLY the source text — a session transcript (tree-blind, with message ids) or raw notes (the front door's second prompt, above). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD.
|
||||
- **Auditor (LLM) — the global-graph scope.** Sees the extractor's output AND the existing graph. Two jobs: the faithfulness net (every candidate verified against verbatim evidence — unchanged), and the **identity judgment**: link each proposition to an existing node (recording the session's phrasing as an alias) or create it new. Conservative: duplicates over wrong merges.
|
||||
- **Merger (deterministic).** Append-only ledger writes; quiz event kinds/refs rebuilt mechanically from quiz tool parts. Every judgment is an LLM's; everything mechanical stays code.
|
||||
|
||||
**Event ledger (amended):** quiz outcomes are rebuilt deterministically from quiz tool parts (correct / miss / don't-know, plus the learner's note). **There is no `taught` event** — no marking-things-done. The extractor's per-proposition understanding levels join the ledger as session-scoped observations (one closed-scale word per node per session). Identity is no longer exact-string matching — it is the auditor's judgment, with aliases as the phrasing record.
|
||||
|
||||
### The format-agnostic front door — notes imports (DECIDED 2026-09-13)
|
||||
|
||||
The same three-stage pipeline (extractor → auditor → merger) runs over **raw
|
||||
personal notes in arbitrary formats** — an Obsidian vault, plain markdown,
|
||||
anything textual. The extractor is format-blind by construction (it always
|
||||
saw rendered text, never ChatRecord JSON), so the notes path needs only a
|
||||
second extractor prompt (general-purpose extraction; the understanding word
|
||||
judges the **author's** grasp from their own commentary) and a source
|
||||
abstraction: each imported file becomes a `NotesSource` record (deterministic
|
||||
id from the path hash — re-imports are idempotent), playing the `chatId` role
|
||||
in mappings, checkpoints, and observation events.
|
||||
|
||||
Decided edges:
|
||||
|
||||
- **No session shaping.** Notes are never bent into teacher/learner
|
||||
transcripts; quiz outcomes inside notes surface only via the extractor's
|
||||
understanding observations (the deterministic quiz-event rebuild is a
|
||||
session-pipeline feature — it needs structured quiz tool parts).
|
||||
- **Static provenance.** Imported nodes show the file path in the sessions
|
||||
list as a static entry — never a link. Understanding-assessment levels
|
||||
apply to personal notes as to sessions.
|
||||
- **Same single-flight registry and observability** as session runs: live
|
||||
status, usage accounting per stage, true abort. One pipeline instance,
|
||||
two front doors (quiz-triggered sessions; POST /api/graph/import).
|
||||
|
||||
### Understanding is derived (amended 2026-09-02)
|
||||
|
||||
Known vs. understood, event ledger as source of truth, derived state recomputed on read — carried over from the design doc in spirit. The state model itself is now an explicit design discussion: **node states should be session-independent and computed from the depth and strength of dependencies** (the `follows`/`supports` topology) plus quiz events and understanding notes. That redesign is pending ideation; the transitional five-class derivation (taught / known / missed / don't-know / candidate) was **removed** (2026-09-08) — the graph view renders state-neutral nodes until the state model is designed. The anchored calibration rubric stays parked until real data.
|
||||
|
||||
### Clusters — regions of the world model (DECIDED 2026-09-13, ideation)
|
||||
|
||||
When the graph fills up, the canvas needs a readable at-a-glance structure: **clusters** — objective regions of the world model (domains like "Lacanian psychoanalysis" or "photosynthesis"), discovered by **community detection** on the `follows`/`supports` edges. A community = nodes linked to each other far more than to the rest (modularity). Everything about them is **entirely inferred from the graph data** — the extraction/audit pipeline never knows clusters exist.
|
||||
|
||||
- **Algorithm: Louvain (modularity maximization), seeded for determinism.** In-stack via graphology's ecosystem, near-linear, scales far beyond our sizes. Determinism gives reproducibility (same graph → same clusters); the seed/graphology choice also keeps the upgrade path open: Louvain's dendrogram contains **hierarchy** — "math ⊃ differential calculus ⊃ chain rule" is one level down — deliberately NOT built now (multi-scale clusters stay parked). Dismissed: Leiden (better resolution guarantees, no JS port), Infomap (heuristic-strong, heavier, weaker JS support), Girvan–Newman (O(n³)), label propagation (unstable), spectral (needs k handed in).
|
||||
- **Structure, not layout.** Clusters are a pure function of the *edges*, never of the visual layout. The force-directed layout already places structurally-close nodes together, so cluster hulls will look contiguous — but cluster identity must not inherit the layout's seed dependence and drift.
|
||||
- **Derived, cached, auto-updated.** Clusters recompute when the ledger changes, cached against a graph version stamp — same doctrine as states and strengths: derived never stored, nothing hand-maintained.
|
||||
- **Size-gated meaning.** Only communities ≥ ~3 nodes render as clusters (threshold tunable on real data) — a visible blob always means "a real domain"; lone nodes and tiny communities stay plain.
|
||||
- **Naming: one cheap LLM call per new/changed cluster** — a single call (no agent loop) with the members' names + statements, producing a short domain label. Never part of extraction; a separate, cached seam.
|
||||
- **Labeling persistence — the core-survival rule.** Each labeling record (append-only, like everything we persist) stores the label AND the member set it was produced with (the **core**). A recomputed community **keeps** its label while ≥ half its members are from that core; when under half are, it **relabels** with all current nodes (a new record is appended — old records stay as history: an audit trail of how the domains were named over time). Single successor: on splits/merges the strongest-overlap community inherits, the others get fresh labelings; a core with no ≥50% successor simply has no living heir. Cluster color keys to the labeling record — color survives exactly as long as the label does.
|
||||
- **Rendering: glow blobs behind everything.** A dashed boundary encloses the cluster's hull; the fill is blurred, low-opacity, and sits *behind* the whole graph (edges and nodes float above it, so the blur naturally fogs the inter-cluster space). Per-cluster stable hue. **Cluster labels are inverse-gated** against the existing camera-ratio span: zoomed out = **domain view** (cluster labels on, node labels off), zoomed in = **proposition view** (node labels on, cluster labels faded) — one symmetric transition between the two views of one canvas.
|
||||
- **Boundary geometry: computed at draw time, never stored** — a pure function of the member positions. This is the constraint that keeps future physics/moving nodes (likely not implemented soon) from invalidating anything: convex hull + padding + rounded corners is cheap per-frame; organic metaball hulls are the same family. Secondary until then.
|
||||
- **Inter-cluster edges stay visible.** Bridges between domains are meaningful (Nietzsche's will-to-power *follows* Schopenhauer's will) — one of the most informative reads of the clustered view, not clutter.
|
||||
- **Honest risk: sparsity.** Community detection wants density contrast, and the current real graph is thin — the 2k-node synthetic benchmark (open question 1) is the *primary* validation of clustering quality, not just of layout/render speed.
|
||||
|
||||
### Rendering (DECIDED — research report 2026-08-22)
|
||||
|
||||
**Render settings (2026-09-13):** four basic controls in the graph view — node size, edge width, arrowhead length, and **the zoom-level control** (the label-visibility threshold; 1× = fit zoom) — persisted per browser. The zoom control is deliberately THE single zoom knob: cluster labels gate inversely on it over the same fade span (implemented: node labels fade out over the span as cluster labels fade in — they never coexist). Settings changes rebuild only the renderer: layout positions and the camera never move. (Edge-length / spacing controls deliberately parked until a cluster-aware spacing design exists.)
|
||||
|
||||
- **Library: sigma.js v3 + graphology + graphology-layout-forceatlas2.** ~40 KB gzipped all-in — half of vis-network, a third of cytoscape, far below react-force-graph (~250 KB deps) and cosmos.gl (luma.gl stack).
|
||||
- **Layout: synchronous run-then-freeze.** `forceAtlas2.assign(graph, { iterations: 100, settings: inferSettings(graph) })` runs once, writes x/y, and no animation loop ever starts — near-zero idle CPU (sigma only redraws on camera/graph events). <150 ms at hundreds of nodes; at ~2k enable `barnesHutOptimize`, then the bundled Web Worker, then server-cached positions — in that order, only if latency bites.
|
||||
- **Labels at zoom:** sigma's `labelRenderedSizeThreshold` replicates Obsidian's "text fade threshold" exactly.
|
||||
@@ -73,7 +115,7 @@ Every session starts with the learner's knowledge graph as base context for the
|
||||
|
||||
### Home page
|
||||
|
||||
The graph view replaces the current placeholder. v1: all nodes from all sessions, node color/shape by state class (the current five classes are transitional — see "Understanding is derived"), pan/zoom, labels at zoom. Clicking a node → the exact (session, message) points where it came up (message-granular provenance, deep-linked). Everything richer (frontier glow, rot map, blast radius) comes after the state model exists.
|
||||
The graph view replaces the current placeholder. v1: all nodes from all sessions, pan/zoom, labels at zoom, state-neutral nodes (see "Understanding is derived"), and — once implemented — cluster glow-blobs with dashed hulls behind the graph, domain↔proposition as the two zoom-gated views (see "Clusters"). Clicking a node → the exact (session, message) points where it came up (message-granular provenance, deep-linked). Everything richer (frontier glow, rot map, blast radius) comes after the state model exists.
|
||||
|
||||
---
|
||||
|
||||
@@ -213,6 +255,8 @@ The teacher's context window and attention are precious — it has the lesson to
|
||||
4. Home-graph v1 interactions beyond pan/zoom (node click → session list? — presumed cheap yes).
|
||||
5. Grain mismatch frequency (`subsumes`-style relations) — per original doc, measured by data, not pre-solved.
|
||||
6. Aggregation semantics for understanding (min / mean / two-level) — fit on real data, per original doc.
|
||||
7. Cluster visual details — boundary aesthetic (convex vs. organic hull), label placement/typography, palette — decide during visual testing with real blobs (§1 Clusters; theory DECIDED 2026-09-13).
|
||||
8. Cluster size threshold (~3) — tune on the synthetic benchmark + real data (§1 Clusters).
|
||||
|
||||
---
|
||||
|
||||
|
||||
+7
-1
@@ -4,7 +4,13 @@ const nextConfig: NextConfig = {
|
||||
// Framework-free core lives in src/ (plain TS, imported by routes and CLI alike).
|
||||
// The pi engine must stay external: its providers are runtime-loaded Node
|
||||
// code (undici streams, catalog data) that breaks when bundled into routes.
|
||||
serverExternalPackages: ["@earendil-works/pi-ai", "@earendil-works/pi-agent-core"],
|
||||
// transformers.js (embeddings) is runtime-loaded ONNX — same reason as
|
||||
// the pi engine: bundling onnxruntime-node breaks route builds.
|
||||
serverExternalPackages: [
|
||||
"@earendil-works/pi-ai",
|
||||
"@earendil-works/pi-agent-core",
|
||||
"@huggingface/transformers",
|
||||
],
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
Generated
+771
-72
File diff suppressed because it is too large
Load Diff
+4
-3
@@ -16,6 +16,7 @@
|
||||
"@ai-sdk/react": "^4.0.79",
|
||||
"@earendil-works/pi-agent-core": "^0.85.1",
|
||||
"@earendil-works/pi-ai": "^0.85.1",
|
||||
"@huggingface/transformers": "^4.2.0",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"@radix-ui/react-use-controllable-state": "^1.2.6",
|
||||
"@rive-app/react-webgl2": "^4.32.1",
|
||||
@@ -29,9 +30,10 @@
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "^1.1.1",
|
||||
"d3-force": "^3.0.0",
|
||||
"d3-selection": "^3.0.0",
|
||||
"d3-zoom": "^3.0.0",
|
||||
"embla-carousel-react": "^8.6.0",
|
||||
"graphology": "^0.26.0",
|
||||
"graphology-layout-forceatlas2": "^0.10.1",
|
||||
"katex": "^0.16.47",
|
||||
"linkedom": "^0.18.13",
|
||||
"lucide-react": "^1.33.0",
|
||||
@@ -47,7 +49,6 @@
|
||||
"react-dom": "^19.2.8",
|
||||
"react-jsx-parser": "^2.4.1",
|
||||
"remark-breaks": "^4.0.0",
|
||||
"sigma": "^3.0.3",
|
||||
"streamdown": "^2.5.0",
|
||||
"tailwind-merge": "^3.6.0",
|
||||
"tokenlens": "^1.3.1",
|
||||
|
||||
@@ -8,13 +8,14 @@
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
|
||||
import { embedNewNodes } from "../src/server/embeddings.ts";
|
||||
import { getGraph } from "../src/core/graph.ts";
|
||||
|
||||
async function main() {
|
||||
const only = process.argv[2]; // optional: extract just one chat id
|
||||
const chats = only ? chatStore.listChats().filter((c) => c.id === only) : chatStore.listChats();
|
||||
console.log(`[backfill] ${chats.length} chats to extract`);
|
||||
const pipeline = createExtractionPipeline({ chatStore, graphStore });
|
||||
const pipeline = createExtractionPipeline({ chatStore, graphStore, embedder: embedNewNodes });
|
||||
for (const chat of chats) {
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
/**
|
||||
* Layout benchmark (W4; vision §1 "Rendering (DECIDED)" — "Pre-implementation
|
||||
* validation: 2k-node synthetic benchmark to confirm layout time").
|
||||
*
|
||||
* npx tsx scripts/bench-graph.ts
|
||||
*
|
||||
* Two scales: the vision's target regime (hundreds of nodes, <150 ms budget
|
||||
* with a synchronous run-then-freeze pass) and the stress case (2,000 nodes,
|
||||
* ~4,000 edges), the latter timed with and without barnesHutOptimize.
|
||||
*
|
||||
* VERDICT (2026-08-22, Apple silicon, Node 24): hundreds of nodes are
|
||||
* comfortably inside the budget (see the measured lines — ~25 ms at 100,
|
||||
* ~90 ms at 300). At 2k nodes the naive O(n²) pass takes seconds, and
|
||||
* barnesHutOptimize cuts it roughly in half — still over budget, which is
|
||||
* exactly the vision's escalation ladder: Barnes-Hut is already switched on
|
||||
* automatically above ~500 nodes in components/graph-view.tsx, and the
|
||||
* bundled Web Worker / server-cached positions stay available as later
|
||||
* rungs if real graphs ever get there. v1 learner graphs are dozens of
|
||||
* nodes; the stress case is preparation, not expectation.
|
||||
*/
|
||||
|
||||
import Graph from "graphology";
|
||||
import forceAtlas2 from "graphology-layout-forceatlas2";
|
||||
|
||||
const ITERATIONS = 100;
|
||||
|
||||
/** A plausible learning graph: mostly-local chains plus long-range cross-links. */
|
||||
function synthesize(nodeCount: number, edgeCount: number): Graph {
|
||||
const graph = new Graph();
|
||||
for (let i = 0; i < nodeCount; i++) {
|
||||
graph.addNode(`n${i}`, {
|
||||
label: `node ${i}`,
|
||||
size: 6,
|
||||
x: Math.random() * 100,
|
||||
y: Math.random() * 100,
|
||||
});
|
||||
}
|
||||
let added = 0;
|
||||
for (let i = 0; added < edgeCount && i < edgeCount * 3; i++) {
|
||||
const from = Math.floor(Math.random() * nodeCount);
|
||||
const span = Math.random() < 0.8 ? 1 + Math.floor(Math.random() * 10) : Math.floor(Math.random() * nodeCount);
|
||||
const to = (from + span) % nodeCount;
|
||||
if (from === to || graph.hasEdge(`n${from}`, `n${to}`)) continue;
|
||||
graph.addEdge(`n${from}`, `n${to}`, { type: "arrow", size: 1 });
|
||||
added++;
|
||||
}
|
||||
return graph;
|
||||
}
|
||||
|
||||
function bench(label: string, nodeCount: number, edgeCount: number, settings: Record<string, unknown>): void {
|
||||
const graph = synthesize(nodeCount, edgeCount);
|
||||
const start = performance.now();
|
||||
forceAtlas2.assign(graph, {
|
||||
iterations: ITERATIONS,
|
||||
settings: { ...forceAtlas2.inferSettings(graph), ...settings },
|
||||
});
|
||||
const elapsed = performance.now() - start;
|
||||
console.log(
|
||||
`${label}: ${elapsed.toFixed(1)} ms (${graph.order} nodes, ${graph.size} edges, ${ITERATIONS} iterations)`,
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`Target regime — synchronous run-then-freeze (budget: <150 ms):\n`);
|
||||
bench(" 100 nodes, ~200 edges", 100, 200, {});
|
||||
bench(" 300 nodes, ~600 edges", 300, 600, {});
|
||||
bench(" 500 nodes, ~1000 edges", 500, 1000, {});
|
||||
console.log(`\nStress case — 2k nodes (~4000 edges), the vision's escalation ladder:\n`);
|
||||
bench(" naive repulsion (O(n²)) ", 2000, 4000, {});
|
||||
bench(" barnesHutOptimize ", 2000, 4000, { barnesHutOptimize: true });
|
||||
bench(" barnesHutOptimize (warmed JIT)", 2000, 4000, { barnesHutOptimize: true });
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* Embed the whole graph (backfill / cold start): one-time pass that
|
||||
* fills the embedding ledger for every node — the same embedNewNodes the
|
||||
* post-merge hook runs, just with an empty (or stale-model) ledger so
|
||||
* every node is "new". First run downloads the model weights (~300MB)
|
||||
* to the local HF cache; subsequent runs embed only what's missing.
|
||||
*
|
||||
* npx tsx scripts/embed-graph.ts
|
||||
*/
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
import { embedNewNodes, EMBEDDING_MODEL, EMBEDDING_DIM } from "../src/server/embeddings.ts";
|
||||
|
||||
async function main() {
|
||||
const t0 = Date.now();
|
||||
console.log(`[embed] model: ${EMBEDDING_MODEL} (dim ${EMBEDDING_DIM}, q8)`);
|
||||
const { embedded, total } = await embedNewNodes(graphStore);
|
||||
console.log(`[embed] embedded ${embedded} nodes → ledger holds ${total} vectors (${((Date.now() - t0) / 1000).toFixed(1)}s)`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("[embed] failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Import raw notes files through the format-agnostic front door (vision §1,
|
||||
* 2026-09-13) — every *.md / *.txt file in a directory becomes one
|
||||
* extraction run (extractor → auditor → merger) over its raw text. Run
|
||||
* from this worktree only — data never crosses trees:
|
||||
*
|
||||
* npx tsx --env-file=.env.local scripts/import-notes.ts <dir> [concurrency]
|
||||
*
|
||||
* Default dir: real-data-extraction/learn-sessions (the prototype
|
||||
* learning-session exports — deliberately NOT imported as chats: they run
|
||||
* through the notes path, raw). Runs in this process: the dev server's
|
||||
* status chips do NOT see them (registry is per-process) — the dev server
|
||||
* sees runs triggered through /api/graph/import.
|
||||
*/
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
|
||||
import { embedNewNodes } from "../src/server/embeddings.ts";
|
||||
import { clusterNamingSettled } from "../src/agent/subagents/cluster-namer.ts";
|
||||
import { getGraph } from "../src/core/graph.ts";
|
||||
import { stores } from "../src/server/stores.ts";
|
||||
|
||||
async function main() {
|
||||
const dir = process.argv[2] ?? "real-data-extraction/learn-sessions";
|
||||
const limit = Number(process.argv[3] ?? 4);
|
||||
const files = fs
|
||||
.readdirSync(dir)
|
||||
.filter((f) => f.endsWith(".md") || f.endsWith(".markdown") || f.endsWith(".txt"))
|
||||
.sort();
|
||||
console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`);
|
||||
|
||||
const pipeline = createExtractionPipeline({ ...stores, embedder: embedNewNodes });
|
||||
const queue = [...files];
|
||||
const totals = { nodes: 0, edges: 0, events: 0, failed: 0 };
|
||||
const workers = Array.from({ length: Math.min(limit, files.length) }, async () => {
|
||||
for (;;) {
|
||||
const file = queue.shift();
|
||||
if (!file) return;
|
||||
const filePath = path.join(dir, file);
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const stats = await pipeline.importNotes({ filePath, text: fs.readFileSync(filePath, "utf-8") });
|
||||
totals.nodes += stats.nodes;
|
||||
totals.edges += stats.edges;
|
||||
totals.events += stats.events;
|
||||
console.log(
|
||||
`[import-notes] ${file}: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events in ${((Date.now() - t0) / 1000).toFixed(1)}s`,
|
||||
);
|
||||
} catch (err) {
|
||||
totals.failed++;
|
||||
console.error(`[import-notes] ${file} FAILED:`, err);
|
||||
}
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
|
||||
// The post-merge cluster naming is fire-and-forget inside the pipeline;
|
||||
// settle it before the process exits so labels actually land.
|
||||
await clusterNamingSettled();
|
||||
|
||||
const graph = getGraph(stores.graphStore);
|
||||
const labels = stores.graphStore.readClusterLabels().length;
|
||||
console.log(
|
||||
`[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` +
|
||||
`graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events, ` +
|
||||
`${labels} named clusters`,
|
||||
);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* Option A simulation: replay the CURRENT graph through the incremental
|
||||
* layout as if it had arrived one imported-notes file at a time — the
|
||||
* organic growth path the user described (one dependency structure /
|
||||
* cluster at a time, orphans collecting on the rim).
|
||||
*
|
||||
* Reads ONLY the ledgers (nodes / edges / mappings / sources); writes ONLY
|
||||
* data/graph/positions.json — no LLM calls, no extraction, no chat records.
|
||||
*
|
||||
* npx tsx scripts/replay-layout.ts [--depth=vertical|radial]
|
||||
*
|
||||
* --depth regenerates the ledger WITH inference-depth anchoring (the
|
||||
* playground mode baked into the rest state); omit it for the free layout.
|
||||
*/
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
// --depth=vertical|radial: bake the playground depth mode into the ledger.
|
||||
const depthArg = process.argv.find((a) => a.startsWith("--depth="));
|
||||
const depthMode: "off" | "vertical" | "radial" =
|
||||
depthArg === "--depth=vertical" ? "vertical" : depthArg === "--depth=radial" ? "radial" : "off";
|
||||
import { incrementalLayout } from "../src/core/layout.ts";
|
||||
import type { GraphNode, NodeMapping, Position } from "../src/core/contracts.ts";
|
||||
|
||||
function main() {
|
||||
const t0 = Date.now();
|
||||
|
||||
// Unique nodes (lines may repeat an id — first defines).
|
||||
const seen = new Set<string>();
|
||||
const nodes: GraphNode[] = [];
|
||||
for (const line of graphStore.readNodes()) {
|
||||
if (seen.has(line.id)) continue;
|
||||
seen.add(line.id);
|
||||
nodes.push(line);
|
||||
}
|
||||
const edges = graphStore.readEdges();
|
||||
// The semantic-topic source: the embedding ledger (run
|
||||
// scripts/embed-graph.ts first if missing/stale).
|
||||
const embeddings = graphStore.readEmbeddings();
|
||||
if (Object.keys(embeddings.vectors).length === 0) {
|
||||
console.warn("[replay] embedding ledger is empty — clusters will be absent (run scripts/embed-graph.ts)");
|
||||
}
|
||||
|
||||
// Batches: each imported file, in real import order (sources.json
|
||||
// importedAt). A node belongs to the file whose mapping first claims it
|
||||
// (mapping line order = arrival); nodes no mapping covers (legacy) ride
|
||||
// the final batch.
|
||||
const sources = [...graphStore.readSources()].sort((a, b) => a.importedAt.localeCompare(b.importedAt));
|
||||
const mappings: NodeMapping[] = graphStore.readMappings();
|
||||
const batchOf = new Map<string, string>(); // nodeId → srcId
|
||||
for (const m of mappings) {
|
||||
if (!batchOf.has(m.graphNodeId)) batchOf.set(m.graphNodeId, m.chatId);
|
||||
}
|
||||
|
||||
const delivered = new Set<string>();
|
||||
let positions = new Map<string, Position>(Object.entries(graphStore.readPositions()));
|
||||
if (positions.size > 0) {
|
||||
console.log("[replay] discarding existing positions ledger — full replay from empty");
|
||||
positions = new Map();
|
||||
}
|
||||
|
||||
const byId = new Map(nodes.map((n) => [n.id, n]));
|
||||
const unmappedFallback = sources[sources.length - 1]?.id ?? "";
|
||||
|
||||
console.log(`[replay] ${nodes.length} unique nodes, ${edges.length} edge lines, ${sources.length} sources`);
|
||||
let maxOldShift = 0;
|
||||
let shiftReport = "";
|
||||
for (const source of sources) {
|
||||
const batch = nodes.filter((n) => batchOf.get(n.id) === source.id);
|
||||
const isLast = source.id === sources[sources.length - 1]!.id;
|
||||
if (isLast) {
|
||||
// Sweep in any never-mapped nodes with the last batch.
|
||||
for (const n of nodes) {
|
||||
if (!batchOf.has(n.id) && !delivered.has(n.id)) {
|
||||
batch.push(n);
|
||||
batchOf.set(n.id, unmappedFallback);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const n of batch) delivered.add(n.id);
|
||||
|
||||
// Edges activate once BOTH endpoints exist (a later file can claim
|
||||
// one endpoint — the edge lights up then).
|
||||
const activeEdges = edges.filter((e) => delivered.has(e.from) && delivered.has(e.to));
|
||||
|
||||
const before = new Map(positions);
|
||||
const batchNodes = nodes.filter((n) => delivered.has(n.id));
|
||||
positions = incrementalLayout(batchNodes, activeEdges, positions, {
|
||||
// The same partition the real settle would see for this batch:
|
||||
// the persisted semantic topics, filtered to delivered nodes
|
||||
// (never recomputed — the agglomerative pass is ~1.5s at scale).
|
||||
clusters: embeddings.clusters
|
||||
.map((c) => ({ members: c.members.filter((id) => delivered.has(id)) }))
|
||||
.filter((c) => c.members.length > 0),
|
||||
...(depthMode === "off"
|
||||
? {}
|
||||
: {
|
||||
depthAnchor: {
|
||||
mode: depthMode,
|
||||
strength: 0.8,
|
||||
spacing: 60,
|
||||
...(depthMode === "radial" ? { baseRadius: 220 } : {}),
|
||||
},
|
||||
}),
|
||||
});
|
||||
for (const [id, prev] of before) {
|
||||
const now = positions.get(id)!;
|
||||
const shift = Math.hypot(prev.x - now.x, prev.y - now.y);
|
||||
if (shift > maxOldShift) {
|
||||
maxOldShift = shift;
|
||||
shiftReport = `${id} ${prev.x.toFixed(0)},${prev.y.toFixed(0)} -> ${now.x.toFixed(0)},${now.y.toFixed(0)} (batch ${batch.length} new, ${activeEdges.length} edges)`;
|
||||
}
|
||||
}
|
||||
const label = source.filePath.split("/").pop() ?? source.id;
|
||||
console.log(
|
||||
`[replay] +${String(batch.length).padStart(3)} nodes ${String(activeEdges.length).padStart(4)} edges active (${label})`,
|
||||
);
|
||||
}
|
||||
|
||||
graphStore.writePositions(Object.fromEntries(positions));
|
||||
console.log(
|
||||
`[replay] done: ${positions.size} positions written in ${((Date.now() - t0) / 1000).toFixed(1)}s` +
|
||||
` (max old-node shift within a batch: ${maxOldShift.toFixed(1)}: ${shiftReport})`,
|
||||
);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -66,7 +66,10 @@ export function toPiTool(def: LhTool<any, any>): AgentTool<any, any> {
|
||||
name: def.name,
|
||||
label: def.name,
|
||||
description: def.description,
|
||||
parameters: z.toJSONSchema(def.schema as z.ZodType) as never,
|
||||
// io: "input" — the declaration describes what the MODEL must send,
|
||||
// so .default() fields are optional in the schema (omission = the
|
||||
// default); required-empty arrays are a known weak-model trap.
|
||||
parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never,
|
||||
// Question tools opt into SEQUENTIAL execution: pi runs the whole
|
||||
// batch in call order when any called tool opts in, so the
|
||||
// tool-input-available chunks arrive in the model's call order — the
|
||||
@@ -110,6 +113,9 @@ export function piToolDeclarations(registry: Record<string, LhTool<any, any>>) {
|
||||
name,
|
||||
label: name,
|
||||
description: def.description,
|
||||
parameters: z.toJSONSchema(def.schema as z.ZodType) as never,
|
||||
// io: "input" — the declaration describes what the MODEL must send,
|
||||
// so .default() fields are optional in the schema (omission = the
|
||||
// default); required-empty arrays are a known weak-model trap.
|
||||
parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never,
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
/**
|
||||
* CLUSTER NAMING (vision §1, DECIDED 2026-09-13): the garnish layer over
|
||||
* community detection. Detection is purely structural and deterministic
|
||||
* (core/clusters.ts); naming is ONE singular cheap LLM call that labels
|
||||
* every NEW/CHANGED community at once, given member names + statements.
|
||||
*
|
||||
* Core-survival: a community keeps an existing labeling while at least
|
||||
* half its core survives in it (matchClusterLabelings); only communities
|
||||
* with no surviving label AND at least MIN_CLUSTER_SIZE members are sent
|
||||
* to the namer. Records are append-only and idempotent (content-hashed
|
||||
* ids dedupe at the store seam), so a naming race or a re-run appends
|
||||
* nothing new.
|
||||
*
|
||||
* Runs POST-EXTRACTION (user-decided 2026-09-13): both front doors call
|
||||
* refreshClusterLabels after a successful merge. Failure is NON-FATAL —
|
||||
* the graph and hulls render fine unlabeled; the next run retries.
|
||||
*/
|
||||
import { z } from "zod";
|
||||
import { lhTool, toPiTools } from "../lh-tool.ts";
|
||||
import type { PiEngine } from "../models.ts";
|
||||
import { runSubagent } from "./pi-run.ts";
|
||||
import type { GraphNode } from "../../core/contracts.ts";
|
||||
import { getGraph, clusterContentId } from "../../core/graph.ts";
|
||||
import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../../core/clusters.ts";
|
||||
import type { GraphStore } from "../../core/ports.ts";
|
||||
|
||||
/** How many member propositions the namer sees per cluster — enough to
|
||||
* recognize the domain without paying for the whole region. */
|
||||
const MAX_MEMBERS_PER_CLUSTER = 12;
|
||||
|
||||
const CLUSTER_NAMER_INSTRUCTIONS = `You name communities of propositions in a personal knowledge graph.
|
||||
|
||||
You are given clusters. Each cluster is a set of related propositions
|
||||
(theory fragments the author is learning) plus, for context, a sketch of
|
||||
every OTHER cluster already named. For each UNNAMED cluster, mint ONE
|
||||
short DOMAIN LABEL: 1-4 words naming the region of the world model those
|
||||
propositions belong to together — like a section heading in a textbook.
|
||||
Use the members' own vocabulary ("Differential forms", "Sales
|
||||
psychology", "Server architecture"), not generic words ("Topics",
|
||||
"Various concepts"). If the propositions are unrelated, that is fine:
|
||||
name the best common denominator anyway — a reader should recognize the
|
||||
region at a glance.
|
||||
|
||||
Your entire deliverable is ONE submit_labels call. Include EVERY
|
||||
top-level field even when empty (labels: [] when nothing to name).`;
|
||||
|
||||
const labelsSchema = z.object({
|
||||
labels: z
|
||||
.array(
|
||||
z.object({
|
||||
/** The cluster key from the prompt ("c1", "c2", ...). */
|
||||
key: z.string(),
|
||||
label: z.string(),
|
||||
}),
|
||||
)
|
||||
.default([]),
|
||||
});
|
||||
|
||||
export interface UnnamedCluster {
|
||||
key: string;
|
||||
members: string[];
|
||||
nodes: Pick<GraphNode, "name" | "statement">[];
|
||||
}
|
||||
|
||||
/** One single-submit namer run (the same shape as every extraction stage):
|
||||
* the whole deliverable is ONE zod-validated submit call. */
|
||||
async function nameClusters(
|
||||
engine: PiEngine | undefined,
|
||||
signal: AbortSignal | undefined,
|
||||
unnamed: UnnamedCluster[],
|
||||
): Promise<{ key: string; label: string }[]> {
|
||||
let submitted: { key: string; label: string }[] | undefined;
|
||||
await runSubagent(
|
||||
toPiTools({
|
||||
submit_labels: lhTool({
|
||||
name: "submit_labels",
|
||||
description: "Submit the cluster labels.",
|
||||
schema: labelsSchema,
|
||||
execute: async (labels) => {
|
||||
submitted = labels.labels;
|
||||
return labels;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
{
|
||||
role: "labeler",
|
||||
engine,
|
||||
systemPrompt: CLUSTER_NAMER_INSTRUCTIONS,
|
||||
prompt: JSON.stringify(
|
||||
{
|
||||
unnamedClusters: unnamed.map((c) => ({
|
||||
key: c.key,
|
||||
members: c.nodes.map((n) => ({ name: n.name, statement: n.statement })),
|
||||
})),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
maxTurns: 4,
|
||||
abortSignal: signal,
|
||||
shouldStop: (_message, toolResults) =>
|
||||
toolResults.some((r) => r.toolName === "submit_labels" && !r.isError),
|
||||
},
|
||||
);
|
||||
if (!submitted) throw new Error("the cluster namer never submitted labels");
|
||||
return submitted;
|
||||
}
|
||||
|
||||
export interface ClusterRefreshStats {
|
||||
/** Communities the current graph divides into (all sizes). */
|
||||
detected: number;
|
||||
/** Communities that kept a label via core-survival. */
|
||||
claimed: number;
|
||||
/** Communities sent to the namer (unnamed, size >= MIN_CLUSTER_SIZE). */
|
||||
named: number;
|
||||
/** Labeling records actually appended (idempotent dedupe may shrink this). */
|
||||
appended: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The post-extraction garnish: detect communities on the CURRENT graph,
|
||||
* join persisted labelings (core-survival), and name what is new — one
|
||||
* singular LLM call. Appends ClusterLabeling records; the NEXT snapshot
|
||||
* read joins them (the payload is derived, never stored).
|
||||
*
|
||||
* Serialized: concurrent merges (parallel notes imports) chain here, so
|
||||
* two runs never mint duplicate labels for the same community.
|
||||
*/
|
||||
async function doRefresh(deps: {
|
||||
graphStore: GraphStore;
|
||||
/** Test seam: a scripted engine (production uses the labeler role). */
|
||||
engine?: PiEngine;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<ClusterRefreshStats> {
|
||||
const { graphStore, engine, signal } = deps;
|
||||
const { nodes } = getGraph(graphStore);
|
||||
// Semantic topics: the partition the embedder persisted in the
|
||||
// embedding ledger (same source the snapshot renders — never
|
||||
// recomputed here; the agglomerative pass is ~1.5s at scale).
|
||||
const communities = graphStore.readEmbeddings().clusters;
|
||||
const labelings = graphStore.readClusterLabels();
|
||||
const claims = matchClusterLabelings(communities, labelings);
|
||||
const byId = new Map(nodes.map((n) => [n.id, n]));
|
||||
|
||||
const claimed = claims.filter((c) => c !== null).length;
|
||||
const unnamed = communities
|
||||
.filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE)
|
||||
.map((c: Community, i) => {
|
||||
const nodesFor = c.members
|
||||
.map((id) => byId.get(id))
|
||||
.filter((n): n is GraphNode => n !== undefined)
|
||||
.slice(0, MAX_MEMBERS_PER_CLUSTER)
|
||||
.map((n) => ({ name: n.name, statement: n.statement }));
|
||||
return { key: `c${i}`, members: c.members, nodes: nodesFor };
|
||||
});
|
||||
|
||||
if (unnamed.length === 0) return { detected: communities.length, claimed, named: 0, appended: 0 };
|
||||
|
||||
const labels = await nameClusters(engine, signal, unnamed);
|
||||
let appended = 0;
|
||||
const seen = new Set<string>();
|
||||
for (const { key, members } of unnamed) {
|
||||
if (seen.has(key)) continue;
|
||||
const label = labels.find((l) => l.key === key)?.label?.trim();
|
||||
if (!label) continue;
|
||||
seen.add(key);
|
||||
graphStore.appendClusterLabel({
|
||||
id: clusterContentId(members),
|
||||
label,
|
||||
core: members,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
appended++;
|
||||
}
|
||||
return { detected: communities.length, claimed, named: unnamed.length, appended };
|
||||
}
|
||||
|
||||
let namingTail: Promise<unknown> = Promise.resolve();
|
||||
|
||||
/**
|
||||
* Fire the refresh, serialized behind any refresh already running (a
|
||||
* parallel import's merge waits its turn; each run reads fresh ledgers).
|
||||
* Waivable: call sites may fire-and-forget; scripts await the return.
|
||||
*/
|
||||
export function refreshClusterLabels(deps: {
|
||||
graphStore: GraphStore;
|
||||
engine?: PiEngine;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<ClusterRefreshStats> {
|
||||
const run = namingTail.then(() => doRefresh(deps));
|
||||
namingTail = run.catch(() => {});
|
||||
return run;
|
||||
}
|
||||
|
||||
/** Await every refresh fired so far (scripts: don't exit mid-naming). */
|
||||
export async function clusterNamingSettled(): Promise<void> {
|
||||
await namingTail;
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
import * as crypto from "node:crypto";
|
||||
import type { UIMessage } from "ai";
|
||||
import type { Usage } from "@earendil-works/pi-ai";
|
||||
import { z } from "zod";
|
||||
import { lhTool, toPiTools } from "../lh-tool.ts";
|
||||
import type { ModelRole } from "../../../config.ts";
|
||||
import type { PiEngine } from "../models.ts";
|
||||
import { runSubagent } from "./pi-run.ts";
|
||||
import type {
|
||||
@@ -14,6 +16,9 @@ import type {
|
||||
SessionFindings,
|
||||
} from "../../core/contracts.ts";
|
||||
import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
|
||||
import type { GraphStore } from "../../core/ports.ts";
|
||||
import { settleGraphLayout } from "../../core/layout-settle.ts";
|
||||
import { refreshClusterLabels } from "./cluster-namer.ts";
|
||||
|
||||
/**
|
||||
* The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the
|
||||
@@ -42,6 +47,17 @@ import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../
|
||||
* whatever the in-flight run missed). On failure nothing is checkpointed —
|
||||
* the next trigger re-runs the whole thing.
|
||||
*
|
||||
* The FORMAT-AGNOSTIC FRONT DOOR (2026-09-13): importNotes() runs the SAME
|
||||
* three stages over raw personal notes (arbitrary format — no chat, no
|
||||
* session, no message ids). A notes-mode extractor prompt replaces the
|
||||
* session one; the auditor and merger are shared (identity + deterministic
|
||||
* appends don't care where text came from). Provenance is the file itself:
|
||||
* source ids ("src_" + path hash) play the chatId role in mappings, events
|
||||
* and checkpoints, and the source record (graphStore sources seam) carries
|
||||
* the file path + the run's usage. No quiz events are ever built from
|
||||
* notes — quiz outcomes inside the text surface only through the
|
||||
* extractor's understanding observations (user-decided 2026-09-13).
|
||||
*
|
||||
* Observability (§5 baseline): every stage logs one line per step, its
|
||||
* start, and its duration; the pipeline logs a failure line before
|
||||
* rethrowing, so the server log reads start → stage timings → merged
|
||||
@@ -154,7 +170,9 @@ const findingsSchema = z.object({
|
||||
.describe("Every distinct proposition the session touched — true statements, misconception-free."),
|
||||
edges: z
|
||||
.array(edgeSchema)
|
||||
.describe("Relations between propositions: follows (derivable) / supports (grounds without entailing)."),
|
||||
.describe("Relations between propositions: follows (derivable) / supports (grounds without entailing). Pass [] when there are none — never omit the field.")
|
||||
.default([]), // weak models omit required-but-empty arrays; omission means "none"
|
||||
|
||||
attribution: z
|
||||
.object({
|
||||
quiz: z
|
||||
@@ -163,9 +181,51 @@ const findingsSchema = z.object({
|
||||
"toolCallId of EVERY quiz line → the proposition id it was really testing. Attribute RETROACTIVELY — the whole transcript is visible; map each quiz to the proposition it tests (often articulated elsewhere in the session). null when genuinely unattributable.",
|
||||
),
|
||||
})
|
||||
.describe("Quiz attribution — your only event judgment."),
|
||||
.describe("Quiz attribution — your only event judgment. Pass { quiz: {} } when the session has no quiz lines — never omit the field.")
|
||||
.default({ quiz: {} }), // weak models omit empty objects too
|
||||
});
|
||||
|
||||
const NOTES_EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE source —
|
||||
the user's own NOTES, in an ARBITRARY format (personal notes, an exported conversation, a document — anything) — and
|
||||
report what those notes show the author engaging with, as structured findings for their global knowledge graph.
|
||||
|
||||
There is no session and no chat structure here: the source is raw text, the author's own material. Read it as such.
|
||||
|
||||
Report, with verbatim evidence quotes for everything:
|
||||
|
||||
NODE THEORY — the graph is a set of TRUE propositions about the world, full stop. A node is never anything about the
|
||||
author or the source. The author's relationship to a proposition is expressed ENTIRELY by your understanding note; it
|
||||
never leaks into the node itself. So no "the author understands X", no "mentioned in these notes": the node is just X.
|
||||
A misconception is NOT a node of its own wrong claim — state the TRUE correction as the proposition.
|
||||
|
||||
1. PROPOSITIONS — every distinct reasoning-step-sized proposition the notes actually engage with:
|
||||
- Each has a "name" — a VERY short label, a few words, a compact handle like a glossary headword ("Will to power",
|
||||
"Divine command theory"). Not a sentence; it becomes the node's label in the graph, so keep it glanceable.
|
||||
- And a "statement" — the self-contained TRUE proposition at full precision.
|
||||
- Notes may explain things at length or quote material the author is studying: extract the propositions themselves,
|
||||
one reasoning step each — never a topic blob, never a sub-step.
|
||||
- "messages": ALWAYS an empty list — provenance is the file itself.
|
||||
- "aliases": verbatim phrasings the notes used for the same proposition.
|
||||
- "understanding": ONE word from the fixed scale — "solid", "shaky", "accepted", "resisted", or "not yet engaged" —
|
||||
judging the AUTHOR's grasp of the proposition as the notes show it: their own commentary, self-assessments,
|
||||
recorded quiz outcomes, the way they write about it. Strictly one word, no numbers, no justification (a
|
||||
wrong-shaped note is rejected by the schema). When the notes give no signal, omit the field.
|
||||
|
||||
2. EDGES between propositions:
|
||||
- "follows": to is DERIVABLE from from (understanding from makes understanding to achievable).
|
||||
- "supports": from GROUNDS to without entailing it (helpful context, not a prerequisite).
|
||||
- Only relations the notes actually articulate — with evidence.
|
||||
|
||||
There is no quiz attribution for notes — the source has no quiz events, and the schema has no attribution field.
|
||||
|
||||
Evidence discipline: every quote must appear VERBATIM in the source. The auditor rejects anything unbacked. When in
|
||||
doubt, report; the auditor decides.
|
||||
|
||||
Include EVERY top-level field of the call even when empty — "edges": [] when there are no relations. NEVER omit
|
||||
a field because it is empty.
|
||||
|
||||
Your entire deliverable is ONE submit_findings call.`;
|
||||
|
||||
const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session
|
||||
transcript and report what was learned — as structured findings for the learner's global knowledge graph.
|
||||
|
||||
@@ -208,25 +268,35 @@ quiz outcomes carry that the learner believed the wrong version.
|
||||
Evidence discipline: every quote must appear VERBATIM in the transcript view (a QUIZ line counts as the
|
||||
transcript). The auditor rejects anything unbacked. When in doubt, report; the auditor decides.
|
||||
|
||||
Include EVERY top-level field of the call even when empty — "edges": [], "attribution": { quiz: {} } when there
|
||||
are no quizzes. NEVER omit a field because it is empty.
|
||||
|
||||
Your entire deliverable is ONE submit_findings call.`;
|
||||
|
||||
/** One extractor run: the whole deliverable is ONE zod-validated submit
|
||||
* call (a pi agent purely so the model can retry a schema-invalid call).
|
||||
* Returns the submitted findings — captured from the execute seam, which
|
||||
/** One single-submit agent run — the shared shape of EVERY LLM stage
|
||||
* here (session + notes): the whole deliverable is ONE zod-validated
|
||||
* submit call (a pi agent purely so the model can retry a schema-invalid
|
||||
* call). The submitted value is captured from the execute seam, which
|
||||
* only fires on a schema-valid call. */
|
||||
async function runExtractor(
|
||||
engine: PiEngine | undefined,
|
||||
prompt: string,
|
||||
signal: AbortSignal | undefined,
|
||||
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
|
||||
): Promise<SessionFindings> {
|
||||
let submitted: SessionFindings | undefined;
|
||||
async function runSubmitAgent<T>(opts: {
|
||||
role: ModelRole;
|
||||
toolName: string;
|
||||
toolDescription: string;
|
||||
schema: z.ZodType<T>;
|
||||
systemPrompt: string;
|
||||
prompt: string;
|
||||
engine: PiEngine | undefined;
|
||||
signal: AbortSignal | undefined;
|
||||
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void;
|
||||
neverSubmitted: string;
|
||||
}): Promise<T> {
|
||||
let submitted: T | undefined;
|
||||
const result = await runSubagent(
|
||||
toPiTools({
|
||||
submit_findings: lhTool({
|
||||
name: "submit_findings",
|
||||
description: "Submit the session findings.",
|
||||
schema: findingsSchema,
|
||||
[opts.toolName]: lhTool({
|
||||
name: opts.toolName,
|
||||
description: opts.toolDescription,
|
||||
schema: opts.schema,
|
||||
execute: async (findings) => {
|
||||
submitted = findings;
|
||||
return findings;
|
||||
@@ -234,28 +304,71 @@ async function runExtractor(
|
||||
}),
|
||||
}),
|
||||
{
|
||||
role: "extractor",
|
||||
engine,
|
||||
systemPrompt: EXTRACTOR_INSTRUCTIONS,
|
||||
prompt,
|
||||
role: opts.role,
|
||||
engine: opts.engine,
|
||||
systemPrompt: opts.systemPrompt,
|
||||
prompt: opts.prompt,
|
||||
maxTurns: 4,
|
||||
abortSignal: signal,
|
||||
abortSignal: opts.signal,
|
||||
onAssistantMessage: (message, step) =>
|
||||
onStep(
|
||||
opts.onStep(
|
||||
message.usage,
|
||||
message.model,
|
||||
step,
|
||||
message.stopReason,
|
||||
message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)",
|
||||
),
|
||||
shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_findings" && !r.isError),
|
||||
shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === opts.toolName && !r.isError),
|
||||
},
|
||||
);
|
||||
void result;
|
||||
if (!submitted) throw new Error("the extractor never submitted findings");
|
||||
if (!submitted) throw new Error(opts.neverSubmitted);
|
||||
return submitted;
|
||||
}
|
||||
|
||||
/** The extractor over ONE session transcript (the chat front door). */
|
||||
function runExtractor(
|
||||
engine: PiEngine | undefined,
|
||||
prompt: string,
|
||||
signal: AbortSignal | undefined,
|
||||
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
|
||||
): Promise<SessionFindings> {
|
||||
return runSubmitAgent({
|
||||
role: "extractor",
|
||||
toolName: "submit_findings",
|
||||
toolDescription: "Submit the session findings.",
|
||||
schema: findingsSchema,
|
||||
systemPrompt: EXTRACTOR_INSTRUCTIONS,
|
||||
prompt,
|
||||
engine,
|
||||
signal,
|
||||
onStep,
|
||||
neverSubmitted: "the extractor never submitted findings",
|
||||
});
|
||||
}
|
||||
|
||||
/** The extractor over ONE source of raw personal notes (the format-agnostic
|
||||
* front door): same single-submit shape, notes-mode prompt + schema. */
|
||||
function runNotesExtractor(
|
||||
engine: PiEngine | undefined,
|
||||
prompt: string,
|
||||
signal: AbortSignal | undefined,
|
||||
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
|
||||
): Promise<NotesFindings> {
|
||||
return runSubmitAgent({
|
||||
role: "extractor",
|
||||
toolName: "submit_findings",
|
||||
toolDescription: "Submit the notes findings.",
|
||||
schema: notesFindingsSchema,
|
||||
systemPrompt: NOTES_EXTRACTOR_INSTRUCTIONS,
|
||||
prompt,
|
||||
engine,
|
||||
signal,
|
||||
onStep,
|
||||
neverSubmitted: "the extractor never submitted findings",
|
||||
});
|
||||
}
|
||||
|
||||
// ─── the auditor ───────────────────────────────────────────────────────────
|
||||
|
||||
const auditSchema = findingsSchema.extend({
|
||||
@@ -276,14 +389,25 @@ const auditSchema = findingsSchema.extend({
|
||||
reason: z.string(),
|
||||
}),
|
||||
)
|
||||
.describe("Everything you dropped, with the reason — the audit trail."),
|
||||
.describe("Everything you dropped, with the reason — the audit trail. Pass [] when you dropped nothing — never omit the field.")
|
||||
.default([]), // same weak-model tolerance as findings.edges
|
||||
});
|
||||
|
||||
const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive
|
||||
ONE session transcript, a set of proposed findings (propositions, edges, quiz attribution), and the CURRENT
|
||||
knowledge graph. Your jobs: the faithfulness net, and identity.
|
||||
// The notes front door's shapes: everything session findings have, minus
|
||||
// quiz attribution — raw notes have no quiz tool parts, so there is
|
||||
// nothing to attribute (and no event to build: outcomes inside the text
|
||||
// surface only through understanding observations, user-decided).
|
||||
const notesFindingsSchema = findingsSchema.omit({ attribution: true });
|
||||
const notesAuditSchema = auditSchema.omit({ attribution: true });
|
||||
type NotesFindings = Omit<SessionFindings, "attribution">;
|
||||
type NotesAuditResult = Omit<AuditResult, "attribution">;
|
||||
|
||||
FAITHFULNESS — verify every candidate against the transcript:
|
||||
const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive
|
||||
ONE source — a session transcript, or the author's personal notes in an ARBITRARY format (the notes front door) — a
|
||||
set of proposed findings (propositions, edges, and quiz attribution where the source has quiz lines), and the
|
||||
CURRENT knowledge graph. Your jobs: the faithfulness net, and identity.
|
||||
|
||||
FAITHFULNESS — verify every candidate against the source text:
|
||||
- PROPOSITIONS: every evidence quote must appear (near-)verbatim in the transcript view, and the statement
|
||||
must be a faithful reading of it. The node theory applies: a statement must be a TRUE, self-contained,
|
||||
world-facing fact — reword learner- or session-relative statements ("the learner understands X" → "X"), state
|
||||
@@ -293,8 +417,10 @@ FAITHFULNESS — verify every candidate against the transcript:
|
||||
- EDGES: the evidence must actually articulate the relation, in the direction claimed. Drop
|
||||
direction-swapped or speculative edges.
|
||||
- ATTRIBUTION: the QUIZ line's question must genuinely test the attributed proposition. Drop wrong
|
||||
attributions to null, or re-point them to the proposition the question actually tests.
|
||||
- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence; each note must be
|
||||
attributions to null, or re-point them to the proposition the question actually tests. (Session
|
||||
sources only — notes findings carry no attribution; there is nothing to verify.)
|
||||
- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence — or, for notes,
|
||||
the author's own words; each note must be
|
||||
exactly one word from the fixed scale ("solid", "shaky", "accepted", "resisted", "not yet engaged") —
|
||||
reword or drop anything else. No justifications.
|
||||
|
||||
@@ -311,7 +437,10 @@ Also drop propositions not referenced by anything else ONLY if they have no othe
|
||||
well-evidenced proposition is still knowledge.
|
||||
|
||||
Never invent new candidates. Your entire deliverable is ONE submit_review call containing the verified findings
|
||||
(the same shapes you received, with linkTo added) plus the dropped list.`;
|
||||
(the same shapes you received, with linkTo added) plus the dropped list.
|
||||
|
||||
Include EVERY top-level field of the call even when empty — "edges": [], "dropped": []. NEVER omit a field
|
||||
because it is empty.`;
|
||||
|
||||
/** The current graph, compact — the auditor's identity context. */
|
||||
function graphView(nodes: GraphNode[]): string {
|
||||
@@ -324,47 +453,47 @@ function graphView(nodes: GraphNode[]): string {
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** One auditor run — same single-submit shape as the extractor. */
|
||||
async function runAuditor(
|
||||
/** The auditor over session findings (the chat front door). */
|
||||
function runAuditor(
|
||||
engine: PiEngine | undefined,
|
||||
prompt: string,
|
||||
signal: AbortSignal | undefined,
|
||||
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
|
||||
): Promise<AuditResult> {
|
||||
let submitted: AuditResult | undefined;
|
||||
const result = await runSubagent(
|
||||
toPiTools({
|
||||
submit_review: lhTool({
|
||||
name: "submit_review",
|
||||
description: "Submit the verified findings subset (with identity verdicts) + the dropped list.",
|
||||
schema: auditSchema,
|
||||
execute: async (review) => {
|
||||
submitted = review;
|
||||
return review;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
{
|
||||
role: "auditor",
|
||||
engine,
|
||||
systemPrompt: AUDITOR_INSTRUCTIONS,
|
||||
prompt,
|
||||
maxTurns: 4,
|
||||
abortSignal: signal,
|
||||
onAssistantMessage: (message, step) =>
|
||||
onStep(
|
||||
message.usage,
|
||||
message.model,
|
||||
step,
|
||||
message.stopReason,
|
||||
message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)",
|
||||
),
|
||||
shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_review" && !r.isError),
|
||||
},
|
||||
);
|
||||
void result;
|
||||
if (!submitted) throw new Error("the auditor never submitted a review");
|
||||
return submitted;
|
||||
return runSubmitAgent({
|
||||
role: "auditor",
|
||||
toolName: "submit_review",
|
||||
toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.",
|
||||
schema: auditSchema,
|
||||
systemPrompt: AUDITOR_INSTRUCTIONS,
|
||||
prompt,
|
||||
engine,
|
||||
signal,
|
||||
onStep,
|
||||
neverSubmitted: "the auditor never submitted a review",
|
||||
});
|
||||
}
|
||||
|
||||
/** The auditor over notes findings (the format-agnostic front door): same
|
||||
* faithfulness + identity job, notes-mode schema (no attribution). */
|
||||
function runNotesAuditor(
|
||||
engine: PiEngine | undefined,
|
||||
prompt: string,
|
||||
signal: AbortSignal | undefined,
|
||||
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
|
||||
): Promise<NotesAuditResult> {
|
||||
return runSubmitAgent({
|
||||
role: "auditor",
|
||||
toolName: "submit_review",
|
||||
toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.",
|
||||
schema: notesAuditSchema,
|
||||
systemPrompt: AUDITOR_INSTRUCTIONS,
|
||||
prompt,
|
||||
engine,
|
||||
signal,
|
||||
onStep,
|
||||
neverSubmitted: "the auditor never submitted a review",
|
||||
});
|
||||
}
|
||||
|
||||
// ─── the pipeline status registry ─────────────────────────────────────────────
|
||||
@@ -376,6 +505,13 @@ export type ExtractionPhase = "extracting" | "auditing" | "merging";
|
||||
* Exported for the status payload's consumer typing (the dock). */
|
||||
export interface RunningExtraction {
|
||||
chatId: string;
|
||||
/** Which front door the run came through — label + usage-persistence
|
||||
* depend on it (chats bill their chat record; notes bill the source
|
||||
* record). */
|
||||
kind: "chat" | "notes";
|
||||
/** Display label (the file path for notes imports); the status route
|
||||
* falls back to the chat store for chat runs. */
|
||||
label?: string;
|
||||
phase: ExtractionPhase;
|
||||
startedAt: number;
|
||||
/** Live per-stage usage, accumulated as steps bill (the dock's cost /
|
||||
@@ -388,6 +524,10 @@ export interface RunningExtraction {
|
||||
|
||||
const runningExtractions = new Map<string, RunningExtraction>();
|
||||
|
||||
/** Last minted startedAt — kept monotonic so two runs starting in the same
|
||||
* millisecond never collide on the usage idempotency key. */
|
||||
let lastStartedAt = 0;
|
||||
|
||||
/** In-flight pipeline runs (oldest first) — the /api/graph/status payload. */
|
||||
export function extractionStatuses(): RunningExtraction[] {
|
||||
return [...runningExtractions.values()].sort((a, b) => a.startedAt - b.startedAt);
|
||||
@@ -411,26 +551,51 @@ export interface ExtractionModels {
|
||||
/** Test seam: a scripted engine per stage (production uses the roles). */
|
||||
extractor?: PiEngine;
|
||||
auditor?: PiEngine;
|
||||
/** Test seam for the cluster namer (post-extraction garnish). */
|
||||
labeler?: PiEngine;
|
||||
}
|
||||
|
||||
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
|
||||
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {
|
||||
/** The post-merge embedding batch (semantic clusters run on it).
|
||||
* INJECTED by the composition root (the agent layer never imports
|
||||
* the server layer — architecture law): production passes the real
|
||||
* local-model embedder (server/embeddings.ts embedNewNodes), tests
|
||||
* fake it. Failure inside it is non-fatal by contract. */
|
||||
embedder: (graphStore: GraphStore) => Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface NotesImportInput {
|
||||
/** The source's file path — its identity (deterministic src id) AND its
|
||||
* provenance label in the detail card. */
|
||||
filePath: string;
|
||||
/** The file's raw content, VERBATIM — any format. */
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ExtractionPipeline {
|
||||
/** Extract + audit + merge ONE chat. Single-flight per chat; resolves to
|
||||
* the merger's append stats (or throws — no checkpoint on failure). */
|
||||
runExtraction(chatId: string): Promise<EmissionStats>;
|
||||
/** The format-agnostic front door (2026-09-13): run the same pipeline
|
||||
* over ONE raw notes file. The source id is deterministic per file
|
||||
* path, so re-importing the same file re-runs extraction against the
|
||||
* same source identity (mappings + events dedupe by it). */
|
||||
importNotes(input: NotesImportInput): Promise<EmissionStats>;
|
||||
}
|
||||
|
||||
export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline {
|
||||
const { chatStore, graphStore } = deps;
|
||||
const { chatStore, graphStore, embedder: embed } = deps;
|
||||
const inFlight = new Set<string>();
|
||||
|
||||
function setPhase(chatId: string, phase: ExtractionPhase): void {
|
||||
function setPhase(chatId: string, phase: ExtractionPhase, kind: "chat" | "notes", label?: string): void {
|
||||
const prev = runningExtractions.get(chatId);
|
||||
if (!prev) lastStartedAt = Math.max(Date.now(), lastStartedAt + 1);
|
||||
runningExtractions.set(chatId, {
|
||||
chatId,
|
||||
kind,
|
||||
label: prev?.label ?? label, // phase changes never lose the label
|
||||
phase,
|
||||
startedAt: prev?.startedAt ?? Date.now(),
|
||||
startedAt: prev?.startedAt ?? lastStartedAt,
|
||||
usage: prev?.usage ?? {}, // phase changes never lose usage
|
||||
controller: prev?.controller ?? new AbortController(),
|
||||
});
|
||||
@@ -525,7 +690,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
function persistRunUsage(chatId: string, durationMs: number, stats: EmissionStats | undefined): void {
|
||||
const run = runningExtractions.get(chatId);
|
||||
if (!run) return;
|
||||
chatStore.appendExtractionUsage(chatId, {
|
||||
const entry: ExtractionRunUsage = {
|
||||
startedAt: run.startedAt,
|
||||
at: new Date().toISOString(),
|
||||
durationMs,
|
||||
@@ -534,7 +699,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
extractor: run.usage.extractor ?? { model: "unknown", steps: [] },
|
||||
auditor: run.usage.auditor ?? { model: "unknown", steps: [] },
|
||||
},
|
||||
});
|
||||
};
|
||||
if (run.kind === "notes") graphStore.appendSourceUsage(chatId, entry);
|
||||
else chatStore.appendExtractionUsage(chatId, entry);
|
||||
}
|
||||
|
||||
async function runExtraction(chatId: string): Promise<EmissionStats> {
|
||||
@@ -543,19 +710,41 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
const t0 = performance.now();
|
||||
try {
|
||||
console.info(`[extraction ${chatId}] pipeline start`);
|
||||
setPhase(chatId, "extracting");
|
||||
setPhase(chatId, "extracting", "chat");
|
||||
const findings = await extract(chatId);
|
||||
setPhase(chatId, "auditing");
|
||||
setPhase(chatId, "auditing", "chat");
|
||||
const auditResult = await audit(chatId, findings);
|
||||
setPhase(chatId, "merging");
|
||||
// The merger re-reads the chat record itself — events are rebuilt
|
||||
// from the transcript, never from stale copies.
|
||||
const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore });
|
||||
setPhase(chatId, "merging", "chat");
|
||||
// The merger merges from the CURRENT record — events are rebuilt
|
||||
// from the live transcript, never from stale copies.
|
||||
const record = chatStore.getChat(chatId);
|
||||
if (!record) throw new Error(`Chat not found: ${chatId}`);
|
||||
const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }, {
|
||||
messages: record.messages as UIMessage[],
|
||||
foldedAt: record.updatedAt,
|
||||
});
|
||||
// Semantic embeddings (DECIDED 2026-09-13): topics come from
|
||||
// MEANING, so embed the new propositions BEFORE the settle and
|
||||
// the namer look at clusters. Failure is non-fatal — clustering
|
||||
// runs without the new nodes until the next merge retries.
|
||||
await embed(graphStore).catch((err) => {
|
||||
console.error(`[extraction ${chatId}] embedding failed (non-fatal):`, err);
|
||||
});
|
||||
// Option A: the layout ledger evolves with the merge (sync — the
|
||||
// next snapshot fetch must already reflect the new nodes).
|
||||
settleGraphLayout(graphStore);
|
||||
persistRunUsage(chatId, Math.round(performance.now() - t0), stats);
|
||||
console.info(
|
||||
`[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
|
||||
`(${Math.round(performance.now() - t0)}ms total)`,
|
||||
);
|
||||
// Post-extraction garnish (DECIDED 2026-09-13): name new/changed
|
||||
// clusters. Fire-and-forget — naming never fails the run.
|
||||
refreshClusterLabels({
|
||||
graphStore,
|
||||
engine: deps.labeler,
|
||||
signal: runningExtractions.get(chatId)?.controller.signal,
|
||||
}).catch(() => {});
|
||||
return stats;
|
||||
} catch (err) {
|
||||
persistRunUsage(chatId, Math.round(performance.now() - t0), undefined);
|
||||
@@ -577,6 +766,125 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
}
|
||||
}
|
||||
|
||||
return { runExtraction };
|
||||
// ── the notes front door (format-agnostic: raw text in, same graph out) ──
|
||||
|
||||
/** The raw source view the notes stages see: the file path + the text
|
||||
* VERBATIM. No message ids — provenance is the file itself. */
|
||||
function notesView(input: { filePath: string; text: string }): string {
|
||||
return `SOURCE (the author's personal notes): ${input.filePath}\n\n${input.text}`;
|
||||
}
|
||||
|
||||
async function notesExtract(input: { id: string; filePath: string; text: string }): Promise<NotesFindings> {
|
||||
const signal = runningExtractions.get(input.id)?.controller.signal;
|
||||
signal?.throwIfAborted(); // an abort during a sync gap ends the run here
|
||||
const view = notesView(input);
|
||||
console.info(`[extraction ${input.id}] notes extractor starting (${view.split("\n").length} source lines)`);
|
||||
const t0 = performance.now();
|
||||
const findings = await runNotesExtractor(
|
||||
deps.extractor,
|
||||
view,
|
||||
signal,
|
||||
(usage, model, step, finishReason, tools) => {
|
||||
recordStep(input.id, "extractor", callUsage(usage), model);
|
||||
console.info(`[extraction] extractor step ${step}: ${finishReason}, tools: ${tools}`);
|
||||
},
|
||||
);
|
||||
console.info(`[extraction ${input.id}] notes extractor done in ${Math.round(performance.now() - t0)}ms`);
|
||||
return findings;
|
||||
}
|
||||
|
||||
async function notesAudit(input: { id: string; filePath: string; text: string }, findings: NotesFindings): Promise<NotesAuditResult> {
|
||||
const signal = runningExtractions.get(input.id)?.controller.signal;
|
||||
signal?.throwIfAborted(); // an abort during the extract→audit gap ends here
|
||||
const graph = getGraph(graphStore);
|
||||
console.info(
|
||||
`[extraction ${input.id}] notes auditor starting (${findings.propositions.length} propositions, ` +
|
||||
`${findings.edges.length} edges, ${graph.nodes.length} graph nodes)`,
|
||||
);
|
||||
const t0 = performance.now();
|
||||
const auditResult = await runNotesAuditor(
|
||||
deps.auditor,
|
||||
`Source:\n\n${notesView(input)}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}\n\n` +
|
||||
`Current knowledge graph (${graph.nodes.length} nodes):\n\n${graphView(graph.nodes)}`,
|
||||
signal,
|
||||
(usage, model, step, finishReason, tools) => {
|
||||
recordStep(input.id, "auditor", callUsage(usage), model);
|
||||
console.info(`[extraction] auditor step ${step}: ${finishReason}, tools: ${tools}`);
|
||||
},
|
||||
);
|
||||
if (auditResult.dropped.length > 0) {
|
||||
console.info(
|
||||
`[extraction ${input.id}] audit dropped ${auditResult.dropped.length}: ` +
|
||||
auditResult.dropped.map((d) => `${d.ref} (${d.reason})`).join("; "),
|
||||
);
|
||||
}
|
||||
console.info(
|
||||
`[extraction ${input.id}] audit kept ${auditResult.propositions.length}/${findings.propositions.length} propositions, ` +
|
||||
`${auditResult.propositions.filter((p) => p.linkTo !== undefined).length} linked to existing nodes`,
|
||||
);
|
||||
console.info(`[extraction ${input.id}] notes auditor done in ${Math.round(performance.now() - t0)}ms`);
|
||||
return auditResult;
|
||||
}
|
||||
|
||||
async function runNotesExtraction(input: { id: string; filePath: string; text: string; importedAt: string }): Promise<EmissionStats> {
|
||||
const { id } = input;
|
||||
if (inFlight.has(id)) return { nodes: 0, edges: 0, events: 0 };
|
||||
inFlight.add(id);
|
||||
const t0 = performance.now();
|
||||
try {
|
||||
console.info(`[extraction ${id}] notes pipeline start (${input.filePath})`);
|
||||
setPhase(id, "extracting", "notes", input.filePath);
|
||||
const findings = await notesExtract(input);
|
||||
setPhase(id, "auditing", "notes", input.filePath);
|
||||
const notesResult = await notesAudit(input, findings);
|
||||
setPhase(id, "merging", "notes", input.filePath);
|
||||
// Normalize to the merger's findings shape: notes carry no quiz
|
||||
// attribution, and the merger scans no quiz parts for notes
|
||||
// (messages: null) — nothing to attribute, nothing to build.
|
||||
const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } };
|
||||
const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt });
|
||||
// Same post-merge chain as the chat path: embeddings first
|
||||
// (topics from meaning), then the settle, then the namer.
|
||||
await embed(graphStore).catch((err) => {
|
||||
console.error(`[extraction ${id}] embedding failed (non-fatal):`, err);
|
||||
});
|
||||
// Option A: same post-merge settle as the chat path.
|
||||
settleGraphLayout(graphStore);
|
||||
persistRunUsage(id, Math.round(performance.now() - t0), stats);
|
||||
console.info(
|
||||
`[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
|
||||
`(${Math.round(performance.now() - t0)}ms total)`,
|
||||
);
|
||||
refreshClusterLabels({
|
||||
graphStore,
|
||||
engine: deps.labeler,
|
||||
signal: runningExtractions.get(id)?.controller.signal,
|
||||
}).catch(() => {});
|
||||
return stats;
|
||||
} catch (err) {
|
||||
persistRunUsage(id, Math.round(performance.now() - t0), undefined);
|
||||
if (err instanceof Error && err.name === "AbortError") {
|
||||
console.info(`[extraction ${id}] aborted (${Math.round(performance.now() - t0)}ms) — nothing merged, nothing checkpointed`);
|
||||
return { nodes: 0, edges: 0, events: 0 };
|
||||
}
|
||||
console.error(`[extraction ${id}] notes pipeline failed (${Math.round(performance.now() - t0)}ms):`, err);
|
||||
throw err;
|
||||
} finally {
|
||||
inFlight.delete(id);
|
||||
runningExtractions.delete(id); // done or failed — no longer running
|
||||
}
|
||||
}
|
||||
|
||||
/** Import ONE raw notes file: register the source (deterministic id per
|
||||
* file path — re-import keeps the same identity and its usage history)
|
||||
* and run the pipeline over it. */
|
||||
async function importNotes(input: NotesImportInput): Promise<EmissionStats> {
|
||||
const id = `src_${crypto.createHash("sha256").update(input.filePath).digest("hex").slice(0, 12)}`;
|
||||
const importedAt = new Date().toISOString();
|
||||
graphStore.upsertSource({ id, filePath: input.filePath, importedAt });
|
||||
return runNotesExtraction({ id, ...input, importedAt });
|
||||
}
|
||||
|
||||
return { runExtraction, importNotes };
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* CLUSTER LABEL MATCHING (vision §1). DETECTION moved to semantic topic
|
||||
* clustering over embeddings (core/semantic-clusters.ts, DECIDED
|
||||
* 2026-09-13) — Louvain over the edge structure is retired: it only ever
|
||||
* found the reasoning chains (the graph is disconnected by nature), and
|
||||
* edge-orphans could never join anything. What REMAINS here is the
|
||||
* cluster contract + core-survival label matching, which is
|
||||
* detection-agnostic: it consumes the same { members } shape semantic
|
||||
* clustering emits.
|
||||
*
|
||||
* Labelings key to sorted MEMBER SETS, never to detection indices —
|
||||
* a labeling record outlives any reshuffle of the partition.
|
||||
*
|
||||
* Browser-safe pure logic (no node builtins, no fs): the server runs
|
||||
* matching for the snapshot payload, tests run it directly.
|
||||
*/
|
||||
import type { ClusterLabeling } from "./contracts.ts";
|
||||
|
||||
/** A detected community. Members are SORTED node ids — the member set IS
|
||||
* the identity a labeling record can outlive a reshuffle with. */
|
||||
export interface Community {
|
||||
members: string[];
|
||||
}
|
||||
|
||||
/** Communities below this size are not rendered, not named — presentation
|
||||
* rule (user-decided 2026-09-13): a region of the world model needs at
|
||||
* least three propositions to be worth a hull and a label. */
|
||||
export const MIN_CLUSTER_SIZE = 3;
|
||||
|
||||
/**
|
||||
* Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED
|
||||
* 2026-09-13): a community keeps a record's label while AT LEAST HALF of
|
||||
* the record's core survives in it (overlap * 2 >= core.length; exactly
|
||||
* half survives). Both sides are single-successor: one community holds at
|
||||
* most one label, one record is claimed by at most one community — on a
|
||||
* split, only the branch carrying the majority of the core keeps the name;
|
||||
* on a merge, the best-overlapping record wins.
|
||||
*
|
||||
* Greedy assignment by overlap size (largest first; ties broken by newer
|
||||
* record). Returns a claim (or null) per community, in input order —
|
||||
* unclaimed communities are the namer's input. Pure: same inputs, same
|
||||
* claims.
|
||||
*/
|
||||
export function matchClusterLabelings(
|
||||
communities: readonly Community[],
|
||||
labelings: readonly ClusterLabeling[],
|
||||
): (ClusterLabeling | null)[] {
|
||||
interface Pair {
|
||||
community: number;
|
||||
labeling: ClusterLabeling;
|
||||
overlap: number;
|
||||
}
|
||||
const pairs: Pair[] = [];
|
||||
communities.forEach((community, ci) => {
|
||||
const members = new Set(community.members);
|
||||
for (const labeling of labelings) {
|
||||
const overlap = labeling.core.reduce((n, id) => n + (members.has(id) ? 1 : 0), 0);
|
||||
if (overlap * 2 >= labeling.core.length && overlap > 0) {
|
||||
pairs.push({ community: ci, labeling, overlap });
|
||||
}
|
||||
}
|
||||
});
|
||||
pairs.sort((a, b) => b.overlap - a.overlap || (a.labeling.createdAt < b.labeling.createdAt ? 1 : -1));
|
||||
const claims: (ClusterLabeling | null)[] = communities.map(() => null);
|
||||
const takenRecords = new Set<string>();
|
||||
for (const pair of pairs) {
|
||||
if (claims[pair.community] || takenRecords.has(pair.labeling.id)) continue;
|
||||
claims[pair.community] = pair.labeling;
|
||||
takenRecords.add(pair.labeling.id);
|
||||
}
|
||||
return claims;
|
||||
}
|
||||
@@ -20,6 +20,8 @@
|
||||
/** Two relations only (DECIDED 2026-09-02): follows = b is derivable from a;
|
||||
* supports = a grounds b without entailing it. No scalars, no encoded
|
||||
* strength — strength is derived at read time from topology. */
|
||||
import type { ExtractionRunUsage } from "./chat.ts";
|
||||
|
||||
export type EdgeType = "follows" | "supports";
|
||||
|
||||
/** What one extraction run merged into the graph ledgers — the merger's
|
||||
@@ -37,10 +39,32 @@ export interface EmissionStats {
|
||||
* edges record every (chat, message) where they came up, deep-linkable to
|
||||
* the exact transcript point. */
|
||||
export interface ProvenancePoint {
|
||||
/** The chat this point anchors to — or a notes source id ("src_…"):
|
||||
* imports carry no message anchors (provenance is the file itself,
|
||||
* recorded through the mapping line + the source record). */
|
||||
chatId: string;
|
||||
messageId: string;
|
||||
}
|
||||
|
||||
/** One imported notes file — the format-agnostic front door's bookkeeping
|
||||
* (DECIDED 2026-09-13): personal notes in ARBITRARY formats run through the
|
||||
* same extractor → auditor → merger pipeline, with the file itself as the
|
||||
* provenance anchor. There is no chat: the detail card shows the file path
|
||||
* as a STATIC entry (not a link), and understanding observations still
|
||||
* apply (judged from the author's own voice in the notes). The run's usage
|
||||
* log lives here, mirroring ChatRecord.extractions. Ids are "src_" + hash
|
||||
* of the file path — deterministic, so re-importing the same file re-runs
|
||||
* extraction against the same source id (mappings/events dedupe by it). */
|
||||
export interface NotesSource {
|
||||
id: string;
|
||||
filePath: string;
|
||||
/** ISO — when this file was imported (the checkpoint's foldedAt). */
|
||||
importedAt: string;
|
||||
/** The run's usage log — same append-only discipline as
|
||||
* ChatRecord.extractions, idempotent per startedAt. */
|
||||
extractions?: ExtractionRunUsage[];
|
||||
}
|
||||
|
||||
export interface GraphNode {
|
||||
/** Opaque stable id, store-assigned: "n_" + nanoid-like. */
|
||||
id: string;
|
||||
@@ -107,6 +131,8 @@ export interface LedgerEvent {
|
||||
/** null = not yet attributed. */
|
||||
nodeId: string | null;
|
||||
kind: LedgerEventKind;
|
||||
/** The chat that produced this evidence — or a notes source id
|
||||
* ("src_…") for imported-notes observations. */
|
||||
chatId: string;
|
||||
at: string;
|
||||
/** Idempotency key: the quiz part's toolCallId, or `obs:<sourceId>` for
|
||||
@@ -244,11 +270,66 @@ export interface AuditResult {
|
||||
}
|
||||
|
||||
/** The GET /api/graph payload — everything the home view's PURE renderer needs. */
|
||||
/** cluster-labels.jsonl line — one LLM-named community (vision §1,
|
||||
* DECIDED 2026-09-13). Append-only history: records are immutable, never
|
||||
* edited or deleted; dead labelings remain as history. IDENTITY is the
|
||||
* sorted member set ("core"), hashed into the id, so re-naming the same
|
||||
* community dedupes and a labeling survives graph edits as long as at
|
||||
* least half its core survives in some single community (core-survival;
|
||||
* under half → the community is renamed, splits can keep only one
|
||||
* successor). */
|
||||
/** The embedding ledger shape (config.EMBEDDINGS_FILE): vectors for
|
||||
* EVERY embedded node, all produced by one model at one dimensionality. */
|
||||
export interface EmbeddingsLedger {
|
||||
/** The embedding model id (e.g. the HF repo) — mismatch = stale
|
||||
* vectors, the writer re-embeds everything. */
|
||||
model: string;
|
||||
/** Vector dimensionality (MRL-truncated on write). */
|
||||
dim: number;
|
||||
/** nodeId → L2-normalized vector (cosine similarity = dot product). */
|
||||
vectors: Record<string, number[]>;
|
||||
/** The topic partition over those vectors, computed by the SAME
|
||||
* embedder run (the user-decided trigger: recluster at the end of
|
||||
* every embedding batch, never per read — the agglomerative pass is
|
||||
* ~1.5s at current scale). Consumers (snapshot, settle, namer) read
|
||||
* this instead of recomputing. Empty when nothing is embedded. */
|
||||
clusters: Array<{ members: string[] }>;
|
||||
/** ISO timestamp of the last write. */
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ClusterLabeling {
|
||||
/** "clu_" + content hash of the sorted core. */
|
||||
id: string;
|
||||
/** The short domain label (1–4 words), e.g. "Differential forms". */
|
||||
label: string;
|
||||
/** The member node ids the label was minted for, sorted — the survival core. */
|
||||
core: string[];
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** One rendered cluster in the snapshot payload (vision §1). Members are
|
||||
* node ids; the renderer draws the hull from member positions at draw
|
||||
* time and gates the label inversely on the node-label zoom control. */
|
||||
export interface Cluster {
|
||||
/** The labeling record id when claimed; unlabeled clusters still
|
||||
* appear (hull, no name) with a member-hash id. */
|
||||
id: string;
|
||||
/** null until the naming call has minted/claimed a labeling. */
|
||||
label: string | null;
|
||||
/** Node ids, sorted. Hull + centroid are renderer concerns. */
|
||||
members: string[];
|
||||
}
|
||||
|
||||
export interface GraphSnapshot {
|
||||
nodes: GraphNode[];
|
||||
edges: GraphEdge[];
|
||||
/** chatId → title, for every chat that touched the graph. */
|
||||
sessions: Record<string, string>;
|
||||
/** srcId → file path, for every IMPORTED NOTES file that touched the
|
||||
* graph (the format-agnostic front door): a static provenance entry —
|
||||
* the detail card shows the path as plain text, not a link. */
|
||||
fileSources: Record<string, string>;
|
||||
/** nodeId → chatIds that touched the node (click-through provenance). */
|
||||
nodeSessions: Record<string, string[]>;
|
||||
/** nodeId → chatId → that session's understanding observation (the
|
||||
@@ -256,6 +337,20 @@ export interface GraphSnapshot {
|
||||
* shown next to the session list — the confidence history). Multiple
|
||||
* notes for one (node, chat) collapse to the latest. */
|
||||
nodeObservations: Record<string, Record<string, string>>;
|
||||
/** Structural communities (Louvain over follows+supports), size ≥ 3,
|
||||
* joined with persisted LLM labelings. Purely structural — never
|
||||
* depends on layout. */
|
||||
clusters: Cluster[];
|
||||
/** Resting layout positions per node id, when a positions ledger
|
||||
* exists (core/layout.ts settle writes it). Absent on a fresh graph —
|
||||
* the view then falls back to its own one-shot forceAtlas2 pass. */
|
||||
positions?: Record<string, Position>;
|
||||
}
|
||||
|
||||
/** A node's resting position in layout space (the positions ledger). */
|
||||
export interface Position {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
/** Derive a short chat title from the learner's first message (pure).
|
||||
|
||||
+85
-15
@@ -38,10 +38,11 @@
|
||||
* adapters in.
|
||||
*/
|
||||
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { isToolUIPart, type UIMessage } from "ai";
|
||||
import type {
|
||||
AuditResult,
|
||||
Cluster,
|
||||
EmissionStats,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
@@ -50,6 +51,7 @@ import type {
|
||||
LedgerEvent,
|
||||
LedgerEventKind,
|
||||
} from "./contracts.ts";
|
||||
import { matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts";
|
||||
import type { QuestionAnswerData } from "./question-core.ts";
|
||||
import type { ChatStore, GraphStore } from "./ports.ts";
|
||||
|
||||
@@ -64,6 +66,17 @@ function newGraphNodeId(): string {
|
||||
return `n_${randomBytes(6).toString("hex")}`;
|
||||
}
|
||||
|
||||
/** What one merge run merges FROM (the caller supplies it — the merger never
|
||||
* reads the chat store): a session's transcript + its `updatedAt`
|
||||
* checkpoint, or — for the format-agnostic notes front door (2026-09-13) —
|
||||
* `messages: null`: raw notes carry no tool parts, so no quiz events are
|
||||
* rebuilt (quiz outcomes inside notes surface only through the extractor's
|
||||
* understanding observations), and the checkpoint stamps the import. */
|
||||
export interface MergeSource {
|
||||
messages: UIMessage[] | null;
|
||||
foldedAt: string;
|
||||
}
|
||||
|
||||
// ─── transcript scanning (shared by the merger + trigger) ─────────────────────
|
||||
|
||||
/** A flattened, order-preserving view of every tool part in a transcript. */
|
||||
@@ -126,11 +139,15 @@ const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean =>
|
||||
* parts, nodeId from the extractor's attribution through this run's
|
||||
* mapping) plus one observation event per proposition carrying an
|
||||
* understanding note (ref `obs:<sourceId>`). No taught events, ever.
|
||||
*
|
||||
* `chatId` is the run's source key: a real chat, or a notes source id
|
||||
* ("src_…") for the format-agnostic import path — same ledgers, same
|
||||
* idempotency, one graph. The `source` parameter carries the transcript
|
||||
* being merged (null messages = raw notes: no quiz events) and the
|
||||
* checkpoint stamp; the merger itself is chat-store-free.
|
||||
*/
|
||||
export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps): EmissionStats {
|
||||
const { graphStore: ledgers, chatStore } = deps;
|
||||
const record = chatStore.getChat(chatId);
|
||||
if (!record) return { nodes: 0, edges: 0, events: 0 };
|
||||
export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps, source: MergeSource): EmissionStats {
|
||||
const { graphStore: ledgers } = deps;
|
||||
const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 };
|
||||
|
||||
// 1. Resolution: this run's propositions → graph node ids. Existing
|
||||
@@ -214,7 +231,7 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
|
||||
for (const event of ledgers.readEvents()) {
|
||||
if (event.chatId === chatId && event.ref) seenRefs.add(event.ref);
|
||||
}
|
||||
for (const part of toolParts(record.messages)) {
|
||||
for (const part of source.messages ? toolParts(source.messages) : []) {
|
||||
if (part.name !== "quiz" || part.state !== "output-available") continue;
|
||||
const event = quizEvent(part, chatId, audit, mapping, now);
|
||||
if (event?.ref !== undefined && !seenRefs.has(event.ref)) {
|
||||
@@ -241,8 +258,9 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
|
||||
stats.events++;
|
||||
}
|
||||
|
||||
// 4. Checkpoint: this chat is extracted through its current updatedAt.
|
||||
ledgers.appendCheckpoint({ chatId, foldedAt: record.updatedAt });
|
||||
// 4. Checkpoint: this source is extracted through its foldedAt (the
|
||||
// chat's updatedAt, or the import time for notes).
|
||||
ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt });
|
||||
return stats;
|
||||
}
|
||||
|
||||
@@ -314,6 +332,16 @@ export function pendingQuizAnswers(chatId: string, deps: GraphDeps): number {
|
||||
|
||||
// ─── derivation (reads only — everything below is recomputable) ─────────────
|
||||
|
||||
/** The content identity of a community: "clu_" + sha256 of the sorted
|
||||
* member ids (12 hex). Deterministic — the same member set always mints
|
||||
* the same id — so re-naming races and idempotent re-runs dedupe at the
|
||||
* store seam (appendClusterLabel). Lives here (server-only core), not in
|
||||
* clusters.ts, because clusters.ts is barrel-exported and must stay
|
||||
* browser-safe (no node:crypto). */
|
||||
export function clusterContentId(members: readonly string[]): string {
|
||||
return "clu_" + createHash("sha256").update([...members].sort().join("\n")).digest("hex").slice(0, 12);
|
||||
}
|
||||
|
||||
export interface DerivedGraph {
|
||||
nodes: GraphNode[];
|
||||
edges: GraphEdge[];
|
||||
@@ -423,20 +451,28 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
|
||||
if (event.nodeId !== null) touch(event.nodeId, event.chatId);
|
||||
}
|
||||
|
||||
// Titles for every chat the graph can point back at.
|
||||
const chatIds = new Set<string>();
|
||||
// Labels for every source the graph can point back at: real chats
|
||||
// (title, deep-linkable) and imported notes files (path, static — the
|
||||
// format-agnostic front door's provenance).
|
||||
const sourceIds = new Set<string>();
|
||||
for (const chats of nodeChats.values()) {
|
||||
for (const id of chats) chatIds.add(id);
|
||||
for (const id of chats) sourceIds.add(id);
|
||||
}
|
||||
const sessions: Record<string, string> = {};
|
||||
for (const id of chatIds) {
|
||||
const fileSources: Record<string, string> = {};
|
||||
for (const id of sourceIds) {
|
||||
const chat = deps.chatStore.getChat(id);
|
||||
if (chat) sessions[id] = chat.title;
|
||||
if (chat) {
|
||||
sessions[id] = chat.title;
|
||||
continue;
|
||||
}
|
||||
const source = deps.graphStore.readSources().find((s) => s.id === id);
|
||||
if (source) fileSources[id] = source.filePath;
|
||||
}
|
||||
|
||||
const nodeSessions: Record<string, string[]> = {};
|
||||
for (const [nodeId, chats] of nodeChats) {
|
||||
const ids = [...chats].filter((id) => sessions[id] !== undefined);
|
||||
const ids = [...chats].filter((id) => sessions[id] !== undefined || fileSources[id] !== undefined);
|
||||
if (ids.length > 0) nodeSessions[nodeId] = ids;
|
||||
}
|
||||
|
||||
@@ -461,7 +497,41 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
|
||||
(nodeObservations[nodeId] ??= {})[chatId] = note;
|
||||
}
|
||||
|
||||
return { nodes, edges, sessions, nodeSessions, nodeObservations };
|
||||
// Semantic topic communities (DECIDED 2026-09-13, replacing structural
|
||||
// Louvain): average-linkage clustering over embeddings — related
|
||||
// topics, not reasoning chains. Edge-orphans join topics by meaning;
|
||||
// stragglers stay loose (filtered at the presentation threshold
|
||||
// below). The partition is computed ONCE per embedding batch by the
|
||||
// embedder (the agglomerative pass is ~1.5s at current scale) and
|
||||
// PERSISTED in the embedding ledger — reads just load it. Nodes whose
|
||||
// embedding hasn't landed yet simply sit unclustered until the next
|
||||
// read after the embedder runs.
|
||||
const communities = deps.graphStore.readEmbeddings().clusters;
|
||||
const claims = matchClusterLabelings(communities, deps.graphStore.readClusterLabels());
|
||||
const clusters: Cluster[] = communities
|
||||
.map((community, index) => ({ community, claim: claims[index] }))
|
||||
.filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE)
|
||||
.map(({ community, claim }) => ({
|
||||
id: claim?.id ?? clusterContentId(community.members),
|
||||
label: claim?.label ?? null,
|
||||
members: community.members,
|
||||
}));
|
||||
|
||||
// Resting layout (Option A): the positions ledger, when the settle has
|
||||
// ever run. Absent on a fresh graph — the view falls back to its own
|
||||
// one-shot pass until the first merge writes the ledger.
|
||||
const positions = deps.graphStore.readPositions();
|
||||
|
||||
return {
|
||||
nodes,
|
||||
edges,
|
||||
sessions,
|
||||
fileSources,
|
||||
nodeSessions,
|
||||
nodeObservations,
|
||||
clusters,
|
||||
...(Object.keys(positions).length > 0 ? { positions } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
// Re-export: the contract owns the shape; legacy importers still reach it.
|
||||
|
||||
@@ -17,8 +17,15 @@
|
||||
|
||||
export * from "./contracts.ts";
|
||||
export * from "./ports.ts";
|
||||
export * from "./render-settings.ts";
|
||||
|
||||
export * from "./question-core.ts";
|
||||
// clusters.ts (Louvain) is deliberately NOT re-exported: nothing in the
|
||||
// shell runs community detection — the snapshot carries the partition
|
||||
// from the server. Keeping it out keeps the heavy clustering deps out of the
|
||||
// client bundle (verified 2026-09-13: chunks contained it while this
|
||||
// line existed). Same logic as layout-settle.ts below.
|
||||
export * from "./layout.ts";
|
||||
export * from "./plan-tree.ts";
|
||||
export * from "./phases.ts";
|
||||
export * from "./pending.ts";
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { EmbeddingsLedger, GraphEdge, GraphNode } from "./contracts.ts";
|
||||
import { incrementalLayout } from "./layout.ts";
|
||||
import type { Position } from "./contracts.ts";
|
||||
|
||||
/**
|
||||
* The merge-time settle: recompute the resting layout from the CURRENT
|
||||
* graph + the positions ledger, and replace the ledger. Called by the
|
||||
* extraction pipeline after every merge (the sole writer) and by the
|
||||
* positions-commit endpoint when the graph grew under a live-physics
|
||||
* commit (client positions become the anchor set). Synchronous — it must
|
||||
* finish before the next /api/graph fetch reflects the change.
|
||||
*
|
||||
* SPLIT FROM layout.ts deliberately (2026-09-13): layout.ts is in the
|
||||
* browser barrel, and the settle needs the Louvain partition for the
|
||||
* cluster centroid pull — the heavy deps must not ship in the client
|
||||
* bundle. Server code imports this module DIRECTLY; it is never
|
||||
* re-exported through src/core/index.ts.
|
||||
*/
|
||||
export function settleGraphLayout(graphStore: {
|
||||
readNodes(): GraphNode[];
|
||||
readEdges(): GraphEdge[];
|
||||
readPositions(): Record<string, Position>;
|
||||
writePositions(positions: Record<string, Position>): void;
|
||||
readEmbeddings(): EmbeddingsLedger;
|
||||
}): void {
|
||||
const nodes = graphStore.readNodes();
|
||||
if (nodes.length === 0) return;
|
||||
// Node LINES may repeat an id (link-appends) — dedupe, first defines.
|
||||
const seen = new Set<string>();
|
||||
const unique = nodes.filter((n) => {
|
||||
if (seen.has(n.id)) return false;
|
||||
seen.add(n.id);
|
||||
return true;
|
||||
});
|
||||
const edges = graphStore.readEdges();
|
||||
const previous = new Map(Object.entries(graphStore.readPositions()));
|
||||
// The cluster partition for the centroid pull: the SAME persisted
|
||||
// semantic topics the snapshot renders (computed by the embedder
|
||||
// once per batch, stored in the embedding ledger) — one cluster
|
||||
// concept everywhere.
|
||||
const communities = graphStore.readEmbeddings().clusters;
|
||||
const next = incrementalLayout(unique, edges, previous, { clusters: communities });
|
||||
graphStore.writePositions(Object.fromEntries(next));
|
||||
}
|
||||
@@ -0,0 +1,629 @@
|
||||
/**
|
||||
* The layout engine — d3-force, everywhere (DECIDED 2026-09-13).
|
||||
*
|
||||
* One engine owns the equilibrium: the merge-time incremental settle AND
|
||||
* the live drag physics run the SAME force set (links as springs, charge
|
||||
* repulsion, weak gravity), so the positions ledger and the transient
|
||||
* simulation agree on what "resting" means. (FA2 was replaced after a
|
||||
* research round: it has no annealing/pinning/settle model and mixing it
|
||||
* with a different live engine visibly re-flows the graph on every drag.)
|
||||
*
|
||||
* Layout EVOLVES instead of reshuffling (Option A): each settle warm-starts
|
||||
* from the positions ledger and spring-ties every previously-positioned
|
||||
* node toward its anchor (a per-node anchor force). New nodes seed near
|
||||
* their positioned neighbors; orphans seed on the rim OUTSIDE the core and
|
||||
* are anchored there, so they read as "not yet connected knowledge".
|
||||
*
|
||||
* Design constraints (kept deliberately, for live physics):
|
||||
* - PURE: no fs, no server types — this module runs server-side (the
|
||||
* merge settle) and in the browser (the live simulation, via the core
|
||||
* barrel). A pinned node is just fx/fy; an anchored node is a spring.
|
||||
* - DETERMINISTIC: seeds derive from node id; d3-force ticks are
|
||||
* deterministic — same inputs, same layout.
|
||||
*/
|
||||
|
||||
import {
|
||||
forceSimulation,
|
||||
forceLink,
|
||||
forceManyBody,
|
||||
forceX,
|
||||
forceY,
|
||||
type Simulation,
|
||||
type SimulationLinkDatum,
|
||||
type SimulationNodeDatum,
|
||||
} from "d3-force";
|
||||
import type { GraphEdge, GraphNode, Position } from "./contracts.ts";
|
||||
import { seededRandom } from "./question-core.ts";
|
||||
|
||||
/** A node the d3 simulation mutates (x/y/vx/vy; fx/fy = pinned). */
|
||||
export interface LayoutNode extends SimulationNodeDatum {
|
||||
id: string;
|
||||
}
|
||||
|
||||
/** A link as d3 sees it (id strings resolve to node refs on init). */
|
||||
type SimLink = SimulationLinkDatum<LayoutNode>;
|
||||
|
||||
/** How the inference rank is expressed spatially (playground, 2026-09-13):
|
||||
* the graph's `follows`/`supports` edges encode derivation (premise →
|
||||
* conclusion), so depth-0 nodes (no incoming edges — the fundamentals)
|
||||
* can anchor a shared baseline and every derivation level rises from it.
|
||||
* `vertical`: levels are horizontal rows, arrows point up.
|
||||
* `radial`: levels are concentric rings, foundations at the center, the
|
||||
* outer boundary IS the edge of the user's knowledge.
|
||||
* `off`: today's free layout. */
|
||||
export type DepthAnchorMode = "off" | "vertical" | "radial";
|
||||
|
||||
export interface DepthAnchor {
|
||||
mode: DepthAnchorMode;
|
||||
/** Anchor pull toward the level (same units as clusterPull). */
|
||||
strength: number;
|
||||
/** Distance between consecutive inference levels (world units). */
|
||||
spacing: number;
|
||||
/** Baseline point: vertical mode uses .y as the depth-0 line; radial
|
||||
* mode uses the full point as the ring center. Default (0,0). */
|
||||
center?: { x: number; y: number };
|
||||
/** Inner ring radius (radial only; depth-0 nodes sit at this ring). */
|
||||
baseRadius?: number;
|
||||
}
|
||||
|
||||
/** Inference depth per node: the longest chain of directed edges from a
|
||||
* base node. CYCLE-SAFE (2026-09-13, user requirement — the DAG property
|
||||
* is typical, not guaranteed): Tarjan SCC condensation collapses each
|
||||
* strongly connected component to one super-node, ranks the (always
|
||||
* acyclic) condensation, and every cycle member shares its component's
|
||||
* depth — mutual support reads as equal footing. Iterative Tarjan (no
|
||||
* recursion depth limits), O(V+E), deterministic. */
|
||||
export function inferenceDepths(
|
||||
nodes: ReadonlyArray<Pick<GraphNode, "id">>,
|
||||
edges: ReadonlyArray<LayoutEdge>,
|
||||
): Map<string, number> {
|
||||
const valid = new Set(nodes.map((n) => n.id));
|
||||
const adj = new Map<string, string[]>();
|
||||
for (const e of edges) {
|
||||
if (e.from === e.to || !valid.has(e.from) || !valid.has(e.to)) continue;
|
||||
let list = adj.get(e.from);
|
||||
if (!list) adj.set(e.from, (list = []));
|
||||
list.push(e.to);
|
||||
}
|
||||
|
||||
// Iterative Tarjan SCC.
|
||||
const num = new Map<string, number>();
|
||||
const low = new Map<string, number>();
|
||||
const onStack = new Set<string>();
|
||||
const stack: string[] = [];
|
||||
const comp = new Map<string, number>();
|
||||
let counter = 0;
|
||||
let compCount = 0;
|
||||
for (const start of valid) {
|
||||
if (num.has(start)) continue;
|
||||
num.set(start, counter);
|
||||
low.set(start, counter);
|
||||
counter++;
|
||||
stack.push(start);
|
||||
onStack.add(start);
|
||||
const work: [string, number][] = [[start, 0]];
|
||||
while (work.length > 0) {
|
||||
const frame = work[work.length - 1]!;
|
||||
const v = frame[0];
|
||||
const outs = adj.get(v) ?? [];
|
||||
if (frame[1] < outs.length) {
|
||||
const i = frame[1];
|
||||
frame[1] = i + 1;
|
||||
const w = outs[i]!;
|
||||
if (!num.has(w)) {
|
||||
num.set(w, counter);
|
||||
low.set(w, counter);
|
||||
counter++;
|
||||
stack.push(w);
|
||||
onStack.add(w);
|
||||
work.push([w, 0]);
|
||||
} else if (onStack.has(w)) {
|
||||
low.set(v, Math.min(low.get(v)!, num.get(w)!));
|
||||
}
|
||||
} else {
|
||||
work.pop();
|
||||
if (low.get(v) === num.get(v)) {
|
||||
let w: string;
|
||||
do {
|
||||
w = stack.pop()!;
|
||||
onStack.delete(w);
|
||||
comp.set(w, compCount);
|
||||
} while (w !== v);
|
||||
compCount++;
|
||||
}
|
||||
if (work.length > 0) {
|
||||
const parent = work[work.length - 1]![0];
|
||||
low.set(parent, Math.min(low.get(parent)!, low.get(v)!));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Condensation predecessors, then longest path. Tarjan emits sinks
|
||||
// first, so descending comp index = topological order (sources first).
|
||||
const preds: Set<number>[] = Array.from({ length: compCount }, () => new Set<number>());
|
||||
for (const e of edges) {
|
||||
const a = comp.get(e.from);
|
||||
const b = comp.get(e.to);
|
||||
if (a == null || b == null || a === b) continue;
|
||||
preds[b]!.add(a);
|
||||
}
|
||||
const depth = new Array<number>(compCount).fill(0);
|
||||
for (let c = compCount - 1; c >= 0; c--) {
|
||||
let d = 0;
|
||||
for (const p of preds[c]!) d = Math.max(d, depth[p]! + 1);
|
||||
depth[c] = d;
|
||||
}
|
||||
const out = new Map<string, number>();
|
||||
for (const n of nodes) out.set(n.id, depth[comp.get(n.id)!] ?? 0);
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface SimOptions {
|
||||
/** Link spring rest length (world units). */
|
||||
linkDistance?: number;
|
||||
/** Link spring stiffness MULTIPLIER (1 = d3's default
|
||||
* 1/min(degree) stiffness). */
|
||||
linkStrength?: number;
|
||||
/** Many-body repulsion strength (negative). */
|
||||
chargeStrength?: number;
|
||||
/** Center-of-mass gravity: pulls loose components inward. Weak —
|
||||
* anchors, not gravity, hold the evolved layout's frame. */
|
||||
gravity?: { x: number; y: number; strength: number };
|
||||
/** Alpha decay per tick — higher = snappier settle. The merge settle
|
||||
* uses d3's default 0.0228 (~300 ticks); the live sim passes ~0.05. */
|
||||
alphaDecay?: number;
|
||||
/** Velocity damping per tick (d3 default 0.4). Higher = more friction. */
|
||||
velocityDecay?: number;
|
||||
/** Inference-depth anchoring (see DepthAnchor). Off by default. */
|
||||
depthAnchor?: DepthAnchor;
|
||||
/** Cluster centroid pull (the "virtual node" without the node):
|
||||
* members of each cluster accelerate toward their own cluster's
|
||||
* centroid. Two O(n) passes per tick — cheaper than a real virtual
|
||||
* node (which would need one extra link spring per member plus a
|
||||
* maintained position). Strength 0 or absent = force off.
|
||||
* Callers supply the partition (Louvain output shape). */
|
||||
clusterPull?: {
|
||||
clusters: ReadonlyArray<{ members: ReadonlyArray<string> }>;
|
||||
strength: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** Link input: edges are undirected for layout purposes. */
|
||||
export interface LayoutEdge {
|
||||
from: string;
|
||||
to: string;
|
||||
}
|
||||
|
||||
export const LINK_DISTANCE = 40;
|
||||
export const CHARGE_STRENGTH = -120;
|
||||
/** Per-tick anchor re-blend: fraction of an anchored node's displacement
|
||||
* from its previous position that SURVIVES each tick. 0.85 = gentle
|
||||
* evolution where forces are strong, stillness where they are not —
|
||||
* the exact FA2-era constant. Position-space (alpha-independent), so it
|
||||
* cannot be overpowered by d3's long-range link springs the way a force-
|
||||
* space anchor can (forceLink is a Hooke spring: strength*(d-L)*alpha). */
|
||||
const ANCHOR_RETENTION = 0.85;
|
||||
/** Gravity strengths. d3 manyBody repulsion falls off as 1/d, so two
|
||||
* 20-node blobs at distance d push with ~n1*n2*|charge|/d; cluster
|
||||
* spacing settles where that equals g*d*n/2 — g=0.03 lands clusters
|
||||
* ~400 world units apart (the FA2-era density), and the anchor re-blend
|
||||
* (position-space, alpha-independent) keeps gravity's transient sag
|
||||
* from ever moving an anchored node at rest. */
|
||||
const BOOTSTRAP_GRAVITY = 0.2;
|
||||
/** Exported: the live sim reuses this so its rest state matches the
|
||||
* server settle's equilibrium (one engine, one rest). */
|
||||
export const INCREMENTAL_GRAVITY = 0.12;
|
||||
/** Exported: default cluster-centroid pull strength (tuned by replay —
|
||||
* compact enough to read as blobs, weak enough that cluster shape is
|
||||
* still the edge network's). */
|
||||
export const CLUSTER_PULL_STRENGTH = 0.15;
|
||||
|
||||
/**
|
||||
* The one force set. Returns a STOPPED simulation — callers drive it
|
||||
* manually (tick-per-frame live, or a bounded tick loop when settling).
|
||||
* Deterministic; mutates `nodes` in place (d3 semantics: impure by design,
|
||||
* zero per-tick allocation).
|
||||
*/
|
||||
export function createGraphSimulation(
|
||||
nodes: LayoutNode[],
|
||||
edges: ReadonlyArray<LayoutEdge>,
|
||||
options: SimOptions = {},
|
||||
): Simulation<LayoutNode, SimLink> {
|
||||
// Link stiffness: d3's default is 1/min(deg) (degree-scaled springs).
|
||||
// linkStrength multiplies it, so 1 = unchanged default behavior.
|
||||
const degree = new Map<string, number>();
|
||||
for (const e of edges) {
|
||||
degree.set(e.from, (degree.get(e.from) ?? 0) + 1);
|
||||
degree.set(e.to, (degree.get(e.to) ?? 0) + 1);
|
||||
}
|
||||
const linkStrength = options.linkStrength ?? 1;
|
||||
|
||||
const sim = forceSimulation<LayoutNode, SimLink>(nodes)
|
||||
.force(
|
||||
"link",
|
||||
forceLink<LayoutNode, SimLink>(
|
||||
edges.map((e) => ({ source: e.from, target: e.to })),
|
||||
)
|
||||
.id((d) => d.id)
|
||||
.distance(options.linkDistance ?? LINK_DISTANCE)
|
||||
.strength(
|
||||
(l) =>
|
||||
(linkStrength * 1) /
|
||||
Math.min(degree.get((l.source as LayoutNode).id) ?? 1, degree.get((l.target as LayoutNode).id) ?? 1),
|
||||
),
|
||||
)
|
||||
.force("charge", forceManyBody<LayoutNode>().strength(options.chargeStrength ?? CHARGE_STRENGTH))
|
||||
.alphaDecay(options.alphaDecay ?? 0.0228)
|
||||
.velocityDecay(options.velocityDecay ?? 0.4)
|
||||
.stop();
|
||||
|
||||
// Gravity, mode-projected: in vertical mode the depth force OWNS the
|
||||
// y axis, so gravity goes x-only (it would fight the level lines); in
|
||||
// radial mode a center pull is purely radial and would fight the
|
||||
// rings, so it is dropped entirely (rings already bound the radius).
|
||||
// Depth mode projections (DECIDED 2026-09-13), with the ORPHAN RULE
|
||||
// (user-decided, same day): a node with no edges at all has no
|
||||
// dependency structure — depth is a property of inference chains, so
|
||||
// stamping unconnected propositions onto the baseline / base ring
|
||||
// assigns them meaning they don't have. Orphans are exempt from the
|
||||
// depth force and keep the REGULAR view's forces instead (full center
|
||||
// gravity + full cluster pull, unprojected).
|
||||
const mode = options.depthAnchor?.mode ?? "off";
|
||||
let orphan: Uint8Array | null = null;
|
||||
if (mode !== "off") {
|
||||
const connected = new Set<string>();
|
||||
const valid = new Set(nodes.map((n) => n.id));
|
||||
for (const e of edges) {
|
||||
// Self-loops and dangling endpoints are not a structure (same
|
||||
// guards as inferenceDepths).
|
||||
if (e.from === e.to || !valid.has(e.from) || !valid.has(e.to)) continue;
|
||||
connected.add(e.from);
|
||||
connected.add(e.to);
|
||||
}
|
||||
orphan = new Uint8Array(nodes.length);
|
||||
nodes.forEach((n, i) => {
|
||||
if (!connected.has(n.id)) orphan![i] = 1;
|
||||
});
|
||||
}
|
||||
const g = options.gravity;
|
||||
if (g && mode === "off") {
|
||||
sim.force("x", forceX<LayoutNode>(g.x).strength(g.strength));
|
||||
sim.force("y", forceY<LayoutNode>(g.y).strength(g.strength));
|
||||
} else if (g) {
|
||||
// Mode-projected gravity, EXACTLY d3's forceX/forceY math (restoring
|
||||
// force × strength × alpha): structured nodes get x-only in vertical
|
||||
// mode (the depth force owns y) and none in radial mode (the rings
|
||||
// own the radius); ORPHANS get the full regular-view pull in every
|
||||
// mode.
|
||||
const orphanMask = orphan!;
|
||||
sim.force("gravity", (alpha: number) => {
|
||||
for (let i = 0; i < nodes.length; i++) {
|
||||
const n = nodes[i]!;
|
||||
if (orphanMask[i]) {
|
||||
n.vx = (n.vx ?? 0) + (g.x - (n.x ?? 0)) * g.strength * alpha;
|
||||
n.vy = (n.vy ?? 0) + (g.y - (n.y ?? 0)) * g.strength * alpha;
|
||||
} else if (mode === "vertical") {
|
||||
n.vx = (n.vx ?? 0) + (g.x - (n.x ?? 0)) * g.strength * alpha;
|
||||
}
|
||||
// radial + structured: nothing — the rings own the radius.
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Cluster centroid pull: the membership index is resolved ONCE (nodes
|
||||
// don't change clusters mid-sim); each tick is two allocation-free
|
||||
// O(n) passes (sum, then pull). Singletons (<2 members) skip themselves.
|
||||
const cp = options.clusterPull;
|
||||
if (cp && cp.strength > 0 && cp.clusters.length > 0) {
|
||||
const clusterIdx = new Map<string, number>();
|
||||
cp.clusters.forEach((c, i) => {
|
||||
for (const id of c.members) clusterIdx.set(id, i);
|
||||
});
|
||||
const nodeCluster = nodes.map((n) => clusterIdx.get(n.id) ?? -1);
|
||||
const k = cp.clusters.length;
|
||||
const sumX = new Float64Array(k);
|
||||
const sumY = new Float64Array(k);
|
||||
const cnt = new Float64Array(k);
|
||||
const strength = cp.strength;
|
||||
// The cluster centroid pull also respects the depth mode: vertical
|
||||
// → x-only (the depth force owns y); radial → tangential only
|
||||
// (the radial component would drag members off their rings; the
|
||||
// tangential component still tightens each cluster's angular
|
||||
// spread, which is all clustering needs to do there).
|
||||
const ringCx = options.depthAnchor?.center?.x ?? 0;
|
||||
const ringCy = options.depthAnchor?.center?.y ?? 0;
|
||||
sim.force("clusterPull", (alpha: number) => {
|
||||
sumX.fill(0);
|
||||
sumY.fill(0);
|
||||
cnt.fill(0);
|
||||
for (let i = 0; i < nodes.length; i++) {
|
||||
const ci = nodeCluster[i]!;
|
||||
if (ci < 0) continue;
|
||||
const n = nodes[i]!;
|
||||
sumX[ci] = sumX[ci]! + (n.x ?? 0);
|
||||
sumY[ci] = sumY[ci]! + (n.y ?? 0);
|
||||
cnt[ci] = cnt[ci]! + 1;
|
||||
}
|
||||
for (let i = 0; i < nodes.length; i++) {
|
||||
const ci = nodeCluster[i]!;
|
||||
if (ci < 0 || (cnt[ci] ?? 0) < 2) continue;
|
||||
const n = nodes[i]!;
|
||||
const px = n.x ?? 0;
|
||||
const py = n.y ?? 0;
|
||||
const ux = sumX[ci]! / cnt[ci]! - px;
|
||||
const uy = sumY[ci]! / cnt[ci]! - py;
|
||||
if (orphan && orphan[i]) {
|
||||
// Orphans: full regular-view pull (see the ORPHAN RULE above).
|
||||
n.vx = (n.vx ?? 0) + ux * strength * alpha;
|
||||
n.vy = (n.vy ?? 0) + uy * strength * alpha;
|
||||
} else if (mode === "vertical") {
|
||||
n.vx = (n.vx ?? 0) + ux * strength * alpha;
|
||||
} else if (mode === "radial") {
|
||||
const rx = px - ringCx;
|
||||
const ry = py - ringCy;
|
||||
const r = Math.hypot(rx, ry);
|
||||
if (r > 1e-9) {
|
||||
// Tangential component: u − (u·r̂)r̂
|
||||
const dot = (ux * rx + uy * ry) / (r * r);
|
||||
n.vx = (n.vx ?? 0) + (ux - dot * rx) * strength * alpha;
|
||||
n.vy = (n.vy ?? 0) + (uy - dot * ry) * strength * alpha;
|
||||
}
|
||||
} else {
|
||||
n.vx = (n.vx ?? 0) + ux * strength * alpha;
|
||||
n.vy = (n.vy ?? 0) + uy * strength * alpha;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ── Inference-depth anchoring ────────────────────────────────────────
|
||||
// The rank is COMPUTED (not simulated): per-node target scalar derived
|
||||
// from inferenceDepths, then one O(n) pull pass per tick. Soft knob —
|
||||
// high strength approaches pinned rows/rings while low strength leaves
|
||||
// organic sag.
|
||||
const da = options.depthAnchor;
|
||||
if (da && da.mode !== "off" && da.strength > 0) {
|
||||
const depths = inferenceDepths(nodes, edges);
|
||||
const strength = da.strength;
|
||||
const spacing = da.spacing;
|
||||
const c = da.center ?? { x: 0, y: 0 };
|
||||
const orphanMask = orphan; // ORPHAN RULE: depth skips unstructured nodes
|
||||
if (da.mode === "vertical") {
|
||||
const baseY = c.y;
|
||||
sim.force("depth", (alpha: number) => {
|
||||
for (let i = 0; i < nodes.length; i++) {
|
||||
if (orphanMask?.[i]) continue;
|
||||
const n = nodes[i]!;
|
||||
const targetY = baseY - (depths.get(n.id) ?? 0) * spacing;
|
||||
n.vy = (n.vy ?? 0) + (targetY - (n.y ?? 0)) * strength * alpha;
|
||||
}
|
||||
});
|
||||
} else {
|
||||
const baseR = da.baseRadius ?? 0;
|
||||
const cx0 = c.x;
|
||||
const cy0 = c.y;
|
||||
sim.force("depth", (alpha: number) => {
|
||||
for (let i = 0; i < nodes.length; i++) {
|
||||
if (orphanMask?.[i]) continue;
|
||||
const n = nodes[i]!;
|
||||
const dx = (n.x ?? 0) - cx0;
|
||||
const dy = (n.y ?? 0) - cy0;
|
||||
// Degenerate direction at the exact ring center: nudge
|
||||
// outward on a deterministic axis.
|
||||
const r = Math.hypot(dx, dy);
|
||||
const ux = r > 1e-9 ? dx / r : 1;
|
||||
const uy = r > 1e-9 ? dy / r : 0;
|
||||
const target = baseR + (depths.get(n.id) ?? 0) * spacing;
|
||||
const mag = (target - r) * strength * alpha;
|
||||
n.vx = (n.vx ?? 0) + ux * mag;
|
||||
n.vy = (n.vy ?? 0) + uy * mag;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
return sim;
|
||||
}
|
||||
|
||||
export interface IncrementalLayoutOptions {
|
||||
/** Manual ticks for the settle (default 300 — d3's anneal to alphaMin). */
|
||||
ticks?: number;
|
||||
/** Per-tick anchor re-blend (default 0.85; 1 = fully pinned). */
|
||||
anchorRetention?: number;
|
||||
/** Center-of-mass gravity strength override (tuning/A-B experiments;
|
||||
* defaults to the internal bootstrap/incremental constants). */
|
||||
gravityStrength?: number;
|
||||
/** Cluster partition (Louvain output shape). Supplied by callers so
|
||||
* this module stays louvain-free (the browser bundle must not carry
|
||||
* embedding ledger; absent = no cluster pull. */
|
||||
clusters?: ReadonlyArray<{ members: ReadonlyArray<string> }>;
|
||||
/** Cluster centroid pull strength (default CLUSTER_PULL_STRENGTH). */
|
||||
clusterPullStrength?: number;
|
||||
/** Inference-depth anchoring (see DepthAnchor). `center` defaults to
|
||||
* the anchored core's centroid. */
|
||||
depthAnchor?: DepthAnchor;
|
||||
}
|
||||
|
||||
/**
|
||||
* One incremental settle. `nodes`/`edges` describe the CURRENT graph;
|
||||
* `previous` is the positions ledger (empty on the very first settle —
|
||||
* then this degrades to a seeded-scatter bootstrap). Returns positions
|
||||
* for EVERY node in `nodes`.
|
||||
*/
|
||||
export function incrementalLayout(
|
||||
nodes: ReadonlyArray<Pick<GraphNode, "id">>,
|
||||
edges: ReadonlyArray<LayoutEdge>,
|
||||
previous: ReadonlyMap<string, Position>,
|
||||
options: IncrementalLayoutOptions = {},
|
||||
): Map<string, Position> {
|
||||
const ticks = options.ticks ?? 300;
|
||||
const retention = options.anchorRetention ?? ANCHOR_RETENTION;
|
||||
|
||||
// Anchors: the warm-start set (the previous rest state).
|
||||
const anchors = new Map<string, Position>();
|
||||
for (const node of nodes) {
|
||||
const prev = previous.get(node.id);
|
||||
if (prev) anchors.set(node.id, { x: prev.x, y: prev.y });
|
||||
}
|
||||
|
||||
// The anchored core's bounding box + centroid, for rim seeds.
|
||||
let minX = Infinity;
|
||||
let maxX = -Infinity;
|
||||
let minY = Infinity;
|
||||
let maxY = -Infinity;
|
||||
let cx = 0;
|
||||
let cy = 0;
|
||||
for (const a of anchors.values()) {
|
||||
minX = Math.min(minX, a.x);
|
||||
maxX = Math.max(maxX, a.x);
|
||||
minY = Math.min(minY, a.y);
|
||||
maxY = Math.max(maxY, a.y);
|
||||
cx += a.x;
|
||||
cy += a.y;
|
||||
}
|
||||
const hasCore = anchors.size > 0;
|
||||
if (hasCore) {
|
||||
cx /= anchors.size;
|
||||
cy /= anchors.size;
|
||||
}
|
||||
const coreRadius = hasCore ? Math.max(maxX - minX, maxY - minY) / 2 : 1;
|
||||
|
||||
// Adjacency for neighbor seeding (undirected: edges include assertions).
|
||||
const adjacency = new Map<string, string[]>();
|
||||
const linkTo = (a: string, b: string): void => {
|
||||
if (a === b) return;
|
||||
let list = adjacency.get(a);
|
||||
if (!list) adjacency.set(a, (list = []));
|
||||
list.push(b);
|
||||
};
|
||||
for (const e of edges) {
|
||||
linkTo(e.from, e.to);
|
||||
linkTo(e.to, e.from);
|
||||
}
|
||||
|
||||
// The rim radius derives from the CONNECTED core's mean radius — orphans
|
||||
// never inflate it (a max-span-based radius feeds back: each batch's
|
||||
// rim pushes the next batch's rim farther out, and the layout runs
|
||||
// away). 1.7x matches the density the FA2-era settle produced
|
||||
// (orphan mean r ~338 vs connected ~194).
|
||||
let rimRadius = 0;
|
||||
let connectedAnchored = 0;
|
||||
for (const [id, a] of anchors) {
|
||||
if ((adjacency.get(id)?.length ?? 0) === 0) continue;
|
||||
connectedAnchored++;
|
||||
rimRadius += Math.hypot(a.x - cx, a.y - cy);
|
||||
}
|
||||
if (connectedAnchored > 0) rimRadius = (rimRadius / connectedAnchored) * 1.7;
|
||||
else if (hasCore) rimRadius = coreRadius * 1.15; // no connected anchors yet: bbox fallback
|
||||
rimRadius += 4;
|
||||
|
||||
// Seed every node.
|
||||
const simNodes: LayoutNode[] = nodes.map((node) => {
|
||||
const rand = seededRandom(node.id);
|
||||
const anchor = anchors.get(node.id);
|
||||
let x: number;
|
||||
let y: number;
|
||||
if (anchor) {
|
||||
// Warm start: resume from the previous rest position.
|
||||
x = anchor.x;
|
||||
y = anchor.y;
|
||||
} else {
|
||||
// New node: seed near its positioned neighbors (the cluster it
|
||||
// arrived with), else on the rim outside the core, else scatter.
|
||||
let sx = 0;
|
||||
let sy = 0;
|
||||
let found = 0;
|
||||
for (const nbr of adjacency.get(node.id) ?? []) {
|
||||
const na = anchors.get(nbr);
|
||||
if (na) {
|
||||
sx += na.x;
|
||||
sy += na.y;
|
||||
found++;
|
||||
}
|
||||
}
|
||||
if (found > 0) {
|
||||
x = sx / found + (rand() - 0.5) * 8;
|
||||
y = sy / found + (rand() - 0.5) * 8;
|
||||
} else if (hasCore) {
|
||||
const angle = rand() * Math.PI * 2;
|
||||
const radius = rimRadius + rand() * 8;
|
||||
x = cx + Math.cos(angle) * radius;
|
||||
y = cy + Math.sin(angle) * radius;
|
||||
// An orphan's intended rest spot IS the rim seed: anchor it
|
||||
// there so gravity cannot sink it into the core. (A later
|
||||
// batch linking it to the graph pulls it in naturally —
|
||||
// but then it also has a previous position, and its anchor
|
||||
// becomes that previous position, exactly like any node.)
|
||||
anchors.set(node.id, { x, y });
|
||||
} else {
|
||||
// Bootstrap: the classic seeded scatter.
|
||||
x = (rand() - 0.5) * 100;
|
||||
y = (rand() - 0.5) * 100;
|
||||
}
|
||||
}
|
||||
// d3 is fine with coincident seeds (no division), but nudge exact
|
||||
// overlaps of a brand-new node onto its only neighbor anyway.
|
||||
if (x === 0 && y === 0) x = 0.5;
|
||||
return { id: node.id, x, y };
|
||||
});
|
||||
|
||||
const sim = createGraphSimulation(simNodes, edges, {
|
||||
gravity: {
|
||||
x: cx,
|
||||
y: cy,
|
||||
strength: options.gravityStrength ?? (hasCore ? INCREMENTAL_GRAVITY : BOOTSTRAP_GRAVITY),
|
||||
},
|
||||
clusterPull: options.clusters
|
||||
? { clusters: options.clusters, strength: options.clusterPullStrength ?? CLUSTER_PULL_STRENGTH }
|
||||
: undefined,
|
||||
depthAnchor:
|
||||
options.depthAnchor && options.depthAnchor.mode !== "off"
|
||||
? { ...options.depthAnchor, center: options.depthAnchor.center ?? { x: cx, y: cy } }
|
||||
: options.depthAnchor,
|
||||
});
|
||||
const dMode = options.depthAnchor?.mode ?? "off";
|
||||
const dCenter = options.depthAnchor?.center ?? { x: cx, y: cy };
|
||||
const byId = new Map(simNodes.map((n) => [n.id, n]));
|
||||
for (let i = 0; i < ticks; i++) {
|
||||
sim.tick();
|
||||
// Anchor re-blend (position space, alpha-independent — see the
|
||||
// ANCHOR_RETENTION note). Pinned nodes (fx/fy) are left to d3.
|
||||
// DEPTH-MODE PROJECTION: the re-blend preserves the user's
|
||||
// arrangement in the FREE dimension only — x in vertical mode, the
|
||||
// angle in radial mode — because the depth force owns the
|
||||
// constrained dimension (a node's level/ring is meaning, not
|
||||
// memory). Off-mode: full 2D blend as before.
|
||||
if (hasCore && retention < 1) {
|
||||
for (const [id, a] of anchors) {
|
||||
const n = byId.get(id);
|
||||
if (!n || n.fx != null || n.fy != null) continue;
|
||||
const x = n.x ?? 0;
|
||||
const y = n.y ?? 0;
|
||||
if (dMode === "vertical") {
|
||||
n.x = a.x + (x - a.x) * retention;
|
||||
} else if (dMode === "radial") {
|
||||
const rx = x - dCenter.x;
|
||||
const ry = y - dCenter.y;
|
||||
const r = Math.hypot(rx, ry);
|
||||
const ax = a.x - dCenter.x;
|
||||
const ay = a.y - dCenter.y;
|
||||
const ar = Math.hypot(ax, ay);
|
||||
if (r > 1e-9 && ar > 1e-9) {
|
||||
const theta = Math.atan2(ry, rx);
|
||||
const thetaA = Math.atan2(ay, ax);
|
||||
const blended = thetaA + Math.atan2(Math.sin(theta - thetaA), Math.cos(theta - thetaA)) * retention;
|
||||
n.x = dCenter.x + r * Math.cos(blended);
|
||||
n.y = dCenter.y + r * Math.sin(blended);
|
||||
}
|
||||
} else {
|
||||
n.x = a.x + (x - a.x) * retention;
|
||||
n.y = a.y + (y - a.y) * retention;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const out = new Map<string, Position>();
|
||||
for (const n of simNodes) out.set(n.id, { x: n.x ?? 0, y: n.y ?? 0 });
|
||||
return out;
|
||||
}
|
||||
@@ -22,8 +22,12 @@ import type {
|
||||
ExtractionRunUsage,
|
||||
NodeMapping,
|
||||
PlanTree,
|
||||
Position,
|
||||
SessionCheckpoint,
|
||||
TurnUsage,
|
||||
NotesSource,
|
||||
ClusterLabeling,
|
||||
EmbeddingsLedger,
|
||||
} from "./contracts.ts";
|
||||
|
||||
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
|
||||
@@ -80,6 +84,34 @@ export interface GraphStore {
|
||||
/** Append one emission checkpoint. */
|
||||
appendCheckpoint(checkpoint: SessionCheckpoint): void;
|
||||
readCheckpoints(): SessionCheckpoint[];
|
||||
|
||||
/** Append one cluster labeling — idempotent by id (the id hashes the
|
||||
* sorted core, so identical member sets dedupe). */
|
||||
appendClusterLabel(labeling: ClusterLabeling): void;
|
||||
readClusterLabels(): ClusterLabeling[];
|
||||
|
||||
/** Upsert one imported-notes source record (config.SOURCES_FILE — a
|
||||
* JSON map, not a ledger; an upsert keeps the existing usage log). */
|
||||
upsertSource(source: NotesSource): void;
|
||||
/** Every notes source record, by id order-insensitive. */
|
||||
readSources(): NotesSource[];
|
||||
/** Append one run's usage to a source record — idempotent per
|
||||
* startedAt, mirroring ChatRecord.extractions. */
|
||||
appendSourceUsage(id: string, entry: ExtractionRunUsage): void;
|
||||
|
||||
/** The layout ledger (config.POSITIONS_FILE): nodeId → resting {x,y}.
|
||||
* WRITTEN ONLY by the incremental settle after a merge (core/layout.ts
|
||||
* settleGraphLayout) — read-only for everything else. */
|
||||
readPositions(): Record<string, Position>;
|
||||
/** Replace the whole ledger (the settle is a full-graph computation). */
|
||||
writePositions(positions: Record<string, Position>): void;
|
||||
|
||||
/** The embedding ledger (config.EMBEDDINGS_FILE). WRITTEN ONLY by
|
||||
* the embedder (server/embeddings.ts) after a merge — read-only for
|
||||
* clustering consumers (snapshot, settle, namer). */
|
||||
readEmbeddings(): EmbeddingsLedger;
|
||||
/** Replace the whole ledger (embedding is a full-batch computation). */
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void;
|
||||
}
|
||||
|
||||
// ─── JobStore — background sub-agent jobs (data/jobs) ─────────────────────────
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* Graph render settings: the shape, the defaults, the sanitizer, and the
|
||||
* slider mapping (log-scaled, default-centered) — shared by the client (components/graph-canvas.ts,
|
||||
* graph-view.tsx), the server store (src/store/render-settings.ts, which
|
||||
* persists them to data/render-settings.json), and the API route.
|
||||
*
|
||||
* This module is pure: no DOM, no d3, no node APIs — safe everywhere.
|
||||
*
|
||||
* Settings are server-persisted (render-settings.json); the localStorage era
|
||||
* is over. Whatever arrives from disk or the network is passed through
|
||||
* sanitizeRenderSettings before use — never trusted raw.
|
||||
*
|
||||
* The labelZoom semantics: the sigma-compatible "camera ratio" threshold
|
||||
* (1 = fit zoom, SMALLER = zoomed in). Node labels show at ratio <= labelZoom
|
||||
* and fade over LABELS_FADE_SPAN; cluster region labels fade in inversely.
|
||||
* Persisted values keep these units across the renderer swap.
|
||||
*/
|
||||
|
||||
import {
|
||||
CHARGE_STRENGTH,
|
||||
CLUSTER_PULL_STRENGTH,
|
||||
INCREMENTAL_GRAVITY,
|
||||
LINK_DISTANCE,
|
||||
type DepthAnchorMode,
|
||||
} from "./layout.ts";
|
||||
|
||||
export interface GraphRenderSettings {
|
||||
nodeSize: number;
|
||||
edgeWidth: number;
|
||||
arrowLength: number;
|
||||
/** Camera-ratio threshold for node labels (see module doc). */
|
||||
labelZoom: number;
|
||||
/** Node + hover label font size, px at fit zoom (scales with camera). */
|
||||
labelSize: number;
|
||||
/** Toggle for the cluster region labels (hulls always draw). */
|
||||
clusterLabels: boolean;
|
||||
/** Region rendering (the blurred cluster tints). Server-persisted
|
||||
* since 2026-09-13 (user request) — was session-only dev state. */
|
||||
regions: RegionTuning;
|
||||
/** Live-sim physics + inference-depth anchoring. Server-persisted
|
||||
* since 2026-09-13 (user request) — was session-only dev state. */
|
||||
physics: PhysicsTuning;
|
||||
}
|
||||
|
||||
|
||||
const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max);
|
||||
|
||||
/**
|
||||
* Coerce arbitrary JSON (server file, network body, localStorage leftovers)
|
||||
* into valid settings: missing keys fall back to defaults, out-of-range
|
||||
* values clamp, wrong types reset. Never throws.
|
||||
*/
|
||||
export function sanitizeRenderSettings(raw: unknown): GraphRenderSettings {
|
||||
const r = (raw ?? {}) as Partial<GraphRenderSettings>;
|
||||
const num = (v: unknown, fallback: number, min: number, max: number) => {
|
||||
const n = typeof v === "number" && Number.isFinite(v) ? v : fallback;
|
||||
return clamp(n, min, max);
|
||||
};
|
||||
return {
|
||||
nodeSize: num(r.nodeSize, DEFAULT_RENDER_SETTINGS.nodeSize, NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max),
|
||||
edgeWidth: num(r.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth, EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max),
|
||||
arrowLength: num(r.arrowLength, DEFAULT_RENDER_SETTINGS.arrowLength, ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max),
|
||||
// Range matches the log slider's span [LABEL_ZOOM_MIN, LABEL_ZOOM_MAX].
|
||||
labelZoom: num(r.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom, LABEL_ZOOM_MIN, LABEL_ZOOM_MAX),
|
||||
labelSize: num(r.labelSize, DEFAULT_RENDER_SETTINGS.labelSize, LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max),
|
||||
clusterLabels:
|
||||
typeof r.clusterLabels === "boolean" ? r.clusterLabels : DEFAULT_RENDER_SETTINGS.clusterLabels,
|
||||
// Nested sections (2026-09-13): old files lack them → defaults.
|
||||
regions: sanitizeRegionTuning(r.regions),
|
||||
physics: sanitizePhysicsTuning(r.physics),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Region + physics tuning (moved from the renderer, 2026-09-13) ─────────────
|
||||
//
|
||||
// The renderer's region/physics controls became USER-FACING persisted
|
||||
// settings: one shape + one sanitizer in core, consumed by the settings
|
||||
// store (this file's schema), the panel, and the renderer's
|
||||
// setRegionTuning/setPhysicsTuning. Every clamp below is a SWEEP-VALIDATED
|
||||
// safe range (2026-09-13, real graph, renderer's exact sim): each field at
|
||||
// both range ends AND all four range corners settles cleanly — no NaN, no
|
||||
// explosion (linkStrength ≥ 8 explodes the layout, 5 wobbles; cap 4),
|
||||
// no non-settle (velocityDecay 0 never damps; floor 0.1), no 3x sprawl
|
||||
// (gravity 0 / charge -1000 excluded).
|
||||
|
||||
/** Region rendering knobs (the blurred cluster tints). */
|
||||
export interface RegionTuning {
|
||||
/** Dilation half-width of the round-cap edge strokes, screen px at
|
||||
* fit zoom (the expansion of the cluster skeleton). */
|
||||
dilation: number;
|
||||
/** Box-blur radius in bitmap px (3 passes ≈ Gaussian). Fixed set:
|
||||
* see REGION_BLUR_OPTIONS (dropdown in the panel — non-integer
|
||||
* radii broke the rendering, so the control is integer-only). */
|
||||
blur: number;
|
||||
/** Max bitmap dimension (cell = worldSpan / this). Fixed set:
|
||||
* see REGION_BITMAP_OPTIONS (dropdown in the panel). */
|
||||
bitmap: number;
|
||||
/** Region alpha, applied ONCE after the blur (dilate at full
|
||||
* opacity → blur → scale). Constant interior, soft edges; overlaps
|
||||
* of any count never stack (DECIDED 2026-09-14 — per-stroke alpha
|
||||
* accumulated through source-over and double-counted orphans). */
|
||||
edgeAlpha: number;
|
||||
}
|
||||
|
||||
export const REGION_BITMAP_OPTIONS = [128, 192, 256, 384, 512] as const;
|
||||
export const REGION_BLUR_OPTIONS = [2, 4, 6, 8, 10] as const;
|
||||
|
||||
export const DEFAULT_REGION_TUNING: RegionTuning = {
|
||||
dilation: 28,
|
||||
blur: 6,
|
||||
bitmap: 384,
|
||||
edgeAlpha: 0.35,
|
||||
};
|
||||
|
||||
/** Nearest member of a fixed option set (dropdowns have no free values). */
|
||||
function snapToOption(value: number, options: readonly number[]): number {
|
||||
return options.reduce((best, o) => (Math.abs(o - value) < Math.abs(best - value) ? o : best), options[0] as number);
|
||||
}
|
||||
|
||||
/** Clamp/validate a region tuning. `bitmap` and `blur` snap to the nearest
|
||||
* allowed option (they're fixed sets — dropdowns, not free numbers). */
|
||||
export function sanitizeRegionTuning(input: Partial<RegionTuning> | null | undefined): RegionTuning {
|
||||
const d = DEFAULT_REGION_TUNING;
|
||||
const num = (v: unknown, fallback: number): number =>
|
||||
typeof v === "number" && Number.isFinite(v) ? v : fallback;
|
||||
const bitmap = num(input?.bitmap, d.bitmap);
|
||||
return {
|
||||
dilation: Math.min(Math.max(num(input?.dilation, d.dilation), REGION_DILATION_RANGE.min), REGION_DILATION_RANGE.max),
|
||||
blur: snapToOption(num(input?.blur, d.blur), REGION_BLUR_OPTIONS),
|
||||
bitmap: snapToOption(bitmap, REGION_BITMAP_OPTIONS),
|
||||
edgeAlpha: Math.min(Math.max(num(input?.edgeAlpha, d.edgeAlpha), REGION_EDGE_ALPHA_RANGE.min), 1),
|
||||
};
|
||||
}
|
||||
|
||||
/** Live-sim physics controls (user-facing since 2026-09-13): every value
|
||||
* feeds createGraphSimulation — the SAME engine the server settle runs —
|
||||
* so tuning the feel changes exactly what a drag relaxes toward. */
|
||||
export interface PhysicsTuning {
|
||||
/** Node repulsion (d3 charge, negative = push apart). */
|
||||
chargeStrength: number;
|
||||
/** Edge attraction: link-spring stiffness multiplier (1 = d3 default). */
|
||||
linkStrength: number;
|
||||
/** Link spring rest length (world units). */
|
||||
linkDistance: number;
|
||||
/** Central pull toward the graph centroid. */
|
||||
gravity: number;
|
||||
/** Per-cluster pull toward the cluster centroid. 0 = off. */
|
||||
clusterPull: number;
|
||||
/** Per-tick velocity damping (d3 default 0.4). */
|
||||
velocityDecay: number;
|
||||
/** Inference-depth anchoring mode (see DepthAnchor in core). */
|
||||
depthMode: DepthAnchorMode;
|
||||
/** Depth-anchor pull strength. */
|
||||
depthStrength: number;
|
||||
/** Spacing between inference levels (world units). */
|
||||
depthSpacing: number;
|
||||
/** Inner ring radius, radial mode (world units). */
|
||||
depthBaseRadius: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_PHYSICS_TUNING: PhysicsTuning = {
|
||||
chargeStrength: CHARGE_STRENGTH,
|
||||
linkStrength: 1,
|
||||
linkDistance: LINK_DISTANCE,
|
||||
gravity: INCREMENTAL_GRAVITY,
|
||||
clusterPull: CLUSTER_PULL_STRENGTH,
|
||||
velocityDecay: 0.4,
|
||||
depthMode: "off",
|
||||
// 0.8 by sweep on the real graph: 215/216 arrows read upward with
|
||||
// organic sag (mean |dev| from level ~34u at spacing 60); 1.5 gives
|
||||
// crisp rows (dev ~18) if the promoted default wants them.
|
||||
depthStrength: 0.8,
|
||||
depthSpacing: 60,
|
||||
depthBaseRadius: 220,
|
||||
};
|
||||
|
||||
/** Clamp anything into a usable tuning (sliders/JSON can be empty/NaN). */
|
||||
export function sanitizePhysicsTuning(input: Partial<PhysicsTuning> | null | undefined): PhysicsTuning {
|
||||
const d = DEFAULT_PHYSICS_TUNING;
|
||||
const num = (v: unknown, fallback: number): number =>
|
||||
typeof v === "number" && Number.isFinite(v) ? v : fallback;
|
||||
return {
|
||||
chargeStrength: Math.min(Math.max(num(input?.chargeStrength, d.chargeStrength), CHARGE_RANGE.min), CHARGE_RANGE.max),
|
||||
linkStrength: Math.min(Math.max(num(input?.linkStrength, d.linkStrength), LINK_STRENGTH_RANGE.min), LINK_STRENGTH_RANGE.max),
|
||||
linkDistance: Math.min(Math.max(num(input?.linkDistance, d.linkDistance), LINK_DISTANCE_RANGE.min), LINK_DISTANCE_RANGE.max),
|
||||
gravity: Math.min(Math.max(num(input?.gravity, d.gravity), GRAVITY_RANGE.min), GRAVITY_RANGE.max),
|
||||
clusterPull: Math.min(Math.max(num(input?.clusterPull, d.clusterPull), 0), CLUSTER_PULL_RANGE.max),
|
||||
velocityDecay: Math.min(Math.max(num(input?.velocityDecay, d.velocityDecay), VELOCITY_DECAY_RANGE.min), VELOCITY_DECAY_RANGE.max),
|
||||
depthMode: input?.depthMode === "vertical" || input?.depthMode === "radial" ? input.depthMode : "off",
|
||||
depthStrength: Math.min(Math.max(num(input?.depthStrength, d.depthStrength), DEPTH_STRENGTH_RANGE.min), DEPTH_STRENGTH_RANGE.max),
|
||||
depthSpacing: Math.min(Math.max(num(input?.depthSpacing, d.depthSpacing), DEPTH_SPACING_RANGE.min), DEPTH_SPACING_RANGE.max),
|
||||
depthBaseRadius: Math.min(Math.max(num(input?.depthBaseRadius, d.depthBaseRadius), DEPTH_BASE_RADIUS_RANGE.min), DEPTH_BASE_RADIUS_RANGE.max),
|
||||
};
|
||||
}
|
||||
|
||||
// Sizes are screen px AT FIT ZOOM (the world-space renderer scales them
|
||||
// with the camera, so a label reads at labelSize x k/kFit on screen).
|
||||
// Defaults were re-tuned for world-scaling on 2026-09-13: the old values
|
||||
// were calibrated in the screen-fixed era, so everything rendered ~2x
|
||||
// too big once sizes started scaling with the graph.
|
||||
export const DEFAULT_RENDER_SETTINGS: GraphRenderSettings = {
|
||||
nodeSize: 2,
|
||||
edgeWidth: 1,
|
||||
arrowLength: 2,
|
||||
labelZoom: 0.5,
|
||||
labelSize: 2,
|
||||
clusterLabels: true,
|
||||
regions: DEFAULT_REGION_TUNING,
|
||||
physics: DEFAULT_PHYSICS_TUNING,
|
||||
};
|
||||
|
||||
// ─── The slider mapping ────────────────────────────────────────────────────────
|
||||
//
|
||||
// EVERY slider is log-scaled over its setting's range with the DEFAULT at
|
||||
// position 50 (dead center), per the user's spec: centered defaults, coarse
|
||||
// steps (the UI steps by 5 → 21 stops), no fine-grained linear tracks.
|
||||
// Ranges are chosen so each default is the geometric mean of its ends.
|
||||
|
||||
export const NODE_SIZE_RANGE = { min: 0.5, max: 8 } as const;
|
||||
export const EDGE_WIDTH_RANGE = { min: 0.25, max: 4 } as const;
|
||||
export const ARROW_LENGTH_RANGE = { min: 0.5, max: 8 } as const;
|
||||
export const LABEL_SIZE_RANGE = { min: 0.5, max: 8 } as const;
|
||||
export const LABEL_ZOOM_MIN = 0.0625; // slider far left: labels only very deep in
|
||||
export const LABEL_ZOOM_MAX = 4; // slider far right: labels visible at fit zoom
|
||||
|
||||
// ─── Region + physics slider ranges ──────────────────────────────────────────
|
||||
//
|
||||
// Same laws as the size sliders: log-scaled, DEFAULT AT 50 (each default is
|
||||
// the geometric mean of its ends), coarse 5-step tracks. Signed/zeroed
|
||||
// fields get special treatment:
|
||||
// chargeStrength is NEGATIVE — the slider maps |charge| and displays "-x";
|
||||
// clusterPull has an OFF zone — position 0 is exactly 0 (see
|
||||
// toOffSliderPosition), the log range starts one step in. Blur used to
|
||||
// have one too; it's a dropdown since 2026-09-13 (non-integers broke).
|
||||
|
||||
export const REGION_DILATION_RANGE = { min: 7, max: 112 } as const; // gm = 28
|
||||
export const REGION_EDGE_ALPHA_RANGE = { min: 0.12, max: 1 } as const; // gm ≈ 0.35
|
||||
|
||||
export const CHARGE_RANGE = { min: -480, max: -30 } as const; // |gm| = 120
|
||||
export const LINK_STRENGTH_RANGE = { min: 0.25, max: 4 } as const; // gm = 1
|
||||
export const LINK_DISTANCE_RANGE = { min: 10, max: 160 } as const; // gm = 40
|
||||
export const GRAVITY_RANGE = { min: 0.03, max: 0.48 } as const; // gm = 0.12
|
||||
export const CLUSTER_PULL_RANGE = { min: 0.0225, max: 1 } as const; // gm = 0.15; 0 = off
|
||||
export const VELOCITY_DECAY_RANGE = { min: 0.18, max: 0.9 } as const; // gm ≈ 0.40
|
||||
export const DEPTH_STRENGTH_RANGE = { min: 0.2, max: 3.2 } as const; // gm = 0.8
|
||||
export const DEPTH_SPACING_RANGE = { min: 15, max: 240 } as const; // gm = 60
|
||||
export const DEPTH_BASE_RADIUS_RANGE = { min: 55, max: 880 } as const; // gm = 220
|
||||
|
||||
/** Off-aware slider mapping (position 0 → value 0 = OFF; the log range
|
||||
* starts one step in). Inverse of fromOffSliderPosition. */
|
||||
export function toOffSliderPosition(value: number, min: number, max: number): number {
|
||||
if (value <= 0) return 0;
|
||||
return toSliderPosition(value, min, max);
|
||||
}
|
||||
|
||||
/** Off-aware slider mapping: position 0 → 0 (off), else log-scaled. */
|
||||
export function fromOffSliderPosition(position: number, min: number, max: number): number {
|
||||
if (position <= 0) return 0;
|
||||
return fromSliderPosition(position, min, max);
|
||||
}
|
||||
|
||||
/** Setting value → slider position [0, 100] (log scale, default at 50). */
|
||||
export function toSliderPosition(value: number, min: number, max: number): number {
|
||||
const v = clamp(value, min, max);
|
||||
const logSpan = Math.log(max / min);
|
||||
return (100 * Math.log(v / min)) / logSpan;
|
||||
}
|
||||
|
||||
/** Slider position [0, 100] → setting value (inverse of toSliderPosition). */
|
||||
export function fromSliderPosition(position: number, min: number, max: number): number {
|
||||
const p = clamp(position, 0, 100) / 100;
|
||||
const logSpan = Math.log(max / min);
|
||||
return min * Math.exp(p * logSpan);
|
||||
}
|
||||
|
||||
/** Display units for labelZoom: normalized so the default (0.5) reads as 1. */
|
||||
export function labelZoomDisplay(labelZoom: number): string {
|
||||
const v = Math.round((labelZoom / DEFAULT_RENDER_SETTINGS.labelZoom) * 100) / 100;
|
||||
return `${v}×`;
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// Semantic topic clustering — the REPLACEMENT for Louvain community
|
||||
// detection (DECIDED 2026-09-13): structural clusters were just the
|
||||
// reasoning chains (the graph is disconnected by nature — every connected
|
||||
// component became its own "cluster"), and edge-orphans could never join
|
||||
// anything. Clusters are now RELATED TOPICS: nodes whose proposition
|
||||
// MEANING is close, computed over the embedding ledger
|
||||
// (config.EMBEDDINGS_FILE). This pulls orphans into topics by meaning
|
||||
// and groups small chains with the large chains they discuss.
|
||||
//
|
||||
// Algorithm: threshold AGGLOMERATIVE (average linkage over cosine
|
||||
// similarity). No JS HDBSCAN is trustworthy (the npm package is an
|
||||
// abandoned alpha), and at proposition scale (~100s-1000s of short texts)
|
||||
// brute-force pairwise cosine is sub-second — so the honest ~60-line
|
||||
// hand-roll wins. "Stragglers" (nodes that fit nothing) fall out
|
||||
// naturally: they simply never merge, and stay singletons.
|
||||
|
||||
/** The default similarity threshold, SWEPT ON THE REAL GRAPH
|
||||
* (2026-09-13): 0.60 → 26 topics of ~20 propositions, 74 of the 90
|
||||
* edge-orphans land in real topics, stragglers 10. Higher (0.65-0.68)
|
||||
* gives tighter micro-topics with far more stragglers; lower (0.55)
|
||||
* collapses into ~13 mega-topics. */
|
||||
export const SEMANTIC_SIM_THRESHOLD = 0.6;
|
||||
|
||||
/** Cosine similarity of two vectors (defensively normalizes inputs;
|
||||
* ledger vectors are already L2-normalized, so this is a dot product). */
|
||||
export function cosineSimilarity(a: readonly number[], b: readonly number[]): number {
|
||||
let dot = 0;
|
||||
let ma = 0;
|
||||
let mb = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i]! * b[i]!;
|
||||
ma += a[i]! * a[i]!;
|
||||
mb += b[i]! * b[i]!;
|
||||
}
|
||||
if (ma === 0 || mb === 0) return 0;
|
||||
return dot / (Math.sqrt(ma) * Math.sqrt(mb));
|
||||
}
|
||||
|
||||
export interface SemanticClusterOptions {
|
||||
/** Merge while the best cluster pair's average-linkage similarity is
|
||||
* >= this (default SEMANTIC_SIM_THRESHOLD). */
|
||||
similarityThreshold?: number;
|
||||
/** After the cut, force every singleton into its nearest topic
|
||||
* ("all-in" mode) instead of leaving stragglers loose. Default
|
||||
* false: some propositions genuinely fit nothing. */
|
||||
attachStragglers?: boolean;
|
||||
}
|
||||
|
||||
/** Average-linkage agglomerative clustering over embedding vectors.
|
||||
* Pure and deterministic (stable Map iteration, greedy by best
|
||||
* similarity, first-found wins ties). Returns clusters size-desc — the
|
||||
* same { members } shape Louvain produced, so label matching and the
|
||||
* renderer contract are unchanged. An empty/missing ledger returns []
|
||||
* (nothing embedded yet: no clusters, every node loose — the snapshot
|
||||
* degrades gracefully until the first embed). */
|
||||
export function semanticClusters(
|
||||
vectors: Readonly<Record<string, readonly number[]>>,
|
||||
options: SemanticClusterOptions = {},
|
||||
): Array<{ members: string[] }> {
|
||||
const ids = Object.keys(vectors);
|
||||
if (ids.length === 0) return [];
|
||||
const simThreshold = options.similarityThreshold ?? SEMANTIC_SIM_THRESHOLD;
|
||||
const attach = options.attachStragglers ?? false;
|
||||
|
||||
// Sum vectors per cluster: average-linkage similarity between clusters
|
||||
// A and B is dot(sumA, sumB) / (|A|·|B|) — the exact mean of pairwise
|
||||
// dots — so the merge loop never revisits the raw O(n²) matrix.
|
||||
const sums = new Map<number, number[]>();
|
||||
const members = new Map<number, string[]>();
|
||||
ids.forEach((id, i) => {
|
||||
sums.set(i, [...vectors[id]!]);
|
||||
members.set(i, [id]);
|
||||
});
|
||||
|
||||
// Greedy merge: repeatedly join the pair with the highest
|
||||
// average-linkage similarity until it drops below the threshold.
|
||||
// O(k²) pair scan per merge — trivial at proposition scale.
|
||||
for (;;) {
|
||||
let bestA = -1;
|
||||
let bestB = -1;
|
||||
let bestSim = -Infinity;
|
||||
for (const [a, sumA] of sums) {
|
||||
for (const [b, sumB] of sums) {
|
||||
if (a >= b) continue;
|
||||
let dot = 0;
|
||||
for (let i = 0; i < sumA.length; i++) dot += sumA[i]! * sumB[i]!;
|
||||
const sim = dot / (members.get(a)!.length * members.get(b)!.length);
|
||||
if (sim > bestSim) {
|
||||
bestSim = sim;
|
||||
bestA = a;
|
||||
bestB = b;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bestA < 0 || bestSim < simThreshold) break;
|
||||
// Merge B into A (deterministic direction: lower cluster index).
|
||||
const sumA = sums.get(bestA)!;
|
||||
const sumB = sums.get(bestB)!;
|
||||
for (let i = 0; i < sumA.length; i++) sumA[i] = sumA[i]! + sumB[i]!;
|
||||
members.get(bestA)!.push(...members.get(bestB)!);
|
||||
sums.delete(bestB);
|
||||
members.delete(bestB);
|
||||
}
|
||||
|
||||
// Optional all-in pass: singletons join their nearest topic regardless
|
||||
// of threshold — exactly the "forced membership" mode. The difference
|
||||
// from stragglers mode is this one pass. Singleton-vs-cluster
|
||||
// average-linkage similarity is dot(sumS, sumK) / |K|.
|
||||
if (attach) {
|
||||
const singletons = [...members.entries()].filter(([, m]) => m.length === 1).map(([k]) => k);
|
||||
for (const s of singletons) {
|
||||
if (!members.has(s)) continue; // absorbed by an earlier attach
|
||||
let bestTarget = -1;
|
||||
let bestSim = -Infinity;
|
||||
for (const [k, m] of members) {
|
||||
if (k === s || m.length < 2) continue;
|
||||
let dot = 0;
|
||||
const sumS = sums.get(s)!;
|
||||
const sumK = sums.get(k)!;
|
||||
for (let i = 0; i < sumS.length; i++) dot += sumS[i]! * sumK[i]!;
|
||||
const sim = dot / m.length;
|
||||
if (sim > bestSim) {
|
||||
bestSim = sim;
|
||||
bestTarget = k;
|
||||
}
|
||||
}
|
||||
if (bestTarget >= 0) {
|
||||
const sumT = sums.get(bestTarget)!;
|
||||
const sumS = sums.get(s)!;
|
||||
for (let i = 0; i < sumT.length; i++) sumT[i] = sumT[i]! + sumS[i]!;
|
||||
members.get(bestTarget)!.push(...members.get(s)!);
|
||||
sums.delete(s);
|
||||
members.delete(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...members.values()]
|
||||
.map((m) => ({ members: [...m].sort() }))
|
||||
.sort((a, b) => b.members.length - a.members.length || (a.members[0] ?? "").localeCompare(b.members[0] ?? ""));
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { createProberAgent, createTeacherAgent, type TeachingAgent } from "../ag
|
||||
import { piEngineFor } from "../agent/models.ts";
|
||||
import { createJobRunner } from "../agent/subagents/runner.ts";
|
||||
import { createExtractionPipeline } from "../agent/subagents/extraction.ts";
|
||||
import { embedNewNodes } from "./embeddings.ts";
|
||||
import type { ModelRole } from "../../config.ts";
|
||||
import { claimSteers, deliverJobResult, setJobRespawn, setTurnRunner } from "./jobs.ts";
|
||||
import { recordToPiMessages } from "./pi/record-adapter.ts";
|
||||
@@ -37,7 +38,15 @@ const jobRunner = createJobRunner({
|
||||
jobStore: stores.jobStore,
|
||||
deliver: deliverJobResult,
|
||||
});
|
||||
const extraction = createExtractionPipeline(stores);
|
||||
/** The app's ONE extraction pipeline instance — both front doors share it:
|
||||
* quiz-triggered session runs (turn-service) and notes imports
|
||||
* (/api/graph/import) hit the same single-flight registry. */
|
||||
export const extraction = createExtractionPipeline({
|
||||
...stores,
|
||||
// The REAL embedder (local transformer model): post-merge embedding
|
||||
// batch — semantic clusters are computed and persisted here.
|
||||
embedder: embedNewNodes,
|
||||
});
|
||||
|
||||
// Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline
|
||||
// fires its server-side teacher turns through THIS composition root, and
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Server-side commit for the live physics drag (PUT /api/graph/positions).
|
||||
* The client's renderer anneals a drag to rest and PUTs the new rest
|
||||
* state; the server trusts it for the nodes it knows about and, if the
|
||||
* graph grew while the user was dragging (a merge landed), runs ONE
|
||||
* incremental settle (settleGraphLayout) so the fresh nodes are absorbed
|
||||
* into the frame the user just arranged — client positions act as
|
||||
* anchors, never as a clobber.
|
||||
*
|
||||
* Every body is sanitized: garbage coordinates clamp or drop, unknown
|
||||
* ids are discarded, nothing throws.
|
||||
*/
|
||||
import type { Position } from "../core/contracts.ts";
|
||||
import { getGraph } from "../core/graph.ts";
|
||||
import { settleGraphLayout } from "../core/layout-settle.ts";
|
||||
import { stores } from "./stores.ts";
|
||||
|
||||
/** Max |x| or |y| accepted — NaN/Infinity clamp here, absurd too. */
|
||||
const MAX_COORD = 1e6;
|
||||
|
||||
/**
|
||||
* Accepts the raw PUT body (a Record of id → {x, y}) and returns the
|
||||
* sanitized positions actually persisted (empty object = nothing written,
|
||||
* e.g. a body with no known ids — the caller reports but never fails).
|
||||
*/
|
||||
export function commitPositions(input: unknown): Record<string, Position> {
|
||||
if (typeof input !== "object" || input === null) return {};
|
||||
const raw = input as Record<string, unknown>;
|
||||
|
||||
// Only ids the graph actually has (first line wins, like getGraph).
|
||||
const known = new Map<string, boolean>();
|
||||
for (const line of stores.graphStore.readNodes()) {
|
||||
if (!known.has(line.id)) known.set(line.id, true);
|
||||
}
|
||||
|
||||
const positions: Record<string, Position> = {};
|
||||
for (const [id, value] of Object.entries(raw)) {
|
||||
if (!known.has(id)) continue;
|
||||
if (typeof value !== "object" || value === null) continue;
|
||||
const { x, y } = value as Record<string, unknown>;
|
||||
if (typeof x !== "number" || typeof y !== "number" || !Number.isFinite(x) || !Number.isFinite(y))
|
||||
continue;
|
||||
positions[id] = {
|
||||
x: Math.min(Math.max(x, -MAX_COORD), MAX_COORD),
|
||||
y: Math.min(Math.max(y, -MAX_COORD), MAX_COORD),
|
||||
};
|
||||
}
|
||||
if (Object.keys(positions).length === 0) return {};
|
||||
|
||||
stores.graphStore.writePositions(positions);
|
||||
|
||||
// The graph grew past what the client knew (a merge landed mid-drag):
|
||||
// absorb the new nodes with the client's rest state as anchors.
|
||||
const unique = getGraph(stores.graphStore).nodes.length;
|
||||
if (unique > Object.keys(positions).length) settleGraphLayout(stores.graphStore);
|
||||
return positions;
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* THE EMBEDDER (DECIDED 2026-09-13): semantic topic clustering needs
|
||||
* vectors, and vectors need a model running ON the user's machine — no
|
||||
* external services, no API bills, no network dependency for re-embedding.
|
||||
*
|
||||
* Runtime: in-process transformers.js (@huggingface/transformers v4 over
|
||||
* onnxruntime-node — native Apple Silicon acceleration). The model
|
||||
* weights download ONCE to the local HF cache on first run (~300MB).
|
||||
*
|
||||
* Model: EmbeddingGemma-300M (q8) — best quality-per-megabyte small
|
||||
* embedding model as of late 2025 (MTEB-en 68.36), built for on-device
|
||||
* use, with MRL truncation so we store 512 of its 768 dims. Every input
|
||||
* uses the model card's document format ("title: none | text: …"),
|
||||
* consistently — mixing prefixed and unprefixed inputs degrades quality.
|
||||
*
|
||||
* Model swaps re-embed EVERYTHING: vectors from different models live in
|
||||
* incompatible spaces. The ledger records model + dim; a mismatch marks
|
||||
* the whole ledger stale and this module re-embeds from scratch (minutes
|
||||
* at current scale).
|
||||
*/
|
||||
import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers";
|
||||
import type { EmbeddingsLedger, GraphNode } from "../core/contracts.ts";
|
||||
import { semanticClusters, SEMANTIC_SIM_THRESHOLD } from "../core/semantic-clusters.ts";
|
||||
|
||||
export const EMBEDDING_MODEL = "onnx-community/embeddinggemma-300m-ONNX";
|
||||
/** MRL truncation of the native 768 — negligible quality loss, half the
|
||||
* ledger size (2026-09-13 researcher sweep). */
|
||||
export const EMBEDDING_DIM = 512;
|
||||
/** Batch size bounds peak memory; propositions are tiny so 32 per call
|
||||
* keeps the pipeline warm without thrashing. */
|
||||
const EMBED_BATCH = 32;
|
||||
|
||||
let pipelinePromise: Promise<FeatureExtractionPipeline> | null = null;
|
||||
|
||||
function getPipeline(): Promise<FeatureExtractionPipeline> {
|
||||
pipelinePromise ??= pipeline("feature-extraction", EMBEDDING_MODEL, { dtype: "q8" });
|
||||
return pipelinePromise;
|
||||
}
|
||||
|
||||
/** EmbeddingGemma's required document format (model card). */
|
||||
function embedInput(statement: string): string {
|
||||
return `title: none | text: ${statement}`;
|
||||
}
|
||||
|
||||
/** Truncate an L2-normalized vector to EMBEDDING_DIM (MRL: leading dims
|
||||
* carry most of the signal) and re-normalize so cosine = dot product. */
|
||||
function truncateNormalize(vec: readonly number[]): number[] {
|
||||
const out = vec.slice(0, EMBEDDING_DIM);
|
||||
let mag = 0;
|
||||
for (const x of out) mag += x * x;
|
||||
mag = Math.sqrt(mag);
|
||||
return mag > 0 ? out.map((x) => x / mag) : out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed every node the ledger is missing — the post-merge hook (new
|
||||
* propositions only, typically a handful) AND the cold-start backfill
|
||||
* (empty ledger = every node) in one function. Model/dim mismatch voids
|
||||
* the ledger first (full re-embed). Also prunes vectors for nodes that
|
||||
* no longer exist. Writes the whole ledger and returns counts for logs.
|
||||
* Non-fatal by contract: the caller treats failure as "clustering runs
|
||||
* without the new nodes until the next merge retries".
|
||||
*/
|
||||
export async function embedNewNodes(
|
||||
graphStore: {
|
||||
readNodes(): GraphNode[];
|
||||
readEmbeddings(): EmbeddingsLedger;
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void;
|
||||
},
|
||||
): Promise<{ embedded: number; total: number }> {
|
||||
const ledger = graphStore.readEmbeddings();
|
||||
const stale = ledger.model !== EMBEDDING_MODEL || ledger.dim !== EMBEDDING_DIM;
|
||||
const vectors: Record<string, number[]> = stale ? {} : { ...ledger.vectors };
|
||||
|
||||
// Node LINES may repeat an id — dedupe, first defines (getGraph rule).
|
||||
const seen = new Set<string>();
|
||||
const todo: GraphNode[] = [];
|
||||
for (const node of graphStore.readNodes()) {
|
||||
if (seen.has(node.id)) continue;
|
||||
seen.add(node.id);
|
||||
if (!vectors[node.id]) todo.push(node);
|
||||
}
|
||||
|
||||
if (todo.length > 0) {
|
||||
const extractor = await getPipeline();
|
||||
for (let i = 0; i < todo.length; i += EMBED_BATCH) {
|
||||
const batch = todo.slice(i, i + EMBED_BATCH);
|
||||
const output = await extractor(batch.map((n) => embedInput(n.statement)), {
|
||||
pooling: "mean",
|
||||
normalize: true,
|
||||
});
|
||||
const rows = output.tolist() as number[][];
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
vectors[batch[j]!.id] = truncateNormalize(rows[j]!);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prune ghosts: vectors for node ids that no longer exist.
|
||||
let pruned = 0;
|
||||
for (const id of Object.keys(vectors)) {
|
||||
if (!seen.has(id)) {
|
||||
delete vectors[id];
|
||||
pruned++;
|
||||
}
|
||||
}
|
||||
|
||||
// RECLUSTER AT THE END OF THE BATCH (user-decided 2026-09-13): the
|
||||
// agglomerative pass is ~1.5s at current scale, so it runs HERE —
|
||||
// once per embedding batch — and consumers (snapshot, settle, namer)
|
||||
// read the persisted partition instead of recomputing per read.
|
||||
const clusters = semanticClusters(vectors, { similarityThreshold: SEMANTIC_SIM_THRESHOLD });
|
||||
|
||||
const wrote = todo.length > 0 || pruned > 0 || stale || ledger.clusters.length !== clusters.length;
|
||||
if (wrote) {
|
||||
graphStore.writeEmbeddings({
|
||||
model: EMBEDDING_MODEL,
|
||||
dim: EMBEDDING_DIM,
|
||||
vectors,
|
||||
clusters,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
return { embedded: todo.length, total: Object.keys(vectors).length };
|
||||
}
|
||||
@@ -7,8 +7,10 @@
|
||||
export * from "./chat-handler.ts";
|
||||
export * from "./chats.ts";
|
||||
export * from "./graph.ts";
|
||||
export * from "./commit-positions.ts";
|
||||
export * from "./jobs.ts";
|
||||
export * from "./stores.ts";
|
||||
export { readRenderSettings, writeRenderSettings } from "../store/render-settings.ts";
|
||||
export * from "./turn-liveness.ts";
|
||||
export * from "./observability.ts";
|
||||
export * from "./model-catalog.ts";
|
||||
|
||||
+185
-1
@@ -15,6 +15,10 @@
|
||||
* (chatId, sourceId) → graphNodeId
|
||||
* sessions.jsonl SessionCheckpoint— emission checkpoints (foldedAt)
|
||||
*
|
||||
* Plus config.SOURCES_FILE (sources.json): the imported-notes bookkeeping
|
||||
* (NOT a ledger) — one JSON map, src-id → NotesSource, read-modify-write
|
||||
* like a chat file. The format-agnostic front door (2026-09-13).
|
||||
*
|
||||
* Implements core/ports.ts GraphStore. data/graph/* is exclusively this
|
||||
* adapter's. Reads are strict against the contracts.ts shapes (the 2026-09-02
|
||||
* graph redesign): corrupt lines are skipped, never guessed at — the ledgers
|
||||
@@ -25,13 +29,18 @@
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { LEDGER_FILES } from "../../config.ts";
|
||||
import { EMBEDDINGS_FILE, LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts";
|
||||
import type {
|
||||
ClusterLabeling,
|
||||
EmbeddingsLedger,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
ExtractionRunUsage,
|
||||
LedgerEvent,
|
||||
LedgerEventKind,
|
||||
NotesSource,
|
||||
NodeMapping,
|
||||
Position,
|
||||
ProvenancePoint,
|
||||
SessionCheckpoint,
|
||||
} from "../core/contracts.ts";
|
||||
@@ -80,6 +89,14 @@ function readValid<T>(file: string, isValid: (v: unknown) => v is T): T[] {
|
||||
const isProvenancePoint = (v: unknown): v is ProvenancePoint =>
|
||||
isRecord(v) && isString(v.chatId) && isString(v.messageId);
|
||||
|
||||
const isClusterLabeling = (v: unknown): v is ClusterLabeling =>
|
||||
isRecord(v) &&
|
||||
isString(v.id) &&
|
||||
isString(v.label) &&
|
||||
Array.isArray(v.core) &&
|
||||
v.core.every(isString) &&
|
||||
isString(v.createdAt);
|
||||
|
||||
const isGraphNode = (v: unknown): v is GraphNode =>
|
||||
isRecord(v) &&
|
||||
isString(v.id) &&
|
||||
@@ -157,4 +174,171 @@ export const graphStore: GraphStore = {
|
||||
readCheckpoints(): SessionCheckpoint[] {
|
||||
return readValid(LEDGER_FILES.sessions, isCheckpoint);
|
||||
},
|
||||
|
||||
// ── cluster labelings (config.LEDGER_FILES.clusterLabels) ──
|
||||
|
||||
/** Idempotent by id: the id is a hash of the sorted core, so re-appending
|
||||
* the same member set (re-naming races, idempotent re-runs) is a no-op. */
|
||||
appendClusterLabel(labeling: ClusterLabeling): void {
|
||||
if (this.readClusterLabels().some((l) => l.id === labeling.id)) return;
|
||||
appendLine(LEDGER_FILES.clusterLabels, labeling);
|
||||
},
|
||||
readClusterLabels(): ClusterLabeling[] {
|
||||
return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling);
|
||||
},
|
||||
|
||||
// ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ──
|
||||
|
||||
/** Upsert: keeps the existing record's usage log when re-importing. */
|
||||
upsertSource(source: NotesSource): void {
|
||||
const all = readSourceMap();
|
||||
const prev = all[source.id];
|
||||
all[source.id] = prev ? { ...source, extractions: prev.extractions } : source;
|
||||
writeSourceMap(all);
|
||||
},
|
||||
readSources(): NotesSource[] {
|
||||
return Object.values(readSourceMap());
|
||||
},
|
||||
/** Idempotent per startedAt — same discipline as chats.appendExtractionUsage. */
|
||||
appendSourceUsage(id: string, entry: ExtractionRunUsage): void {
|
||||
const all = readSourceMap();
|
||||
const source = all[id];
|
||||
if (!source) return; // no such source — nothing to bill against
|
||||
source.extractions = source.extractions?.some((r) => r.startedAt === entry.startedAt)
|
||||
? source.extractions
|
||||
: [...(source.extractions ?? []), entry];
|
||||
writeSourceMap(all);
|
||||
},
|
||||
|
||||
// ── the layout ledger (config.POSITIONS_FILE — a JSON map) ──
|
||||
readPositions(): Record<string, Position> {
|
||||
return readPositionMap();
|
||||
},
|
||||
writePositions(positions: Record<string, Position>): void {
|
||||
writePositionMap(positions);
|
||||
},
|
||||
readEmbeddings(): EmbeddingsLedger {
|
||||
return readEmbeddingsLedger();
|
||||
},
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void {
|
||||
writeEmbeddingsLedger(ledger);
|
||||
},
|
||||
};
|
||||
|
||||
/** positions.json — { positions, updatedAt }. Read-modify-write, NOT
|
||||
* append-only: the settle is a full-graph computation, so it replaces the
|
||||
* whole map. The updatedAt stamp exists for future live-physics conflict
|
||||
* handling (a browser commit must be able to detect "the graph changed
|
||||
* underneath me"); reads just return the map. */
|
||||
function readPositionMap(): Record<string, Position> {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(POSITIONS_FILE, "utf-8");
|
||||
} catch {
|
||||
return {}; // no ledger yet — nothing has ever settled
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!isRecord(parsed) || !isRecord(parsed.positions)) return {};
|
||||
const out: Record<string, Position> = {};
|
||||
for (const [id, value] of Object.entries(parsed.positions)) {
|
||||
if (isPosition(value)) out[id] = { x: value.x, y: value.y };
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function writePositionMap(positions: Record<string, Position>): void {
|
||||
fs.mkdirSync(path.dirname(POSITIONS_FILE), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
POSITIONS_FILE,
|
||||
JSON.stringify({ positions, updatedAt: new Date().toISOString() }, null, "\t"),
|
||||
"utf-8",
|
||||
);
|
||||
}
|
||||
|
||||
/** embeddings.json — the whole EmbeddingsLedger at once. Missing/corrupt
|
||||
* file reads as an EMPTY ledger (model "") so consumers see "nothing
|
||||
* embedded yet" rather than an error; the embedder backfills. */
|
||||
function readEmbeddingsLedger(): EmbeddingsLedger {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(EMBEDDINGS_FILE, "utf-8");
|
||||
} catch {
|
||||
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!isRecord(parsed)) return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
const model = typeof parsed.model === "string" ? parsed.model : "";
|
||||
const dim = typeof parsed.dim === "number" && Number.isFinite(parsed.dim) ? parsed.dim : 0;
|
||||
const vectors: Record<string, number[]> = {};
|
||||
if (isRecord(parsed.vectors)) {
|
||||
for (const [id, value] of Object.entries(parsed.vectors)) {
|
||||
if (
|
||||
Array.isArray(value) &&
|
||||
value.length > 0 &&
|
||||
value.every((x) => typeof x === "number" && Number.isFinite(x))
|
||||
) {
|
||||
vectors[id] = value as number[];
|
||||
}
|
||||
}
|
||||
}
|
||||
const clusters: Array<{ members: string[] }> = [];
|
||||
if (Array.isArray(parsed.clusters)) {
|
||||
for (const c of parsed.clusters) {
|
||||
if (
|
||||
isRecord(c) &&
|
||||
Array.isArray(c.members) &&
|
||||
c.members.length > 0 &&
|
||||
c.members.every((m) => typeof m === "string")
|
||||
) {
|
||||
clusters.push({ members: [...(c.members as string[])] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { model, dim, vectors, clusters, updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "" };
|
||||
} catch {
|
||||
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
}
|
||||
}
|
||||
|
||||
function writeEmbeddingsLedger(ledger: EmbeddingsLedger): void {
|
||||
fs.mkdirSync(path.dirname(EMBEDDINGS_FILE), { recursive: true });
|
||||
fs.writeFileSync(EMBEDDINGS_FILE, JSON.stringify(ledger, null, "\t"), "utf-8");
|
||||
}
|
||||
|
||||
const isPosition = (v: unknown): v is Position =>
|
||||
isRecord(v) && typeof v.x === "number" && Number.isFinite(v.x) && typeof v.y === "number" && Number.isFinite(v.y);
|
||||
|
||||
/** sources.json — { id: NotesSource }. Read-modify-write, NOT append-only:
|
||||
* the usage log lives inside the record (like a chat's JSON file). */
|
||||
function readSourceMap(): Record<string, NotesSource> {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(SOURCES_FILE, "utf-8");
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!isRecord(parsed)) return {};
|
||||
const out: Record<string, NotesSource> = {};
|
||||
for (const [id, value] of Object.entries(parsed)) {
|
||||
if (isNotesSource(value)) out[id] = value;
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function writeSourceMap(map: Record<string, NotesSource>): void {
|
||||
fs.mkdirSync(path.dirname(SOURCES_FILE), { recursive: true });
|
||||
fs.writeFileSync(SOURCES_FILE, JSON.stringify(map, null, "\t"), "utf-8");
|
||||
}
|
||||
|
||||
const isNotesSource = (v: unknown): v is NotesSource =>
|
||||
isRecord(v) && isString(v.id) && isString(v.filePath) && isString(v.importedAt);
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* Server-side persistence for the graph render settings (the settings-panel
|
||||
* controls): data/render-settings.json — read-modify-write, NOT a ledger.
|
||||
*
|
||||
* The shape, defaults, and sanitizer live in src/core/render-settings.ts
|
||||
* (pure, shared with the client); this module only moves bytes. Whatever
|
||||
* comes off disk is sanitized by the caller on load — and we sanitize on
|
||||
* write too, so the file can never hold an out-of-range value that would
|
||||
* silently shift a slider.
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { RENDER_SETTINGS_FILE } from "../../config.ts";
|
||||
import { DEFAULT_RENDER_SETTINGS, sanitizeRenderSettings, type GraphRenderSettings } from "../core/render-settings.ts";
|
||||
|
||||
/** The persisted settings, or null when no file exists yet (first boot). */
|
||||
export function readRenderSettings(): GraphRenderSettings | null {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(RENDER_SETTINGS_FILE, "utf-8");
|
||||
} catch {
|
||||
return null; // never booted / unreadable → caller uses defaults
|
||||
}
|
||||
try {
|
||||
return sanitizeRenderSettings(JSON.parse(raw));
|
||||
} catch {
|
||||
return null; // corrupt → defaults, never crash the view
|
||||
}
|
||||
}
|
||||
|
||||
/** Persist settings (sanitized). Returns what was written. */
|
||||
export function writeRenderSettings(raw: unknown): GraphRenderSettings {
|
||||
const settings = sanitizeRenderSettings(raw);
|
||||
fs.mkdirSync(path.dirname(RENDER_SETTINGS_FILE), { recursive: true });
|
||||
fs.writeFileSync(RENDER_SETTINGS_FILE, JSON.stringify(settings, null, "\t"), "utf-8");
|
||||
return settings;
|
||||
}
|
||||
|
||||
export { DEFAULT_RENDER_SETTINGS };
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Community detection tests (vision §1, DECIDED 2026-09-13) — pure
|
||||
* structural Louvain over follows+supports, deterministic via seed.
|
||||
* Assertions are on SHAPE and MEMBER SETS, never community indices
|
||||
* (indices reshuffle; member sets are what labelings key to).
|
||||
*/
|
||||
import { strict as assert } from "node:assert";
|
||||
import { describe, test } from "node:test";
|
||||
import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../src/core/clusters.ts";
|
||||
import { clusterHue } from "../components/cluster-geometry.ts";
|
||||
import type { ClusterLabeling } from "../src/core/contracts.ts";
|
||||
|
||||
const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`);
|
||||
|
||||
/** A clique on the given ids. */
|
||||
const clique = (nodeIds: string[]) =>
|
||||
nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b })));
|
||||
|
||||
const labeling = (id: string, core: string[], createdAt = "2026-09-01T00:00:00.000Z"): ClusterLabeling => ({
|
||||
id,
|
||||
label: `label-${id}`,
|
||||
core,
|
||||
createdAt,
|
||||
});
|
||||
|
||||
describe("matchClusterLabelings", () => {
|
||||
test("claims a labeling while at least half its core survives in the community", () => {
|
||||
const community: Community = { members: ["a", "b", "c", "d", "e"] };
|
||||
const records = [labeling("clu_1", ["a", "b", "x", "y"])]; // 2 of 4 survive — exactly half
|
||||
const [claim] = matchClusterLabelings([community], records);
|
||||
assert.equal(claim?.id, "clu_1");
|
||||
// One fewer survivor: under half → relabel.
|
||||
const [dead] = matchClusterLabelings([{ members: ["a", "c", "d", "e", "f"] }], records);
|
||||
assert.equal(dead, null);
|
||||
});
|
||||
|
||||
test("single successor: a split core keeps its label in only one community", () => {
|
||||
const records = [labeling("clu_1", ["a", "b", "c", "d"])];
|
||||
const left: Community = { members: ["a", "b", "z"] }; // 2 of 4 — half
|
||||
const right: Community = { members: ["c", "d", "w"] }; // 2 of 4 — half
|
||||
// Tie: larger overlap wins; equal → both eligible, greedy takes first come.
|
||||
const claims = matchClusterLabelings([left, right], records);
|
||||
const claimed = claims.filter((c) => c !== null);
|
||||
assert.equal(claimed.length, 1);
|
||||
assert.equal(claimed[0]?.id, "clu_1");
|
||||
});
|
||||
|
||||
test("greedy: the best-overlapping community wins the record", () => {
|
||||
const records = [labeling("clu_1", ["a", "b", "c", "d"])];
|
||||
const weak: Community = { members: ["a", "b", "q", "r", "s"] }; // 2/4
|
||||
const strong: Community = { members: ["a", "b", "c", "x", "y"] }; // 3/4
|
||||
const claims = matchClusterLabelings([weak, strong], records);
|
||||
assert.equal(claims[0], null);
|
||||
assert.equal(claims[1]?.id, "clu_1");
|
||||
});
|
||||
|
||||
test("one record per community; newest record breaks ties", () => {
|
||||
const oldR = labeling("clu_old", ["a", "b"], "2026-09-01T00:00:00.000Z");
|
||||
const newR = labeling("clu_new", ["a", "b"], "2026-09-02T00:00:00.000Z");
|
||||
const community: Community = { members: ["a", "b", "c"] };
|
||||
const [claim] = matchClusterLabelings([community], [oldR, newR]);
|
||||
assert.equal(claim?.id, "clu_new");
|
||||
});
|
||||
|
||||
test("no labelings: every community unclaimed; empty communities array ok", () => {
|
||||
assert.deepEqual(matchClusterLabelings([{ members: ["a", "b"] }], []), [null]);
|
||||
assert.deepEqual(matchClusterLabelings([], [labeling("clu_1", ["a"])]), []);
|
||||
});
|
||||
|
||||
test("MIN_CLUSTER_SIZE is 3 (the render/naming threshold)", () => {
|
||||
assert.equal(MIN_CLUSTER_SIZE, 3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("clusterHue", () => {
|
||||
test("is stable per id and spreads", () => {
|
||||
// Same id → same hue, always; different ids → usually different hues.
|
||||
assert.equal(clusterHue("cluster-a"), clusterHue("cluster-a"));
|
||||
const hues = new Set<number>();
|
||||
for (let i = 0; i < 40; i++) hues.add(clusterHue(`cluster-${i}`));
|
||||
assert.ok(hues.size > 20, `expected spread, got ${hues.size} distinct hues for 40 ids`);
|
||||
for (const h of hues) assert.ok(h >= 0 && h < 360);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -27,15 +27,20 @@ import type { UIMessage } from "ai";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
import type {
|
||||
AuditResult,
|
||||
ClusterLabeling,
|
||||
ExtractionRunUsage,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
LedgerEvent,
|
||||
NodeMapping,
|
||||
NotesSource,
|
||||
SessionCheckpoint,
|
||||
SessionFindings,
|
||||
} from "../src/core/contracts.ts";
|
||||
import type { GraphStore } from "../src/core/ports.ts";
|
||||
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
|
||||
import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
|
||||
import { semanticClusters } from "../src/core/semantic-clusters.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
|
||||
@@ -60,6 +65,16 @@ function memoryLedgers() {
|
||||
const events: LedgerEvent[] = [];
|
||||
const mappings: NodeMapping[] = [];
|
||||
const checkpoints: SessionCheckpoint[] = [];
|
||||
const sources = new Map<string, NotesSource>();
|
||||
const clusterLabels: ClusterLabeling[] = [];
|
||||
const positions = new Map<string, { x: number; y: number }>();
|
||||
const embeddings = {
|
||||
model: "",
|
||||
dim: 0,
|
||||
updatedAt: "",
|
||||
vectors: new Map<string, number[]>(),
|
||||
clusters: new Array<Array<string>>(),
|
||||
};
|
||||
return {
|
||||
appendNode: (n: GraphNode) => void nodes.push(n),
|
||||
readNodes: () => nodes.map((n) => ({ ...n })),
|
||||
@@ -71,6 +86,40 @@ function memoryLedgers() {
|
||||
readMappings: () => mappings.map((m) => ({ ...m })),
|
||||
appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c),
|
||||
readCheckpoints: () => checkpoints.map((c) => ({ ...c })),
|
||||
upsertSource: (s: NotesSource) => {
|
||||
// Mirrors the real store: an upsert KEEPS the existing usage log.
|
||||
const prev = sources.get(s.id);
|
||||
sources.set(s.id, { ...s, extractions: [...(prev?.extractions ?? [])] });
|
||||
},
|
||||
readSources: () => [...sources.values()].map((s) => ({ ...s, extractions: [...(s.extractions ?? [])] })),
|
||||
appendSourceUsage: (id: string, entry: ExtractionRunUsage) => {
|
||||
const source = sources.get(id);
|
||||
if (!source) return;
|
||||
if (!source.extractions?.some((r) => r.startedAt === entry.startedAt)) {
|
||||
source.extractions = [...(source.extractions ?? []), entry];
|
||||
}
|
||||
},
|
||||
// Cluster labelings — mirrors the real store: append dedupes by id.
|
||||
appendClusterLabel: (l: ClusterLabeling) => {
|
||||
if (!clusterLabels.some((x) => x.id === l.id)) clusterLabels.push(l);
|
||||
},
|
||||
readClusterLabels: () => clusterLabels.map((l) => ({ ...l, core: [...l.core] })),
|
||||
// The layout ledger — settleGraphLayout writes it after each merge.
|
||||
readPositions: () => Object.fromEntries([...positions].map(([id, p]) => [id, { ...p }])),
|
||||
writePositions: (next) => {
|
||||
positions.clear();
|
||||
for (const [id, p] of Object.entries(next)) positions.set(id, { x: p.x, y: p.y });
|
||||
},
|
||||
// The embedding ledger — the embedder writes it after a merge.
|
||||
readEmbeddings: () => ({ model: embeddings.model, dim: embeddings.dim, vectors: Object.fromEntries([...embeddings.vectors].map(([id, v]) => [id, [...v]])), clusters: [...embeddings.clusters.map((c) => ({ members: [...c] }))], updatedAt: embeddings.updatedAt }),
|
||||
writeEmbeddings: (next) => {
|
||||
embeddings.model = next.model;
|
||||
embeddings.dim = next.dim;
|
||||
embeddings.updatedAt = next.updatedAt;
|
||||
embeddings.vectors.clear();
|
||||
for (const [id, v] of Object.entries(next.vectors)) embeddings.vectors.set(id, [...v]);
|
||||
embeddings.clusters = next.clusters.map((c) => [...c.members]);
|
||||
},
|
||||
} satisfies GraphStore;
|
||||
}
|
||||
|
||||
@@ -288,8 +337,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: submitModel("submit_findings", FINDINGS),
|
||||
auditor: submitModel("submit_review", verdict),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const stats = await pipeline.runExtraction(chatId);
|
||||
|
||||
@@ -328,8 +379,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: submitModel("submit_findings", FINDINGS),
|
||||
auditor: silentModel(), // never submits the review
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/);
|
||||
assert.equal(ledgers.readNodes().length, 0);
|
||||
@@ -338,8 +391,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const failingExtractor = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: silentModel(),
|
||||
auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/);
|
||||
assert.equal(ledgers.readCheckpoints().length, 0);
|
||||
@@ -368,10 +423,12 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
// Each stage's first generate is the observation point: the
|
||||
// registry must already carry this chat at the right phase.
|
||||
extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())),
|
||||
auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await pipeline.runExtraction(chatId);
|
||||
assert.deepEqual(seen, ["extracting", "auditing"]);
|
||||
@@ -400,12 +457,14 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
// During the auditor's generate, the registry must already carry
|
||||
// the extractor's fully accumulated live usage (2 steps × 10/5).
|
||||
extractor: submitModel("submit_findings", FINDINGS),
|
||||
auditor: submitModel("submit_review", verdict, () =>
|
||||
usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens),
|
||||
),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await pipeline.runExtraction(chatId);
|
||||
|
||||
@@ -440,8 +499,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: hangingModel(), // hangs until aborted — no step finishes
|
||||
auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const runPromise = pipeline.runExtraction(chatId);
|
||||
|
||||
@@ -490,8 +551,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: gatedExtractor,
|
||||
auditor: silentModel(),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
|
||||
const first = pipeline.runExtraction(chatId);
|
||||
@@ -507,9 +570,224 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: silentModel(),
|
||||
auditor: silentModel(),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── the notes front door (format-agnostic import) ────────────────────────────
|
||||
|
||||
/** Raw personal notes — arbitrary format, no session, no message ids. */
|
||||
const NOTES_TEXT = `> [!quote] YOU
|
||||
> teach me vim
|
||||
|
||||
> [!abstract] PI
|
||||
> Vim always starts in Normal mode — keys are commands, not text.`;
|
||||
|
||||
/** What the notes-mode extractor submits: same shapes minus attribution,
|
||||
* and messages always empty (the file is the provenance). */
|
||||
const NOTES_FINDINGS = {
|
||||
propositions: [
|
||||
{
|
||||
id: "p1",
|
||||
name: "Vim normal mode",
|
||||
statement: "Vim starts in Normal mode, where keys are commands.",
|
||||
aliases: [],
|
||||
messages: [],
|
||||
evidence: ["Vim always starts in Normal mode — keys are commands, not text."],
|
||||
understanding: "solid",
|
||||
},
|
||||
],
|
||||
edges: [],
|
||||
} as const;
|
||||
|
||||
const NOTES_VERDICT = {
|
||||
propositions: [NOTES_FINDINGS.propositions[0]],
|
||||
edges: [],
|
||||
dropped: [],
|
||||
} as const;
|
||||
|
||||
const CLUSTER_FINDINGS = {
|
||||
propositions: [
|
||||
{ id: "p1", name: "Modal editing", statement: "Vim edits text through modal command modes.", aliases: [], messages: [], evidence: ["Normal mode keys are commands."], understanding: "solid" },
|
||||
{ id: "p2", name: "Verb-object grammar", statement: "Vim commands compose as verb and object.", aliases: [], messages: [], evidence: ["d then motion deletes."], understanding: "solid" },
|
||||
{ id: "p3", name: "Operator-pending", statement: "Vim enters operator-pending mode between verb and object.", aliases: [], messages: [], evidence: ["After d, a motion is expected."], understanding: "solid" },
|
||||
],
|
||||
edges: [
|
||||
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
|
||||
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
|
||||
],
|
||||
} as const;
|
||||
|
||||
const CLUSTER_VERDICT = {
|
||||
propositions: CLUSTER_FINDINGS.propositions,
|
||||
edges: CLUSTER_FINDINGS.edges,
|
||||
dropped: [],
|
||||
} as const;
|
||||
|
||||
/** Test seam for the embedder: fakes the post-merge embedding batch —
|
||||
* one identical unit vector per node → semanticClusters merges them all
|
||||
* into ONE topic — then persists the partition like the real embedder. */
|
||||
const fakeEmbedder = async (store: GraphStore) => {
|
||||
const seen = new Set<string>();
|
||||
const ids: string[] = [];
|
||||
for (const n of store.readNodes()) {
|
||||
if (seen.has(n.id)) continue;
|
||||
seen.add(n.id);
|
||||
ids.push(n.id);
|
||||
}
|
||||
const vectors = Object.fromEntries(ids.map((id) => [id, [1, 0, 0]]));
|
||||
store.writeEmbeddings({
|
||||
model: "test/embedder",
|
||||
dim: 3,
|
||||
vectors,
|
||||
clusters: semanticClusters(vectors),
|
||||
updatedAt: "2026-09-13T00:00:00.000Z",
|
||||
});
|
||||
};
|
||||
|
||||
describe("cluster naming (the post-extraction garnish)", () => {
|
||||
it("names a new community after a notes merge — one record, claimed on the next refresh", async () => {
|
||||
const ledgers = memoryLedgers();
|
||||
const namerCalls: number[] = [];
|
||||
const labeler = submitModel("submit_labels", { labels: [{ key: "c0", label: "Vim modal grammar" }] }, () => namerCalls.push(1));
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
|
||||
auditor: submitModel("submit_review", CLUSTER_VERDICT),
|
||||
labeler,
|
||||
});
|
||||
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes, any format" });
|
||||
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 });
|
||||
// The refresh is fire-and-forget inside the pipeline — settle it.
|
||||
await clusterNamingSettled();
|
||||
assert.equal(namerCalls.length, 1, "ONE singular naming call");
|
||||
const records = ledgers.readClusterLabels();
|
||||
assert.equal(records.length, 1);
|
||||
assert.equal(records[0]!.label, "Vim modal grammar");
|
||||
assert.equal(records[0]!.core.length, 3);
|
||||
assert.match(records[0]!.id, /^clu_/);
|
||||
// Second refresh: core-survival claims it — nothing new to name.
|
||||
const again = await refreshClusterLabels({ graphStore: ledgers, engine: labeler });
|
||||
assert.equal(again.named, 0);
|
||||
assert.equal(again.appended, 0);
|
||||
assert.ok(again.claimed >= 1);
|
||||
assert.equal(ledgers.readClusterLabels().length, 1, "no duplicate records");
|
||||
});
|
||||
|
||||
it("naming failure never fails the run", async () => {
|
||||
const ledgers = memoryLedgers();
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
|
||||
auditor: submitModel("submit_review", CLUSTER_VERDICT),
|
||||
labeler: silentModel(), // never submits — the namer throws
|
||||
});
|
||||
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes" });
|
||||
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }, "the merge itself succeeded");
|
||||
await clusterNamingSettled();
|
||||
assert.equal(ledgers.readClusterLabels().length, 0, "nothing named, nothing appended");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the notes front door (importNotes — format-agnostic)", () => {
|
||||
it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
|
||||
const ledgers = memoryLedgers();
|
||||
const seen: { kind?: string; label?: string }[] = [];
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: submitModel("submit_findings", NOTES_FINDINGS),
|
||||
// During the auditor's call the registry must show the notes run.
|
||||
auditor: submitModel("submit_review", NOTES_VERDICT, () => {
|
||||
const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
|
||||
seen.push({ kind: run?.kind, label: run?.label });
|
||||
}),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||
|
||||
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
|
||||
// The run surfaced as a NOTES run labeled with the file path.
|
||||
assert.deepEqual(seen, [{ kind: "notes", label: "/vault/learn vim.md" }]);
|
||||
assert.equal(extractionStatuses().length, 0, "the registry clears when the run ends");
|
||||
|
||||
// The source record: deterministic src id, the file path, the usage log.
|
||||
const source = ledgers.readSources()[0]!;
|
||||
assert.match(source.id, /^src_/);
|
||||
assert.equal(source.filePath, "/vault/learn vim.md");
|
||||
assert.ok(Date.parse(source.importedAt));
|
||||
assert.equal(source.extractions?.length, 1, "the run's usage persisted into the source record");
|
||||
assert.equal(source.extractions![0]!.stats?.nodes, 1);
|
||||
assert.ok(source.extractions![0]!.stages.extractor.steps.length > 0, "faux usage captured");
|
||||
|
||||
// File-scoped provenance: the mapping line carries the source id;
|
||||
// the node carries NO message anchors.
|
||||
const node = ledgers.readNodes()[0]!;
|
||||
assert.equal(node.statement, "Vim starts in Normal mode, where keys are commands.");
|
||||
assert.deepEqual(node.provenance, []);
|
||||
assert.deepEqual(ledgers.readMappings(), [{ chatId: source.id, sourceId: "p1", graphNodeId: node.id }]);
|
||||
|
||||
// One observation event, zero quiz events — outcomes inside notes
|
||||
// surface only through understanding observations (user-decided).
|
||||
const events = ledgers.readEvents();
|
||||
assert.deepEqual(
|
||||
events.map((e) => [e.kind, e.chatId, e.ref, e.note]),
|
||||
[["observation", source.id, "obs:p1", "solid"]],
|
||||
);
|
||||
// The checkpoint stamps the import.
|
||||
assert.equal(ledgers.readCheckpoints()[0]!.chatId, source.id);
|
||||
});
|
||||
|
||||
it("re-importing the same file is idempotent: same source identity, refs dedupe, usage history grows", async () => {
|
||||
const ledgers = memoryLedgers();
|
||||
const mk = () =>
|
||||
createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: submitModel("submit_findings", NOTES_FINDINGS),
|
||||
auditor: submitModel("submit_review", NOTES_VERDICT),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||
assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
|
||||
|
||||
const second = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||
assert.deepEqual(second, { nodes: 0, edges: 0, events: 0 }, "the same source id dedupes everything");
|
||||
|
||||
assert.equal(ledgers.readSources().length, 1, "one source identity — the path hashes to the same id");
|
||||
assert.equal(ledgers.readSources()[0]!.extractions?.length, 2, "both runs billed their usage");
|
||||
assert.equal(ledgers.readNodes().length, 1, "no duplicate node");
|
||||
});
|
||||
|
||||
it("aborts like a chat run: true cancellation, nothing merged, partial usage in the source record", async () => {
|
||||
const ledgers = memoryLedgers();
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
embedder: fakeEmbedder,
|
||||
extractor: hangingModel(),
|
||||
auditor: submitModel("submit_review", NOTES_VERDICT),
|
||||
labeler: submitModel("submit_labels", { labels: [] }),
|
||||
});
|
||||
const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||
|
||||
const srcId = extractionStatuses().find((r) => r.chatId.startsWith("src_"))!.chatId;
|
||||
assert.equal(abortExtraction(srcId), true);
|
||||
assert.deepEqual(await runPromise, { nodes: 0, edges: 0, events: 0 });
|
||||
assert.equal(ledgers.readNodes().length, 0, "nothing merged");
|
||||
assert.equal(ledgers.readCheckpoints().length, 0, "nothing checkpointed");
|
||||
assert.equal(ledgers.readSources()[0]!.extractions?.length, 1, "partial usage still persisted");
|
||||
assert.equal(ledgers.readSources()[0]!.extractions![0]!.stats, undefined, "never merged → no stats");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
/**
|
||||
* Pure-helper tests for the hand-rolled canvas renderer
|
||||
* (components/graph-canvas.ts). The renderer class itself is DOM-bound
|
||||
* (canvas, rAF, ResizeObserver) and is verified live in the browser; the
|
||||
* math it runs on every frame must be exact, so it is pinned here.
|
||||
*
|
||||
* The camera-ratio convention is the load-bearing contract with the
|
||||
* persisted settings: sigma's ratio (1 = fit, SMALLER = zoomed in) is kept
|
||||
* verbatim so stored labelZoom values survive the renderer swap.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it as test } from "node:test";
|
||||
import {
|
||||
arrowGeometry,
|
||||
boxBlurRound,
|
||||
fitTransform,
|
||||
labelAlpha,
|
||||
pickNearest,
|
||||
zoomRatioOf,
|
||||
LABELS_FADE_SPAN,
|
||||
} from "../components/graph-canvas.ts";
|
||||
import {
|
||||
sanitizePhysicsTuning,
|
||||
sanitizeRegionTuning,
|
||||
DEFAULT_PHYSICS_TUNING,
|
||||
DEFAULT_REGION_TUNING,
|
||||
REGION_BITMAP_OPTIONS,
|
||||
} from "../src/core/render-settings.ts";
|
||||
|
||||
describe("zoomRatioOf (the sigma-compatible camera ratio)", () => {
|
||||
test("1 at fit zoom, smaller when zoomed in, larger when zoomed out", () => {
|
||||
assert.equal(zoomRatioOf(2, 2), 1); // at fit
|
||||
assert.equal(zoomRatioOf(2, 4), 0.5); // zoomed in 2x -> ratio halves
|
||||
assert.equal(zoomRatioOf(2, 1), 2); // zoomed out 2x -> ratio doubles
|
||||
});
|
||||
});
|
||||
|
||||
describe("labelAlpha (the single zoom gate)", () => {
|
||||
test("fully visible at/below the threshold, faded out after the span", () => {
|
||||
assert.equal(labelAlpha(0.5, 0.5), 1); // at the threshold
|
||||
assert.equal(labelAlpha(0.25, 0.5), 1); // zoomed further in: stays 1
|
||||
assert.equal(labelAlpha(0.75, 0.5), 0.5); // mid-fade
|
||||
assert.equal(labelAlpha(1.0, 0.5), 0); // one span out: gone
|
||||
assert.equal(labelAlpha(2.0, 0.5), 0); // far out: clamped
|
||||
});
|
||||
|
||||
test("node labels and cluster labels cross-fade over the exact same span", () => {
|
||||
// The renderer draws cluster labels at 1 - labelAlpha; the two must
|
||||
// always sum to 1 within the fade and never coexist at full opacity.
|
||||
for (const ratio of [0, 0.25, 0.5, 0.6, 0.75, 0.9, 1, 2, 4]) {
|
||||
const node = labelAlpha(ratio, 0.5);
|
||||
const cluster = 1 - node;
|
||||
assert.ok(node + cluster > 0.999 && node + cluster < 1.001, `sum at ratio ${ratio}`);
|
||||
assert.ok(node <= 1 && cluster <= 1, `clamped at ratio ${ratio}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("the span constant is unchanged from the sigma era (stored settings depend on it)", () => {
|
||||
assert.equal(LABELS_FADE_SPAN, 0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fitTransform", () => {
|
||||
test("centers the bbox and respects padding", () => {
|
||||
// A 100x50 graph in a 500x300 viewport, 40px padding:
|
||||
// available 420x220 -> k = min(4.2, 4.4) = 4.2
|
||||
const t = fitTransform({ minX: -50, maxX: 50, minY: -25, maxY: 25 }, 500, 300, 40);
|
||||
assert.ok(Math.abs(t.k - 4.2) < 1e-9, `k=${t.k}`);
|
||||
assert.ok(Math.abs(t.x - 250) < 1e-9, `x=${t.x}`); // centered
|
||||
assert.ok(Math.abs(t.y - 150) < 1e-9, `y=${t.y}`);
|
||||
// World center maps to viewport center:
|
||||
assert.ok(Math.abs(0 * t.k + t.x - 250) < 1e-9);
|
||||
});
|
||||
|
||||
test("never returns a non-positive scale for degenerate bboxes", () => {
|
||||
const t = fitTransform({ minX: 5, maxX: 5, minY: 5, maxY: 5 }, 100, 100, 10);
|
||||
assert.ok(t.k > 0 && Number.isFinite(t.k));
|
||||
});
|
||||
});
|
||||
|
||||
describe("arrowGeometry", () => {
|
||||
test("the tip lands exactly on the target disc edge", () => {
|
||||
const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5);
|
||||
assert.ok(g, "non-degenerate edge");
|
||||
assert.ok(Math.abs(g!.tip.x - 94) < 1e-9 && Math.abs(g!.tip.y) < 1e-9);
|
||||
});
|
||||
|
||||
test("the shaft starts outside the source disc", () => {
|
||||
const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5)!;
|
||||
assert.ok(Math.abs(g.shaftFrom.x - 6) < 1e-9 && Math.abs(g.shaftFrom.y) < 1e-9);
|
||||
// Shaft ends at the head base; head is drawn from base to tip.
|
||||
assert.ok(Math.abs(g.shaftTo.x - 89) < 1e-9);
|
||||
});
|
||||
|
||||
test("the head triangle is symmetric around the shaft axis (0.8x wideness)", () => {
|
||||
const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5)!;
|
||||
// halfW = 0.4 x headLength = 2; both base corners share x, mirrored y.
|
||||
assert.ok(Math.abs(Math.abs(g.left.y) - 2) < 1e-9 && Math.abs(Math.abs(g.right.y) - 2) < 1e-9);
|
||||
assert.ok(Math.abs(g.left.y + g.right.y) < 1e-9); // mirror
|
||||
assert.ok(Math.abs(g.left.x - g.right.x) < 1e-9);
|
||||
});
|
||||
|
||||
test("works at arbitrary angles (diagonal stays normalized)", () => {
|
||||
const a = { x: 0, y: 0 };
|
||||
const b = { x: 60, y: 80 }; // len 100, dir (0.6, 0.8)
|
||||
const g = arrowGeometry(a, b, 10, 10, 8)!;
|
||||
assert.ok(Math.abs(g.tip.x - (60 - 0.6 * 10)) < 1e-9 && Math.abs(g.tip.y - (80 - 0.8 * 10)) < 1e-9);
|
||||
});
|
||||
|
||||
test("null for coincident endpoints (never a NaN arrow)", () => {
|
||||
assert.equal(arrowGeometry({ x: 5, y: 5 }, { x: 5, y: 5 }, 6, 6, 5), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pickNearest (distance-check picking)", () => {
|
||||
const nodes = [
|
||||
{ id: "a", x: 10, y: 10 },
|
||||
{ id: "b", x: 50, y: 50 },
|
||||
{ id: "c", x: 90, y: 10 },
|
||||
];
|
||||
|
||||
test("picks the nearest node within the pad", () => {
|
||||
assert.equal(pickNearest(nodes, 12, 12, 16)?.id, "a");
|
||||
assert.equal(pickNearest(nodes, 48, 52, 16)?.id, "b");
|
||||
assert.equal(pickNearest(nodes, 88, 8, 16)?.id, "c");
|
||||
});
|
||||
|
||||
test("the picking pad extends the hitbox beyond the disc", () => {
|
||||
// 20px away, only reachable because the pad covers it.
|
||||
assert.equal(pickNearest(nodes, 28, 18, 25)?.id, "a");
|
||||
// Beyond the pad: nothing.
|
||||
assert.equal(pickNearest(nodes, 30, 40, 3), null);
|
||||
});
|
||||
|
||||
test("ties resolve deterministically (first scanned wins)", () => {
|
||||
const pair = [
|
||||
{ id: "first", x: 0, y: 0 },
|
||||
{ id: "second", x: 10, y: 0 },
|
||||
];
|
||||
assert.equal(pickNearest(pair, 5, 0, 10)?.id, "first");
|
||||
});
|
||||
|
||||
test("empty node list picks nothing", () => {
|
||||
assert.equal(pickNearest([], 0, 0, 100), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sanitizeRegionTuning (dev number inputs)", () => {
|
||||
test("defaults survive untouched; partial input keeps the rest", () => {
|
||||
const t = sanitizeRegionTuning(undefined);
|
||||
assert.deepEqual(t, DEFAULT_REGION_TUNING);
|
||||
const half = sanitizeRegionTuning({ blur: 7.2 });
|
||||
assert.equal(half.blur, 8); // snaps into the fixed set
|
||||
assert.equal(half.dilation, DEFAULT_REGION_TUNING.dilation);
|
||||
});
|
||||
|
||||
test("garbage clamps: NaN falls back, out-of-range clamps", () => {
|
||||
const t = sanitizeRegionTuning({ dilation: Number.NaN, blur: -5, bitmap: 99999, edgeAlpha: 42 });
|
||||
assert.equal(t.dilation, DEFAULT_REGION_TUNING.dilation); // NaN -> default
|
||||
assert.equal(t.blur, 2); // snaps into the fixed set
|
||||
assert.equal(t.bitmap, REGION_BITMAP_OPTIONS[REGION_BITMAP_OPTIONS.length - 1]); // 99999 snaps to the largest option
|
||||
assert.equal(t.edgeAlpha, 1);
|
||||
});
|
||||
|
||||
test("empty string / null / undefined values fall back per-field", () => {
|
||||
const t = sanitizeRegionTuning({ edgeAlpha: Number.NaN });
|
||||
assert.equal(t.edgeAlpha, DEFAULT_REGION_TUNING.edgeAlpha);
|
||||
});
|
||||
});
|
||||
|
||||
describe("boxBlurRound (premultiplied region blur)", () => {
|
||||
test("spreads a single opaque px into its neighborhood and conserves sum", () => {
|
||||
// 5x5 RGBA, one white opaque pixel at (2,2)
|
||||
const w = 5, h = 5;
|
||||
const a = new Float32Array(w * h * 4);
|
||||
const center = (2 * w + 2) * 4;
|
||||
a[center] = 255; a[center + 1] = 255; a[center + 2] = 255; a[center + 3] = 255;
|
||||
const tmp = new Float32Array(a.length);
|
||||
const b = new Float32Array(a.length);
|
||||
boxBlurRound(a, tmp, b, w, h, 1);
|
||||
// Alpha conservation: total alpha unchanged.
|
||||
let sumA = 0;
|
||||
for (let i = 3; i < b.length; i += 4) sumA += b[i]!;
|
||||
assert.ok(Math.abs(sumA - 255) < 1e-3, `alpha sum ${sumA}`);
|
||||
// The center dimmed, the 8 neighbors picked some up.
|
||||
assert.ok(b[center + 3]! < 255 && b[center + 3]! > 0);
|
||||
const up = ((1 * w) + 2) * 4 + 3;
|
||||
assert.ok(b[up]! > 0, "neighbor got alpha");
|
||||
// Far corner (4,4) untouched.
|
||||
assert.equal(b[(4 * 5 + 4) * 4 + 3], 0);
|
||||
});
|
||||
|
||||
test("radius 0 is a plain copy", () => {
|
||||
const a = new Float32Array([1, 2, 3, 4, 5, 6, 7, 8]);
|
||||
const tmp = new Float32Array(8);
|
||||
const b = new Float32Array(8);
|
||||
boxBlurRound(a, tmp, b, 1, 2, 0);
|
||||
assert.deepEqual([...b], [1, 2, 3, 4, 5, 6, 7, 8]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sanitizePhysicsTuning (physics controls)", () => {
|
||||
test("defaults survive untouched; partial input keeps the rest", () => {
|
||||
assert.deepEqual(sanitizePhysicsTuning(undefined), DEFAULT_PHYSICS_TUNING);
|
||||
const half = sanitizePhysicsTuning({ clusterPull: 0.6 });
|
||||
assert.equal(half.clusterPull, 0.6);
|
||||
assert.equal(half.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength);
|
||||
});
|
||||
|
||||
test("garbage clamps: NaN falls back, out-of-range clamps", () => {
|
||||
const t = sanitizePhysicsTuning({
|
||||
chargeStrength: Number.NaN,
|
||||
linkStrength: 99,
|
||||
linkDistance: -5,
|
||||
gravity: 7,
|
||||
clusterPull: -1,
|
||||
velocityDecay: 3,
|
||||
});
|
||||
assert.equal(t.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength);
|
||||
assert.equal(t.linkStrength, 4); // sweep-validated cap (8 explodes, 5 wobbles)
|
||||
assert.equal(t.linkDistance, 10);
|
||||
assert.equal(t.gravity, 0.48);
|
||||
assert.equal(t.clusterPull, 0);
|
||||
assert.equal(t.velocityDecay, 0.9);
|
||||
});
|
||||
});
|
||||
+153
-11
@@ -20,17 +20,27 @@ import { after, describe, it } from "node:test";
|
||||
import type { UIMessage } from "ai";
|
||||
import type { AuditResult } from "../src/core/contracts.ts";
|
||||
import type { QuestionAnswerData } from "../src/core/question-core.ts";
|
||||
import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
|
||||
import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { LEDGER_FILES, PATHS } from "../config.ts";
|
||||
import { LEDGER_FILES, PATHS, SOURCES_FILE, EMBEDDINGS_FILE } from "../config.ts";
|
||||
|
||||
/** The wired fs adapters, exactly what src/server/stores.ts passes in production. */
|
||||
const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps;
|
||||
|
||||
/** The session merge source for a chat — what the pipeline passes: the live
|
||||
* transcript + its updatedAt checkpoint stamp. (Notes imports pass
|
||||
* messages: null — see the notes-merger test below.) */
|
||||
function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: string } {
|
||||
const record = chatStore.getChat(chatId)!;
|
||||
return { messages: record.messages as UIMessage[], foldedAt: record.updatedAt };
|
||||
}
|
||||
|
||||
// ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ──────────────
|
||||
|
||||
const LEDGER_PATHS = Object.values(LEDGER_FILES);
|
||||
// EMBEDDINGS_FILE too: the semantic-cluster source (read by graphSnapshot)
|
||||
// — isolated like the rest so this suite never touches the real vectors.
|
||||
const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE, EMBEDDINGS_FILE];
|
||||
const backups: Array<{ file: string; bak: string }> = [];
|
||||
const createdChats: string[] = [];
|
||||
|
||||
@@ -233,7 +243,7 @@ function scopedGraph(chatIds: string[]) {
|
||||
|
||||
describe("the merger (mergeFindings)", () => {
|
||||
it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => {
|
||||
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps);
|
||||
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
|
||||
assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 });
|
||||
|
||||
const graph = scopedGraph([chat1.id]);
|
||||
@@ -302,7 +312,7 @@ describe("the merger (mergeFindings)", () => {
|
||||
});
|
||||
|
||||
it("is idempotent — re-merging the same verdicts appends nothing", () => {
|
||||
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps);
|
||||
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
|
||||
assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 });
|
||||
const graph = scopedGraph([chat1.id]);
|
||||
assert.equal(graph.nodes.length, 3);
|
||||
@@ -346,6 +356,7 @@ describe("the merger (mergeFindings)", () => {
|
||||
dropped: [],
|
||||
},
|
||||
graphDeps,
|
||||
sessionSource(chat2.id),
|
||||
);
|
||||
assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 });
|
||||
|
||||
@@ -363,6 +374,100 @@ describe("the merger (mergeFindings)", () => {
|
||||
assert.deepEqual(m, { chatId: chat2.id, sourceId: "q1", graphNodeId: alphaId });
|
||||
});
|
||||
|
||||
it("resolves imported-notes ids to their file paths in the snapshot (static provenance)", () => {
|
||||
// The notes front door in miniature: a source record + a merge under
|
||||
// its src id — the snapshot must label it with the file path (a
|
||||
// static entry), never as a chat session.
|
||||
graphStore.upsertSource({ id: "src_snapfixture", filePath: "/vault/learn vim.md", importedAt: "2026-09-13T00:00:00.000Z" });
|
||||
mergeFindings(
|
||||
"src_snapfixture",
|
||||
{
|
||||
propositions: [
|
||||
{
|
||||
id: "p1",
|
||||
name: "Vim registers",
|
||||
statement: "Vim registers map single keys to editing commands.",
|
||||
aliases: [],
|
||||
messages: [],
|
||||
evidence: ["fixture evidence"],
|
||||
understanding: "solid",
|
||||
},
|
||||
],
|
||||
edges: [],
|
||||
attribution: { quiz: {} },
|
||||
dropped: [],
|
||||
},
|
||||
graphDeps,
|
||||
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
|
||||
);
|
||||
const snap = graphSnapshot(graphDeps);
|
||||
const node = snap.nodes.find((n) => n.statement === "Vim registers map single keys to editing commands.")!;
|
||||
assert.equal(snap.fileSources["src_snapfixture"], "/vault/learn vim.md");
|
||||
assert.deepEqual(snap.nodeSessions[node.id], ["src_snapfixture"], "the file entry joins the node's sessions list");
|
||||
assert.equal(snap.sessions["src_snapfixture"], undefined, "a notes source is never a chat session");
|
||||
// The understanding observation joins under the source id like any session's.
|
||||
assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
|
||||
});
|
||||
|
||||
it("derives semantic clusters in graphSnapshot, joined with persisted labelings", () => {
|
||||
// Three propositions; the cluster comes from the EMBEDDING ledger
|
||||
// (semantic topics, DECIDED 2026-09-13) — the embedder persists the
|
||||
// partition, the snapshot just reads it. Here we play the embedder.
|
||||
const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const;
|
||||
const merge = mergeFindings(
|
||||
"src_clusterfixture",
|
||||
{
|
||||
propositions: tri.map((statement, i) => ({
|
||||
id: `p${i + 1}`,
|
||||
name: `Cluster node ${i + 1}`,
|
||||
statement,
|
||||
aliases: [],
|
||||
messages: [],
|
||||
evidence: ["fixture evidence"],
|
||||
})),
|
||||
edges: [
|
||||
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
|
||||
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
|
||||
{ from: "p1", to: "p3", type: "follows", messages: [], evidence: ["e"] },
|
||||
],
|
||||
attribution: { quiz: {} },
|
||||
dropped: [],
|
||||
},
|
||||
graphDeps,
|
||||
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
|
||||
);
|
||||
assert.equal(merge.nodes, 3);
|
||||
// The embedder's contract: recluster at the end of the batch and
|
||||
// PERSIST the partition in the embeddings ledger.
|
||||
const ids = tri.map((s) => getGraph(graphStore).nodes.find((n) => n.statement === s)!.id);
|
||||
graphStore.writeEmbeddings({
|
||||
model: "test/embedder",
|
||||
dim: 2,
|
||||
vectors: {},
|
||||
clusters: [{ members: ids }],
|
||||
updatedAt: "2026-09-13T00:00:00.000Z",
|
||||
});
|
||||
const snap = graphSnapshot(graphDeps);
|
||||
// Presentation threshold: every cluster is >= 3 members.
|
||||
assert.ok(snap.clusters.every((c) => c.members.length >= 3));
|
||||
// Our trio is exactly one cluster, unlabeled until the namer runs.
|
||||
const mine = snap.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap.nodes.find((n) => n.statement === s)!.id)));
|
||||
assert.ok(mine, "the 3-clique is a cluster");
|
||||
assert.equal(mine!.label, null);
|
||||
assert.match(mine!.id, /^clu_/);
|
||||
// Mint the labeling the namer would append — the next read joins it.
|
||||
graphStore.appendClusterLabel({
|
||||
id: clusterContentId(mine!.members),
|
||||
label: "Test region",
|
||||
core: mine!.members,
|
||||
createdAt: "2026-09-13T00:00:00.000Z",
|
||||
});
|
||||
const snap2 = graphSnapshot(graphDeps);
|
||||
const claimed = snap2.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap2.nodes.find((n) => n.statement === s)!.id)))!;
|
||||
assert.equal(claimed.label, "Test region");
|
||||
assert.equal(claimed.id, clusterContentId(mine!.members), "the claimed cluster carries the record's id");
|
||||
});
|
||||
|
||||
it("derives sessions and per-session understanding observations in graphSnapshot", () => {
|
||||
const snap = graphSnapshot(graphDeps);
|
||||
const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));
|
||||
@@ -408,17 +513,54 @@ describe("the merger (mergeFindings)", () => {
|
||||
dropped: [],
|
||||
},
|
||||
graphDeps,
|
||||
sessionSource(chat3.id),
|
||||
);
|
||||
assert.equal(stats.nodes, 1, "created — the merger does not match strings");
|
||||
assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2);
|
||||
});
|
||||
|
||||
it("returns zeros for unknown chats", () => {
|
||||
assert.deepEqual(mergeFindings("chat_does_not_exist", fixtureAudit(), graphDeps), {
|
||||
nodes: 0,
|
||||
edges: 0,
|
||||
events: 0,
|
||||
it("merges the notes front door: source id as chatId, no quiz events, checkpoint at import time", () => {
|
||||
// Raw personal notes carry no transcript — the merger gets no
|
||||
// message parts, so NO quiz events are rebuilt; the understanding
|
||||
// observation still lands, keyed to the source id.
|
||||
const stats = mergeFindings(
|
||||
"src_notesfixture",
|
||||
{
|
||||
propositions: [
|
||||
{
|
||||
id: "p1",
|
||||
name: "Vim normal mode",
|
||||
statement: "Vim starts in Normal mode, where keys are commands.",
|
||||
aliases: [],
|
||||
messages: [], // notes carry no message anchors — the file is the provenance
|
||||
evidence: ["fixture evidence"],
|
||||
understanding: "solid",
|
||||
},
|
||||
],
|
||||
edges: [],
|
||||
attribution: { quiz: {} }, // normalized by the notes pipeline — nothing to attribute
|
||||
dropped: [],
|
||||
},
|
||||
graphDeps,
|
||||
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
|
||||
);
|
||||
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
|
||||
|
||||
const node = getGraph(graphStore).nodes.find((n) => n.statement.startsWith("Vim starts"))!;
|
||||
assert.deepEqual(node.provenance, [], "no message anchors — the mapping line carries the source");
|
||||
assert.deepEqual(graphStore.readMappings().find((m) => m.chatId === "src_notesfixture"), {
|
||||
chatId: "src_notesfixture",
|
||||
sourceId: "p1",
|
||||
graphNodeId: node.id,
|
||||
});
|
||||
// The observation event is file-scoped; no quiz events exist.
|
||||
const events = graphStore.readEvents().filter((e) => e.chatId === "src_notesfixture");
|
||||
assert.deepEqual(
|
||||
events.map((e) => [e.kind, e.ref, e.note]),
|
||||
[["observation", "obs:p1", "solid"]],
|
||||
);
|
||||
// The checkpoint stamps the import time.
|
||||
assert.equal(graphStore.readCheckpoints().find((c) => c.chatId === "src_notesfixture")?.foldedAt, "2026-09-13T00:00:00.000Z");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -452,7 +594,7 @@ describe("the trigger helper (pendingQuizAnswers)", () => {
|
||||
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending");
|
||||
assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0);
|
||||
|
||||
mergeFindings(chat.id, fixtureAudit(), graphDeps);
|
||||
mergeFindings(chat.id, fixtureAudit(), graphDeps, sessionSource(chat.id));
|
||||
// call-s is unattributed here (fixtureAudit has no call-s attribution)
|
||||
// — but it still becomes an event, so nothing stays pending.
|
||||
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending");
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
/**
|
||||
* Option A — incremental anchored force layout (core/layout.ts). The
|
||||
* properties the user asked for: the layout EVOLVES (old placements stay
|
||||
* recognizable), new nodes land near their cluster, orphans collect on
|
||||
* the rim, and everything is deterministic.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { createGraphSimulation, incrementalLayout, inferenceDepths } from "../src/core/layout.ts";
|
||||
import { settleGraphLayout } from "../src/core/layout-settle.ts";
|
||||
import type { GraphNode, GraphEdge, Position } from "../src/core/contracts.ts";
|
||||
|
||||
const node = (id: string): GraphNode => ({
|
||||
id,
|
||||
name: id,
|
||||
statement: id,
|
||||
aliases: [],
|
||||
createdAt: "2026-09-13T00:00:00.000Z",
|
||||
provenance: [],
|
||||
});
|
||||
const edge = (from: string, to: string): GraphEdge => ({ from, to, type: "supports", provenance: [] });
|
||||
|
||||
const mean = (positions: Map<string, Position>, ids: string[]): { x: number; y: number } => ({
|
||||
x: ids.reduce((s, id) => s + positions.get(id)!.x, 0) / ids.length,
|
||||
y: ids.reduce((s, id) => s + positions.get(id)!.y, 0) / ids.length,
|
||||
});
|
||||
|
||||
describe("incrementalLayout", () => {
|
||||
it("bootstraps deterministically from an empty ledger", () => {
|
||||
const nodes = ["a", "b", "c", "d"].map(node);
|
||||
const edges = [edge("a", "b"), edge("b", "c"), edge("c", "d")];
|
||||
const first = incrementalLayout(nodes, edges, new Map());
|
||||
const second = incrementalLayout(nodes, edges, new Map());
|
||||
assert.equal(first.size, 4);
|
||||
for (const [id, p] of first) {
|
||||
assert.ok(Number.isFinite(p.x) && Number.isFinite(p.y), `finite coords for ${id}`);
|
||||
assert.deepEqual(p, second.get(id), `deterministic position for ${id}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("rest state is stable: a re-settle with nothing new barely moves nodes", () => {
|
||||
const nodes = ["a", "b", "c", "d", "e"].map(node);
|
||||
const edges = [edge("a", "b"), edge("b", "c"), edge("c", "d"), edge("d", "e"), edge("a", "e")];
|
||||
const rest = incrementalLayout(nodes, edges, new Map());
|
||||
const span =
|
||||
Math.max(...[...rest.values()].map((p) => p.x)) - Math.min(...[...rest.values()].map((p) => p.x));
|
||||
const again = incrementalLayout(nodes, edges, rest);
|
||||
let maxShift = 0;
|
||||
for (const [id, p] of rest) {
|
||||
const q = again.get(id)!;
|
||||
maxShift = Math.max(maxShift, Math.hypot(p.x - q.x, p.y - q.y));
|
||||
}
|
||||
assert.ok(maxShift < span * 0.1, `re-settle drift ${maxShift.toFixed(2)} should be < 10% of span ${span.toFixed(2)}`);
|
||||
});
|
||||
|
||||
it("evolves: an anchored node stays near its anchor while a new neighbor joins it", () => {
|
||||
const core = ["a", "b", "c"].map(node);
|
||||
const coreEdges = [edge("a", "b"), edge("b", "c")];
|
||||
const previous = incrementalLayout(core, coreEdges, new Map());
|
||||
const anchor = previous.get("b")!;
|
||||
const dist = Math.hypot;
|
||||
const nearby = ["d", "e"].map(node);
|
||||
const next = incrementalLayout(
|
||||
[...core, ...nearby],
|
||||
[...coreEdges, edge("b", "d"), edge("d", "e")],
|
||||
previous,
|
||||
);
|
||||
// The anchor keeps its neighborhood (bounded drift), and the new
|
||||
// nodes landed near it rather than being flung elsewhere.
|
||||
const coreSpanBefore =
|
||||
Math.max(...[...previous.values()].map((p) => p.x)) - Math.min(...[...previous.values()].map((p) => p.x));
|
||||
assert.ok(
|
||||
dist(next.get("b")!.x - anchor.x, next.get("b")!.y - anchor.y) < coreSpanBefore,
|
||||
"anchored node b stays near its previous position",
|
||||
);
|
||||
const centroid = mean(next, ["b", "d", "e"]);
|
||||
assert.ok(
|
||||
dist(next.get("d")!.x - centroid.x, next.get("d")!.y - centroid.y) < coreSpanBefore * 2,
|
||||
"new neighbor d landed near the cluster it arrived with",
|
||||
);
|
||||
});
|
||||
|
||||
it("rim: an orphan ends up outside the anchored core", () => {
|
||||
const core = ["a", "b", "c", "d", "e"].map(node);
|
||||
const coreEdges = [
|
||||
edge("a", "b"),
|
||||
edge("b", "c"),
|
||||
edge("c", "d"),
|
||||
edge("d", "e"),
|
||||
edge("a", "e"),
|
||||
edge("b", "d"),
|
||||
];
|
||||
const previous = incrementalLayout(core, coreEdges, new Map());
|
||||
const next = incrementalLayout([...core, node("orphan")], coreEdges, previous);
|
||||
const c = mean(previous, ["a", "b", "c", "d", "e"]);
|
||||
const coreMax = Math.max(
|
||||
...[...next.entries()].filter(([id]) => id !== "orphan").map(([, p]) => Math.hypot(p.x - c.x, p.y - c.y)),
|
||||
);
|
||||
const orphanR = Math.hypot(next.get("orphan")!.x - c.x, next.get("orphan")!.y - c.y);
|
||||
assert.ok(orphanR > coreMax, `orphan at r=${orphanR.toFixed(1)} should sit outside core max r=${coreMax.toFixed(1)}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("settleGraphLayout", () => {
|
||||
it("dedupes repeated node lines, runs, and writes the whole ledger", () => {
|
||||
const written: Record<string, Position>[] = [];
|
||||
const lines = [node("a"), node("b"), node("a")]; // "a" appended twice
|
||||
const store = {
|
||||
readNodes: () => lines.map((l) => ({ ...l })),
|
||||
readEdges: () => [edge("a", "b")],
|
||||
readPositions: () => ({} as Record<string, Position>),
|
||||
writePositions: (p: Record<string, Position>) => void written.push(p),
|
||||
readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }),
|
||||
};
|
||||
settleGraphLayout(store);
|
||||
assert.equal(written.length, 1);
|
||||
assert.deepEqual(Object.keys(written[0]!).sort(), ["a", "b"], "one position per UNIQUE node");
|
||||
});
|
||||
|
||||
it("does nothing on an empty graph", () => {
|
||||
const written: Record<string, Position>[] = [];
|
||||
settleGraphLayout({
|
||||
readNodes: () => [],
|
||||
readEdges: () => [],
|
||||
readPositions: () => ({} as Record<string, Position>),
|
||||
writePositions: (p) => void written.push(p),
|
||||
readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }),
|
||||
});
|
||||
assert.equal(written.length, 0);
|
||||
});
|
||||
});
|
||||
describe("clusterPull (the centroid force)", () => {
|
||||
it("compacts each cluster toward its OWN centroid; strength 0 = off", () => {
|
||||
// Two loose chains, far apart, edges only WITHIN each chain.
|
||||
const mk = (p: string, baseX: number) =>
|
||||
Array.from({ length: 8 }, (_, i) => ({ id: `${p}${i}`, x: baseX + i * 40, y: i * 15 }));
|
||||
const a = mk("a", -400);
|
||||
const b = mk("b", 400);
|
||||
const nodes = [...a, ...b];
|
||||
const edges = [
|
||||
...a.slice(0, -1).map((n, i) => ({ from: n.id, to: a[i + 1]!.id })),
|
||||
...b.slice(0, -1).map((n, i) => ({ from: n.id, to: b[i + 1]!.id })),
|
||||
];
|
||||
const clusters = [{ members: a.map((n) => n.id) }, { members: b.map((n) => n.id) }];
|
||||
|
||||
const spreadOf = (ns: { id: string; x: number; y: number }[]): number => {
|
||||
let sum = 0;
|
||||
for (const group of [ns.filter((n) => n.id[0] === "a"), ns.filter((n) => n.id[0] === "b")]) {
|
||||
const cx = group.reduce((s, n) => s + n.x, 0) / group.length;
|
||||
const cy = group.reduce((s, n) => s + n.y, 0) / group.length;
|
||||
sum += group.reduce((s, n) => s + Math.hypot(n.x - cx, n.y - cy), 0) / group.length;
|
||||
}
|
||||
return sum;
|
||||
};
|
||||
|
||||
const run = (strength: number) => {
|
||||
const sim = createGraphSimulation(
|
||||
nodes.map((n) => ({ ...n })),
|
||||
edges,
|
||||
{ clusterPull: { clusters, strength } },
|
||||
);
|
||||
for (let i = 0; i < 150; i++) sim.tick();
|
||||
return spreadOf(sim.nodes() as unknown as { id: string; x: number; y: number }[]);
|
||||
};
|
||||
const pulled = run(0.15);
|
||||
const unpulled = run(0);
|
||||
// The force compacts (well under the unpulled baseline)...
|
||||
assert.ok(pulled < unpulled * 0.8, `pulled spread ${pulled.toFixed(1)} should be well under ${unpulled.toFixed(1)}`);
|
||||
// ...and both groups survive as separate blobs (no merge).
|
||||
assert.ok(pulled > 0, "spread is finite/positive");
|
||||
});
|
||||
});
|
||||
|
||||
describe("inferenceDepths (SCC-safe ranking)", () => {
|
||||
const nodes = (ids: string[]) => ids.map((id) => ({ id }));
|
||||
|
||||
it("ranks a chain, a diamond takes the longest path, orphans are depth 0", () => {
|
||||
// chain a->b->c; diamond a->d, b->d, c->d (d reachable at 1 and 2 and 3)
|
||||
const ns = nodes(["a", "b", "c", "d", "loner"]);
|
||||
const edges = [
|
||||
{ from: "a", to: "b" },
|
||||
{ from: "b", to: "c" },
|
||||
{ from: "a", to: "d" },
|
||||
{ from: "b", to: "d" },
|
||||
{ from: "c", to: "d" },
|
||||
];
|
||||
const d = inferenceDepths(ns, edges);
|
||||
assert.equal(d.get("a"), 0);
|
||||
assert.equal(d.get("b"), 1);
|
||||
assert.equal(d.get("c"), 2);
|
||||
assert.equal(d.get("d"), 3); // longest path, not shortest
|
||||
assert.equal(d.get("loner"), 0);
|
||||
});
|
||||
|
||||
it("cycles share a depth (mutual support = equal footing)", () => {
|
||||
// x <-> y (a cycle), fed by s; z follows x.
|
||||
const ns = nodes(["s", "x", "y", "z"]);
|
||||
const edges = [
|
||||
{ from: "s", to: "x" },
|
||||
{ from: "x", to: "y" },
|
||||
{ from: "y", to: "x" },
|
||||
{ from: "x", to: "z" },
|
||||
];
|
||||
const d = inferenceDepths(ns, edges);
|
||||
assert.equal(d.get("s"), 0);
|
||||
assert.equal(d.get("x"), d.get("y"), "cycle members share a depth");
|
||||
assert.equal(d.get("x"), 1);
|
||||
assert.equal(d.get("z"), 2);
|
||||
// Deterministic across calls.
|
||||
assert.deepEqual([...inferenceDepths(ns, edges).entries()], [...d.entries()]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("depthAnchor force", () => {
|
||||
const nodes = (ids: string[]) => ids.map((id) => ({ id, x: 0, y: 0 }));
|
||||
|
||||
it("vertical: the premise lands below its conclusion", () => {
|
||||
const ns = nodes(["base", "derived"]);
|
||||
const edges = [{ from: "base", to: "derived" }];
|
||||
const sim = createGraphSimulation(ns, edges, {
|
||||
depthAnchor: { mode: "vertical", strength: 1.2, spacing: 100, center: { x: 0, y: 0 } },
|
||||
});
|
||||
for (let i = 0; i < 300; i++) sim.tick();
|
||||
const byId = new Map(sim.nodes().map((n) => [n.id, n]));
|
||||
const baseY = byId.get("base")!.y!;
|
||||
const derivedY = byId.get("derived")!.y!;
|
||||
assert.ok(baseY > derivedY, `base (${baseY.toFixed(0)}) should sit BELOW derived (${derivedY.toFixed(0)})`);
|
||||
assert.ok(
|
||||
baseY - derivedY > 60,
|
||||
`level separation should be substantial (got ${(baseY - derivedY).toFixed(0)})`,
|
||||
);
|
||||
});
|
||||
|
||||
it("radial: inference depth becomes radius", () => {
|
||||
const ns = nodes(["base", "mid", "top"]);
|
||||
const edges = [
|
||||
{ from: "base", to: "mid" },
|
||||
{ from: "mid", to: "top" },
|
||||
];
|
||||
const sim = createGraphSimulation(ns, edges, {
|
||||
depthAnchor: { mode: "radial", strength: 1.2, spacing: 80, baseRadius: 100, center: { x: 0, y: 0 } },
|
||||
});
|
||||
for (let i = 0; i < 300; i++) sim.tick();
|
||||
const r = (id: string) => {
|
||||
const n = sim.nodes().find((x) => x.id === id)!;
|
||||
return Math.hypot(n.x!, n.y!);
|
||||
};
|
||||
assert.ok(r("base") < r("mid"), `base ring (${r("base").toFixed(0)}) inside mid (${r("mid").toFixed(0)})`);
|
||||
assert.ok(r("mid") < r("top"), `mid ring (${r("mid").toFixed(0)}) inside top (${r("top").toFixed(0)})`);
|
||||
});
|
||||
|
||||
it("settle with depthAnchor: old nodes keep x memory but snap to their level", () => {
|
||||
// Node "old" was previously placed at y=300 (off-level). With
|
||||
// vertical anchoring a re-settle must move it to the baseline
|
||||
// while preserving (roughly) its x arrangement.
|
||||
const previous = new Map([["old", { x: 250, y: 300 }], ["peer", { x: 260, y: 290 }]]);
|
||||
const next = incrementalLayout(
|
||||
[{ id: "old" }, { id: "peer" }, { id: "child" }],
|
||||
[
|
||||
{ from: "old", to: "child" },
|
||||
{ from: "peer", to: "child" },
|
||||
],
|
||||
previous,
|
||||
{ depthAnchor: { mode: "vertical", strength: 0.6, spacing: 80 } },
|
||||
);
|
||||
const old = next.get("old")!;
|
||||
// The baseline defaults to the anchor centroid: (300+290)/2 = 295.
|
||||
assert.ok(Math.abs(old.y - 295) < 20, `old node snaps to the baseline (y=${old.y.toFixed(1)})`);
|
||||
assert.ok(old.x > 150, `x memory preserved (x=${old.x.toFixed(1)})`);
|
||||
const child = next.get("child")!;
|
||||
assert.ok(child.y < old.y - 40, `child rises above its premises (child=${child.y.toFixed(1)}, old=${old.y.toFixed(1)})`);
|
||||
});
|
||||
|
||||
it("orphans are exempt from depth anchoring — regular forces govern them (ORPHAN RULE)", () => {
|
||||
// A premise→conclusion chain plus an unconnected loner. Gravity's
|
||||
// center is FAR from the baseline, so the two are distinguishable:
|
||||
// a depth-stamped orphan would sit on the baseline; an exempt one
|
||||
// follows gravity instead.
|
||||
const ns = [
|
||||
{ id: "base", x: 0, y: 0 },
|
||||
{ id: "derived", x: 10, y: 0 },
|
||||
{ id: "loner", x: 0, y: 0 },
|
||||
];
|
||||
const sim = createGraphSimulation(ns, [{ from: "base", to: "derived" }], {
|
||||
gravity: { x: 0, y: 500, strength: 0.1 },
|
||||
depthAnchor: { mode: "vertical", strength: 1.5, spacing: 120, center: { x: 0, y: 0 } },
|
||||
});
|
||||
for (let i = 0; i < 400; i++) sim.tick();
|
||||
const byId = new Map(sim.nodes().map((n) => [n.id, n]));
|
||||
assert.ok(Math.abs(byId.get("base")!.y!) < 40, "structured base sits on the baseline");
|
||||
assert.ok(byId.get("derived")!.y! < -60, "derived rises a level");
|
||||
const lonerY = byId.get("loner")!.y!;
|
||||
assert.ok(lonerY > 250, `the orphan follows gravity, not the baseline (y=${lonerY.toFixed(0)})`);
|
||||
assert.ok(Math.abs(lonerY - 500) < 120, `the orphan converges on gravity's center (y=${lonerY.toFixed(0)})`);
|
||||
});
|
||||
|
||||
it("radial: orphans are exempt — they gather at the center, not on the base ring", () => {
|
||||
const ns = [
|
||||
{ id: "base", x: 0, y: 0 },
|
||||
{ id: "mid", x: 10, y: 0 },
|
||||
{ id: "loner", x: 300, y: 300 },
|
||||
];
|
||||
const sim = createGraphSimulation(ns, [{ from: "base", to: "mid" }], {
|
||||
gravity: { x: 0, y: 0, strength: 0.1 },
|
||||
depthAnchor: { mode: "radial", strength: 1.5, spacing: 80, baseRadius: 150, center: { x: 0, y: 0 } },
|
||||
});
|
||||
for (let i = 0; i < 400; i++) sim.tick();
|
||||
const r = (id: string) => {
|
||||
const n = sim.nodes().find((x) => x.id === id)!;
|
||||
return Math.hypot(n.x!, n.y!);
|
||||
};
|
||||
assert.ok(Math.abs(r("base") - 150) < 40, `base sits on the base ring (r=${r("base").toFixed(0)})`);
|
||||
assert.ok(r("loner") < 60, `the orphan follows gravity to the center, not the ring (r=${r("loner").toFixed(0)})`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,217 @@
|
||||
/**
|
||||
* Tests for src/core/render-settings.ts — the shared settings contract
|
||||
* (shape, defaults, sanitizer, label-zoom slider mapping). Pure module,
|
||||
* pure tests: whatever JSON arrives from disk or the network goes through
|
||||
* sanitizeRenderSettings and must come out valid, never throw.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it as test } from "node:test";
|
||||
import {
|
||||
ARROW_LENGTH_RANGE,
|
||||
CHARGE_RANGE,
|
||||
CLUSTER_PULL_RANGE,
|
||||
DEFAULT_RENDER_SETTINGS,
|
||||
DEPTH_BASE_RADIUS_RANGE,
|
||||
DEPTH_SPACING_RANGE,
|
||||
DEPTH_STRENGTH_RANGE,
|
||||
EDGE_WIDTH_RANGE,
|
||||
GRAVITY_RANGE,
|
||||
LABEL_SIZE_RANGE,
|
||||
LABEL_ZOOM_MAX,
|
||||
LABEL_ZOOM_MIN,
|
||||
LINK_DISTANCE_RANGE,
|
||||
LINK_STRENGTH_RANGE,
|
||||
NODE_SIZE_RANGE,
|
||||
REGION_BLUR_OPTIONS,
|
||||
sanitizeRegionTuning,
|
||||
REGION_DILATION_RANGE,
|
||||
REGION_EDGE_ALPHA_RANGE,
|
||||
VELOCITY_DECAY_RANGE,
|
||||
DEFAULT_PHYSICS_TUNING,
|
||||
DEFAULT_REGION_TUNING,
|
||||
fromOffSliderPosition,
|
||||
fromSliderPosition,
|
||||
labelZoomDisplay,
|
||||
sanitizeRenderSettings,
|
||||
toOffSliderPosition,
|
||||
toSliderPosition,
|
||||
} from "../src/core/render-settings.ts";
|
||||
|
||||
describe("sanitizeRenderSettings", () => {
|
||||
test("garbage in → full defaults out", () => {
|
||||
assert.deepEqual(sanitizeRenderSettings(null), DEFAULT_RENDER_SETTINGS);
|
||||
assert.deepEqual(sanitizeRenderSettings(undefined), DEFAULT_RENDER_SETTINGS);
|
||||
assert.deepEqual(sanitizeRenderSettings("junk"), DEFAULT_RENDER_SETTINGS);
|
||||
assert.deepEqual(sanitizeRenderSettings(42), DEFAULT_RENDER_SETTINGS);
|
||||
});
|
||||
|
||||
test("partial input → defaults fill the gaps", () => {
|
||||
const s = sanitizeRenderSettings({ nodeSize: 6 });
|
||||
assert.equal(s.nodeSize, 6);
|
||||
assert.equal(s.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth);
|
||||
assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize);
|
||||
assert.equal(s.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom);
|
||||
assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels);
|
||||
});
|
||||
|
||||
test("out-of-range values clamp; wrong types reset", () => {
|
||||
const s = sanitizeRenderSettings({
|
||||
nodeSize: 999,
|
||||
edgeWidth: 0,
|
||||
arrowLength: -5,
|
||||
labelZoom: 100,
|
||||
labelSize: "big" as unknown as number,
|
||||
clusterLabels: "yes" as unknown as boolean,
|
||||
});
|
||||
assert.equal(s.nodeSize, NODE_SIZE_RANGE.max); // clamped to max
|
||||
assert.equal(s.edgeWidth, EDGE_WIDTH_RANGE.min); // clamped to min
|
||||
assert.equal(s.arrowLength, ARROW_LENGTH_RANGE.min);
|
||||
assert.equal(s.labelZoom, LABEL_ZOOM_MAX);
|
||||
assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); // NaN-safe reset
|
||||
assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels);
|
||||
});
|
||||
|
||||
test("the low ends accept small sizes (log-slider floors)", () => {
|
||||
const s = sanitizeRenderSettings({ nodeSize: 0.5, edgeWidth: 0.25, arrowLength: 0.5, labelSize: 0.5 });
|
||||
assert.deepEqual(
|
||||
{ nodeSize: s.nodeSize, edgeWidth: s.edgeWidth, arrowLength: s.arrowLength, labelSize: s.labelSize },
|
||||
{ nodeSize: 0.5, edgeWidth: 0.25, arrowLength: 0.5, labelSize: 0.5 },
|
||||
);
|
||||
});
|
||||
|
||||
test("a fully valid object passes through unchanged", () => {
|
||||
const valid = {
|
||||
nodeSize: 4,
|
||||
edgeWidth: 2,
|
||||
arrowLength: 3,
|
||||
labelZoom: 0.25,
|
||||
labelSize: 4,
|
||||
clusterLabels: false,
|
||||
regions: DEFAULT_REGION_TUNING,
|
||||
physics: DEFAULT_PHYSICS_TUNING,
|
||||
};
|
||||
assert.deepEqual(sanitizeRenderSettings(valid), valid);
|
||||
});
|
||||
|
||||
test("the pre-2026-09-13 stored shape (flat, no regions/physics) still loads", () => {
|
||||
// data/render-settings.json on disk predates the nested sections —
|
||||
// those loads must get defaults, not crash, not shift anything.
|
||||
const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true });
|
||||
assert.deepEqual(s.regions, DEFAULT_REGION_TUNING);
|
||||
assert.deepEqual(s.physics, DEFAULT_PHYSICS_TUNING);
|
||||
assert.equal(s.nodeSize, 4);
|
||||
});
|
||||
|
||||
test("nested sections sanitize: garbage clamps inside them without touching the rest", () => {
|
||||
const s = sanitizeRenderSettings({
|
||||
nodeSize: 3,
|
||||
regions: { blur: -5, bitmap: 999, edgeAlpha: 9 },
|
||||
physics: { linkStrength: 99, gravity: 7, depthMode: "sideways" },
|
||||
});
|
||||
assert.equal(s.nodeSize, 3);
|
||||
assert.equal(s.regions.blur, 2); // snaps into the fixed set {2,4,6,8,10}
|
||||
assert.equal(s.regions.bitmap, 512); // snaps to the nearest fixed option
|
||||
assert.equal(s.regions.edgeAlpha, 1);
|
||||
assert.equal(s.regions.dilation, DEFAULT_REGION_TUNING.dilation);
|
||||
assert.equal(s.physics.linkStrength, 4); // sweep-validated cap
|
||||
assert.equal(s.physics.gravity, 0.48);
|
||||
assert.equal(s.physics.depthMode, "off"); // unknown mode → off
|
||||
assert.equal(s.physics.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength);
|
||||
});
|
||||
|
||||
test("the pre-wave stored shape (no labelSize) still loads", () => {
|
||||
// localStorage-era records lacked labelSize — must not shift anything.
|
||||
const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true });
|
||||
assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize);
|
||||
assert.equal(s.nodeSize, 4);
|
||||
});
|
||||
});
|
||||
|
||||
describe("slider mapping (log scale, default-centered)", () => {
|
||||
const RANGES: [string, number, number, number][] = [
|
||||
["nodeSize", NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max, DEFAULT_RENDER_SETTINGS.nodeSize],
|
||||
["edgeWidth", EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max, DEFAULT_RENDER_SETTINGS.edgeWidth],
|
||||
["arrowLength", ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max, DEFAULT_RENDER_SETTINGS.arrowLength],
|
||||
["labelSize", LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max, DEFAULT_RENDER_SETTINGS.labelSize],
|
||||
["labelZoom", LABEL_ZOOM_MIN, LABEL_ZOOM_MAX, DEFAULT_RENDER_SETTINGS.labelZoom],
|
||||
];
|
||||
|
||||
test("every default sits dead-center at position 50", () => {
|
||||
for (const [name, min, max, dflt] of RANGES)
|
||||
assert.equal(Math.round(toSliderPosition(dflt, min, max)), 50, name);
|
||||
});
|
||||
|
||||
test("round-trips both directions across the whole span", () => {
|
||||
for (const [name, min, max] of RANGES) {
|
||||
for (let p = 0; p <= 100; p += 5) {
|
||||
const v = fromSliderPosition(p, min, max);
|
||||
assert.ok(Math.abs(toSliderPosition(v, min, max) - p) < 1e-9, `${name} @ ${p}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("slider extremes map to the range ends and clamp beyond", () => {
|
||||
for (const [name, min, max] of RANGES) {
|
||||
assert.equal(fromSliderPosition(0, min, max), min, name);
|
||||
assert.ok(Math.abs(fromSliderPosition(100, min, max) - max) < 1e-9, name); // exp/log float noise
|
||||
assert.equal(toSliderPosition(min * 0.1, min, max), 0, name); // clamps
|
||||
assert.equal(toSliderPosition(max * 10, min, max), 100, name);
|
||||
}
|
||||
});
|
||||
|
||||
test("display units normalize the default to 1", () => {
|
||||
assert.equal(labelZoomDisplay(0.5), "1×");
|
||||
assert.equal(labelZoomDisplay(0.25), "0.5×");
|
||||
assert.equal(labelZoomDisplay(4), "8×");
|
||||
assert.equal(labelZoomDisplay(0.0625), "0.13×"); // rounded to 2 decimals
|
||||
});
|
||||
|
||||
test("every region/physics default sits dead-center at position 50", () => {
|
||||
// The same law for the 2026-09-13 sliders: default = geometric mean
|
||||
// of the range ends (repulsion maps |charge|; friction tolerates
|
||||
// the sub-0.5° drift).
|
||||
const pairs: [number, number, number][] = [
|
||||
[REGION_DILATION_RANGE.min, REGION_DILATION_RANGE.max, DEFAULT_REGION_TUNING.dilation],
|
||||
[REGION_EDGE_ALPHA_RANGE.min, REGION_EDGE_ALPHA_RANGE.max, DEFAULT_REGION_TUNING.edgeAlpha],
|
||||
// |charge| in [30, 480] (the range stores negative ends).
|
||||
[-CHARGE_RANGE.max, -CHARGE_RANGE.min, -DEFAULT_PHYSICS_TUNING.chargeStrength],
|
||||
[LINK_STRENGTH_RANGE.min, LINK_STRENGTH_RANGE.max, DEFAULT_PHYSICS_TUNING.linkStrength],
|
||||
[LINK_DISTANCE_RANGE.min, LINK_DISTANCE_RANGE.max, DEFAULT_PHYSICS_TUNING.linkDistance],
|
||||
[GRAVITY_RANGE.min, GRAVITY_RANGE.max, DEFAULT_PHYSICS_TUNING.gravity],
|
||||
[VELOCITY_DECAY_RANGE.min, VELOCITY_DECAY_RANGE.max, DEFAULT_PHYSICS_TUNING.velocityDecay],
|
||||
[DEPTH_STRENGTH_RANGE.min, DEPTH_STRENGTH_RANGE.max, DEFAULT_PHYSICS_TUNING.depthStrength],
|
||||
[DEPTH_SPACING_RANGE.min, DEPTH_SPACING_RANGE.max, DEFAULT_PHYSICS_TUNING.depthSpacing],
|
||||
[DEPTH_BASE_RADIUS_RANGE.min, DEPTH_BASE_RADIUS_RANGE.max, DEFAULT_PHYSICS_TUNING.depthBaseRadius],
|
||||
];
|
||||
for (const [min, max, d] of pairs) {
|
||||
assert.ok(Math.abs(toSliderPosition(d, min, max) - 50) < 0.6, `default ${d} in [${min}, ${max}]`);
|
||||
}
|
||||
// Off-zone fields: the default sits at 50 INSIDE the on-part.
|
||||
assert.ok(Math.abs(toOffSliderPosition(DEFAULT_PHYSICS_TUNING.clusterPull, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - 50) < 0.6);
|
||||
});
|
||||
|
||||
test("off-slider mapping: position 0 ↔ value 0; the log part round-trips", () => {
|
||||
assert.equal(toOffSliderPosition(0, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), 0);
|
||||
assert.equal(fromOffSliderPosition(0, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), 0);
|
||||
const v = fromOffSliderPosition(toOffSliderPosition(0.15, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max);
|
||||
assert.ok(Math.abs(v - 0.15) < 1e-9);
|
||||
// One step in (position 5) sits just above the log floor:
|
||||
// min * (max/min)^(5/100), NOT the floor itself (that's position 0,
|
||||
// the off zone).
|
||||
const stepIn = CLUSTER_PULL_RANGE.min * (CLUSTER_PULL_RANGE.max / CLUSTER_PULL_RANGE.min) ** (5 / 100);
|
||||
assert.ok(Math.abs(fromOffSliderPosition(5, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - stepIn) < 1e-9);
|
||||
});
|
||||
|
||||
test("blur is a fixed set: the default is 6, non-members snap to the nearest", () => {
|
||||
// Non-integer radii broke the rendering, so the control became a
|
||||
// dropdown — the sanitizer enforces the set for ANY input.
|
||||
assert.equal(DEFAULT_REGION_TUNING.blur, 6);
|
||||
assert.equal(sanitizeRegionTuning({ blur: 3.7 }).blur, 4);
|
||||
assert.equal(sanitizeRegionTuning({ blur: 7.2 }).blur, 8);
|
||||
assert.equal(sanitizeRegionTuning({ blur: -5 }).blur, 2);
|
||||
assert.equal(sanitizeRegionTuning({ blur: 99 }).blur, 10);
|
||||
// Stored blur 0 (the old off zone) snaps back into the set.
|
||||
assert.equal(sanitizeRegionTuning({ blur: 0 }).blur, 2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* Semantic topic clustering (the Louvain REPLACEMENT, DECIDED
|
||||
* 2026-09-13): average-linkage agglomerative clustering over the
|
||||
* embedding ledger — related TOPICS, not reasoning chains. Pure and
|
||||
* deterministic, so these tests are exact.
|
||||
*/
|
||||
import { strict as assert } from "node:assert";
|
||||
import { describe, test } from "node:test";
|
||||
import { semanticClusters, SEMANTIC_SIM_THRESHOLD, cosineSimilarity } from "../src/core/semantic-clusters.ts";
|
||||
|
||||
/** Two clearly-related one-hot-ish vectors (high cosine similarity). */
|
||||
function near(vec: number[], target: number[], amount = 0.2): number[] {
|
||||
let dot = 0;
|
||||
for (let i = 0; i < vec.length; i++) dot += vec[i]! * target[i]!;
|
||||
const mixed = vec.map((x, i) => x + amount * target[i]!);
|
||||
let mag = 0;
|
||||
for (const x of mixed) mag += x * x;
|
||||
mag = Math.sqrt(mag);
|
||||
return mixed.map((x) => x / mag);
|
||||
}
|
||||
|
||||
function axis(dim: number, k: number): number[] {
|
||||
return Array.from({ length: dim }, (_, i) => (i === k ? 1 : 0));
|
||||
}
|
||||
|
||||
describe("cosineSimilarity", () => {
|
||||
test("identical unit vectors → 1; orthogonal → 0; defensive normalization", () => {
|
||||
const a = axis(4, 0);
|
||||
const b = axis(4, 1);
|
||||
assert.ok(cosineSimilarity(a, a) > 0.999);
|
||||
assert.ok(Math.abs(cosineSimilarity(a, b)) < 1e-9);
|
||||
// Unnormalized twins still match.
|
||||
assert.ok(cosineSimilarity([3, 0], [5, 0]) > 0.999);
|
||||
});
|
||||
});
|
||||
|
||||
describe("semanticClusters", () => {
|
||||
test("empty ledger: zero clusters (nothing embedded yet)", () => {
|
||||
assert.deepEqual(semanticClusters({}), []);
|
||||
});
|
||||
|
||||
test("related vectors merge; distant ones stay separate (threshold)", () => {
|
||||
const dim = 8;
|
||||
const topicA = axis(dim, 0);
|
||||
const topicB = axis(dim, 1);
|
||||
const vectors = {
|
||||
a1: near(topicA, topicB, 0.1),
|
||||
a2: near(topicA, topicB, 0.15),
|
||||
a3: near(topicA, topicB, 0.05),
|
||||
b1: near(topicB, topicA, 0.1),
|
||||
b2: near(topicB, topicA, 0.12),
|
||||
};
|
||||
const clusters = semanticClusters(vectors);
|
||||
// Both topics form; average-linkage similarity within each is well
|
||||
// above the default threshold, across is well below.
|
||||
const big = clusters.filter((c) => c.members.length >= 2);
|
||||
assert.equal(big.length, 2);
|
||||
assert.deepEqual(
|
||||
big.map((c) => c.members.sort()).sort((x, y) => x[0]!.localeCompare(y[0]!)),
|
||||
[["a1", "a2", "a3"], ["b1", "b2"]],
|
||||
);
|
||||
});
|
||||
|
||||
test("threshold is the granularity knob: higher splits, lower merges", () => {
|
||||
const dim = 6;
|
||||
const vectors = {
|
||||
x: near(axis(dim, 0), axis(dim, 1), 0.3),
|
||||
y: near(axis(dim, 1), axis(dim, 0), 0.3),
|
||||
};
|
||||
// Similarity ~0.72: merges below that, splits above.
|
||||
const sim = cosineSimilarity(vectors.x!, vectors.y!);
|
||||
assert.equal(semanticClusters(vectors, { similarityThreshold: sim - 0.01 }).length, 1);
|
||||
assert.equal(semanticClusters(vectors, { similarityThreshold: sim + 0.01 }).length, 2);
|
||||
});
|
||||
|
||||
test("stragglers stay singletons by default; attachStragglers forces membership", () => {
|
||||
const dim = 6;
|
||||
const topic = axis(dim, 0);
|
||||
const vectors = {
|
||||
t1: near(topic, axis(dim, 1), 0.1),
|
||||
t2: near(topic, axis(dim, 2), 0.1),
|
||||
t3: near(topic, axis(dim, 3), 0.1),
|
||||
loner: axis(dim, 5), // orthogonal: fits nothing
|
||||
};
|
||||
const loose = semanticClusters(vectors);
|
||||
const looseLoner = loose.find((c) => c.members.includes("loner"));
|
||||
assert.equal(looseLoner?.members.length, 1, "default: loner is a straggler");
|
||||
assert.equal(loose.find((c) => c.members.length >= 3)?.members.length, 3);
|
||||
|
||||
const allIn = semanticClusters(vectors, { attachStragglers: true });
|
||||
assert.equal(allIn.length, 1, "attach: the loner joins the nearest topic");
|
||||
assert.ok(allIn[0]!.members.includes("loner"));
|
||||
});
|
||||
|
||||
test("members sorted; clusters ordered by size then id (deterministic)", () => {
|
||||
const dim = 4;
|
||||
const vectors = {
|
||||
solo: axis(dim, 3),
|
||||
p1: near(axis(dim, 0), axis(dim, 1), 0.05),
|
||||
p2: near(axis(dim, 0), axis(dim, 2), 0.05),
|
||||
q1: near(axis(dim, 1), axis(dim, 0), 0.05),
|
||||
q2: near(axis(dim, 1), axis(dim, 2), 0.05),
|
||||
};
|
||||
const a = semanticClusters(vectors);
|
||||
const b = semanticClusters(vectors);
|
||||
assert.deepEqual(a, b, "same input → identical partition");
|
||||
for (let i = 1; i < a.length; i++) {
|
||||
assert.ok(a[i - 1]!.members.length >= a[i]!.members.length, "size-descending");
|
||||
}
|
||||
for (const c of a) {
|
||||
assert.deepEqual(c.members, [...c.members].sort(), "members sorted");
|
||||
}
|
||||
});
|
||||
|
||||
test("default threshold is the swept real-graph value", () => {
|
||||
assert.equal(SEMANTIC_SIM_THRESHOLD, 0.6);
|
||||
});
|
||||
});
|
||||
+39
-1
@@ -10,7 +10,8 @@ import test from "node:test";
|
||||
import * as fs from "node:fs";
|
||||
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
import { PATHS, SOURCES_FILE } from "../config.ts";
|
||||
import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts";
|
||||
import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts";
|
||||
|
||||
@@ -170,3 +171,40 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl
|
||||
|
||||
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
|
||||
});
|
||||
|
||||
test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => {
|
||||
// The real fs graphStore — sources.json backed up and restored around
|
||||
// this test (same discipline as the chat fixture files).
|
||||
fs.mkdirSync(PATHS.graph, { recursive: true });
|
||||
let bak: string | undefined;
|
||||
if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`;
|
||||
if (bak) fs.renameSync(SOURCES_FILE, bak);
|
||||
try {
|
||||
const runEntry = (startedAt: number): ExtractionRunUsage => ({
|
||||
startedAt,
|
||||
at: "2026-09-13T00:00:00Z",
|
||||
durationMs: 42_000,
|
||||
stats: { nodes: 1, edges: 2, events: 3 },
|
||||
stages: {
|
||||
extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
},
|
||||
});
|
||||
const id = "src_usagetest01";
|
||||
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" });
|
||||
graphStore.appendSourceUsage(id, runEntry(1));
|
||||
graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op
|
||||
graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend
|
||||
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
|
||||
|
||||
// An upsert (re-import) KEEPS the usage history.
|
||||
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" });
|
||||
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
|
||||
|
||||
// Unknown ids are a silent no-op (nothing to bill against).
|
||||
graphStore.appendSourceUsage("src_nope", runEntry(1));
|
||||
} finally {
|
||||
fs.rmSync(SOURCES_FILE, { force: true });
|
||||
if (bak) fs.renameSync(bak, SOURCES_FILE);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user