amosblomqvist
f5edc7f0a5
Louvain only ever found the reasoning chains (the graph is disconnected by nature — every connected component became its own cluster) and edge-orphans could never join anything. Clusters are now RELATED TOPICS: propositions grouped by meaning. - Embedder (src/server/embeddings.ts): in-process transformers.js v4 running EmbeddingGemma-300M q8 (MTEB 68.4, ~300MB one-time download), MRL-truncated to 512d, "title: none | text:" prefix on 100% of inputs, L2-normalized so cosine = dot. Model/dim mismatch voids the ledger (full re-embed). Injected at the composition root — the agent layer never imports the server layer (architecture law). - Ledger (data/graph/embeddings.json): model + dim + vectors + the persisted cluster partition. Reclustering runs at the END of every embedding batch (post-merge hook, both front doors) — never per read (the agglomerative pass is ~1.5s at 338 nodes). - Clustering (src/core/semantic-clusters.ts): hand-rolled average-linkage agglomerative over cosine (npm HDBSCAN is an abandoned alpha). Stragglers stay singletons by default; attachStragglers is the all-in toggle. Threshold 0.60 swept on the real graph: 26 topics, 74/90 edge-orphans land in real topics, 10 stragglers. - Consumers (snapshot, settle, namer, replay) read the persisted partition — one cluster concept everywhere; core-survival label matching unchanged (20/26 topics kept labels). - Louvain retired: detectCommunities + graphology* deps deleted (graphology-louvain had leaked into the client bundle since the clusters wave); scripts/bench-graph.ts (dead FA2 bench) removed. - scripts/embed-graph.ts: backfill/cold-start (338 nodes in 150s including the one-time model download).
103 lines
4.9 KiB
TypeScript
103 lines
4.9 KiB
TypeScript
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
/** Repository root (this file is at <root>/config.ts). */
|
|
export const ROOT = path.dirname(fileURLToPath(import.meta.url));
|
|
|
|
export const PATHS = {
|
|
root: ROOT,
|
|
/** Teacher system prompt — the agent's instructions (src/agent/). */
|
|
teacherPrompt: path.join(ROOT, "src", "agent", "system-prompt.md"),
|
|
data: path.join(ROOT, "data"),
|
|
/** Chat sessions (AI SDK UIMessage transcripts, one JSON file per chat). */
|
|
chats: path.join(ROOT, "data", "chats"),
|
|
graph: path.join(ROOT, "data", "graph"),
|
|
} as const;
|
|
|
|
// ─── Models ────────────────────────────────────────────────────────────────
|
|
// THE single place every LLM is chosen and tuned. .env stays for secrets
|
|
// (API keys) only — model choice and thinking budget are product tuning,
|
|
// not deployment environment, and this section is the one place they grow.
|
|
// `effort` is pi's thinking-level scale (off|minimal|low|medium|high|xhigh
|
|
// |max): the per-model truth lives in pi-ai's catalog (thinkingLevelMap) —
|
|
// getSupportedThinkingLevels(model) lists what a model actually accepts,
|
|
// and streamSimple clamps any other value to the nearest supported one
|
|
// (e.g. GLM-5.3-flash supports exactly low|high|max).
|
|
|
|
export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
|
|
|
|
export interface ModelSpec {
|
|
/** OpenRouter model id (e.g. "z-ai/glm-5.3-flash") — must exist in pi-ai's catalog. */
|
|
model: string;
|
|
/** Thinking budget — what the model may spend on reasoning tokens. */
|
|
effort: ThinkingLevel;
|
|
}
|
|
|
|
export const MODELS = {
|
|
/** The prober (agent A) — edge-bracketing questions; keep it lively and adaptive. */
|
|
prober: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
|
/** The teacher (agent B) — planning + streaming, lesson-facing: smart, but keep it lively. */
|
|
teacher: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
|
/** Background sub-agents: fast and cheap beats deep thought. */
|
|
researcher: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
|
demoBuilder: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
|
/** Must be vision-capable: it looks at rendered SVGs. */
|
|
visualMaker: { model: "qwen/qwen3.8-flash", effort: "low" },
|
|
/** Extraction is judgment over given evidence — light thinking suffices. */
|
|
extractor: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
|
/** The auditor verifies evidence exists; keep it cheap. */
|
|
auditor: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
|
/** Names detected communities — one singular call, given member names
|
|
* + statements. Purely a labeling flourish; keep it cheap. */
|
|
labeler: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
|
} satisfies Record<string, ModelSpec>;
|
|
|
|
export type ModelRole = keyof typeof MODELS;
|
|
|
|
/**
|
|
* The extraction pipeline (vision §1 construction): the sole graph writer.
|
|
* Triggered per chat once this many NEW quiz answers have landed since the
|
|
* chat's last extraction checkpoint. The trigger mechanism is a v1
|
|
* placeholder — deliberately isolated behind one seam
|
|
* (server/turn-service.ts) so it can be swapped without touching the
|
|
* pipeline.
|
|
*/
|
|
export const EXTRACTION = { everyNQuizAnswers: 3 } as const;
|
|
|
|
/**
|
|
* The LIVE graph store (src/store/graph.ts) — owned by the app's extraction
|
|
* pipeline. Line shapes: src/core/contracts.ts.
|
|
*/
|
|
export const LEDGER_FILES = {
|
|
nodes: path.join(PATHS.graph, "nodes.jsonl"),
|
|
edges: path.join(PATHS.graph, "edges.jsonl"),
|
|
events: path.join(PATHS.graph, "events.jsonl"),
|
|
mappings: path.join(PATHS.graph, "mappings.jsonl"),
|
|
sessions: path.join(PATHS.graph, "sessions.jsonl"),
|
|
/** LLM-named community records (contracts/graph.ts ClusterLabeling). */
|
|
clusterLabels: path.join(PATHS.graph, "cluster-labels.jsonl"),
|
|
} as const;
|
|
|
|
/**
|
|
* Notes-import bookkeeping (NOT a ledger): one JSON map, src-id → NotesSource
|
|
* record (contracts/graph.ts) — the file path + import time + the run's usage
|
|
* log. Read-modify-write like a chat file, not append-only like the ledgers.
|
|
*/
|
|
export const SOURCES_FILE = path.join(PATHS.graph, "sources.json");
|
|
/** The layout ledger (read-modify-write like sources.json): nodeId →
|
|
* {x,y} resting positions from the incremental settle (core/layout.ts).
|
|
* Carries an updatedAt stamp for future live-physics conflict handling. */
|
|
export const POSITIONS_FILE = path.join(PATHS.graph, "positions.json");
|
|
|
|
/** The embedding ledger: one L2-normalized vector per node id, all from
|
|
* ONE model (swapping models re-embeds everything — vectors from
|
|
* different models live in incompatible spaces). Read-modify-write
|
|
* JSON like positions/sources. */
|
|
export const EMBEDDINGS_FILE = path.join(PATHS.graph, "embeddings.json");
|
|
|
|
/**
|
|
* Graph render settings (the settings-panel controls): one small JSON blob,
|
|
* read-modify-write. Shared shape/sanitizer: src/core/render-settings.ts.
|
|
*/
|
|
export const RENDER_SETTINGS_FILE = path.join(PATHS.graph, "render-settings.json");
|