amosblomqvist
078635e61c
Wave 7 chunk A (950ddfd) replaced semantic clustering with birth-session clusters labeled by session names — the LLM namer became dead code, but extraction still called it, minting naming runs nobody reads on every merge. Real model spend wasted per extraction. Removed (agent band): - src/agent/subagents/cluster-namer.ts — the naming sub-agent, deleted whole (refreshClusterLabels, clusterNamingSettled, the naming loop). - extraction.ts: both post-merge garnish blocks (chat + notes paths), the labeler test seam on ExtractionModels, and the import. The labeler role's only consumer is gone; the 'labeler' entry in BACKGROUND_MODEL_ROLES/ROLE_SLOT (core band) is now consumer-less — flagged to Platform, not crossed. - scripts/import-notes.ts: the settle-before-exit line + the named- clusters count in the summary. - tests: the cluster-naming describe (naming + failure-non-fatal tests) and the dead CLUSTER_* fixtures; the labeler-less BYOK test repurposed as the kill pin — a merge mints NO naming runs and NO label records. NOT touched (flagged, outside the agent band): matchClusterLabelings (core/clusters.ts), clusterContentId (core/graph.ts — now consumer- less), the appendClusterLabel/readClusterLabels port seams + store impls + ClusterLabeling type (store/core band), tests/clusters.test.ts's matchClusterLabelings describe. Cost-summary's naming stage untouched per Platform's ruling (historical spend display reads readNamingRuns, which stays live). Gates: typecheck, 921/921, lint, build.
63 lines
2.6 KiB
TypeScript
63 lines
2.6 KiB
TypeScript
/**
|
|
* Import raw notes files through the format-agnostic front door (vision §1,
|
|
* 2026-09-13) — every *.md / *.txt file in a directory becomes one
|
|
* extraction run (extractor → auditor → merger) over its raw text. Run
|
|
* from this worktree only — data never crosses trees:
|
|
*
|
|
* npx tsx --env-file=.env.local scripts/import-notes.ts <dir> [concurrency]
|
|
*
|
|
* Default dir: real-data-extraction/learn-sessions (the prototype
|
|
* learning-session exports — deliberately NOT imported as chats: they run
|
|
* through the notes path, raw). Runs in this process: the dev server's
|
|
* status chips do NOT see them (registry is per-process) — the dev server
|
|
* sees runs triggered through /api/graph/import.
|
|
*/
|
|
import * as fs from "node:fs";
|
|
import * as path from "node:path";
|
|
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
|
|
import { embedNewNodes } from "../src/server/embeddings.ts";
|
|
import { getGraph } from "../src/core/graph.ts";
|
|
import { stores } from "../src/server/stores.ts";
|
|
|
|
async function main() {
|
|
const dir = process.argv[2] ?? "real-data-extraction/learn-sessions";
|
|
const limit = Number(process.argv[3] ?? 4);
|
|
const files = fs
|
|
.readdirSync(dir)
|
|
.filter((f) => f.endsWith(".md") || f.endsWith(".markdown") || f.endsWith(".txt"))
|
|
.sort();
|
|
console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`);
|
|
|
|
const pipeline = createExtractionPipeline({ ...stores, embedder: embedNewNodes });
|
|
const queue = [...files];
|
|
const totals = { nodes: 0, edges: 0, events: 0, failed: 0 };
|
|
const workers = Array.from({ length: Math.min(limit, files.length) }, async () => {
|
|
for (;;) {
|
|
const file = queue.shift();
|
|
if (!file) return;
|
|
const filePath = path.join(dir, file);
|
|
const t0 = Date.now();
|
|
try {
|
|
const stats = await pipeline.importNotes({ filePath, text: fs.readFileSync(filePath, "utf-8") });
|
|
totals.nodes += stats.nodes;
|
|
totals.edges += stats.edges;
|
|
totals.events += stats.events;
|
|
console.log(
|
|
`[import-notes] ${file}: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events in ${((Date.now() - t0) / 1000).toFixed(1)}s`,
|
|
);
|
|
} catch (err) {
|
|
totals.failed++;
|
|
console.error(`[import-notes] ${file} FAILED:`, err);
|
|
}
|
|
}
|
|
});
|
|
await Promise.all(workers);
|
|
|
|
const graph = getGraph(stores.graphStore);
|
|
console.log(
|
|
`[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` +
|
|
`graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events`,
|
|
);
|
|
}
|
|
|
|
main(); |