Merge branch 'feat/platform'
# Conflicts: # scripts/import-notes.ts
This commit is contained in:
@@ -8,14 +8,13 @@
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
|
||||
import { embedNewNodes } from "../src/server/embeddings.ts";
|
||||
import { getGraph } from "../src/core/graph.ts";
|
||||
|
||||
async function main() {
|
||||
const only = process.argv[2]; // optional: extract just one chat id
|
||||
const chats = only ? chatStore.listChats().filter((c) => c.id === only) : chatStore.listChats();
|
||||
console.log(`[backfill] ${chats.length} chats to extract`);
|
||||
const pipeline = createExtractionPipeline({ chatStore, graphStore, embedder: embedNewNodes });
|
||||
const pipeline = createExtractionPipeline({ chatStore, graphStore, embedder: async () => ({ embedded: 0, total: 0, skipped: "embeddings removed (wave 7)" }) });
|
||||
for (const chat of chats) {
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
/**
|
||||
* Embed the whole graph (backfill / cold start / model change): one-time
|
||||
* pass that fills the embedding ledger for every node — the same
|
||||
* embedNewNodes the post-merge hook runs, just with an empty (or
|
||||
* stale-model) ledger so every node is "new".
|
||||
*
|
||||
* BYOK (wave 6, 2026-09-23): vectors come from the configured provider
|
||||
* (graph.embeddingModel, keys from the auth store) — this script resolves
|
||||
* the live state up front so an unconfigured machine says so before
|
||||
* anything bills. A model CHANGE re-embeds everything (ledger staleness).
|
||||
*
|
||||
* npx tsx scripts/embed-graph.ts
|
||||
*/
|
||||
import { embedNewNodes, embeddingPort } from "../src/server/embeddings.ts";
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
|
||||
async function main() {
|
||||
const resolved = embeddingPort();
|
||||
if (!resolved.ok) {
|
||||
console.error(`[embed] cannot run — ${resolved.reason}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const t0 = Date.now();
|
||||
console.log(`[embed] model: ${resolved.port.model} (provider embeddings, batch 32)`);
|
||||
const { embedded, total, usage, failed } = await embedNewNodes(graphStore);
|
||||
const secs = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
if (usage) {
|
||||
const tokens = usage.steps.reduce((sum, s) => sum + (s.inputTokens ?? 0), 0);
|
||||
console.log(`[embed] embedding spend: ${tokens} prompt tokens over ${usage.steps.length} call(s)`);
|
||||
}
|
||||
// Usage-on-error + pay-once (owner ruling 2026-09-23): a mid-run provider
|
||||
// failure still reports the landed batches' spend above; the landed
|
||||
// vectors are persisted, so the honest count is what LANDED — the
|
||||
// in-memory `total` would read as a ledger claim it can't make.
|
||||
if (failed) {
|
||||
console.error(
|
||||
`[embed] failed after ${embedded} node(s) landed (${secs}s) — the ledger keeps exactly those (a retry embeds only the rest): ${failed}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`[embed] embedded ${embedded} nodes → ledger holds ${total} vectors (${secs}s)`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("[embed] failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -15,7 +15,6 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
|
||||
import { embedNewNodes } from "../src/server/embeddings.ts";
|
||||
import { getGraph } from "../src/core/graph.ts";
|
||||
import { stores } from "../src/server/stores.ts";
|
||||
|
||||
@@ -28,7 +27,7 @@ async function main() {
|
||||
.sort();
|
||||
console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`);
|
||||
|
||||
const pipeline = createExtractionPipeline({ ...stores, embedder: embedNewNodes });
|
||||
const pipeline = createExtractionPipeline({ ...stores, embedder: async () => ({ embedded: 0, total: 0, skipped: "embeddings removed (wave 7)" }) });
|
||||
const queue = [...files];
|
||||
const totals = { nodes: 0, edges: 0, events: 0, failed: 0 };
|
||||
const workers = Array.from({ length: Math.min(limit, files.length) }, async () => {
|
||||
|
||||
@@ -18,6 +18,7 @@ const depthArg = process.argv.find((a) => a.startsWith("--depth="));
|
||||
const depthMode: "off" | "vertical" | "radial" =
|
||||
depthArg === "--depth=vertical" ? "vertical" : depthArg === "--depth=radial" ? "radial" : "off";
|
||||
import { incrementalLayout } from "../src/core/layout.ts";
|
||||
import { birthSessionPartition } from "../src/core/session-clusters.ts";
|
||||
import type { GraphNode, NodeMapping, Position } from "../src/core/contracts.ts";
|
||||
|
||||
function main() {
|
||||
@@ -32,12 +33,9 @@ function main() {
|
||||
nodes.push(line);
|
||||
}
|
||||
const edges = graphStore.readEdges();
|
||||
// The semantic-topic source: the embedding ledger (run
|
||||
// scripts/embed-graph.ts first if missing/stale).
|
||||
const embeddings = graphStore.readEmbeddings();
|
||||
if (Object.keys(embeddings.vectors).length === 0) {
|
||||
console.warn("[replay] embedding ledger is empty — clusters will be absent (run scripts/embed-graph.ts)");
|
||||
}
|
||||
// The cluster source (wave 7): the birth-session partition, derived on
|
||||
// read — same as the real settle (core/layout-settle.ts).
|
||||
const birthClusters = birthSessionPartition(nodes);
|
||||
|
||||
// Batches: each imported file, in real import order (sources.json
|
||||
// importedAt). A node belongs to the file whose mapping first claims it
|
||||
@@ -85,9 +83,8 @@ function main() {
|
||||
const batchNodes = nodes.filter((n) => delivered.has(n.id));
|
||||
positions = incrementalLayout(batchNodes, activeEdges, positions, {
|
||||
// The same partition the real settle would see for this batch:
|
||||
// the persisted semantic topics, filtered to delivered nodes
|
||||
// (never recomputed — the agglomerative pass is ~1.5s at scale).
|
||||
clusters: embeddings.clusters
|
||||
// the birth-session clusters, filtered to delivered nodes.
|
||||
clusters: birthClusters
|
||||
.map((c) => ({ members: c.members.filter((id) => delivered.has(id)) }))
|
||||
.filter((c) => c.members.length > 0),
|
||||
...(depthMode === "off"
|
||||
|
||||
Reference in New Issue
Block a user