BYOK provider embeddings (wave 6 chunk B): kill local transformers
Replace the @huggingface/transformers local embedder (~316MB ONNX cache) with an EmbeddingPort seam: graph.embeddingModel (nullable "provider:id" slug) selects an OpenRouter/OpenAI/Gemini adapter, resolved fresh per call from settings + the auth store so key rotations land immediately. - src/server/embeddings.ts rewritten: slug grammar, injected-fetch adapters (OpenRouter input_type, Gemini taskType batching, OpenAI index-ordered rows), honest resolution (null = skip-with-info, set model with missing key = honest error), embedNewNodes with model-change re-embed wipe + ghost pruning + usage steps, federated catalog (OpenRouter /api/v1/embeddings/models — VERIFIED live, the inverse path 404s; OpenAI text-embedding- filter; Gemini embedContent filter). - GET /api/models/embeddings: federated catalog, empty when no keys. - Extraction tail records stages.embedding usage into the run's money channel; EMBEDDING_PRICES overlay in core/prices.ts (outside the sync-generated block), callCost falls back to it; cost-summary folds the fourth stage; graph-context injection resolves its embedder per run, fails honestly when unconfigured, records query tokens. - Settings: GraphSettings.embeddingModel (null factory), section parser validates slug shape (id may contain colons — OpenRouter :free suffixes, live catalog fact). - Removal: @huggingface/transformers uninstalled, PATHS.modelCache and onnx tracing entries deleted, @huggingface tracing-excludes guard added, stale sharp/libvips comments updated to historical. Live-verified on port 3004: catalog federates 36 rows (33 OpenRouter + 3 OpenAI), embedNewNodes embedded 10 nodes via the free liquid/lfm-2.5 model (dim 1024, 435 prompt tokens, 1 call, $0), settings PUT rejects bad slugs with 400. Gates: typecheck, lint, 895/895 tests, build.
This commit is contained in:
amosblomqvist
@@ -96,6 +96,39 @@ describe("graphCostSummary (the cost pill's fold)", () => {
|
||||
assert.equal(s.unpricedRuns, 1); // run-level: only the both-stages run
|
||||
});
|
||||
|
||||
test("the embedding stage (wave 6): folds when present, absent when the run embedded nothing", () => {
|
||||
const EMBED_MODEL = "openai:text-embedding-3-small";
|
||||
const embStep = callCost(step(5000, 0), EMBED_MODEL)!;
|
||||
assert.ok(embStep > 0, "the overlay prices the known embedding models");
|
||||
const withEmbed = extractionRun(100, PRICED, PRICED);
|
||||
withEmbed.stages.embedding = { model: EMBED_MODEL, steps: [step(5000, 0)] };
|
||||
const s = graphCostSummary({
|
||||
chats: [{ title: "a", extractions: [withEmbed] }, { title: "b", extractions: [extractionRun(200, PRICED, PRICED)] }],
|
||||
sources: [],
|
||||
naming: [],
|
||||
});
|
||||
assert.equal(s.stages.embedding.runs, 1, "only the run that embedded");
|
||||
assert.equal(s.stages.embedding.dollars, embStep);
|
||||
assert.equal(s.stages.embedding.tokens.inputTokens, 5000);
|
||||
assert.equal(s.stages.extraction.runs, 2, "the two-stage era still folds");
|
||||
assert.equal(s.stages.extraction.runs, 2);
|
||||
// Run-level: the embedding dollars ride the run's total.
|
||||
const extStep = callCost(step(1000, 200), PRICED)!;
|
||||
const audStep = callCost(step(2000, 100), PRICED)!;
|
||||
assert.equal(s.runs.find((r) => r.startedAt === 100)!.dollars, extStep + audStep + embStep);
|
||||
assert.equal(s.runs.find((r) => r.startedAt === 200)!.dollars, extStep + audStep);
|
||||
assert.equal(s.totalDollars, (extStep + audStep) * 2 + embStep);
|
||||
});
|
||||
|
||||
test("an UNPRICED embedding model stays honest: counted, never a silent zero; a priced sibling saves the run total", () => {
|
||||
const unpricedEmbed = extractionRun(100, PRICED, "unknown-model");
|
||||
unpricedEmbed.stages.embedding = { model: "some/new-embed", steps: [step(900, 0)] };
|
||||
const s = graphCostSummary({ chats: [{ title: "x", extractions: [unpricedEmbed] }], sources: [], naming: [] });
|
||||
assert.equal(s.stages.embedding.unpricedRuns, 1);
|
||||
assert.equal(s.stages.embedding.dollars, 0);
|
||||
assert.notEqual(s.runs[0]!.dollars, null, "the priced extractor saves the run total");
|
||||
});
|
||||
|
||||
test("recent runs cap: newest survive", () => {
|
||||
const extractions = [1, 2, 3, 4].map((n) => extractionRun(n * 100, PRICED, PRICED));
|
||||
const s = graphCostSummary({
|
||||
|
||||
@@ -19,7 +19,7 @@ const REPO_ROOT = path.resolve(import.meta.dirname, "..");
|
||||
const TSX_LOADER = path.join(REPO_ROOT, "node_modules", "tsx", "dist", "loader.mjs");
|
||||
|
||||
describe("LH_DATA_DIR seam", () => {
|
||||
it("relocates data/chats/graph + modelCache under LH_DATA_DIR, keeps prompts ROOT-anchored", async () => {
|
||||
it("relocates data/chats/graph under LH_DATA_DIR, keeps prompts ROOT-anchored", async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lh-seam-"));
|
||||
process.env.LH_DATA_DIR = dir;
|
||||
try {
|
||||
@@ -27,7 +27,6 @@ describe("LH_DATA_DIR seam", () => {
|
||||
assert.equal(PATHS.data, dir, "PATHS.data resolves the env value (resolved absolute)");
|
||||
assert.equal(PATHS.chats, path.join(dir, "chats"));
|
||||
assert.equal(PATHS.graph, path.join(dir, "graph"));
|
||||
assert.equal(PATHS.modelCache, path.join(dir, "models"), "the HF model cache follows the seam");
|
||||
assert.equal(
|
||||
PATHS.teacherPrompt,
|
||||
path.join(ROOT, "src", "agent", "system-prompt.md"),
|
||||
@@ -41,12 +40,11 @@ describe("LH_DATA_DIR seam", () => {
|
||||
it("unset = the repo data/ exactly as before (fresh child process, no env)", () => {
|
||||
const out = execFileSync(
|
||||
process.execPath,
|
||||
["--import", TSX_LOADER, "--eval", `const m = await import(${JSON.stringify(path.join(REPO_ROOT, "config.ts"))}); console.log(JSON.stringify({ data: m.PATHS.data, chats: m.PATHS.chats, modelCache: m.PATHS.modelCache }));`],
|
||||
["--import", TSX_LOADER, "--eval", `const m = await import(${JSON.stringify(path.join(REPO_ROOT, "config.ts"))}); console.log(JSON.stringify({ data: m.PATHS.data, chats: m.PATHS.chats }));`],
|
||||
{ cwd: REPO_ROOT, encoding: "utf-8" },
|
||||
);
|
||||
const parsed = JSON.parse(out) as { data: string; chats: string | null; modelCache: string | null };
|
||||
const parsed = JSON.parse(out) as { data: string; chats: string | null };
|
||||
assert.equal(parsed.data, path.join(REPO_ROOT, "data"));
|
||||
assert.equal(parsed.chats, path.join(REPO_ROOT, "data", "chats"));
|
||||
assert.equal(parsed.modelCache, null, "dev keeps the transformers package default cache");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,401 @@
|
||||
/**
|
||||
* BYOK provider embeddings (wave 6, 2026-09-23): the adapters' wire
|
||||
* protocols (injected fetch — never the network), the slug/keys
|
||||
* resolution rules (null = honestly off; a set model with a missing key
|
||||
* is an honest error), the post-merge batch's staleness/re-embed/skip
|
||||
* semantics (scripted port — never the machine's settings/auth files),
|
||||
* and the federated catalog's federation rules.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { callCost, EMBEDDING_PRICES } from "../src/core/prices.ts";
|
||||
import {
|
||||
embedNewNodes,
|
||||
embeddingModelCatalog,
|
||||
googleEmbeddingProvider,
|
||||
openaiEmbeddingProvider,
|
||||
openrouterEmbeddingProvider,
|
||||
parseEmbeddingModelSlug,
|
||||
resolveEmbeddingPortFor,
|
||||
type EmbeddingResolution,
|
||||
} from "../src/server/embeddings.ts";
|
||||
import type { EmbeddingPort, EmbeddingTask } from "../src/core/ports.ts";
|
||||
import type { EmbeddingsLedger, GraphNode } from "../src/core/contracts/graph.ts";
|
||||
|
||||
// ─── scripted fetch + in-memory graph (no fs, no network) ────────────────────
|
||||
|
||||
type RecordedCall = { url: string; init: RequestInit };
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function recordingFetch(
|
||||
route: (url: string, body: unknown) => Response | Promise<Response>,
|
||||
): { calls: RecordedCall[]; fetch: (input: string, init: RequestInit) => Promise<Response> } {
|
||||
const calls: RecordedCall[] = [];
|
||||
const fetchFn = async (input: string, init: RequestInit): Promise<Response> => {
|
||||
calls.push({ url: input, init });
|
||||
const body = init.body ? (JSON.parse(String(init.body)) as unknown) : null;
|
||||
return route(input, body);
|
||||
};
|
||||
return { calls, fetch: fetchFn };
|
||||
}
|
||||
|
||||
function node(id: string, statement = `statement of ${id}`): GraphNode {
|
||||
return { id, name: id, statement, aliases: [], provenance: [], createdAt: "2026-09-23T00:00:00.000Z" };
|
||||
}
|
||||
|
||||
function fakeGraphStore(start?: Partial<EmbeddingsLedger>): {
|
||||
ledger: EmbeddingsLedger;
|
||||
writes: EmbeddingsLedger[];
|
||||
store: {
|
||||
readNodes(): GraphNode[];
|
||||
readEmbeddings(): EmbeddingsLedger;
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void;
|
||||
};
|
||||
nodes: GraphNode[];
|
||||
} {
|
||||
const nodes = [node("n1", "alpha fact"), node("n2", "beta fact")];
|
||||
const ledger: EmbeddingsLedger = {
|
||||
model: start?.model ?? "test-embed",
|
||||
dim: start?.dim ?? 2,
|
||||
vectors: { ...start?.vectors },
|
||||
clusters: [...start?.clusters ?? []],
|
||||
updatedAt: start?.updatedAt ?? "2026-09-23T00:00:00.000Z",
|
||||
};
|
||||
const writes: EmbeddingsLedger[] = [];
|
||||
return {
|
||||
ledger,
|
||||
writes,
|
||||
nodes,
|
||||
store: {
|
||||
readNodes: () => nodes,
|
||||
readEmbeddings: () => ledger,
|
||||
writeEmbeddings: (next) => {
|
||||
writes.push(next);
|
||||
Object.assign(ledger, next);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** A scripted port: unit vectors by text hash, records (texts, task) calls. */
|
||||
function scriptedPort(model = "openai:test-embed", dim = 2, tokensPerCall = 120): {
|
||||
calls: Array<{ texts: string[]; task: EmbeddingTask }>;
|
||||
port: EmbeddingPort;
|
||||
} {
|
||||
const calls: Array<{ texts: string[]; task: EmbeddingTask }> = [];
|
||||
return {
|
||||
calls,
|
||||
port: {
|
||||
model,
|
||||
async embed(texts, task) {
|
||||
calls.push({ texts: [...texts], task });
|
||||
return {
|
||||
vectors: texts.map((_, i) => Array.from({ length: dim }, (_, j) => (j === i ? 1 : 0))),
|
||||
model,
|
||||
dim,
|
||||
promptTokens: tokensPerCall,
|
||||
};
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const okPort = (port: EmbeddingPort): EmbeddingResolution => ({ ok: true, port });
|
||||
|
||||
// ─── the money channel's embedding overlay ─────────────────────────────────
|
||||
|
||||
describe("the embedding price overlay (hand-maintained, outside the sync block)", () => {
|
||||
it("prices the known embedding models at their verified input rates", () => {
|
||||
// USD per 1M input tokens, verified against provider pages 2026-09-23.
|
||||
assert.equal(callCost({ inputTokens: 1_000_000, outputTokens: 0 }, "openai:text-embedding-3-small"), 0.02);
|
||||
assert.equal(callCost({ inputTokens: 1_000_000, outputTokens: 0 }, "openai:text-embedding-3-large"), 0.13);
|
||||
assert.equal(callCost({ inputTokens: 1_000_000, outputTokens: 0 }, "google:gemini-embedding-001"), 0.15);
|
||||
assert.equal(callCost({ inputTokens: 1_000_000, outputTokens: 0 }, "google:gemini-embedding-2"), 0.2);
|
||||
assert.equal(callCost({ inputTokens: 1_000_000, outputTokens: 0 }, "openrouter:openai/text-embedding-3-small"), 0.02);
|
||||
});
|
||||
|
||||
it("unknown embedding models stay unpriced (—, never zero), tokens still captured", () => {
|
||||
assert.equal(callCost({ inputTokens: 100, outputTokens: 0 }, "openrouter:mystery/embed"), null);
|
||||
assert.equal(!("openrouter:mystery/embed" in EMBEDDING_PRICES), true);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the slug grammar ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("parseEmbeddingModelSlug", () => {
|
||||
it("parses the three shipped providers", () => {
|
||||
assert.deepEqual(parseEmbeddingModelSlug("openai:text-embedding-3-small"), { provider: "openai", id: "text-embedding-3-small" });
|
||||
assert.deepEqual(parseEmbeddingModelSlug("openrouter:openai/text-embedding-3-large"), { provider: "openrouter", id: "openai/text-embedding-3-large" });
|
||||
assert.deepEqual(parseEmbeddingModelSlug("google:gemini-embedding-001"), { provider: "google", id: "gemini-embedding-001" });
|
||||
// OpenRouter ids may carry a :free suffix (live catalog fact) — the
|
||||
// provider is the part before the FIRST colon.
|
||||
assert.deepEqual(parseEmbeddingModelSlug("openrouter:liquid/lfm-2.5-embedding-350m:free"), { provider: "openrouter", id: "liquid/lfm-2.5-embedding-350m:free" });
|
||||
});
|
||||
|
||||
it("rejects malformed slugs (no guessing)", () => {
|
||||
assert.equal(parseEmbeddingModelSlug("text-embedding-3-small"), null, "no provider prefix");
|
||||
assert.equal(parseEmbeddingModelSlug("anthropic:embed"), null, "unshipped provider");
|
||||
assert.equal(parseEmbeddingModelSlug("openai:"), null, "empty id");
|
||||
assert.equal(parseEmbeddingModelSlug(":id"), null, "empty provider");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── resolution: BYOK honesty rules ───────────────────────────────────────────
|
||||
|
||||
describe("resolveEmbeddingPortFor", () => {
|
||||
it("null/absent setting = honestly off", () => {
|
||||
for (const slug of [null, undefined, ""]) {
|
||||
const res = resolveEmbeddingPortFor(slug, () => "key");
|
||||
assert.equal(res.ok, false);
|
||||
assert.match((res as { reason: string }).reason, /no embedding model configured/);
|
||||
}
|
||||
});
|
||||
|
||||
it("a set model with no key is an honest error naming the key", () => {
|
||||
const res = resolveEmbeddingPortFor("openai:text-embedding-3-small", () => undefined);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match((res as { reason: string }).reason, /OPENAI_API_KEY/);
|
||||
});
|
||||
|
||||
it("an invalid slug never resolves", () => {
|
||||
const res = resolveEmbeddingPortFor("bedrock:whatever", () => "key");
|
||||
assert.equal(res.ok, false);
|
||||
assert.match((res as { reason: string }).reason, /invalid embedding model slug/);
|
||||
});
|
||||
|
||||
it("a keyed model resolves to its provider adapter", () => {
|
||||
const res = resolveEmbeddingPortFor("openai:text-embedding-3-small", () => "sk-test");
|
||||
assert.equal(res.ok, true);
|
||||
assert.equal((res as { port: EmbeddingPort }).port.model, "openai:text-embedding-3-small");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the adapters' wire protocols (injected fetch) ────────────────────────────
|
||||
|
||||
describe("openrouterEmbeddingProvider", () => {
|
||||
it("POSTs /api/v1/embeddings with Bearer auth and the task input_type", async () => {
|
||||
const { calls, fetch } = recordingFetch(() =>
|
||||
jsonResponse({ data: [{ embedding: [1, 0], index: 0 }], usage: { prompt_tokens: 42 } }),
|
||||
);
|
||||
const port = openrouterEmbeddingProvider("openrouter:openai/text-embedding-3-small", "k", fetch);
|
||||
const result = await port.embed(["hello"], "document");
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0]!.url, "https://openrouter.ai/api/v1/embeddings");
|
||||
assert.equal((calls[0]!.init.headers as Record<string, string>).authorization, "Bearer k");
|
||||
assert.deepEqual(JSON.parse(String(calls[0]!.init.body)), {
|
||||
model: "openai/text-embedding-3-small",
|
||||
input: ["hello"],
|
||||
input_type: "search_document",
|
||||
});
|
||||
assert.deepEqual(result.vectors, [[1, 0]]);
|
||||
assert.equal(result.promptTokens, 42);
|
||||
|
||||
await port.embed(["q"], "query");
|
||||
assert.equal((JSON.parse(String(calls[1]!.init.body)) as { input_type: string }).input_type, "search_query");
|
||||
});
|
||||
});
|
||||
|
||||
describe("openaiEmbeddingProvider", () => {
|
||||
it("POSTs /v1/embeddings, Bearer auth, no task hint; index ordering honored", async () => {
|
||||
const { calls, fetch } = recordingFetch(() =>
|
||||
jsonResponse({
|
||||
data: [{ embedding: [0, 1], index: 1 }, { embedding: [1, 0], index: 0 }],
|
||||
usage: { prompt_tokens: 7, total_tokens: 7 },
|
||||
}),
|
||||
);
|
||||
const port = openaiEmbeddingProvider("openai:text-embedding-3-small", "sk", fetch);
|
||||
const result = await port.embed(["a", "b"], "document");
|
||||
assert.equal(calls[0]!.url, "https://api.openai.com/v1/embeddings");
|
||||
assert.equal((calls[0]!.init.headers as Record<string, string>).authorization, "Bearer sk");
|
||||
const body = JSON.parse(String(calls[0]!.init.body)) as Record<string, unknown>;
|
||||
assert.deepEqual(body.input, ["a", "b"]);
|
||||
assert.equal("input_type" in body, false, "OpenAI takes no task hint");
|
||||
// Rows arrive out of order — the adapter must restore input order.
|
||||
assert.deepEqual(result.vectors, [[1, 0], [0, 1]]);
|
||||
assert.equal(result.promptTokens, 7);
|
||||
});
|
||||
});
|
||||
|
||||
describe("googleEmbeddingProvider", () => {
|
||||
it("POSTs :batchEmbedContents with x-goog-api-key and per-text taskType", async () => {
|
||||
const { calls, fetch } = recordingFetch((_url, body) => {
|
||||
const n = ((body as { requests?: unknown[] } | null)?.requests ?? []).length;
|
||||
return jsonResponse({ embeddings: Array.from({ length: n }, (_, i) => ({ values: i === 0 ? [1, 0] : [0, 1] })) });
|
||||
});
|
||||
const port = googleEmbeddingProvider("google:gemini-embedding-001", "gk", fetch);
|
||||
const result = await port.embed(["a", "b"], "document");
|
||||
assert.equal(calls[0]!.url, "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents");
|
||||
assert.equal((calls[0]!.init.headers as Record<string, string>)["x-goog-api-key"], "gk");
|
||||
assert.deepEqual(JSON.parse(String(calls[0]!.init.body)), {
|
||||
requests: [
|
||||
{ model: "models/gemini-embedding-001", content: { parts: [{ text: "a" }] }, taskType: "RETRIEVAL_DOCUMENT" },
|
||||
{ model: "models/gemini-embedding-001", content: { parts: [{ text: "b" }] }, taskType: "RETRIEVAL_DOCUMENT" },
|
||||
],
|
||||
});
|
||||
assert.deepEqual(result.vectors, [[1, 0], [0, 1]]);
|
||||
assert.equal(result.promptTokens, undefined, "batch responses carry no token count we can trust");
|
||||
|
||||
await port.embed(["q"], "query");
|
||||
const second = JSON.parse(String(calls[1]!.init.body)) as { requests: Array<{ taskType: string }> };
|
||||
assert.equal(second.requests[0]!.taskType, "RETRIEVAL_QUERY");
|
||||
});
|
||||
|
||||
it("a short/missing vector is an honest error, never a silent misalignment", async () => {
|
||||
const { fetch } = recordingFetch(() => jsonResponse({ embeddings: [{ values: [1, 0] }] }));
|
||||
const port = googleEmbeddingProvider("google:gemini-embedding-001", "gk", fetch);
|
||||
await assert.rejects(port.embed(["a", "b"], "document"), /1\/2 vectors/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("adapter error mapping (all providers)", () => {
|
||||
it("HTTP failures become clear Errors carrying the status", async () => {
|
||||
for (const factory of [openrouterEmbeddingProvider, openaiEmbeddingProvider, googleEmbeddingProvider]) {
|
||||
const { fetch } = recordingFetch(() => jsonResponse({ error: { message: "bad key" } }, 401));
|
||||
const port = factory("openai:text-embedding-3-small", "k", fetch);
|
||||
await assert.rejects(port.embed(["x"], "document"), /API 401/);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the post-merge batch (scripted port — the machine's files stay unread) ───
|
||||
|
||||
describe("embedNewNodes (the extraction tail)", () => {
|
||||
it("embeds missing nodes, stamps model/dim, records one usage step per API call", async () => {
|
||||
const g = fakeGraphStore();
|
||||
const scripted = scriptedPort("openai:text-embedding-3-small", 2, 120);
|
||||
const result = await embedNewNodes(g.store, () => okPort(scripted.port));
|
||||
assert.deepEqual(result, {
|
||||
embedded: 2,
|
||||
total: 2,
|
||||
usage: {
|
||||
model: "openai:text-embedding-3-small",
|
||||
steps: [
|
||||
{ inputTokens: 120, outputTokens: 0, cacheReadTokens: undefined, cacheWriteTokens: undefined, reasoningTokens: undefined },
|
||||
],
|
||||
},
|
||||
});
|
||||
assert.equal(g.writes.length, 1, "the ledger is written once per batch");
|
||||
assert.equal(g.ledger.model, "openai:text-embedding-3-small");
|
||||
assert.equal(g.ledger.dim, 2);
|
||||
assert.ok(g.ledger.vectors.n1 && g.ledger.vectors.n2);
|
||||
assert.deepEqual(scripted.calls[0]!.texts, ["alpha fact", "beta fact"]);
|
||||
assert.equal(scripted.calls[0]!.task, "document");
|
||||
assert.equal(result.usage?.steps.length, 1);
|
||||
});
|
||||
|
||||
it("a MODEL CHANGE voids the ledger and re-embeds everything", async () => {
|
||||
// The ledger holds a stale-model vector for a node that still exists.
|
||||
const g = fakeGraphStore({ model: "openai:old-embed", vectors: { n1: [9, 9] } });
|
||||
const scripted = scriptedPort("openai:new-embed", 2);
|
||||
const result = await embedNewNodes(g.store, () => okPort(scripted.port));
|
||||
assert.equal(result.embedded, 2, "the stale vector dies — BOTH nodes re-embed");
|
||||
assert.deepEqual(scripted.calls[0]!.texts, ["alpha fact", "beta fact"]);
|
||||
assert.equal(g.ledger.model, "openai:new-embed");
|
||||
assert.ok(!g.ledger.vectors.n1!.every((x, i) => x === [9, 9][i]), "the old vector was replaced");
|
||||
});
|
||||
|
||||
it("null model = info skip: nothing written, nothing embedded, the reason rides the result", async () => {
|
||||
const g = fakeGraphStore({ vectors: { n1: [1, 0] } });
|
||||
const result = await embedNewNodes(g.store, () => ({
|
||||
ok: false,
|
||||
reason: "no embedding model configured (graph.embeddingModel is null)",
|
||||
}));
|
||||
assert.deepEqual(result, { embedded: 0, total: 1, skipped: "no embedding model configured (graph.embeddingModel is null)" });
|
||||
assert.equal(g.writes.length, 0, "a skipped batch never touches the ledger");
|
||||
});
|
||||
|
||||
it("a set model with a missing key is the same skip path (its reason says so)", async () => {
|
||||
const g = fakeGraphStore();
|
||||
const result = await embedNewNodes(g.store, () => ({
|
||||
ok: false,
|
||||
reason: 'embedding model is set to "openai:x" but its OPENAI_API_KEY key is missing',
|
||||
}));
|
||||
assert.match(result.skipped ?? "", /OPENAI_API_KEY/);
|
||||
});
|
||||
|
||||
it("prunes ghost vectors for nodes that no longer exist", async () => {
|
||||
const g = fakeGraphStore({ vectors: { n1: [1, 0], ghost: [0, 1] } });
|
||||
const scripted = scriptedPort(g.ledger.model, 2);
|
||||
const result = await embedNewNodes(g.store, () => okPort(scripted.port));
|
||||
assert.equal(result.embedded, 1, "n1 already has a vector; only n2 embeds");
|
||||
assert.ok(!("ghost" in g.ledger.vectors), "the ghost vector is pruned");
|
||||
});
|
||||
|
||||
it("batches at 32 (a 40-node backfill = two API calls)", async () => {
|
||||
const many: GraphNode[] = Array.from({ length: 40 }, (_, i) => node(`n${i}`, `statement ${i}`));
|
||||
const ledger: EmbeddingsLedger = { model: "m", dim: 2, vectors: {}, clusters: [], updatedAt: "" };
|
||||
const writes: EmbeddingsLedger[] = [];
|
||||
const scripted = scriptedPort("m", 2);
|
||||
const result = await embedNewNodes(
|
||||
{
|
||||
readNodes: () => many,
|
||||
readEmbeddings: () => ledger,
|
||||
writeEmbeddings: (next) => writes.push(next),
|
||||
},
|
||||
() => okPort(scripted.port),
|
||||
);
|
||||
assert.equal(result.embedded, 40);
|
||||
assert.equal(scripted.calls.length, 2);
|
||||
assert.equal(scripted.calls[0]!.texts.length, 32);
|
||||
assert.equal(scripted.calls[1]!.texts.length, 8);
|
||||
assert.equal(writes.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the federated catalog (scripted keys + fetch) ────────────────────────────
|
||||
|
||||
describe("embeddingModelCatalog", () => {
|
||||
it("no keys = empty catalog", async () => {
|
||||
const models = await embeddingModelCatalog(
|
||||
recordingFetch(() => jsonResponse({})).fetch,
|
||||
() => undefined,
|
||||
);
|
||||
assert.deepEqual(models, []);
|
||||
});
|
||||
|
||||
it("federates every keyed provider with its filter rules", async () => {
|
||||
const { fetch, calls } = recordingFetch((url) => {
|
||||
if (url.includes("openrouter.ai/api/v1/embeddings/models")) {
|
||||
return jsonResponse({ data: [{ id: "openai/text-embedding-3-small", name: "Text Embedding 3 Small" }, { id: "other/embed" }] });
|
||||
}
|
||||
if (url.includes("api.openai.com")) {
|
||||
return jsonResponse({ data: [{ id: "text-embedding-3-large" }, { id: "text-embedding-3-small" }, { id: "gpt-6-sol" }] });
|
||||
}
|
||||
if (url.includes("generativelanguage")) {
|
||||
return jsonResponse({
|
||||
models: [
|
||||
{ name: "models/gemini-embedding-001", displayName: "Gemini Embedding", supportedGenerationMethods: ["embedContent", "generateContent"] },
|
||||
{ name: "models/gemini-6-pro", supportedGenerationMethods: ["generateContent"] },
|
||||
],
|
||||
});
|
||||
}
|
||||
return jsonResponse({});
|
||||
});
|
||||
const models = await embeddingModelCatalog(fetch, (id) => (id === "OPENROUTER_API_KEY" || id === "OPENAI_API_KEY" || id === "GEMINI_API_KEY" ? "k" : undefined));
|
||||
assert.equal(calls[0]!.url, "https://openrouter.ai/api/v1/embeddings/models?limit=500", "the real catalog path (the inverse 404s — verified live)");
|
||||
assert.deepEqual(models, [
|
||||
{ id: "openrouter:openai/text-embedding-3-small", name: "Text Embedding 3 Small" },
|
||||
{ id: "openrouter:other/embed", name: "other/embed" },
|
||||
{ id: "openai:text-embedding-3-large", name: "text-embedding-3-large" },
|
||||
{ id: "openai:text-embedding-3-small", name: "text-embedding-3-small" },
|
||||
{ id: "google:gemini-embedding-001", name: "Gemini Embedding" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("a failing provider never poisons the rest", async () => {
|
||||
const { fetch } = recordingFetch((url) => {
|
||||
if (url.includes("openrouter.ai")) return jsonResponse({}, 500); // their side broke
|
||||
if (url.includes("api.openai.com")) return jsonResponse({ data: [{ id: "text-embedding-3-small" }] });
|
||||
return jsonResponse({});
|
||||
});
|
||||
const models = await embeddingModelCatalog(fetch, (id) => (id === "OPENROUTER_API_KEY" || id === "OPENAI_API_KEY" ? "k" : undefined));
|
||||
assert.deepEqual(models, [{ id: "openai:text-embedding-3-small", name: "text-embedding-3-small" }]);
|
||||
});
|
||||
});
|
||||
@@ -15,7 +15,7 @@ import { after, describe, it } from "node:test";
|
||||
import { deliverJobResult, setTurnRunner } from "../src/server/jobs.ts";
|
||||
import { abortJob } from "../src/server/jobs.ts";
|
||||
import { persistTurn } from "../src/server/turn-service.ts";
|
||||
import { startGraphContextInjection, runGraphContextJob, type GraphContextDeps } from "../src/agent/subagents/graph-context.ts";
|
||||
import { startGraphContextInjection, runGraphContextJob, type GraphContextDeps, type GraphContextEmbedder } from "../src/agent/subagents/graph-context.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { jobStore } from "../src/store/jobs.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
@@ -104,17 +104,18 @@ function newTopicChat(label: string, settings?: Record<string, unknown>): { id:
|
||||
return { id: chat.id };
|
||||
}
|
||||
|
||||
/** The scripted embedder: deterministic query vectors, recorded topics. */
|
||||
/** The scripted embedder: deterministic query vectors, recorded topics.
|
||||
* Wave 6 shape: the query carries its (absent) prompt-token count; the
|
||||
* resolver seam wraps the static fake (fresh resolution per run). */
|
||||
function fakeEmbedder(vec: number[] = unit(0)) {
|
||||
const topics: string[] = [];
|
||||
return {
|
||||
topics,
|
||||
embedder: {
|
||||
model: "test-embed",
|
||||
dim: 8,
|
||||
embedQuery: async (text: string) => {
|
||||
topics.push(text);
|
||||
return vec;
|
||||
return { vector: vec, promptTokens: undefined };
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -122,10 +123,10 @@ function fakeEmbedder(vec: number[] = unit(0)) {
|
||||
|
||||
function deps(
|
||||
graph: Pick<GraphContextDeps["graphStore"], "readNodes" | "readEvents" | "readEmbeddings">,
|
||||
embedder: GraphContextDeps["embedder"],
|
||||
embedder: GraphContextEmbedder,
|
||||
deliver: (steer: SteerMessage) => Promise<void>,
|
||||
): GraphContextDeps {
|
||||
return { chatStore, jobStore, graphStore: graph, embedder, deliver };
|
||||
return { chatStore, jobStore, graphStore: graph, embedder: () => embedder, deliver };
|
||||
}
|
||||
|
||||
describe("startGraphContextInjection — the one-shot gates", () => {
|
||||
@@ -205,7 +206,9 @@ describe("runGraphContextJob — failure modes are honest errors, never phantom
|
||||
trackChat(chat.id);
|
||||
const job = jobStore.createJob({ kind: "graph-context", chatId, toolCallId: `ctx_${chatId}` });
|
||||
trackJob(job.id);
|
||||
// Ledger stamped with a DIFFERENT model than the embedder.
|
||||
// Ledger stamped with a DIFFERENT model than the embedder (wave 6:
|
||||
// staleness is the model identity — dim is checked against the
|
||||
// served vector below).
|
||||
const d = deps(fakeGraphStore({ vectors: { n1: unit(0) }, model: "other-model" }), fakeEmbedder().embedder, async () => {});
|
||||
await runGraphContextJob(d, job.id);
|
||||
const got = jobStore.getJob(job.id)!;
|
||||
@@ -280,8 +283,7 @@ describe("the injection + the wake pipeline (end to end, fake turn runner)", ()
|
||||
});
|
||||
const embedder = {
|
||||
model: "test-embed",
|
||||
dim: 8,
|
||||
embedQuery: () => hanging.then(() => unit(0)),
|
||||
embedQuery: () => hanging.then(() => ({ vector: unit(0), promptTokens: undefined })),
|
||||
};
|
||||
const fired: number[] = [];
|
||||
setTurnRunner(async () => {
|
||||
|
||||
@@ -11,9 +11,12 @@ import { providerSafePng } from "../src/agent/subagents/headless.ts";
|
||||
* passed. providerSafePng merges the split back into the canonical ONE-IDAT
|
||||
* form (PNG spec: the IDAT chunks are an arbitrary split of ONE zlib stream —
|
||||
* the merged form carries the byte-identical stream). Pure Node on purpose:
|
||||
* the process already loads transformers' nested sharp/libvips; a second copy
|
||||
* (root sharp) triggered macOS's duplicate-libvips ObjC warning. These tests
|
||||
* pin the property with no native deps at all.
|
||||
* historically the process already carried a native image lib via
|
||||
* @huggingface/transformers' nested sharp/libvips, and a second copy (root
|
||||
* sharp) triggered macOS's duplicate-libvips ObjC warning — the transformers
|
||||
* embedder died with wave 6 (2026-09-23) but the no-native-deps rule lives
|
||||
* on: app code carries no image library. These tests pin the property with
|
||||
* no native deps at all.
|
||||
*/
|
||||
|
||||
// ── a tiny PNG toolchain (spec-level, no image library) ─────────────────────
|
||||
|
||||
@@ -156,6 +156,30 @@ describe("parseGraphSettingsBody", () => {
|
||||
const r = parseGraphSettingsBody({ enabled: false, clustering: true, labeling: true });
|
||||
assert.ok(r.ok);
|
||||
});
|
||||
|
||||
it("embeddingModel (wave 6): null or a shipped-provider slug; anything else is a 400", () => {
|
||||
const nulled = parseGraphSettingsBody({ embeddingModel: null });
|
||||
assert.ok(nulled.ok);
|
||||
assert.deepEqual(nulled.settings, { embeddingModel: null });
|
||||
for (const slug of [
|
||||
"openai:text-embedding-3-small",
|
||||
"openrouter:openai/text-embedding-3-large",
|
||||
"google:gemini-embedding-001",
|
||||
// OpenRouter ids may carry a :free-style suffix — the provider is
|
||||
// the part before the FIRST colon (live catalog fact, 2026-09-23).
|
||||
"openrouter:liquid/lfm-2.5-embedding-350m:free",
|
||||
]) {
|
||||
const r = parseGraphSettingsBody({ embeddingModel: slug });
|
||||
assert.ok(r.ok, slug);
|
||||
assert.deepEqual(r.settings, { embeddingModel: slug });
|
||||
}
|
||||
// Bad shapes never become a live model.
|
||||
assert.equal(parseGraphSettingsBody({ embeddingModel: "text-embedding-3-small" }).ok, false, "no provider prefix");
|
||||
assert.equal(parseGraphSettingsBody({ embeddingModel: "bedrock:whatever" }).ok, false, "unshipped provider");
|
||||
assert.equal(parseGraphSettingsBody({ embeddingModel: "openai:" }).ok, false, "empty id");
|
||||
assert.equal(parseGraphSettingsBody({ embeddingModel: 42 }).ok, false, "non-string");
|
||||
assert.equal(parseGraphSettingsBody({ embeddingModel: "openai:has spaces" }).ok, false, "whitespace id");
|
||||
});
|
||||
});
|
||||
|
||||
describe("SETTINGS_SECTIONS (the route's section map)", () => {
|
||||
|
||||
@@ -42,7 +42,7 @@ describe("resolveSettings", () => {
|
||||
assert.equal(eff.appearance.proseFontSize, 16);
|
||||
assert.equal(eff.appearance.sessionNameSerif, true);
|
||||
assert.equal(eff.appearance.mdRendering.headingWeight, 900);
|
||||
assert.deepEqual(eff.graph, { enabled: true, embedding: true, clustering: true, labeling: true });
|
||||
assert.deepEqual(eff.graph, { enabled: true, embedding: true, clustering: true, labeling: true, embeddingModel: null }); // BYOK wave 6: no factory embedding model
|
||||
});
|
||||
|
||||
it("explicit record values win over every factory default", () => {
|
||||
@@ -77,7 +77,7 @@ describe("resolveSettings", () => {
|
||||
assert.equal(eff.appearance.mdRendering.headingWeight, 600);
|
||||
// graph: enabled=false explicit; absent rungs stay factory-on (stored
|
||||
// but inert — the ladder grays them out in the UI).
|
||||
assert.deepEqual(eff.graph, { enabled: false, embedding: true, clustering: true, labeling: true });
|
||||
assert.deepEqual(eff.graph, { enabled: false, embedding: true, clustering: true, labeling: true, embeddingModel: null }); // wave 6: unset embeddingModel resolves to the null floor
|
||||
});
|
||||
|
||||
it("partial role override: thinkingLevel alone borrows the SLOT's model (or nothing, without one)", () => {
|
||||
@@ -224,15 +224,15 @@ describe("resolveConversationSpec (the turn gate's chain)", () => {
|
||||
|
||||
describe("the graph channel gates (the wave's seam honoring)", () => {
|
||||
it("WRITE channel (extraction): dies with the master switch only", () => {
|
||||
assert.equal(extractionEnabled({ enabled: true, embedding: true, clustering: true, labeling: true }), true);
|
||||
assert.equal(extractionEnabled({ enabled: true, embedding: false, clustering: false, labeling: false }), true);
|
||||
assert.equal(extractionEnabled({ enabled: false, embedding: true, clustering: true, labeling: true }), false);
|
||||
assert.equal(extractionEnabled({ enabled: true, embedding: true, clustering: true, labeling: true, embeddingModel: null }), true);
|
||||
assert.equal(extractionEnabled({ enabled: true, embedding: false, clustering: false, labeling: false, embeddingModel: null }), true);
|
||||
assert.equal(extractionEnabled({ enabled: false, embedding: true, clustering: true, labeling: true, embeddingModel: null }), false);
|
||||
});
|
||||
|
||||
it("READ channel (graph-context): needs master AND the embedding rung", () => {
|
||||
assert.equal(graphContextEnabled({ enabled: true, embedding: true, clustering: false, labeling: false }), true);
|
||||
assert.equal(graphContextEnabled({ enabled: true, embedding: false, clustering: true, labeling: true }), false);
|
||||
assert.equal(graphContextEnabled({ enabled: false, embedding: true, clustering: true, labeling: true }), false);
|
||||
assert.equal(graphContextEnabled({ enabled: true, embedding: true, clustering: false, labeling: false, embeddingModel: null }), true);
|
||||
assert.equal(graphContextEnabled({ enabled: true, embedding: false, clustering: true, labeling: true, embeddingModel: null }), false);
|
||||
assert.equal(graphContextEnabled({ enabled: false, embedding: true, clustering: true, labeling: true, embeddingModel: null }), false);
|
||||
});
|
||||
|
||||
it("both gates ride resolveSettings' factory defaults (all on → both open)", () => {
|
||||
|
||||
@@ -21,7 +21,7 @@ describe("FACTORY_SETTINGS (the compiled factory layer)", () => {
|
||||
it("pins the owner-decided defaults", () => {
|
||||
assert.equal(CURRENT_SETTINGS_VERSION, 1);
|
||||
assert.deepEqual(FACTORY_SETTINGS.general, { autoUpdate: true });
|
||||
assert.deepEqual(FACTORY_SETTINGS.graph, { enabled: true, embedding: true, clustering: true, labeling: true });
|
||||
assert.deepEqual(FACTORY_SETTINGS.graph, { enabled: true, embedding: true, clustering: true, labeling: true, embeddingModel: null }); // BYOK wave 6: no factory embedding model
|
||||
const a = FACTORY_SETTINGS.appearance;
|
||||
assert.equal(a.mode, "dark");
|
||||
assert.equal(a.background, null); // the mode's own default, not a factory hex
|
||||
|
||||
Reference in New Issue
Block a user