feat(models): strong/cheap/vision model slots + multi-provider engine federation
Model slots wave (2026-09-20), chunk B:
- settings: new `models` section {strong, cheap, vision} slots (each
field independent, same shape as a role override); ROLE_SLOT maps all
nine roles to tiers (strong=prober+teacher, cheap=researcher/
demoBuilder/extractor/auditor/labeler, vision=visualMaker+pdfReader);
resolution chain per role: override → slot → factory pin.
- config: ModelSpec gains provider (all pins openrouter, the BYOK
default); MODEL_PINS in stores.ts now spans all nine roles (the strong
slot's factory rep IS the prober pin).
- engine: piModels() registers EVERY builtin provider catalog;
piEngineForSpec(resolved spec) is the new engine factory (pi-ai
resolves per-provider keys from env itself); piEngineFor stays as the
config-pin fallback. chat-handler resolves the conversation chain
(chat override → strong slot → pin) and injects engineFor into the job
runner, closing the old gap where per-role overrides never reached
job engines.
- /api/models: models federate ONLY keyed providers (providers surface
with keyPresent, env-wins, fresh per call); slots (resolved tier pins)
join rolePins; catalogDefaultsFor inherits the strong slot; the models
settings section gets the same catalog gate as chat.
- pricing: PRICES/CONTEXT_LIMITS uniformly keyed provider:id
(regenerated from ALL pi-ai catalogs, 1354 keys); usage stamps
(TurnUsage, JobUsage, extraction stages) switch to provider:id —
owner decision: no legacy compat, old stamps render unpriced.
This commit is contained in:
@@ -16,7 +16,7 @@ import type { CallUsage, ExtractionRunUsage, NamingRunUsage } from "../src/core/
|
||||
import { PRICES, callCost } from "../src/core/prices.ts";
|
||||
|
||||
/** A model with full published pricing (from the generated registry). */
|
||||
const PRICED = "z-ai/glm-5.3-flash";
|
||||
const PRICED = "openrouter:z-ai/glm-5.3-flash";
|
||||
assert.ok(PRICES[PRICED], "fixture model must exist in PRICES");
|
||||
|
||||
const step = (input: number, output: number): CallUsage => ({
|
||||
|
||||
@@ -553,7 +553,8 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const run = record.extractions![0]!;
|
||||
assert.equal(run.stats?.nodes, 0); // linked, not created
|
||||
assert.equal(run.stats?.events, 3);
|
||||
assert.equal(run.stages.extractor.model, "faux-1"); // the scripted engine's model id
|
||||
// the scripted engine's model, provider-qualified (model slots wave)
|
||||
assert.equal(run.stages.extractor.model, "faux:faux-1");
|
||||
assert.equal(run.stages.extractor.steps.length, 1); // the submit call — the run STOPS after a clean submit
|
||||
const step = run.stages.extractor.steps[0]!;
|
||||
assert.ok(step.inputTokens! > 0 && step.outputTokens! > 0, "estimated usage captured");
|
||||
|
||||
+3
-1
@@ -1218,7 +1218,9 @@ describe("abort (src/server/jobs.ts abortJob)", () => {
|
||||
assert.equal(job.status, "error");
|
||||
assert.equal(job.error, "Aborted by learner");
|
||||
assert.equal(job.usage!.steps, 1, "usage-so-far persisted by the runner's finally");
|
||||
assert.equal(job.usage!.model, "z-ai/glm-5.3-flash", "the config role's OpenRouter id");
|
||||
// model slots wave: the stamp is the ENGINE's actual model,
|
||||
// provider-qualified — the scripted faux engine here.
|
||||
assert.equal(job.usage!.model, "faux:faux-1");
|
||||
// faux estimates tokens from the prompt/response text — the
|
||||
// pinned fact is that the ONE completed step's spend survived.
|
||||
assert.ok(job.usage!.inputTokens! > 0, "input tokens billed");
|
||||
|
||||
+87
-12
@@ -1,21 +1,27 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { modelCatalog, validateChatSettings, catalogDefaultsFor, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts";
|
||||
import { modelCatalog, validateChatSettings, validateModelsSettings, catalogDefaultsFor, catalogProviders, catalogSlots, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts";
|
||||
import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai";
|
||||
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import { MODELS } from "../config.ts";
|
||||
import { piEngineFor } from "../src/agent/models.ts";
|
||||
import { BACKGROUND_MODEL_ROLES, type BackgroundModelRole } from "../src/core/settings.ts";
|
||||
import { resolveSettings } from "../src/core/settings-resolution.ts";
|
||||
import { BACKGROUND_MODEL_PINS } from "../src/server/stores.ts";
|
||||
import { MODEL_PINS } from "../src/server/stores.ts";
|
||||
|
||||
// The model-control wave (2026-09-12): pi-ai owns the registry — these pins
|
||||
// hold our projection + validation honest against the shipped catalog, and
|
||||
// pin pi's per-model thinking truth so a pi-ai upgrade can't silently change
|
||||
// what we send.
|
||||
|
||||
// The picker federates only providers with a PRESENT key (model slots
|
||||
// wave, 2026-09-20): pin openrouter's env key so its full catalog
|
||||
// federates in tests regardless of the developer shell's exports.
|
||||
process.env.OPENROUTER_API_KEY ??= "test-key-openrouter-catalog";
|
||||
|
||||
describe("the model catalog projection", () => {
|
||||
it("serves pi-ai's OpenRouter catalog with the picker's DTO shape", () => {
|
||||
const { models, defaults } = modelCatalog();
|
||||
it("serves the keyed providers' catalogs with the picker's DTO shape", () => {
|
||||
const { models, defaults, providers, slots } = modelCatalog();
|
||||
// 0.85.1 ships 366 OpenRouter models — the exact count is pi's, but
|
||||
// the shape and the config models' presence are ours to pin.
|
||||
assert.ok(models.length > 300, `expected a full catalog, got ${models.length}`);
|
||||
@@ -32,23 +38,35 @@ describe("the model catalog projection", () => {
|
||||
assert.deepEqual(glm!.input, ["text", "image"]);
|
||||
// pi's current glm price (our old table said 0.15/0.5 — stale)
|
||||
assert.deepEqual(glm!.cost, { input: 0.075, output: 0.25, cacheRead: 0.015, cacheWrite: 0 });
|
||||
// model slots wave: the providers surface + the resolved slot pins
|
||||
const openrouter = providers.find((p) => p.id === "openrouter");
|
||||
assert.ok(openrouter && openrouter.keyPresent, "openrouter federated (key pinned above)");
|
||||
assert.ok(providers.some((p) => !p.keyPresent), "keyless providers listed for discovery");
|
||||
assert.deepEqual(Object.keys(slots), ["strong", "cheap", "vision"]);
|
||||
// a keyless provider's models never reach the picker
|
||||
const keyless = providers.filter((p) => !p.keyPresent).map((p) => p.id);
|
||||
for (const id of keyless) {
|
||||
assert.ok(!models.some((m) => m.provider === id), `keyless ${id} must not federate`);
|
||||
}
|
||||
});
|
||||
|
||||
it("defaults mirror the conversation agents' config pin", () => {
|
||||
const d = catalogDefaultsFor();
|
||||
it("defaults mirror the conversation agents' config pin (strong rep = prober pin)", () => {
|
||||
const strong = { provider: "openrouter", id: MODELS.prober.model, thinkingLevel: MODELS.prober.effort as ThinkingLevel };
|
||||
const d = catalogDefaultsFor(undefined, strong);
|
||||
assert.deepEqual(d, { model: { provider: "openrouter", id: MODELS.prober.model }, thinkingLevel: MODELS.prober.effort });
|
||||
assert.equal(d.model.id, MODELS.teacher.model, "one controller: prober and teacher share the pin");
|
||||
});
|
||||
|
||||
it("a picked default in the record's chat section overrides the pin (catalogDefaultsFor, settings wave)", () => {
|
||||
const d = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" });
|
||||
const strong = { provider: "openrouter", id: MODELS.prober.model, thinkingLevel: MODELS.prober.effort as ThinkingLevel };
|
||||
const d = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" }, strong);
|
||||
assert.deepEqual(d, { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" });
|
||||
// partial: only the model chosen → thinking falls back to the pin
|
||||
const partial = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } });
|
||||
const partial = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }, strong);
|
||||
assert.equal(partial.thinkingLevel, MODELS.prober.effort);
|
||||
assert.equal(partial.model.id, "qwen/qwen3.8-flash");
|
||||
// graphContextTokens-only record: model untouched
|
||||
assert.deepEqual(catalogDefaultsFor({ graphContextTokens: 512 }), {
|
||||
assert.deepEqual(catalogDefaultsFor({ graphContextTokens: 512 }, strong), {
|
||||
model: { provider: "openrouter", id: MODELS.prober.model },
|
||||
thinkingLevel: MODELS.prober.effort,
|
||||
});
|
||||
@@ -67,7 +85,7 @@ describe("the model catalog projection", () => {
|
||||
|
||||
describe("the background roles' resolved pins (rolePins)", () => {
|
||||
it("no overrides → every role pins its config MODELS spec, nested DTO shape, all six roles", () => {
|
||||
const pins = rolePinsFor(resolveSettings(null, BACKGROUND_MODEL_PINS).agent.models);
|
||||
const pins = rolePinsFor(resolveSettings(null, MODEL_PINS).agent.models);
|
||||
assert.deepEqual(Object.keys(pins), [...BACKGROUND_MODEL_ROLES]);
|
||||
for (const role of BACKGROUND_MODEL_ROLES) {
|
||||
assert.deepEqual(pins[role].model, { provider: "openrouter", id: MODELS[role].model });
|
||||
@@ -82,7 +100,7 @@ describe("the background roles' resolved pins (rolePins)", () => {
|
||||
version: 1,
|
||||
agent: { models: { researcher: { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } } } },
|
||||
};
|
||||
const pins = rolePinsFor(resolveSettings(record, BACKGROUND_MODEL_PINS).agent.models);
|
||||
const pins = rolePinsFor(resolveSettings(record, MODEL_PINS).agent.models);
|
||||
assert.equal(pins.researcher.model.id, "qwen/qwen3.8-flash");
|
||||
assert.equal(pins.researcher.thinkingLevel, MODELS.researcher.effort, "level stays the pin unless overridden too");
|
||||
assert.equal(pins.visualMaker.model.id, MODELS.visualMaker.model);
|
||||
@@ -91,12 +109,69 @@ describe("the background roles' resolved pins (rolePins)", () => {
|
||||
|
||||
it("a thinkingLevel-only override keeps the model, changes the level", () => {
|
||||
const record = { version: 1, agent: { models: { extractor: { thinkingLevel: "high" as const } } } };
|
||||
const pins = rolePinsFor(resolveSettings(record, BACKGROUND_MODEL_PINS).agent.models);
|
||||
const pins = rolePinsFor(resolveSettings(record, MODEL_PINS).agent.models);
|
||||
assert.equal(pins.extractor.model.id, MODELS.extractor.model);
|
||||
assert.equal(pins.extractor.thinkingLevel, "high");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the model slots surface (model slots wave, 2026-09-20)", () => {
|
||||
it("catalogSlots reports the factory rep pins when no record exists", () => {
|
||||
const slots = catalogSlots();
|
||||
assert.deepEqual(slots.strong.model, { provider: "openrouter", id: MODELS.prober.model });
|
||||
assert.deepEqual(slots.cheap.model, { provider: "openrouter", id: MODELS.researcher.model });
|
||||
assert.deepEqual(slots.vision.model, { provider: "openrouter", id: MODELS.visualMaker.model });
|
||||
});
|
||||
|
||||
it("catalogProviders reads keys env-wins, fresh per call", () => {
|
||||
const before = catalogProviders().find((p) => p.id === "zai");
|
||||
assert.equal(before?.keyPresent, false);
|
||||
const saved = process.env.ZAI_API_KEY;
|
||||
process.env.ZAI_API_KEY = "test-zai-key-1234567890";
|
||||
try {
|
||||
assert.equal(catalogProviders().find((p) => p.id === "zai")?.keyPresent, true);
|
||||
} finally {
|
||||
if (saved === undefined) delete process.env.ZAI_API_KEY;
|
||||
else process.env.ZAI_API_KEY = saved;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("the models-section validation gate", () => {
|
||||
it("accepts a valid slot and rejects unknown provider/model", () => {
|
||||
assert.equal(
|
||||
validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } } }).ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
validateModelsSettings({ cheap: { model: { provider: "nope", id: "x" } } }).ok,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
validateModelsSettings({ vision: { model: { provider: "openrouter", id: "nope/nope" } } }).ok,
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("thinkingLevel resolves against the slot's model, else the tier's resolved pin", () => {
|
||||
// level checked against the named model
|
||||
assert.equal(
|
||||
validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "max" } }).ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "minimal" } }).ok,
|
||||
false,
|
||||
);
|
||||
// bare level: against the strong slot's resolved model (config pin)
|
||||
assert.equal(validateModelsSettings({ strong: { thinkingLevel: "minimal" } }).ok, false);
|
||||
});
|
||||
|
||||
it("accepts empty settings (the clear-slots write)", () => {
|
||||
assert.deepEqual(validateModelsSettings({}), { ok: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("the settings validation gate", () => {
|
||||
it("accepts a supported level and rejects an unsupported one with the list", () => {
|
||||
const ok = validateChatSettings({ model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "max" });
|
||||
|
||||
@@ -48,7 +48,7 @@ describe("activity dock pure layer", () => {
|
||||
it("folds cost with jobCost; absent usage is null, never zero", () => {
|
||||
const priced = job({
|
||||
usage: {
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
inputTokens: 10_000,
|
||||
outputTokens: 1000,
|
||||
cacheReadTokens: 5000,
|
||||
@@ -67,7 +67,7 @@ describe("activity dock pure layer", () => {
|
||||
|
||||
it("context line = LAST step's tokens vs the model limit, never a cumulative sum", () => {
|
||||
const base = {
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
inputTokens: 400_000, // running total (much bigger — must NOT feed the line)
|
||||
outputTokens: 8_000,
|
||||
cacheReadTokens: 0,
|
||||
@@ -166,7 +166,7 @@ describe("activity section render", () => {
|
||||
|
||||
it("shows total running cost when jobs are priced", () => {
|
||||
const usage = {
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
inputTokens: 10_000,
|
||||
outputTokens: 1000,
|
||||
cacheReadTokens: 0,
|
||||
|
||||
@@ -18,7 +18,7 @@ function turn(over: Partial<TurnUsage> = {}): TurnUsage {
|
||||
id: "t1",
|
||||
messageId: "m1",
|
||||
at: "2026-09-09T12:00:00Z",
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
steps: [step(10_000, 1_000)],
|
||||
...over,
|
||||
};
|
||||
|
||||
@@ -11,6 +11,7 @@ import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
parseAgentSettingsBody,
|
||||
parseModelsSettingsBody,
|
||||
parseAppearanceSettingsBody,
|
||||
parseGeneralSettingsBody,
|
||||
parseGraphSettingsBody,
|
||||
@@ -158,7 +159,47 @@ describe("parseGraphSettingsBody", () => {
|
||||
});
|
||||
|
||||
describe("SETTINGS_SECTIONS (the route's section map)", () => {
|
||||
it("maps exactly the five record sections", () => {
|
||||
assert.deepEqual(Object.keys(SETTINGS_SECTIONS), ["general", "chat", "agent", "appearance", "graph"]);
|
||||
it("maps exactly the six record sections", () => {
|
||||
assert.deepEqual(Object.keys(SETTINGS_SECTIONS), ["general", "chat", "agent", "appearance", "graph", "models"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseModelsSettingsBody (model slots wave, 2026-09-20)", () => {
|
||||
it("parses the three slot tiers, each field independent", () => {
|
||||
const r = parseModelsSettingsBody({
|
||||
strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } },
|
||||
cheap: { thinkingLevel: "high" },
|
||||
vision: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "low" },
|
||||
});
|
||||
assert.ok(r.ok);
|
||||
assert.deepEqual(r.settings, {
|
||||
strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } },
|
||||
cheap: { thinkingLevel: "high" },
|
||||
vision: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "low" },
|
||||
});
|
||||
});
|
||||
|
||||
it("{} is the legal clear-slots write", () => {
|
||||
const r = parseModelsSettingsBody({});
|
||||
assert.ok(r.ok);
|
||||
assert.deepEqual(r.settings, {});
|
||||
});
|
||||
|
||||
it("unknown slots are 400s with the supported list", () => {
|
||||
const bad = parseModelsSettingsBody({ mega: { thinkingLevel: "high" } });
|
||||
assert.equal(bad.ok, false);
|
||||
if (!bad.ok) assert.match(bad.error, /unknown slot "mega" \(supported: strong, cheap, vision\)/);
|
||||
});
|
||||
|
||||
it("a present-but-empty slot object is rejected (absent slot = unset)", () => {
|
||||
const bad = parseModelsSettingsBody({ strong: {} });
|
||||
assert.equal(bad.ok, false);
|
||||
if (!bad.ok) assert.match(bad.error, /slot "strong" must set model or thinkingLevel/);
|
||||
});
|
||||
|
||||
it("malformed slot shapes are 400s", () => {
|
||||
assert.equal(parseModelsSettingsBody({ cheap: { model: { provider: "x" } } }).ok, false);
|
||||
assert.equal(parseModelsSettingsBody({ vision: { thinkingLevel: 7 } }).ok, false);
|
||||
assert.equal(parseModelsSettingsBody("nope").ok, false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,26 +1,42 @@
|
||||
/**
|
||||
* Tests for the resolution layer (settings-storage wave, 2026-09-19):
|
||||
* core/settings-resolution.ts — the delta→concrete projection. The factory
|
||||
* layer is threaded in as a parameter (core stays pure; server/stores.ts
|
||||
* threads config.ts's MODELS), so these tests use a small synthetic pin.
|
||||
* Tests for the resolution layer (settings-storage wave, 2026-09-19;
|
||||
* model-slots wave 2026-09-20 adds the slot tier): core/settings-resolution.ts
|
||||
* — the delta→concrete projection. The factory layer is threaded in as a
|
||||
* parameter (core stays pure; server/stores.ts threads config.ts's MODELS),
|
||||
* so these tests use a small synthetic pin spanning ALL NINE roles (the
|
||||
* factory contract since the slots wave — the strong slot's rep IS the
|
||||
* prober pin).
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
extractionEnabled,
|
||||
factorySlotRep,
|
||||
graphContextEnabled,
|
||||
resolveModelSpec,
|
||||
resolveSettings,
|
||||
resolveSlot,
|
||||
type ModelRoleSpecs,
|
||||
} from "../src/core/settings-resolution.ts";
|
||||
import { BACKGROUND_MODEL_ROLES, type UserSettings } from "../src/core/settings.ts";
|
||||
import { BACKGROUND_MODEL_ROLES, type ModelSlotName, type UserSettings } from "../src/core/settings.ts";
|
||||
|
||||
const FACTORY_PINS = Object.fromEntries(
|
||||
BACKGROUND_MODEL_ROLES.map((role, i) => [
|
||||
[
|
||||
"prober",
|
||||
"teacher",
|
||||
"researcher",
|
||||
"demoBuilder",
|
||||
"visualMaker",
|
||||
"pdfReader",
|
||||
"extractor",
|
||||
"auditor",
|
||||
"labeler",
|
||||
].map((role, i) => [
|
||||
role,
|
||||
{ model: `factory/model-${i}`, effort: i % 2 === 0 ? "low" : "off" },
|
||||
{ provider: i === 5 ? "other" : "openrouter", model: `factory/model-${i}`, effort: i % 2 === 0 ? "low" : "off" },
|
||||
]),
|
||||
) as Record<(typeof BACKGROUND_MODEL_ROLES)[number], { model: string; effort: "low" | "off" }>;
|
||||
) as ModelRoleSpecs;
|
||||
|
||||
describe("resolveSettings", () => {
|
||||
it("null record (first boot): every knob lands on its factory default", () => {
|
||||
@@ -29,11 +45,28 @@ describe("resolveSettings", () => {
|
||||
assert.deepEqual(eff.chat, {}); // BYOK: no default model, ever
|
||||
assert.deepEqual(eff.agent.tools, {});
|
||||
assert.deepEqual(eff.agent.prompts, {});
|
||||
assert.deepEqual(eff.agent.search, {});
|
||||
for (const role of BACKGROUND_MODEL_ROLES) {
|
||||
assert.equal(eff.agent.models[role].id, FACTORY_PINS[role].model);
|
||||
assert.equal(eff.agent.models[role].provider, "openrouter");
|
||||
assert.equal(eff.agent.models[role].thinkingLevel, FACTORY_PINS[role].effort);
|
||||
assert.equal(eff.agent.models[role].id, FACTORY_PINS[role]!.model);
|
||||
assert.equal(eff.agent.models[role].provider, FACTORY_PINS[role]!.provider);
|
||||
assert.equal(eff.agent.models[role].thinkingLevel, FACTORY_PINS[role]!.effort);
|
||||
}
|
||||
// slots with no record: every tier lands on its factory rep pin
|
||||
assert.deepEqual(eff.models.strong, {
|
||||
provider: "openrouter",
|
||||
id: FACTORY_PINS.prober!.model,
|
||||
thinkingLevel: FACTORY_PINS.prober!.effort,
|
||||
});
|
||||
assert.deepEqual(eff.models.cheap, {
|
||||
provider: "openrouter",
|
||||
id: FACTORY_PINS.researcher!.model,
|
||||
thinkingLevel: FACTORY_PINS.researcher!.effort,
|
||||
});
|
||||
assert.deepEqual(eff.models.vision, {
|
||||
provider: "openrouter",
|
||||
id: FACTORY_PINS.visualMaker!.model,
|
||||
thinkingLevel: FACTORY_PINS.visualMaker!.effort,
|
||||
});
|
||||
assert.equal(eff.appearance.mode, "dark");
|
||||
assert.equal(eff.appearance.background, null);
|
||||
assert.equal(eff.appearance.fontFamily, null);
|
||||
@@ -52,6 +85,7 @@ describe("resolveSettings", () => {
|
||||
models: { researcher: { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } } },
|
||||
tools: { research: false },
|
||||
prompts: { "system-prompt.md": true },
|
||||
search: { engine: "tavily" },
|
||||
},
|
||||
appearance: { mode: "light", proseFontSize: 20, mdRendering: { headingWeight: 600 } },
|
||||
graph: { enabled: false },
|
||||
@@ -64,6 +98,7 @@ describe("resolveSettings", () => {
|
||||
id: "qwen/qwen3.8-flash",
|
||||
thinkingLevel: "low", // thinking NOT overridden → factory effort
|
||||
});
|
||||
assert.deepEqual(eff.agent.search, { engine: "tavily" });
|
||||
assert.deepEqual(eff.agent.tools, { research: false });
|
||||
assert.deepEqual(eff.agent.prompts, { "system-prompt.md": true });
|
||||
assert.equal(eff.appearance.mode, "light");
|
||||
@@ -79,29 +114,101 @@ describe("resolveSettings", () => {
|
||||
{ version: 1, agent: { models: { extractor: { thinkingLevel: "medium" } } } },
|
||||
FACTORY_PINS,
|
||||
);
|
||||
const extractor = FACTORY_PINS.extractor;
|
||||
const extractor = FACTORY_PINS.extractor!;
|
||||
assert.deepEqual(eff.agent.models.extractor, {
|
||||
provider: "openrouter",
|
||||
provider: extractor.provider,
|
||||
id: extractor.model,
|
||||
thinkingLevel: "medium",
|
||||
});
|
||||
// sibling roles untouched
|
||||
assert.equal(eff.agent.models.researcher.thinkingLevel, FACTORY_PINS.researcher.effort);
|
||||
assert.equal(eff.agent.models.researcher.thinkingLevel, FACTORY_PINS.researcher!.effort);
|
||||
});
|
||||
|
||||
it("resolveModelSpec is field-independent per override aspect", () => {
|
||||
assert.deepEqual(
|
||||
resolveModelSpec("labeler", { model: { provider: "other", id: "x" } }, FACTORY_PINS),
|
||||
{ provider: "other", id: "x", thinkingLevel: FACTORY_PINS.labeler.effort },
|
||||
{ provider: "other", id: "x", thinkingLevel: FACTORY_PINS.labeler!.effort },
|
||||
);
|
||||
assert.deepEqual(resolveModelSpec("labeler", {}, FACTORY_PINS), {
|
||||
provider: "openrouter",
|
||||
id: FACTORY_PINS.labeler.model,
|
||||
thinkingLevel: FACTORY_PINS.labeler.effort,
|
||||
provider: FACTORY_PINS.labeler!.provider,
|
||||
id: FACTORY_PINS.labeler!.model,
|
||||
thinkingLevel: FACTORY_PINS.labeler!.effort,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("the slot tier (model slots wave, 2026-09-20)", () => {
|
||||
it("slot alone re-tiers every role of its tier (no role overrides present)", () => {
|
||||
const eff = resolveSettings(
|
||||
{ version: 1, models: { cheap: { model: { provider: "zai", id: "glm-5.3-flash" } } } },
|
||||
FACTORY_PINS,
|
||||
);
|
||||
for (const role of ["researcher", "demoBuilder", "extractor", "auditor", "labeler"] as const) {
|
||||
assert.deepEqual(eff.agent.models[role as "researcher"], {
|
||||
provider: "zai",
|
||||
id: "glm-5.3-flash",
|
||||
thinkingLevel: FACTORY_PINS[role]!.effort, // thinking not in slot → factory
|
||||
}, role);
|
||||
}
|
||||
// other tiers untouched
|
||||
assert.equal(eff.agent.models.visualMaker.id, FACTORY_PINS.visualMaker!.model);
|
||||
});
|
||||
|
||||
it("role override still beats the slot; slot beats the factory pin (the chain)", () => {
|
||||
const eff = resolveSettings(
|
||||
{
|
||||
version: 1,
|
||||
agent: { models: { researcher: { model: { provider: "groq", id: "r-override" } } } },
|
||||
models: { cheap: { model: { provider: "zai", id: "r-slot" }, thinkingLevel: "high" } },
|
||||
},
|
||||
FACTORY_PINS,
|
||||
);
|
||||
// researcher: override model + slot thinking (field-independent)
|
||||
assert.deepEqual(eff.agent.models.researcher, {
|
||||
provider: "groq",
|
||||
id: "r-override",
|
||||
thinkingLevel: "high",
|
||||
});
|
||||
// labeler (same tier, no override): full slot
|
||||
assert.deepEqual(eff.agent.models.labeler, { provider: "zai", id: "r-slot", thinkingLevel: "high" });
|
||||
});
|
||||
|
||||
it("vision slot covers visualMaker AND pdfReader; strong's rep is the prober pin", () => {
|
||||
const eff = resolveSettings(
|
||||
{ version: 1, models: { vision: { thinkingLevel: "max" } } },
|
||||
FACTORY_PINS,
|
||||
);
|
||||
assert.equal(eff.agent.models.visualMaker.thinkingLevel, "max");
|
||||
assert.equal(eff.agent.models.visualMaker.id, FACTORY_PINS.visualMaker!.model);
|
||||
// resolved slot views exist for every tier, set or not
|
||||
assert.deepEqual(eff.models.strong, factorySlotRep("strong", FACTORY_PINS));
|
||||
});
|
||||
|
||||
it("resolveSlot: slot over factory rep, per field", () => {
|
||||
assert.deepEqual(resolveSlot("strong", { thinkingLevel: "medium" }, FACTORY_PINS), {
|
||||
provider: FACTORY_PINS.prober!.provider,
|
||||
id: FACTORY_PINS.prober!.model,
|
||||
thinkingLevel: "medium",
|
||||
});
|
||||
const rep = factorySlotRep("vision", FACTORY_PINS);
|
||||
assert.equal(rep.id, FACTORY_PINS.visualMaker!.model);
|
||||
const slot: ModelSlotName = "cheap";
|
||||
assert.deepEqual(resolveSlot(slot, undefined, FACTORY_PINS), {
|
||||
provider: FACTORY_PINS.researcher!.provider,
|
||||
id: FACTORY_PINS.researcher!.model,
|
||||
thinkingLevel: FACTORY_PINS.researcher!.effort,
|
||||
});
|
||||
});
|
||||
|
||||
it("absent provider coordinate anywhere rides the factory provider (legacy ids)", () => {
|
||||
// resolveModelSpec with no slots and a factory provider of "other"
|
||||
// (pdfReader is index 5 in the synthetic pins): a legacy override
|
||||
// with no provider keeps the FACTORY provider, not a hardcoded one.
|
||||
const spec = resolveModelSpec("auditor", {}, FACTORY_PINS);
|
||||
assert.equal(spec.provider, FACTORY_PINS.auditor!.provider);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the graph channel gates (the wave's seam honoring)", () => {
|
||||
it("WRITE channel (extraction): dies with the master switch only", () => {
|
||||
assert.equal(extractionEnabled({ enabled: true, embedding: true, clustering: true, labeling: true }), true);
|
||||
|
||||
@@ -63,6 +63,10 @@ describe("sanitizeUserSettings", () => {
|
||||
prompts: { "system-prompt.md": true, "teacher/20-teaching.md": false },
|
||||
search: { engine: "brave" },
|
||||
},
|
||||
models: {
|
||||
strong: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "low" },
|
||||
cheap: { thinkingLevel: "medium" },
|
||||
},
|
||||
appearance: {
|
||||
mode: "light",
|
||||
background: "#ffffff",
|
||||
|
||||
+13
-13
@@ -40,7 +40,7 @@ test("sumSteps folds steps; absent stays absent until a value appears", () => {
|
||||
});
|
||||
|
||||
test("callCost: glm-5.3-flash math — fresh input, cached read, output", () => {
|
||||
const cost = callCost(call({ inputTokens: 100_000, outputTokens: 20_000, cacheReadTokens: 10_000 }), "z-ai/glm-5.3-flash");
|
||||
const cost = callCost(call({ inputTokens: 100_000, outputTokens: 20_000, cacheReadTokens: 10_000 }), "openrouter:z-ai/glm-5.3-flash");
|
||||
assert.ok(cost !== null);
|
||||
// fresh 90k @ $0.075/M + cached 10k @ $0.015/M + output 20k @ $0.25/M
|
||||
// (pi-ai catalog prices — regenerated tables, scripts/sync-pricing)
|
||||
@@ -50,7 +50,7 @@ test("callCost: glm-5.3-flash math — fresh input, cached read, output", () =>
|
||||
test("callCost: cache writes bill at the cacheWrite rate", () => {
|
||||
const cost = callCost(
|
||||
call({ inputTokens: 1_000_000, outputTokens: 0, cacheWriteTokens: 500_000 }),
|
||||
"qwen/qwen3.8-max-0902",
|
||||
"openrouter:qwen/qwen3.8-max-0902",
|
||||
);
|
||||
assert.ok(cost !== null);
|
||||
// fresh 1M @ $2/M + cache write 500k @ $2.5/M
|
||||
@@ -59,7 +59,7 @@ test("callCost: cache writes bill at the cacheWrite rate", () => {
|
||||
|
||||
test("callCost: unknown model and absent fields are null, never zero", () => {
|
||||
assert.equal(callCost(call({ inputTokens: 100, outputTokens: 5 }), "not/a-model"), null);
|
||||
assert.equal(callCost(call({ inputTokens: 100, outputTokens: undefined }), "z-ai/glm-5.3-flash"), null);
|
||||
assert.equal(callCost(call({ inputTokens: 100, outputTokens: undefined }), "openrouter:z-ai/glm-5.3-flash"), null);
|
||||
});
|
||||
|
||||
test("turnCost sums steps; sessionCost reports unpriced turns", () => {
|
||||
@@ -67,7 +67,7 @@ test("turnCost sums steps; sessionCost reports unpriced turns", () => {
|
||||
id: "t1",
|
||||
messageId: "m1",
|
||||
at: "2026-09-09T00:00:00Z",
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
steps: [call({ inputTokens: 10_000, outputTokens: 2_000 }), call({ inputTokens: 12_000, outputTokens: 1_000 })],
|
||||
};
|
||||
assert.ok(turnCost(turn) !== null);
|
||||
@@ -81,12 +81,12 @@ test("turnCost sums steps; sessionCost reports unpriced turns", () => {
|
||||
test("contextWindow: last call's footprint against the model's limit", () => {
|
||||
const cw = contextWindow({
|
||||
turns: [
|
||||
{ id: "t1", messageId: "m1", at: "x", model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
|
||||
{ id: "t2", messageId: "m2", at: "x", model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
|
||||
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
|
||||
{ id: "t2", messageId: "m2", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
|
||||
],
|
||||
});
|
||||
assert.ok(cw);
|
||||
assert.equal(cw.model, "z-ai/glm-5.3-flash");
|
||||
assert.equal(cw.model, "openrouter:z-ai/glm-5.3-flash");
|
||||
assert.equal(cw.usedTokens, 100_000);
|
||||
assert.equal(cw.limit, 1_048_576);
|
||||
assert.equal(contextWindow({ turns: [] }), null);
|
||||
@@ -96,7 +96,7 @@ test("jobCost: null without usage, computed with it", () => {
|
||||
assert.equal(jobCost({}), null);
|
||||
const job: Pick<JobRecord, "usage"> = {
|
||||
usage: {
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
inputTokens: 1_000_000,
|
||||
outputTokens: 100_000,
|
||||
cacheReadTokens: 0,
|
||||
@@ -117,7 +117,7 @@ test("appendTurnUsage: appends, idempotent per turn id, continuation turns (same
|
||||
id: "turn_1",
|
||||
messageId: "msg_a",
|
||||
at: "2026-09-09T00:00:00Z",
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
steps: [call({ inputTokens: 100, outputTokens: 10 })],
|
||||
};
|
||||
chatStore.appendTurnUsage(record.id, entry);
|
||||
@@ -149,8 +149,8 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl
|
||||
durationMs: 42_000,
|
||||
stats: { nodes: 1, edges: 2, events: 3 },
|
||||
stages: {
|
||||
extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
},
|
||||
});
|
||||
chatStore.appendExtractionUsage(record.id, runEntry(1));
|
||||
@@ -186,8 +186,8 @@ test("appendSourceUsage: appends to the source record, idempotent per startedAt,
|
||||
durationMs: 42_000,
|
||||
stats: { nodes: 1, edges: 2, events: 3 },
|
||||
stages: {
|
||||
extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
},
|
||||
});
|
||||
const id = "src_usagetest01";
|
||||
|
||||
Reference in New Issue
Block a user