feat(models): strong/cheap/vision model slots + multi-provider engine federation
Model slots wave (2026-09-20), chunk B:
- settings: new `models` section {strong, cheap, vision} slots (each
field independent, same shape as a role override); ROLE_SLOT maps all
nine roles to tiers (strong=prober+teacher, cheap=researcher/
demoBuilder/extractor/auditor/labeler, vision=visualMaker+pdfReader);
resolution chain per role: override → slot → factory pin.
- config: ModelSpec gains provider (all pins openrouter, the BYOK
default); MODEL_PINS in stores.ts now spans all nine roles (the strong
slot's factory rep IS the prober pin).
- engine: piModels() registers EVERY builtin provider catalog;
piEngineForSpec(resolved spec) is the new engine factory (pi-ai
resolves per-provider keys from env itself); piEngineFor stays as the
config-pin fallback. chat-handler resolves the conversation chain
(chat override → strong slot → pin) and injects engineFor into the job
runner, closing the old gap where per-role overrides never reached
job engines.
- /api/models: models federate ONLY keyed providers (providers surface
with keyPresent, env-wins, fresh per call); slots (resolved tier pins)
join rolePins; catalogDefaultsFor inherits the strong slot; the models
settings section gets the same catalog gate as chat.
- pricing: PRICES/CONTEXT_LIMITS uniformly keyed provider:id
(regenerated from ALL pi-ai catalogs, 1354 keys); usage stamps
(TurnUsage, JobUsage, extraction stages) switch to provider:id —
owner decision: no legacy compat, old stamps render unpriced.
This commit is contained in:
amosblomqvist
+87
-12
@@ -1,21 +1,27 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { modelCatalog, validateChatSettings, catalogDefaultsFor, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts";
|
||||
import { modelCatalog, validateChatSettings, validateModelsSettings, catalogDefaultsFor, catalogProviders, catalogSlots, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts";
|
||||
import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai";
|
||||
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import { MODELS } from "../config.ts";
|
||||
import { piEngineFor } from "../src/agent/models.ts";
|
||||
import { BACKGROUND_MODEL_ROLES, type BackgroundModelRole } from "../src/core/settings.ts";
|
||||
import { resolveSettings } from "../src/core/settings-resolution.ts";
|
||||
import { BACKGROUND_MODEL_PINS } from "../src/server/stores.ts";
|
||||
import { MODEL_PINS } from "../src/server/stores.ts";
|
||||
|
||||
// The model-control wave (2026-09-12): pi-ai owns the registry — these pins
|
||||
// hold our projection + validation honest against the shipped catalog, and
|
||||
// pin pi's per-model thinking truth so a pi-ai upgrade can't silently change
|
||||
// what we send.
|
||||
|
||||
// The picker federates only providers with a PRESENT key (model slots
|
||||
// wave, 2026-09-20): pin openrouter's env key so its full catalog
|
||||
// federates in tests regardless of the developer shell's exports.
|
||||
process.env.OPENROUTER_API_KEY ??= "test-key-openrouter-catalog";
|
||||
|
||||
describe("the model catalog projection", () => {
|
||||
it("serves pi-ai's OpenRouter catalog with the picker's DTO shape", () => {
|
||||
const { models, defaults } = modelCatalog();
|
||||
it("serves the keyed providers' catalogs with the picker's DTO shape", () => {
|
||||
const { models, defaults, providers, slots } = modelCatalog();
|
||||
// 0.85.1 ships 366 OpenRouter models — the exact count is pi's, but
|
||||
// the shape and the config models' presence are ours to pin.
|
||||
assert.ok(models.length > 300, `expected a full catalog, got ${models.length}`);
|
||||
@@ -32,23 +38,35 @@ describe("the model catalog projection", () => {
|
||||
assert.deepEqual(glm!.input, ["text", "image"]);
|
||||
// pi's current glm price (our old table said 0.15/0.5 — stale)
|
||||
assert.deepEqual(glm!.cost, { input: 0.075, output: 0.25, cacheRead: 0.015, cacheWrite: 0 });
|
||||
// model slots wave: the providers surface + the resolved slot pins
|
||||
const openrouter = providers.find((p) => p.id === "openrouter");
|
||||
assert.ok(openrouter && openrouter.keyPresent, "openrouter federated (key pinned above)");
|
||||
assert.ok(providers.some((p) => !p.keyPresent), "keyless providers listed for discovery");
|
||||
assert.deepEqual(Object.keys(slots), ["strong", "cheap", "vision"]);
|
||||
// a keyless provider's models never reach the picker
|
||||
const keyless = providers.filter((p) => !p.keyPresent).map((p) => p.id);
|
||||
for (const id of keyless) {
|
||||
assert.ok(!models.some((m) => m.provider === id), `keyless ${id} must not federate`);
|
||||
}
|
||||
});
|
||||
|
||||
it("defaults mirror the conversation agents' config pin", () => {
|
||||
const d = catalogDefaultsFor();
|
||||
it("defaults mirror the conversation agents' config pin (strong rep = prober pin)", () => {
|
||||
const strong = { provider: "openrouter", id: MODELS.prober.model, thinkingLevel: MODELS.prober.effort as ThinkingLevel };
|
||||
const d = catalogDefaultsFor(undefined, strong);
|
||||
assert.deepEqual(d, { model: { provider: "openrouter", id: MODELS.prober.model }, thinkingLevel: MODELS.prober.effort });
|
||||
assert.equal(d.model.id, MODELS.teacher.model, "one controller: prober and teacher share the pin");
|
||||
});
|
||||
|
||||
it("a picked default in the record's chat section overrides the pin (catalogDefaultsFor, settings wave)", () => {
|
||||
const d = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" });
|
||||
const strong = { provider: "openrouter", id: MODELS.prober.model, thinkingLevel: MODELS.prober.effort as ThinkingLevel };
|
||||
const d = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" }, strong);
|
||||
assert.deepEqual(d, { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" });
|
||||
// partial: only the model chosen → thinking falls back to the pin
|
||||
const partial = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } });
|
||||
const partial = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }, strong);
|
||||
assert.equal(partial.thinkingLevel, MODELS.prober.effort);
|
||||
assert.equal(partial.model.id, "qwen/qwen3.8-flash");
|
||||
// graphContextTokens-only record: model untouched
|
||||
assert.deepEqual(catalogDefaultsFor({ graphContextTokens: 512 }), {
|
||||
assert.deepEqual(catalogDefaultsFor({ graphContextTokens: 512 }, strong), {
|
||||
model: { provider: "openrouter", id: MODELS.prober.model },
|
||||
thinkingLevel: MODELS.prober.effort,
|
||||
});
|
||||
@@ -67,7 +85,7 @@ describe("the model catalog projection", () => {
|
||||
|
||||
describe("the background roles' resolved pins (rolePins)", () => {
|
||||
it("no overrides → every role pins its config MODELS spec, nested DTO shape, all six roles", () => {
|
||||
const pins = rolePinsFor(resolveSettings(null, BACKGROUND_MODEL_PINS).agent.models);
|
||||
const pins = rolePinsFor(resolveSettings(null, MODEL_PINS).agent.models);
|
||||
assert.deepEqual(Object.keys(pins), [...BACKGROUND_MODEL_ROLES]);
|
||||
for (const role of BACKGROUND_MODEL_ROLES) {
|
||||
assert.deepEqual(pins[role].model, { provider: "openrouter", id: MODELS[role].model });
|
||||
@@ -82,7 +100,7 @@ describe("the background roles' resolved pins (rolePins)", () => {
|
||||
version: 1,
|
||||
agent: { models: { researcher: { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } } } },
|
||||
};
|
||||
const pins = rolePinsFor(resolveSettings(record, BACKGROUND_MODEL_PINS).agent.models);
|
||||
const pins = rolePinsFor(resolveSettings(record, MODEL_PINS).agent.models);
|
||||
assert.equal(pins.researcher.model.id, "qwen/qwen3.8-flash");
|
||||
assert.equal(pins.researcher.thinkingLevel, MODELS.researcher.effort, "level stays the pin unless overridden too");
|
||||
assert.equal(pins.visualMaker.model.id, MODELS.visualMaker.model);
|
||||
@@ -91,12 +109,69 @@ describe("the background roles' resolved pins (rolePins)", () => {
|
||||
|
||||
it("a thinkingLevel-only override keeps the model, changes the level", () => {
|
||||
const record = { version: 1, agent: { models: { extractor: { thinkingLevel: "high" as const } } } };
|
||||
const pins = rolePinsFor(resolveSettings(record, BACKGROUND_MODEL_PINS).agent.models);
|
||||
const pins = rolePinsFor(resolveSettings(record, MODEL_PINS).agent.models);
|
||||
assert.equal(pins.extractor.model.id, MODELS.extractor.model);
|
||||
assert.equal(pins.extractor.thinkingLevel, "high");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the model slots surface (model slots wave, 2026-09-20)", () => {
|
||||
it("catalogSlots reports the factory rep pins when no record exists", () => {
|
||||
const slots = catalogSlots();
|
||||
assert.deepEqual(slots.strong.model, { provider: "openrouter", id: MODELS.prober.model });
|
||||
assert.deepEqual(slots.cheap.model, { provider: "openrouter", id: MODELS.researcher.model });
|
||||
assert.deepEqual(slots.vision.model, { provider: "openrouter", id: MODELS.visualMaker.model });
|
||||
});
|
||||
|
||||
it("catalogProviders reads keys env-wins, fresh per call", () => {
|
||||
const before = catalogProviders().find((p) => p.id === "zai");
|
||||
assert.equal(before?.keyPresent, false);
|
||||
const saved = process.env.ZAI_API_KEY;
|
||||
process.env.ZAI_API_KEY = "test-zai-key-1234567890";
|
||||
try {
|
||||
assert.equal(catalogProviders().find((p) => p.id === "zai")?.keyPresent, true);
|
||||
} finally {
|
||||
if (saved === undefined) delete process.env.ZAI_API_KEY;
|
||||
else process.env.ZAI_API_KEY = saved;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("the models-section validation gate", () => {
|
||||
it("accepts a valid slot and rejects unknown provider/model", () => {
|
||||
assert.equal(
|
||||
validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } } }).ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
validateModelsSettings({ cheap: { model: { provider: "nope", id: "x" } } }).ok,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
validateModelsSettings({ vision: { model: { provider: "openrouter", id: "nope/nope" } } }).ok,
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("thinkingLevel resolves against the slot's model, else the tier's resolved pin", () => {
|
||||
// level checked against the named model
|
||||
assert.equal(
|
||||
validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "max" } }).ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "minimal" } }).ok,
|
||||
false,
|
||||
);
|
||||
// bare level: against the strong slot's resolved model (config pin)
|
||||
assert.equal(validateModelsSettings({ strong: { thinkingLevel: "minimal" } }).ok, false);
|
||||
});
|
||||
|
||||
it("accepts empty settings (the clear-slots write)", () => {
|
||||
assert.deepEqual(validateModelsSettings({}), { ok: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("the settings validation gate", () => {
|
||||
it("accepts a supported level and rejects an unsupported one with the list", () => {
|
||||
const ok = validateChatSettings({ model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "max" });
|
||||
|
||||
Reference in New Issue
Block a user