feat: model + thinking control — the stats bar's picker, on pi-ai's catalog
The model-control wave (user-decided design, 2026-09-12): ONE controller
for the conversation agents (prober/teacher are functionally one agent);
background sub-agents stay config-pinned. pi-ai owns the registry — the
shipped 366-model OpenRouter catalog (cost, context, modality,
thinkingLevelMap), getSupportedThinkingLevels/clampThinkingLevel for the
per-model thinking truth, and streamSimple's automatic clamp. No second
registry to drift out of sync.
Seams:
- config.ts: ReasoningEffort retired for pi's ThinkingLevel
(off|minimal|low|medium|high|xhigh|max). GLM roles go honest at "low" —
pi's catalog says GLM-5.3-flash accepts exactly low|high|max, and the old
"minimal" was silently clamped to low by streamSimple since the swap;
nothing changes behaviorally. visualMaker → qwen/qwen3.8-flash (fixes
the dead qwen/qwen3.8-max slug — the catalog only ships -0902; also the
user's cost pick, 0.15/0.47 per-1M).
- src/agent/models.ts: piEngineFor(role, settings?) — provider+id resolved
from the catalog, thinking level from settings ?? config, CLAMPED to the
model's supported list (a settings change between models can never send
an invalid level). PiEngine gains `level` (observability/tests).
- ChatRecord.settings {model?: {provider, id}, thinkingLevel?} — written
by PUT /api/chats/[id]/settings (replace-whole, no updatedAt churn),
read by chat-handler at turn composition: agents are stateless per turn,
so the change is next-turn-effective by construction. The turns log
still records the ACTUAL billed model (the honest witness).
- GET /api/models: the configured-provider catalog as client-safe DTOs
(core/contracts/models.ts) + config defaults; a second provider joins
the picker by registering it in piModels() — nothing else changes.
- PUT settings validation gate (src/server/model-catalog.ts): unknown
provider/model or unsupported level → 400 carrying the supported list,
never a silent clamp.
- scripts/sync-pricing.ts: regenerated PRICES/CONTEXT_LIMITS from pi-ai's
shipped openrouter.json (offline; same source as the engine + picker).
All 366 models priced; glm's stale 0.15/0.5 → pi's 0.075/0.25, limit
1310720 → 1048576 — cost is inferred at read time, so history re-prices.
- UI (components/model-picker.tsx): stats-bar right side becomes model +
thinking buttons; menus open ABOVE; fuzzy search over "provider id
name" (core/fuzzy.ts — pi's /model pattern); model rows show name/id/
price/context; the thinking menu lists exactly the chosen model's
supported levels (GLM: low|high|max — no dead options, no "off").
Model switch drops an incompatible pinned level (falls back to config,
which clamps honestly) rather than carrying a 400.
Tests (350/350, +15): catalog projection + defaults, the settings gate,
pi thinking truth pinned against upgrades (glm low|high|max; minimal→low;
off→low — GLM cannot disable thinking), the engine factory's
override/fallback/clamp/throw matrix, fuzzy ranking, store round-trip,
and the re-priced folds (usage, stats bar, activity dock).
Live smoke (disposable driver, deleted): catalog route 366 + defaults;
glm+minimal → 400 + supported; turn 1 billed qwen/qwen3.8-flash under
the override; mid-session switch flipped turn 2 to glm; clear-overrides
restored the config pin. ALL CHECKS PASS on real OpenRouter.
This commit is contained in:
@@ -422,6 +422,20 @@ describe("chat store", () => {
|
||||
assert.equal(loaded.messages.length, 1);
|
||||
assert.equal(loaded.title, "Store test");
|
||||
});
|
||||
|
||||
it("persist the per-chat model settings (replace-whole, no updatedAt churn)", async () => {
|
||||
const chat = chatStore.createChat("Settings test");
|
||||
createdChats.push(chat.id);
|
||||
const before = chatStore.getChat(chat.id)!.updatedAt;
|
||||
chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "high" });
|
||||
const saved = chatStore.getChat(chat.id)!;
|
||||
assert.deepEqual(saved.settings, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "high" });
|
||||
assert.equal(saved.updatedAt, before, "a picker write is not conversation activity");
|
||||
// replace-whole: a settings-less write clears the override (an empty
|
||||
// object persists — every field absent = config defaults)
|
||||
chatStore.saveSettings(chat.id, {});
|
||||
assert.deepEqual(chatStore.getChat(chat.id)!.settings, {});
|
||||
});
|
||||
});
|
||||
|
||||
describe("mid-turn navigation (turn liveness + write-ahead)", () => {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { fuzzyScore } from "../src/core/fuzzy.ts";
|
||||
|
||||
// The picker's search ranking (pi's /model pattern): tokens must each match
|
||||
// some field; exact beats prefix beats buried substring; whole-query
|
||||
// contiguity in one field is a bonus.
|
||||
|
||||
describe("fuzzyScore", () => {
|
||||
const fields = (id: string, name: string) => ["openrouter", id, name];
|
||||
|
||||
it("matches across fields case-insensitively", () => {
|
||||
assert.ok(fuzzyScore("GLM", ...fields("z-ai/glm-5.3-flash", "ZAI: GLM 5.3 Flash")) !== null);
|
||||
assert.ok(fuzzyScore("flash", ...fields("z-ai/glm-5.3-flash", "ZAI: GLM 5.3 Flash")) !== null);
|
||||
});
|
||||
|
||||
it("multi-token queries need every token to match somewhere", () => {
|
||||
assert.ok(fuzzyScore("glm flash", ...fields("z-ai/glm-5.3-flash", "ZAI: GLM 5.3 Flash")) !== null);
|
||||
assert.equal(fuzzyScore("glm qwen", ...fields("z-ai/glm-5.3-flash", "ZAI: GLM 5.3 Flash")), null);
|
||||
});
|
||||
|
||||
it("ranks exact field equality above prefix above buried substring", () => {
|
||||
const exact = fuzzyScore("flash", "openrouter", "flash", "other"); // field === token
|
||||
const prefix = fuzzyScore("flash", "openrouter", "flash-2", "other"); // field startsWith
|
||||
const buried = fuzzyScore("flash", "openrouter", "z-ai/glm-5.3-flash", "other");
|
||||
assert.ok(exact !== null && prefix !== null && buried !== null);
|
||||
assert.ok(exact > prefix && prefix > buried, `${exact} > ${prefix} > ${buried}`);
|
||||
});
|
||||
|
||||
it("rewards whole-query contiguity in a single field", () => {
|
||||
const split = fuzzyScore("glm flash", ...fields("z-ai/glm-5.3-flash", "x")); // two fields, no contiguity
|
||||
const joined = fuzzyScore("glm 5.3", ...fields("z-ai/glm-5.3-flash", "x")); // contiguous in the id
|
||||
assert.ok(split !== null && joined !== null);
|
||||
// same per-token points; joined earns the +50 contiguity bonus
|
||||
assert.ok(joined > split, `${joined} > ${split}`);
|
||||
});
|
||||
|
||||
it("empty query matches everything at score 0", () => {
|
||||
assert.equal(fuzzyScore("", "anything"), 0);
|
||||
assert.equal(fuzzyScore(" ", "anything"), 0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,106 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { modelCatalog, validateChatSettings, catalogDefaults, toModelDto } from "../src/server/model-catalog.ts";
|
||||
import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai";
|
||||
import { MODELS } from "../config.ts";
|
||||
import { piEngineFor } from "../src/agent/models.ts";
|
||||
|
||||
// The model-control wave (2026-09-12): pi-ai owns the registry — these pins
|
||||
// hold our projection + validation honest against the shipped catalog, and
|
||||
// pin pi's per-model thinking truth so a pi-ai upgrade can't silently change
|
||||
// what we send.
|
||||
|
||||
describe("the model catalog projection", () => {
|
||||
it("serves pi-ai's OpenRouter catalog with the picker's DTO shape", () => {
|
||||
const { models, defaults } = modelCatalog();
|
||||
// 0.85.1 ships 366 OpenRouter models — the exact count is pi's, but
|
||||
// the shape and the config models' presence are ours to pin.
|
||||
assert.ok(models.length > 300, `expected a full catalog, got ${models.length}`);
|
||||
for (const m of models.slice(0, 25)) {
|
||||
assert.equal(typeof m.id, "string");
|
||||
assert.equal(typeof m.name, "string");
|
||||
assert.ok(Array.isArray(m.supportedLevels));
|
||||
assert.ok(typeof m.cost.input === "number" && typeof m.cost.output === "number");
|
||||
assert.ok(m.contextWindow > 0);
|
||||
}
|
||||
const glm = models.find((m) => m.id === "z-ai/glm-5.3-flash");
|
||||
assert.ok(glm, "glm-5.3-flash in catalog");
|
||||
assert.deepEqual(glm!.supportedLevels, ["low", "high", "max"]);
|
||||
assert.deepEqual(glm!.input, ["text", "image"]);
|
||||
// pi's current glm price (our old table said 0.15/0.5 — stale)
|
||||
assert.deepEqual(glm!.cost, { input: 0.075, output: 0.25, cacheRead: 0.015, cacheWrite: 0 });
|
||||
});
|
||||
|
||||
it("defaults mirror the conversation agents' config pin", () => {
|
||||
const d = catalogDefaults();
|
||||
assert.deepEqual(d, { model: { provider: "openrouter", id: MODELS.prober.model }, thinkingLevel: MODELS.prober.effort });
|
||||
assert.equal(d.model.id, MODELS.teacher.model, "one controller: prober and teacher share the pin");
|
||||
});
|
||||
|
||||
it("toModelDto projects the raw catalog model verbatim", () => {
|
||||
const raw = piEngineFor("prober").model; // the catalog's glm entry
|
||||
const dto = toModelDto(raw);
|
||||
assert.equal(dto.id, raw.id);
|
||||
assert.equal(dto.name, raw.name);
|
||||
assert.deepEqual(dto.supportedLevels, getSupportedThinkingLevels(raw));
|
||||
assert.equal(dto.cost.input, raw.cost.input);
|
||||
assert.equal(dto.contextWindow, raw.contextWindow);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the settings validation gate", () => {
|
||||
it("accepts a supported level and rejects an unsupported one with the list", () => {
|
||||
const ok = validateChatSettings({ model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "max" });
|
||||
assert.deepEqual(ok, { ok: true });
|
||||
const bad = validateChatSettings({ model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "minimal" });
|
||||
assert.equal(bad.ok, false);
|
||||
if (!bad.ok) {
|
||||
assert.match(bad.error, /does not support thinking level/);
|
||||
assert.deepEqual(bad.supported, ["low", "high", "max"]);
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves a bare thinking level against the config default model", () => {
|
||||
assert.deepEqual(validateChatSettings({ thinkingLevel: "high" }), { ok: true });
|
||||
// "minimal" is unsupported on config's glm — rejected even with no model set
|
||||
assert.equal(validateChatSettings({ thinkingLevel: "minimal" }).ok, false);
|
||||
});
|
||||
|
||||
it("rejects unknown providers and models", () => {
|
||||
assert.equal(validateChatSettings({ model: { provider: "nope", id: "x/y" } }).ok, false);
|
||||
assert.equal(validateChatSettings({ model: { provider: "openrouter", id: "nope/nope" } }).ok, false);
|
||||
});
|
||||
|
||||
it("accepts empty settings (the clear-overrides write)", () => {
|
||||
assert.deepEqual(validateChatSettings({}), { ok: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("pi thinking truth (pinned against pi-ai upgrades)", () => {
|
||||
it("glm-5.3-flash: low|high|max; anything else clamps to the nearest", () => {
|
||||
const glm = piEngineFor("prober").model;
|
||||
assert.deepEqual(getSupportedThinkingLevels(glm), ["low", "high", "max"]);
|
||||
assert.equal(clampThinkingLevel(glm, "minimal"), "low");
|
||||
assert.equal(clampThinkingLevel(glm, "off"), "low", "GLM cannot disable thinking (map: off → null)");
|
||||
assert.equal(clampThinkingLevel(glm, "xhigh"), "max");
|
||||
assert.equal(clampThinkingLevel(glm, "high"), "high");
|
||||
});
|
||||
|
||||
it("the engine factory honors settings, falls back to config, clamps", () => {
|
||||
// override both fields
|
||||
const over = piEngineFor("teacher", { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" });
|
||||
assert.equal(over.model.id, "qwen/qwen3.8-flash");
|
||||
assert.equal(over.level, "high");
|
||||
// no settings → config pin
|
||||
const pinned = piEngineFor("prober");
|
||||
assert.equal(pinned.model.id, MODELS.prober.model);
|
||||
assert.equal(pinned.level, MODELS.prober.effort);
|
||||
// settings level unsupported by the settings model → clamped honestly
|
||||
const clamped = piEngineFor("teacher", { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "minimal" });
|
||||
assert.equal(clamped.level, "low");
|
||||
// bare level resolves against the config default model
|
||||
assert.equal(piEngineFor("prober", { thinkingLevel: "max" }).level, "max");
|
||||
// unknown model → honest throw, never a silent fallback
|
||||
assert.throws(() => piEngineFor("teacher", { model: { provider: "openrouter", id: "nope/nope" } }), /catalog has no model/);
|
||||
});
|
||||
});
|
||||
@@ -57,8 +57,9 @@ describe("activity dock pure layer", () => {
|
||||
},
|
||||
});
|
||||
const blob = toBlobs({ jobs: [priced], extractions: [] })[0]!;
|
||||
// (10000-5000)/1e6*0.15 + 5000/1e6*0.03 + 1000/1e6*0.5 = 0.0014
|
||||
assert.ok(blob.cost !== null && blob.cost > 0.0013 && blob.cost < 0.0015);
|
||||
// (10000-5000)/1e6*0.075 + 5000/1e6*0.015 + 1000/1e6*0.25 = 0.0007
|
||||
// (pi-ai catalog prices)
|
||||
assert.ok(blob.cost !== null && blob.cost > 0.0006 && blob.cost < 0.0008);
|
||||
const absent = toBlobs({ jobs: [job()], extractions: [] })[0]!;
|
||||
assert.equal(absent.cost, null);
|
||||
});
|
||||
@@ -78,7 +79,7 @@ describe("activity dock pure layer", () => {
|
||||
});
|
||||
const blob = toBlobs({ jobs: [withLast], extractions: [] })[0]!;
|
||||
assert.equal(blob.context?.used, 92_000);
|
||||
assert.equal(blob.context?.limit, 1310720);
|
||||
assert.equal(blob.context?.limit, 1048576);
|
||||
// No lastStep yet (capture pending) → "—", even though usage exists.
|
||||
const stripped = toBlobs({ jobs: [job({ usage: { ...base } })], extractions: [] })[0]!;
|
||||
assert.equal(stripped.context, null);
|
||||
@@ -167,7 +168,7 @@ describe("activity dock render", () => {
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
// two identical priced jobs → 2 × $0.002 = $0.0040 total
|
||||
assert.match(html, /activity-total-cost[^>]*>\$0\.0040</);
|
||||
// two identical priced jobs → 2 × $0.001 = $0.0020 total (pi prices)
|
||||
assert.match(html, /activity-total-cost[^>]*>\$0\.0020</);
|
||||
});
|
||||
});
|
||||
@@ -3,11 +3,11 @@ import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { StatsBar } from "../components/stats-bar.tsx";
|
||||
import { prettyModel } from "../components/usage-format.ts";
|
||||
import type { TurnUsage } from "../src/core/contracts.ts";
|
||||
import type { ChatSettings, TurnUsage } from "../src/core/contracts.ts";
|
||||
|
||||
// The stats bar: folds only (no cost math in the component), the "—"
|
||||
// doctrine for absent data, honest cost (never a fake $0), and the
|
||||
// model/thinking placeholder chip for the coming control.
|
||||
// doctrine for absent data, honest cost (never a fake $0), and the model +
|
||||
// thinking control (components/model-picker.tsx — pi-ai catalog backed).
|
||||
|
||||
function step(inputTokens: number, outputTokens: number, over: Record<string, number | undefined> = {}) {
|
||||
return { inputTokens, outputTokens, cacheReadTokens: undefined, cacheWriteTokens: undefined, reasoningTokens: undefined, ...over };
|
||||
@@ -24,28 +24,40 @@ function turn(over: Partial<TurnUsage> = {}): TurnUsage {
|
||||
};
|
||||
}
|
||||
|
||||
const SETTINGS_STUB = (_s: ChatSettings) => {};
|
||||
|
||||
describe("stats bar", () => {
|
||||
it("renders — everywhere when the record has no turns yet (doctrine)", () => {
|
||||
const html = renderToStaticMarkup(<StatsBar turns={[]} />);
|
||||
const html = renderToStaticMarkup(
|
||||
<StatsBar turns={[]} chatId="chat_x" onSettingsSaved={SETTINGS_STUB} />,
|
||||
);
|
||||
assert.match(html, /data-testid="stats-bar"/);
|
||||
assert.match(html, /data-testid="stats-context">—</);
|
||||
assert.match(html, /data-testid="stats-cost">—</);
|
||||
assert.match(html, /data-testid="stats-model">—</);
|
||||
// no turns, no settings, catalog not loaded → both control labels "—"
|
||||
assert.match(html, /data-testid="stats-model"[^>]*><span>—<\/span>/);
|
||||
assert.match(html, /Thinking —</);
|
||||
});
|
||||
|
||||
it("folds context + cost from the turns log (core/prices, no local math)", () => {
|
||||
const html = renderToStaticMarkup(<StatsBar turns={[turn(), turn()]} />);
|
||||
// last turn's last call: 10k in + 1k out = 11k; limit 1,310,720 → "1.3M"
|
||||
assert.match(html, /data-testid="stats-context">11k \/ 1\.3M</);
|
||||
// each turn: 10k fresh input ×0.15 + 1k output ×0.5 = $0.002 → two = $0.0040
|
||||
assert.match(html, /data-testid="stats-cost">\$0\.0040</);
|
||||
// model chip: pretty name from the last turn's model id
|
||||
assert.match(html, /data-testid="stats-model">Glm 5\.3 Flash</);
|
||||
const html = renderToStaticMarkup(
|
||||
<StatsBar turns={[turn(), turn()]} chatId="chat_x" onSettingsSaved={SETTINGS_STUB} />,
|
||||
);
|
||||
// last turn's last call: 10k in + 1k out = 11k; pi catalog limit
|
||||
// 1_048_576 → "1M" (tables regenerated from pi-ai's catalog)
|
||||
assert.match(html, /data-testid="stats-context">11k \/ 1M</);
|
||||
// pi catalog price 0.075/0.25 per 1M: each turn 10k×0.075 + 1k×0.25
|
||||
// = $0.001 → two = $0.0020
|
||||
assert.match(html, /data-testid="stats-cost">\$0\.0020</);
|
||||
// model button label falls back to the last turn's actual model
|
||||
// (the catalog + settings are what drive it once loaded)
|
||||
assert.match(html, /data-testid="stats-model"[^>]*><span>Glm 5\.3 Flash<\/span>/);
|
||||
});
|
||||
|
||||
it("shows — for cost when every turn is unpriceable, never a fake $0", () => {
|
||||
const html = renderToStaticMarkup(<StatsBar turns={[turn({ model: "nope/unknown" })]} />);
|
||||
const html = renderToStaticMarkup(
|
||||
<StatsBar turns={[turn({ model: "nope/unknown" })]} chatId="chat_x" onSettingsSaved={SETTINGS_STUB} />,
|
||||
);
|
||||
assert.match(html, /data-testid="stats-cost">—</);
|
||||
// unknown model also kills the context limit
|
||||
assert.match(html, /data-testid="stats-context">—</);
|
||||
|
||||
+4
-3
@@ -41,8 +41,9 @@ test("sumSteps folds steps; absent stays absent until a value appears", () => {
|
||||
test("callCost: glm-5.3-flash math — fresh input, cached read, output", () => {
|
||||
const cost = callCost(call({ inputTokens: 100_000, outputTokens: 20_000, cacheReadTokens: 10_000 }), "z-ai/glm-5.3-flash");
|
||||
assert.ok(cost !== null);
|
||||
// fresh 90k @ $0.15/M + cached 10k @ $0.03/M + output 20k @ $0.50/M
|
||||
assert.ok(Math.abs(cost! - (0.0135 + 0.0003 + 0.01)) < 1e-9);
|
||||
// fresh 90k @ $0.075/M + cached 10k @ $0.015/M + output 20k @ $0.25/M
|
||||
// (pi-ai catalog prices — regenerated tables, scripts/sync-pricing)
|
||||
assert.ok(Math.abs(cost! - (0.00675 + 0.00015 + 0.005)) < 1e-9);
|
||||
});
|
||||
|
||||
test("callCost: cache writes bill at the cacheWrite rate", () => {
|
||||
@@ -86,7 +87,7 @@ test("contextWindow: last call's footprint against the model's limit", () => {
|
||||
assert.ok(cw);
|
||||
assert.equal(cw.model, "z-ai/glm-5.3-flash");
|
||||
assert.equal(cw.usedTokens, 100_000);
|
||||
assert.equal(cw.limit, 1_310_720);
|
||||
assert.equal(cw.limit, 1_048_576);
|
||||
assert.equal(contextWindow({ turns: [] }), null);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user