feat(models): BYOK all the way down — factory pins deleted, per-need refusal gates

Owner decision 2026-09-22: 'it doesn't make sense to have any default
model configured anywhere; the application is bring your own keys.'

config.ts: MODELS deleted; ModelRole becomes an explicit 9-role union;
DEFAULT_THINKING_ORDER (medium, low, high, minimal, xhigh, max) bakes
the unset-level default — first entry the model supports; explicitly
set levels keep pi's native nearest-clamp.

core/settings-resolution: the factory layer dissolves — the chain is
role override → slot → NOTHING. resolveSettings(record) takes no
factory. Empty chains resolve to null/undefined; a thinkingLevel-only
slot is inert (stored, never run). resolveConversationSpec: chat.model
override over the strong slot, null when both empty. resolveRoleSpec
spans all nine roles for the job/extraction engines.

Refusal envelope (core/settings.ts): modelsNotConfigured(missing) —
{error, code: 'models_not_configured', missing: [slot tiers]} +
ModelsNotConfiguredError with .envelope. Per-need gates, each honest:
- chat-handler: universal turn gate (new + continuing turns) returns
  503 before beginTurn — no liveness, no write-ahead persist, no graph
  trigger before the gate fires.
- runner.runJob: unresolvable engine lands the job as error with the
  envelope wording; no phantom loop, no result steer.
- extraction: refuseIfStagesUnconfigured throws before any stage
  (turn-service info-skips so coverage retries next turn; the import
  route surfaces per-file code+missing); labeler-less runs skip
  cluster naming with a console line and still merge.

Write side: slot entries must carry a model — level-only slot writes
are 400s (absent slot key = clear). piEngineForSpec resolves unset
thinking levels via DEFAULT_THINKING_ORDER; piEngineFor now throws
honestly; thinkingLevelFor/modelIdFor deleted; teacher-agent, pi-run,
visual-maker, pdf-reader all read engine.level.

model-catalog: defaults/slots/rolePins null when the chain is empty
(contracts nullable) — no phantom ids in payloads. Stores drop
MODEL_PINS entirely.

Tests: pin-leaners converted to explicit injection (per-chat model
pins on scripted chats — exactly what the setup flow writes; faux
engines for agent-construction tests). New coverage: the 503 gate
envelope, the runner refusal, the extraction refusal + labeler skip.

Gates: typecheck, lint, 815/815, build.
This commit is contained in:
amosblomqvist
2026-09-22 16:59:16 +03:00
parent 8675b19bef
commit bfa906e2f8
26 changed files with 889 additions and 562 deletions
+64 -9
View File
@@ -25,6 +25,7 @@ import {
} from "@earendil-works/pi-ai";
import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
import { handleChatRequest } from "../src/server/chat-handler.ts";
import { effectiveSettings } from "../src/server/stores.ts";
import { isTurnRunning } from "../src/server/turn-liveness.ts";
import { recordToPiMessages } from "../src/server/pi/record-adapter.ts";
import { chatStore } from "../src/store/chats.ts";
@@ -98,6 +99,17 @@ function post(chatId: string, messages: unknown[]): Request {
});
}
/** BYOK turn gate (owner, 2026-09-22): no factory floor — a chat needs a
* conversation model before any turn streams. The scripted tests give
* each chat a per-chat pin exactly like the setup flow does in
* production (the injected faux engine still runs the loop). */
function createConfiguredChat(title: string) {
const chat = chatStore.createChat(title);
chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } });
return chat;
}
const createdChats: string[] = [];
after(() => {
for (const id of createdChats) {
@@ -112,7 +124,7 @@ after(() => {
describe("teacher chat flow (scripted model)", () => {
it("quiz tool call persists as a pending part; the answer continues the lesson", async () => {
const { engine, calls } = scriptedTeacher(defaultLesson);
const chat = chatStore.createChat("Test: solar system");
const chat = createConfiguredChat("Test: solar system");
createdChats.push(chat.id);
// ── Turn 1: learner opens with a topic; teacher probes with a quiz ──
@@ -199,7 +211,7 @@ describe("teacher chat flow (scripted model)", () => {
// tool-input-start chunk. If a transport change ever stopped emitting
// it, the indicator would silently die — pin the protocol.
const { engine } = scriptedTeacher(defaultLesson);
const chat = chatStore.createChat("Test: protocol pin");
const chat = createConfiguredChat("Test: protocol pin");
createdChats.push(chat.id);
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] };
const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)));
@@ -212,7 +224,7 @@ describe("teacher chat flow (scripted model)", () => {
it("persists even when the stream is aborted (resume keeps partial state)", async () => {
const { engine } = scriptedTeacher(defaultLesson);
const chat = chatStore.createChat("Test: abort");
const chat = createConfiguredChat("Test: abort");
createdChats.push(chat.id);
const req = post(chat.id, [{ id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] }]);
@@ -260,7 +272,7 @@ describe("teacher chat flow (scripted model)", () => {
];
const { engine, calls } = scriptedTeacher(steps);
const chat = chatStore.createChat("Test: bad quiz retry");
const chat = createConfiguredChat("Test: bad quiz retry");
createdChats.push(chat.id);
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] };
const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)));
@@ -332,7 +344,7 @@ describe("teacher chat flow (scripted model)", () => {
];
const { engine } = scriptedTeacher(() => steps);
const chat = chatStore.createChat("Test: string options");
const chat = createConfiguredChat("Test: string options");
createdChats.push(chat.id);
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] };
const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)));
@@ -374,7 +386,7 @@ describe("teacher chat flow (scripted model)", () => {
),
]);
const chat = chatStore.createChat("Test: parallel quizzes");
const chat = createConfiguredChat("Test: parallel quizzes");
createdChats.push(chat.id);
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me twice" }] };
const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine)));
@@ -441,7 +453,7 @@ describe("chat store", () => {
describe("mid-turn navigation (turn liveness + write-ahead)", () => {
it("one turn per chat: a concurrent request is 409'd; the next one after finish is accepted", async () => {
const { engine } = scriptedTeacher(defaultLesson);
const chat = chatStore.createChat("Test: turn guard");
const chat = createConfiguredChat("Test: turn guard");
createdChats.push(chat.id);
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] };
@@ -462,7 +474,7 @@ describe("mid-turn navigation (turn liveness + write-ahead)", () => {
it("write-ahead: the learner's answer is durable BEFORE the turn finishes", async () => {
const { engine } = scriptedTeacher(defaultLesson);
const chat = chatStore.createChat("Test: write-ahead");
const chat = createConfiguredChat("Test: write-ahead");
createdChats.push(chat.id);
// Turn 1: the teacher asks the quiz (pending part in the record).
@@ -511,7 +523,7 @@ describe("mid-turn navigation (turn liveness + write-ahead)", () => {
});
it("a turn that fails to start (malformed messages) clears the registry — no lockout", async () => {
const chat = chatStore.createChat("Test: turn failed to start");
const chat = createConfiguredChat("Test: turn failed to start");
createdChats.push(chat.id);
// A text part with no text: fails message validation before the stream
// exists — the path where no onFinish will ever run.
@@ -557,3 +569,46 @@ describe("renameChat (the header's click-to-edit title)", () => {
assert.equal(renamed.title, "Untitled session");
});
});
describe("the BYOK turn gate (no factory floor — owner, 2026-09-22)", () => {
it("a chat with no model refuses the turn with the 503 envelope — nothing lingers", async () => {
const chat = chatStore.createChat("Test: unconfigured chat");
createdChats.push(chat.id);
// The record's strong slot belongs to settings-store.test.ts (the
// one-fs-owner policy) — READ the resolved truth, never write it.
// Refusal holds when the chain is empty; a record-carried strong
// slot legitimately opens the gate (the chain is honest both ways).
const expectRefusal = effectiveSettings().models.strong === null;
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] };
const res = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(scriptedTeacher(defaultLesson).engine));
if (expectRefusal) {
assert.equal(res.status, 503, "the universal turn gate refuses before beginTurn");
const body = (await res.json()) as { error: string; code: string; missing: string[] };
assert.equal(body.code, "models_not_configured");
assert.deepEqual(body.missing, ["strong"]);
assert.match(body.error, /No model configured/);
} else {
assert.equal(res.status, 200, "the record's strong slot satisfies the conversation chain");
await drain(res);
}
// Either way: no phantom liveness, nothing durable from a refused turn.
if (expectRefusal) {
assert.equal(isTurnRunning(chat.id), false, "the refusal never registers a live turn");
const saved = chatStore.getChat(chat.id)!;
assert.equal(saved.messages.length, 0, "no write-ahead persist before the gate");
}
});
it("a per-chat model alone opens the gate (the setup flow's exact write)", async () => {
const chat = createConfiguredChat("Test: per-chat pin only");
createdChats.push(chat.id);
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] };
const res = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(scriptedTeacher(defaultLesson).engine));
assert.equal(res.status, 200, "chat.settings.model satisfies the chain without any record slot");
await drain(res);
});
});
+3
View File
@@ -754,6 +754,9 @@ describe("demo/visual job runners", () => {
it("dispatch through a teacher turn: the call acks mid-turn and the runner completes in the background", async () => {
const chat = chatStore.createChat("Test: demo spawn dispatch");
createdChats.push(chat.id);
// BYOK turn gate (owner, 2026-09-22): the scripted chat needs a
// conversation model — the per-chat pin, exactly like the setup flow.
chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } });
// A teacher that calls demo on turn 1, then continues in the SAME turn.
const teacher = scriptedModel([
+42
View File
@@ -41,6 +41,7 @@ import type {
} from "../src/core/contracts.ts";
import type { GraphStore } from "../src/core/ports.ts";
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
import { ModelsNotConfiguredError } from "../src/core/settings.ts";
import { batchTag, coverageOf, coverageSlots } from "../src/core/coverage.ts";
import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
import { semanticClusters } from "../src/core/semantic-clusters.ts";
@@ -394,6 +395,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
assert.equal(covered.length, record.messages.length);
assert.ok(covered.every((c) => c.chatId === chatId && c.fp.length === 64 && c.tokens > 0));
assert.equal(ledgers.readCheckpoints().length, 0, "the chat path writes coverage, not the legacy checkpoint");
assert.equal(
coverageOf(record.messages as UIMessage[], covered, chatId).uncoveredCount,
0,
@@ -401,6 +403,46 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
);
});
it("BYOK refusal: no stage engines anywhere → the run throws honestly before any stage (owner, 2026-09-22)", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
// No direct engines AND no engineFor — the chain came up empty.
const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, embedder: fakeEmbedder });
await assert.rejects(pipeline.runExtraction(chatId), (err: unknown) => {
assert.ok(err instanceof ModelsNotConfiguredError);
assert.deepEqual(err.envelope.missing, ["cheap"]); // extractor + auditor are the cheap tier
assert.equal(err.envelope.code, "models_not_configured");
return true;
});
// nothing ran: no phase residue, no usage, no coverage writes
assert.equal(ledgers.readCoverage().length, 0);
});
it("a labeler-less run merges fine and SKIPS cluster naming honestly (BYOK, 2026-09-22)", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
// Identity-pass verdict: p1 becomes a new node, p2 dropped (no links).
const verdict: AuditResult = {
propositions: [{ ...FINDINGS.propositions[0]! }],
edges: [],
attribution: { quiz: { "probe-1": "p1" } },
dropped: [{ ref: "p2", reason: "evidence not found" }],
};
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: submitModel("submit_findings", FINDINGS),
auditor: submitModel("submit_review", verdict),
// NO labeler: the cluster-naming garnish skips (console line),
// the merge itself is unaffected.
});
const stats = await pipeline.runExtraction(chatId);
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 3 });
});
it("engineFor fills the stages (model slots wave): the chain replaces config pins", async () => {
const { chatId } = fixtureChat();
const ledgers = memoryLedgers();
+35
View File
@@ -356,6 +356,9 @@ describe("dispatch (async protocol §2)", () => {
it("the research call resolves immediately with an ack; the turn continues and the runner steers", async () => {
const chat = chatStore.createChat("Test: research dispatch");
trackChat(chat.id);
// BYOK turn gate (owner, 2026-09-22): the scripted chat needs a
// conversation model — the per-chat pin, exactly like the setup flow.
chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } });
const delivery = fakeDeliver();
const search = stubSearch();
@@ -1344,3 +1347,35 @@ describe("stale-job recovery sweep (liveness-gated)", () => {
assert.equal(jobStore.getJob(jobId)!.status, "done");
});
});
// ─── BYOK: the unresolvable-engine refusal (owner, 2026-09-22) ──────────────
describe("BYOK job refusal (no factory floor)", () => {
it("runJob with NO resolvable engine lands the job as error — the envelope wording, no phantom run", async () => {
const chat = chatStore.createChat("Test: unconfigured spawn");
trackChat(chat.id);
chatStore.saveMessages(chat.id, [
{ id: "u1", role: "user", parts: [{ type: "text", text: "hi" }] },
{
id: "a1",
role: "assistant",
parts: [{ type: "tool-research", toolCallId: "call-nc1", state: "output-available", input: RESEARCH_INPUT, output: "Dispatched." }],
},
] as never);
const job = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "call-nc1" });
trackJob(job.id);
const delivery = fakeDeliver();
// NO test override, NO engineFor: the chain (override → slot → floor)
// comes up empty — the runner must refuse honestly, not fall back.
const runner = createJobRunner({ chatStore, jobStore, deliver: delivery.deliver });
await runner.runJob(job.id);
const done = jobStore.getJob(job.id)!;
assert.equal(done.status, "error", "the job lands error, never a phantom run");
assert.match(done.error ?? "", /No model configured/);
assert.match(done.error ?? "", /missing: cheap/);
assert.equal(delivery.steers.length, 0, "no result steer — nothing ran");
});
});
+92 -65
View File
@@ -2,23 +2,30 @@ import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { modelCatalog, validateChatSettings, validateModelsSettings, catalogDefaultsFor, catalogProviders, catalogSlots, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts";
import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai";
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
import { MODELS } from "../config.ts";
import { piEngineFor } from "../src/agent/models.ts";
import { BACKGROUND_MODEL_ROLES, type BackgroundModelRole } from "../src/core/settings.ts";
import { DEFAULT_THINKING_ORDER } from "../config.ts";
import { piEngineForSpec, piModels } from "../src/agent/models.ts";
import { BACKGROUND_MODEL_ROLES } from "../src/core/settings.ts";
import { resolveSettings } from "../src/core/settings-resolution.ts";
import { MODEL_PINS } from "../src/server/stores.ts";
// The model-control wave (2026-09-12): pi-ai owns the registry — these pins
// hold our projection + validation honest against the shipped catalog, and
// pin pi's per-model thinking truth so a pi-ai upgrade can't silently change
// what we send.
// what we send. BYOK wave (2026-09-22): no factory pins — unset surfaces
// are NULL/undefined, never phantom defaults.
// The picker federates only providers with a PRESENT key (model slots
// wave, 2026-09-20): pin openrouter's env key so its full catalog
// federates in tests regardless of the developer shell's exports.
process.env.OPENROUTER_API_KEY ??= "test-key-openrouter-catalog";
/** The catalog's glm entry — the old factory pin's model, still the
* best-known thinking-truth specimen (low|high|max). */
function glmModel() {
const glm = piModels().getModel("openrouter", "z-ai/glm-5.3-flash");
assert.ok(glm, "glm-5.3-flash in catalog");
return glm!;
}
describe("the model catalog projection", () => {
it("serves the keyed providers' catalogs with the picker's DTO shape", () => {
const { models, defaults, providers, slots } = modelCatalog();
@@ -48,32 +55,41 @@ describe("the model catalog projection", () => {
for (const id of keyless) {
assert.ok(!models.some((m) => m.provider === id), `keyless ${id} must not federate`);
}
// BYOK: nothing configured in this environment's record → defaults NULL,
// slots NULL — no phantom ids anywhere in the payload
assert.equal(defaults, null);
for (const name of ["strong", "cheap", "vision"] as const) {
assert.equal(slots[name], null, name);
}
});
it("defaults mirror the conversation agents' config pin (strong rep = prober pin)", () => {
const strong = { provider: "openrouter", id: MODELS.prober.model, thinkingLevel: MODELS.prober.effort as ThinkingLevel };
it("defaults: the chat section over the strong slot; null when both empty (BYOK)", () => {
assert.equal(catalogDefaultsFor(undefined, null), null);
const strong = { provider: "openrouter", id: "z-ai/glm-5.3-flash", thinkingLevel: "low" as const };
const d = catalogDefaultsFor(undefined, strong);
assert.deepEqual(d, { model: { provider: "openrouter", id: MODELS.prober.model }, thinkingLevel: MODELS.prober.effort });
assert.equal(d.model.id, MODELS.teacher.model, "one controller: prober and teacher share the pin");
});
it("a picked default in the record's chat section overrides the pin (catalogDefaultsFor, settings wave)", () => {
const strong = { provider: "openrouter", id: MODELS.prober.model, thinkingLevel: MODELS.prober.effort as ThinkingLevel };
const d = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" }, strong);
assert.deepEqual(d, { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" });
// partial: only the model chosen → thinking falls back to the pin
assert.deepEqual(d, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "low" });
assert.deepEqual(catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" }, strong), {
model: { provider: "openrouter", id: "qwen/qwen3.8-flash" },
thinkingLevel: "high",
});
// partial: only the model chosen → level falls back to the slot's
const partial = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }, strong);
assert.equal(partial.thinkingLevel, MODELS.prober.effort);
assert.equal(partial.model.id, "qwen/qwen3.8-flash");
assert.ok(partial);
assert.equal(partial!.thinkingLevel, "low");
// graphContextTokens-only record: model untouched
assert.deepEqual(catalogDefaultsFor({ graphContextTokens: 512 }, strong), {
model: { provider: "openrouter", id: MODELS.prober.model },
thinkingLevel: MODELS.prober.effort,
model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" },
thinkingLevel: "low",
});
// chat.model with NO strong slot: the conversation runs on it alone
assert.deepEqual(
catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }, null),
{ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } },
);
});
it("toModelDto projects the raw catalog model verbatim", () => {
const raw = piEngineFor("prober").model; // the catalog's glm entry
const raw = glmModel();
const dto = toModelDto(raw);
assert.equal(dto.id, raw.id);
assert.equal(dto.name, raw.name);
@@ -84,43 +100,40 @@ describe("the model catalog projection", () => {
});
describe("the background roles' resolved pins (rolePins)", () => {
it("no overrides → every role pins its config MODELS spec, nested DTO shape, all six roles", () => {
const pins = rolePinsFor(resolveSettings(null, MODEL_PINS).agent.models);
it("no record → every role is NULL (BYOK: no phantom defaults)", () => {
const pins = rolePinsFor(resolveSettings(null).agent.models);
assert.deepEqual(Object.keys(pins), [...BACKGROUND_MODEL_ROLES]);
for (const role of BACKGROUND_MODEL_ROLES) {
assert.deepEqual(pins[role].model, { provider: "openrouter", id: MODELS[role].model });
assert.equal(pins[role].thinkingLevel, MODELS[role].effort);
assert.equal(pins[role], null, role);
}
// the vision pin rides along like any other
assert.equal(pins.visualMaker.model.id, "qwen/qwen3.8-flash");
});
it("a model override wins for that role only; others stay pinned", () => {
it("a slot re-tiers its roles; overrides win for that role only", () => {
const record = {
version: 1,
agent: { models: { researcher: { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } } } },
models: { cheap: { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } } },
agent: { models: { researcher: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "high" as const } } },
};
const pins = rolePinsFor(resolveSettings(record, MODEL_PINS).agent.models);
assert.equal(pins.researcher.model.id, "qwen/qwen3.8-flash");
assert.equal(pins.researcher.thinkingLevel, MODELS.researcher.effort, "level stays the pin unless overridden too");
assert.equal(pins.visualMaker.model.id, MODELS.visualMaker.model);
assert.equal(pins.labeler.model.id, MODELS.labeler.model);
const pins = rolePinsFor(resolveSettings(record).agent.models);
assert.deepEqual(pins.researcher, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "high" });
assert.deepEqual(pins.labeler, { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }); // level unset
assert.equal(pins.visualMaker, null, "vision tier untouched");
});
it("a thinkingLevel-only override keeps the model, changes the level", () => {
const record = { version: 1, agent: { models: { extractor: { thinkingLevel: "high" as const } } } };
const pins = rolePinsFor(resolveSettings(record, MODEL_PINS).agent.models);
assert.equal(pins.extractor.model.id, MODELS.extractor.model);
assert.equal(pins.extractor.thinkingLevel, "high");
it("catalogRolePins matches the same resolution the engine wiring uses", () => {
const pins = catalogRolePins();
for (const role of BACKGROUND_MODEL_ROLES) {
assert.ok(pins[role] === null || typeof pins[role]!.model.id === "string");
}
});
});
describe("the model slots surface (model slots wave, 2026-09-20)", () => {
it("catalogSlots reports the factory rep pins when no record exists", () => {
it("catalogSlots reports NULL when a tier holds no model", () => {
const slots = catalogSlots();
assert.deepEqual(slots.strong.model, { provider: "openrouter", id: MODELS.prober.model });
assert.deepEqual(slots.cheap.model, { provider: "openrouter", id: MODELS.researcher.model });
assert.deepEqual(slots.vision.model, { provider: "openrouter", id: MODELS.visualMaker.model });
for (const name of ["strong", "cheap", "vision"] as const) {
assert.equal(slots[name], null, name);
}
});
it("catalogProviders reads keys env-wins, fresh per call", () => {
@@ -153,7 +166,7 @@ describe("the models-section validation gate", () => {
);
});
it("thinkingLevel resolves against the slot's model, else the tier's resolved pin", () => {
it("thinkingLevel resolves against the slot's model; level-only slots check nothing", () => {
// level checked against the named model
assert.equal(
validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "max" } }).ok,
@@ -163,8 +176,9 @@ describe("the models-section validation gate", () => {
validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "minimal" } }).ok,
false,
);
// bare level: against the strong slot's resolved model (config pin)
assert.equal(validateModelsSettings({ strong: { thinkingLevel: "minimal" } }).ok, false);
// (a level-only slot never passes the WRITE parser — owner, 2026-09-22;
// the validation gate tolerates it here with nothing to check against)
assert.equal(validateModelsSettings({ strong: { thinkingLevel: "minimal" } }).ok, true);
});
it("accepts empty settings (the clear-slots write)", () => {
@@ -184,10 +198,8 @@ describe("the settings validation gate", () => {
}
});
it("resolves a bare thinking level against the config default model", () => {
assert.deepEqual(validateChatSettings({ thinkingLevel: "high" }), { ok: true });
// "minimal" is unsupported on config's glm — rejected even with no model set
assert.equal(validateChatSettings({ thinkingLevel: "minimal" }).ok, false);
it("a bare thinking level checks against the chat model, else the strong slot, else rides unchecked", () => {
assert.deepEqual(validateChatSettings({ thinkingLevel: "minimal" }), { ok: true }, "no model to check against → accepted (clamps at run)");
});
it("rejects unknown providers and models", () => {
@@ -202,7 +214,7 @@ describe("the settings validation gate", () => {
describe("pi thinking truth (pinned against pi-ai upgrades)", () => {
it("glm-5.3-flash: low|high|max; anything else clamps to the nearest", () => {
const glm = piEngineFor("prober").model;
const glm = glmModel();
assert.deepEqual(getSupportedThinkingLevels(glm), ["low", "high", "max"]);
assert.equal(clampThinkingLevel(glm, "minimal"), "low");
assert.equal(clampThinkingLevel(glm, "off"), "low", "GLM cannot disable thinking (map: off → null)");
@@ -210,21 +222,36 @@ describe("pi thinking truth (pinned against pi-ai upgrades)", () => {
assert.equal(clampThinkingLevel(glm, "high"), "high");
});
it("the engine factory honors settings, falls back to config, clamps", () => {
// override both fields
const over = piEngineFor("teacher", { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" });
it("the engine factory clamps explicit levels; unknown models throw honestly", () => {
const over = piEngineForSpec({ provider: "openrouter", id: "qwen/qwen3.8-flash", thinkingLevel: "high" });
assert.equal(over.model.id, "qwen/qwen3.8-flash");
assert.equal(over.level, "high");
// no settings → config pin
const pinned = piEngineFor("prober");
assert.equal(pinned.model.id, MODELS.prober.model);
assert.equal(pinned.level, MODELS.prober.effort);
// settings level unsupported by the settings model → clamped honestly
const clamped = piEngineFor("teacher", { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "minimal" });
const clamped = piEngineForSpec({ provider: "openrouter", id: "z-ai/glm-5.3-flash", thinkingLevel: "minimal" });
assert.equal(clamped.level, "low");
// bare level resolves against the config default model
assert.equal(piEngineFor("prober", { thinkingLevel: "max" }).level, "max");
// unknown model → honest throw, never a silent fallback
assert.throws(() => piEngineFor("teacher", { model: { provider: "openrouter", id: "nope/nope" } }), /catalog has no model/);
assert.throws(() => piEngineForSpec({ provider: "openrouter", id: "nope/nope", thinkingLevel: "low" }), /catalog has no model/);
});
it("UNSET levels resolve from the baked DEFAULT_THINKING_ORDER (owner, 2026-09-22)", () => {
assert.deepEqual([...DEFAULT_THINKING_ORDER], ["medium", "low", "high", "minimal", "xhigh", "max"]);
// glm supports low|high|max → first supported in the order is "low"
const glm = piEngineForSpec({ provider: "openrouter", id: "z-ai/glm-5.3-flash" });
assert.equal(glm.level, "low");
// explicit levels never consult the order — pi clamps nearest
assert.equal(piEngineForSpec({ provider: "openrouter", id: "z-ai/glm-5.3-flash", thinkingLevel: "max" }).level, "max");
// a non-reasoning model reports exactly ["off"] → the order lands there
const flat = piModels()
.getProviders()
.flatMap((p) => piModels().getModels(p.id))
.find((m) => !m.reasoning);
if (flat) {
const engine = piEngineForSpec({ provider: flat.provider, id: flat.id });
assert.deepEqual(getSupportedThinkingLevels(flat), ["off"]);
assert.equal(engine.level, "off");
}
});
it("the legacy piEngineFor role entry point throws honestly (BYOK: no pins)", async () => {
const { piEngineFor } = await import("../src/agent/models.ts");
assert.throws(() => piEngineFor("prober"), /no configured model for role "prober"/);
});
});
+15 -2
View File
@@ -16,6 +16,15 @@ import * as teacherAgent from "../src/agent/teacher-agent.ts";
import { allSegmentPaths, effectivePromptSegments } from "../src/agent/prompt-segments.ts";
import { isValidSegmentPath } from "../src/core/prompt-segments.ts";
import { promptOverrides } from "../src/store/prompt-overrides.ts";
import { createFauxCore } from "@earendil-works/pi-ai";
import type { PiEngine } from "../src/agent/models.ts";
/** A scripted pi engine (pi-ai's faux provider) — the agent build needs an
* injected engine since BYOK removed the factory pins (owner, 2026-09-22). */
function fauxEngine(): PiEngine {
const faux = createFauxCore({});
return { model: faux.getModel(), streamFn: faux.streamSimple };
}
after(() => {
fs.rmSync(PROMPTS_OVERRIDE_DIR, { recursive: true, force: true });
@@ -210,9 +219,13 @@ describe("composeInstructions: override honoring + the byte-identical source pat
it("promptSwitches threads through agent construction into the system prompt", () => {
promptOverrides.writeOverride("system-prompt.md", "TOTALLY CUSTOM BASE");
try {
const agent = createTeacherAgent(undefined, { promptSwitches: { "system-prompt.md": true } });
// BYOK (owner, 2026-09-22): no factory pins — the agent build needs
// an injected engine; a scripted faux one stands in (the prompt is
// what's under test here, not the model).
const engine = fauxEngine();
const agent = createTeacherAgent(undefined, { engine, promptSwitches: { "system-prompt.md": true } });
assert.ok(agent.state.systemPrompt.includes("TOTALLY CUSTOM BASE"));
const source = createTeacherAgent(undefined, {});
const source = createTeacherAgent(undefined, { engine });
assert.ok(!source.state.systemPrompt.includes("TOTALLY CUSTOM BASE"), "no switches → cached source");
} finally {
promptOverrides.deleteOverride("system-prompt.md");
+10 -4
View File
@@ -165,18 +165,24 @@ describe("SETTINGS_SECTIONS (the route's section map)", () => {
});
describe("parseModelsSettingsBody (model slots wave, 2026-09-20)", () => {
it("parses the three slot tiers, each field independent", () => {
it("parses the three slot tiers — model required, level rides (BYOK, owner 2026-09-22)", () => {
const r = parseModelsSettingsBody({
strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } },
cheap: { thinkingLevel: "high" },
cheap: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "high" },
vision: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "low" },
});
assert.ok(r.ok);
assert.deepEqual(r.settings, {
strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } },
cheap: { thinkingLevel: "high" },
cheap: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "high" },
vision: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "low" },
});
// a thinkingLevel WITHOUT a model can never run (no factory floor)
const levelOnly = parseModelsSettingsBody({ cheap: { thinkingLevel: "high" } });
assert.equal(levelOnly.ok, false);
if (!levelOnly.ok) {
assert.match(levelOnly.error, /slot "cheap" must set a model/);
}
});
it("{} is the legal clear-slots write", () => {
@@ -194,7 +200,7 @@ describe("parseModelsSettingsBody (model slots wave, 2026-09-20)", () => {
it("a present-but-empty slot object is rejected (absent slot = unset)", () => {
const bad = parseModelsSettingsBody({ strong: {} });
assert.equal(bad.ok, false);
if (!bad.ok) assert.match(bad.error, /slot "strong" must set model or thinkingLevel/);
if (!bad.ok) assert.match(bad.error, /slot "strong" must set a model/);
});
it("malformed slot shapes are 400s", () => {
+131 -118
View File
@@ -1,72 +1,41 @@
/**
* Tests for the resolution layer (settings-storage wave, 2026-09-19;
* model-slots wave 2026-09-20 adds the slot tier): core/settings-resolution.ts
* — the delta→concrete projection. The factory layer is threaded in as a
* parameter (core stays pure; server/stores.ts threads config.ts's MODELS),
* so these tests use a small synthetic pin spanning ALL NINE roles (the
* factory contract since the slots wave — the strong slot's rep IS the
* prober pin).
* model-slots wave 2026-09-20; BYOK all-the-way-down wave 2026-09-22):
* core/settings-resolution.ts — the delta→concrete projection. There are
* NO factory pins anymore: a model chain that ends empty resolves to
* null/undefined, and the per-need refusal gates (turn, job spawn,
* extraction) fire on exactly that.
*/
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
extractionEnabled,
factorySlotRep,
graphContextEnabled,
resolveConversationSpec,
resolveModelSpec,
resolveRoleSpec,
resolveSettings,
resolveSlot,
type ModelRoleSpecs,
} from "../src/core/settings-resolution.ts";
import { BACKGROUND_MODEL_ROLES, type ModelSlotName, type UserSettings } from "../src/core/settings.ts";
const FACTORY_PINS = Object.fromEntries(
[
"prober",
"teacher",
"researcher",
"demoBuilder",
"visualMaker",
"pdfReader",
"extractor",
"auditor",
"labeler",
].map((role, i) => [
role,
{ provider: i === 5 ? "other" : "openrouter", model: `factory/model-${i}`, effort: i % 2 === 0 ? "low" : "off" },
]),
) as ModelRoleSpecs;
import { BACKGROUND_MODEL_ROLES, type UserSettings } from "../src/core/settings.ts";
describe("resolveSettings", () => {
it("null record (first boot): every knob lands on its factory default", () => {
const eff = resolveSettings(null, FACTORY_PINS);
it("null record (first boot): non-model knobs land on factory defaults; every model view is EMPTY", () => {
const eff = resolveSettings(null);
assert.deepEqual(eff.general, { autoUpdate: true });
assert.deepEqual(eff.chat, {}); // BYOK: no default model, ever
assert.deepEqual(eff.agent.tools, {});
assert.deepEqual(eff.agent.prompts, {});
assert.deepEqual(eff.agent.search, {});
// BYOK (owner, 2026-09-22): no factory floor — every role undefined,
// every slot null. No phantom ids anywhere.
for (const role of BACKGROUND_MODEL_ROLES) {
assert.equal(eff.agent.models[role].id, FACTORY_PINS[role]!.model);
assert.equal(eff.agent.models[role].provider, FACTORY_PINS[role]!.provider);
assert.equal(eff.agent.models[role].thinkingLevel, FACTORY_PINS[role]!.effort);
assert.equal(eff.agent.models[role], undefined, role);
}
// slots with no record: every tier lands on its factory rep pin
assert.deepEqual(eff.models.strong, {
provider: "openrouter",
id: FACTORY_PINS.prober!.model,
thinkingLevel: FACTORY_PINS.prober!.effort,
});
assert.deepEqual(eff.models.cheap, {
provider: "openrouter",
id: FACTORY_PINS.researcher!.model,
thinkingLevel: FACTORY_PINS.researcher!.effort,
});
assert.deepEqual(eff.models.vision, {
provider: "openrouter",
id: FACTORY_PINS.visualMaker!.model,
thinkingLevel: FACTORY_PINS.visualMaker!.effort,
});
assert.equal(eff.models.strong, null);
assert.equal(eff.models.cheap, null);
assert.equal(eff.models.vision, null);
assert.equal(eff.appearance.mode, "dark");
assert.equal(eff.appearance.background, null);
assert.equal(eff.appearance.fontFamily, null);
@@ -90,13 +59,15 @@ describe("resolveSettings", () => {
appearance: { mode: "light", proseFontSize: 20, mdRendering: { headingWeight: 600 } },
graph: { enabled: false },
};
const eff = resolveSettings(record, FACTORY_PINS);
const eff = resolveSettings(record);
assert.equal(eff.general.autoUpdate, false);
assert.deepEqual(eff.chat, { thinkingLevel: "high" });
assert.deepEqual(eff.agent.models.researcher, {
provider: "openrouter",
id: "qwen/qwen3.8-flash",
thinkingLevel: "low", // thinking NOT overridden → factory effort
// thinking NOT overridden and no cheap slot → UNSET (the engine
// build resolves it from DEFAULT_THINKING_ORDER, not a pin).
thinkingLevel: undefined,
});
assert.deepEqual(eff.agent.search, { engine: "tavily" });
assert.deepEqual(eff.agent.tools, { research: false });
@@ -109,103 +80,145 @@ describe("resolveSettings", () => {
assert.deepEqual(eff.graph, { enabled: false, embedding: true, clustering: true, labeling: true });
});
it("partial role override: thinkingLevel alone swaps the budget, model stays pinned", () => {
const eff = resolveSettings(
{ version: 1, agent: { models: { extractor: { thinkingLevel: "medium" } } } },
FACTORY_PINS,
);
const extractor = FACTORY_PINS.extractor!;
it("partial role override: thinkingLevel alone borrows the SLOT's model (or nothing, without one)", () => {
const eff = resolveSettings({
version: 1,
models: { cheap: { model: { provider: "zai", id: "glm-5.3-flash" } } },
agent: { models: { extractor: { thinkingLevel: "medium" } } },
});
assert.deepEqual(eff.agent.models.extractor, {
provider: extractor.provider,
id: extractor.model,
provider: "zai",
id: "glm-5.3-flash",
thinkingLevel: "medium",
});
// sibling roles untouched
assert.equal(eff.agent.models.researcher.thinkingLevel, FACTORY_PINS.researcher!.effort);
const bare = resolveSettings({ version: 1, agent: { models: { extractor: { thinkingLevel: "medium" } } } });
assert.equal(bare.agent.models.extractor, undefined); // no model anywhere → chain empty
});
it("resolveModelSpec is field-independent per override aspect", () => {
const slots = { cheap: { model: { provider: "zai", id: "slot-model" }, thinkingLevel: "high" as const } };
assert.deepEqual(
resolveModelSpec("labeler", { model: { provider: "other", id: "x" } }, FACTORY_PINS),
{ provider: "other", id: "x", thinkingLevel: FACTORY_PINS.labeler!.effort },
resolveModelSpec("labeler", { model: { provider: "other", id: "x" } }, slots),
{ provider: "other", id: "x", thinkingLevel: "high" }, // override model + slot level
);
assert.deepEqual(resolveModelSpec("labeler", {}, FACTORY_PINS), {
provider: FACTORY_PINS.labeler!.provider,
id: FACTORY_PINS.labeler!.model,
thinkingLevel: FACTORY_PINS.labeler!.effort,
assert.deepEqual(resolveModelSpec("labeler", { thinkingLevel: "max" }, slots), {
provider: "zai",
id: "slot-model",
thinkingLevel: "max", // override level + slot model
});
assert.equal(resolveModelSpec("labeler", undefined, {}), undefined);
});
});
describe("the slot tier (model slots wave, 2026-09-20)", () => {
describe("the slot tier (BYOK, owner 2026-09-22)", () => {
it("slot alone re-tiers every role of its tier (no role overrides present)", () => {
const eff = resolveSettings(
{ version: 1, models: { cheap: { model: { provider: "zai", id: "glm-5.3-flash" } } } },
FACTORY_PINS,
);
const eff = resolveSettings({ version: 1, models: { cheap: { model: { provider: "zai", id: "glm-5.3-flash" } } } });
for (const role of ["researcher", "demoBuilder", "extractor", "auditor", "labeler"] as const) {
assert.deepEqual(eff.agent.models[role as "researcher"], {
provider: "zai",
id: "glm-5.3-flash",
thinkingLevel: FACTORY_PINS[role]!.effort, // thinking not in slot → factory
}, role);
assert.deepEqual(
eff.agent.models[role],
{ provider: "zai", id: "glm-5.3-flash", thinkingLevel: undefined }, // level unset → default order
role,
);
}
// other tiers untouched
assert.equal(eff.agent.models.visualMaker.id, FACTORY_PINS.visualMaker!.model);
// other tiers untouched (empty)
assert.equal(eff.agent.models.visualMaker, undefined);
});
it("role override still beats the slot; slot beats the factory pin (the chain)", () => {
const eff = resolveSettings(
{
version: 1,
agent: { models: { researcher: { model: { provider: "groq", id: "r-override" } } } },
models: { cheap: { model: { provider: "zai", id: "r-slot" }, thinkingLevel: "high" } },
},
FACTORY_PINS,
);
// researcher: override model + slot thinking (field-independent)
it("role override still beats the slot; each field independent", () => {
const eff = resolveSettings({
version: 1,
agent: { models: { researcher: { model: { provider: "groq", id: "r-override" } } } },
models: { cheap: { model: { provider: "zai", id: "r-slot" }, thinkingLevel: "high" } },
});
assert.deepEqual(eff.agent.models.researcher, {
provider: "groq",
id: "r-override",
thinkingLevel: "high",
thinkingLevel: "high", // slot level rides the override's model
});
// labeler (same tier, no override): full slot
assert.deepEqual(eff.agent.models.labeler, { provider: "zai", id: "r-slot", thinkingLevel: "high" });
});
it("vision slot covers visualMaker AND pdfReader; strong's rep is the prober pin", () => {
const eff = resolveSettings(
{ version: 1, models: { vision: { thinkingLevel: "max" } } },
FACTORY_PINS,
);
assert.equal(eff.agent.models.visualMaker.thinkingLevel, "max");
assert.equal(eff.agent.models.visualMaker.id, FACTORY_PINS.visualMaker!.model);
// resolved slot views exist for every tier, set or not
assert.deepEqual(eff.models.strong, factorySlotRep("strong", FACTORY_PINS));
it("vision slot covers visualMaker AND pdfReader; level-only slots resolve as NULL", () => {
const eff = resolveSettings({ version: 1, models: { vision: { model: { provider: "zai", id: "v-model" }, thinkingLevel: "max" } } });
assert.deepEqual(eff.agent.models.visualMaker, { provider: "zai", id: "v-model", thinkingLevel: "max" });
assert.deepEqual(eff.models.vision, { provider: "zai", id: "v-model", thinkingLevel: "max" });
// a stored level WITHOUT a model is inert (kept on disk, no phantom default)
const inert = resolveSettings({ version: 1, models: { strong: { thinkingLevel: "medium" } } });
assert.equal(inert.models.strong, null);
assert.equal(inert.agent.models.researcher, undefined); // strong's emptiness must not leak into other tiers
});
it("resolveSlot: slot over factory rep, per field", () => {
assert.deepEqual(resolveSlot("strong", { thinkingLevel: "medium" }, FACTORY_PINS), {
provider: FACTORY_PINS.prober!.provider,
id: FACTORY_PINS.prober!.model,
it("resolveSlot: configured iff the model is present", () => {
assert.deepEqual(resolveSlot({ model: { provider: "zai", id: "m" }, thinkingLevel: "medium" }), {
provider: "zai",
id: "m",
thinkingLevel: "medium",
});
const rep = factorySlotRep("vision", FACTORY_PINS);
assert.equal(rep.id, FACTORY_PINS.visualMaker!.model);
const slot: ModelSlotName = "cheap";
assert.deepEqual(resolveSlot(slot, undefined, FACTORY_PINS), {
provider: FACTORY_PINS.researcher!.provider,
id: FACTORY_PINS.researcher!.model,
thinkingLevel: FACTORY_PINS.researcher!.effort,
assert.deepEqual(resolveSlot({ model: { provider: "zai", id: "m" } }), {
provider: "zai",
id: "m",
thinkingLevel: undefined, // unset → resolved from the default order at build
});
assert.equal(resolveSlot({ thinkingLevel: "medium" }), null);
assert.equal(resolveSlot(undefined), null);
});
it("absent provider coordinate anywhere rides the factory provider (legacy ids)", () => {
// resolveModelSpec with no slots and a factory provider of "other"
// (pdfReader is index 5 in the synthetic pins): a legacy override
// with no provider keeps the FACTORY provider, not a hardcoded one.
const spec = resolveModelSpec("auditor", {}, FACTORY_PINS);
assert.equal(spec.provider, FACTORY_PINS.auditor!.provider);
it("resolveRoleSpec: background roles and the conversation pair share the chain", () => {
const record: UserSettings = {
version: 1,
models: { strong: { model: { provider: "zai", id: "strong-model" } } },
};
// conversation pair: slot only (their override is chat-controlled)
assert.deepEqual(resolveRoleSpec("prober", record), {
provider: "zai",
id: "strong-model",
thinkingLevel: undefined, // level unset → default order at build
});
// background role override beats the slot
const withOverride: UserSettings = {
...record,
agent: { models: { researcher: { model: { provider: "groq", id: "r" } } } },
};
assert.deepEqual(resolveRoleSpec("researcher", withOverride), { provider: "groq", id: "r", thinkingLevel: undefined });
// pdfReader rides the vision slot (never a background override)
const visionRecord: UserSettings = {
version: 1,
models: { vision: { model: { provider: "zai", id: "v" }, thinkingLevel: "low" } },
};
assert.deepEqual(resolveRoleSpec("pdfReader", visionRecord), { provider: "zai", id: "v", thinkingLevel: "low" });
assert.equal(resolveRoleSpec("teacher", null), undefined);
});
});
describe("resolveConversationSpec (the turn gate's chain)", () => {
it("chat.model over the strong slot; null when both empty", () => {
assert.equal(resolveConversationSpec(undefined, null), null);
assert.deepEqual(
resolveConversationSpec(undefined, { provider: "zai", id: "s" }),
{ provider: "zai", id: "s", thinkingLevel: undefined },
);
assert.deepEqual(
resolveConversationSpec({ model: { provider: "groq", id: "c" } }, null),
{ provider: "groq", id: "c", thinkingLevel: undefined },
);
});
it("bare chat thinking level rides the strong slot's model", () => {
assert.deepEqual(
resolveConversationSpec({ thinkingLevel: "high" }, { provider: "zai", id: "s", thinkingLevel: "low" }),
{ provider: "zai", id: "s", thinkingLevel: "high" },
);
// chat.model present, strong unset: level may still ride the chat alone
assert.deepEqual(
resolveConversationSpec({ model: { provider: "groq", id: "c" }, thinkingLevel: "max" }, null),
{ provider: "groq", id: "c", thinkingLevel: "max" },
);
// nothing set anywhere: unset level, not a phantom one
assert.deepEqual(resolveConversationSpec({ model: { provider: "groq", id: "c" } }, null), {
provider: "groq",
id: "c",
thinkingLevel: undefined,
});
});
});
@@ -223,11 +236,11 @@ describe("the graph channel gates (the wave's seam honoring)", () => {
});
it("both gates ride resolveSettings' factory defaults (all on → both open)", () => {
const eff = resolveSettings(null, FACTORY_PINS);
const eff = resolveSettings(null);
assert.equal(extractionEnabled(eff.graph), true);
assert.equal(graphContextEnabled(eff.graph), true);
const off = resolveSettings({ version: 1, graph: { enabled: false } }, FACTORY_PINS);
const off = resolveSettings({ version: 1, graph: { enabled: false } });
assert.equal(extractionEnabled(off.graph), false);
assert.equal(graphContextEnabled(off.graph), false);
});
});
});
+12 -2
View File
@@ -18,6 +18,15 @@ import type { ProberToolContext } from "../src/agent/tools.ts";
import { jobStore } from "../src/store/jobs.ts";
import { parseAgentSettingsBody } from "../src/server/settings-body.ts";
import { chatStore } from "../src/store/chats.ts";
import { createFauxCore } from "@earendil-works/pi-ai";
import type { PiEngine } from "../src/agent/models.ts";
/** A scripted pi engine — the agent builds need an injected engine since
* BYOK removed the factory pins (owner, 2026-09-22). */
function fauxEngine(): PiEngine {
const faux = createFauxCore({});
return { model: faux.getModel(), streamFn: faux.streamSimple };
}
import type { AgentSettings } from "../src/core/settings.ts";
describe("AgentBuildOptions.toolToggles (the P2 bridge)", () => {
@@ -27,14 +36,14 @@ describe("AgentBuildOptions.toolToggles (the P2 bridge)", () => {
it("flows into the teacher's registry: a job-tool delta is a no-op on the bare set", () => {
// No ctx → bare agent: quiz/ask only. A job-tool delta names tools
// that don't exist there — the bridge must not crash on it.
const bare = createTeacherAgent(undefined, { toolToggles: { research: false } });
const bare = createTeacherAgent(undefined, { engine: fauxEngine(), toolToggles: { research: false } });
assert.deepEqual(toolNames(bare), ["ask_user_question", "quiz"]);
});
it("flows into the prober's registry; probe_done survives everything", () => {
const chatId = chatStore.createChat("Toggle bridge test").id;
const ctx: ProberToolContext = { chatId, chatStore, deliver: () => {}, jobStore, spawn: () => {} };
const prober = createProberAgent(ctx, { toolToggles: { ask_user_question: false } });
const prober = createProberAgent(ctx, { engine: fauxEngine(), toolToggles: { ask_user_question: false } });
const names = toolNames(prober);
assert.ok(!names.includes("ask_user_question"), "the delta dropped the tool");
assert.ok(names.includes("probe_done"), "probe_done always-on");
@@ -42,6 +51,7 @@ describe("AgentBuildOptions.toolToggles (the P2 bridge)", () => {
it("an explicitly injected opts.tools registry is NEVER toggle-filtered", () => {
const teacher = createTeacherAgent(undefined, {
engine: fauxEngine(),
tools: {
quiz: { name: "quiz", description: "test", clientQuestion: true, schema: z.object({}) },
},