diff --git a/app/api/graph/import/route.ts b/app/api/graph/import/route.ts index 7c44afd..0f29d85 100644 --- a/app/api/graph/import/route.ts +++ b/app/api/graph/import/route.ts @@ -14,7 +14,7 @@ * /api/graph/status, as with session runs. */ import { extraction } from "@/src/server"; -import type { EmissionStats } from "@/src/core"; +import { ModelsNotConfiguredError, type EmissionStats } from "@/src/core"; export const dynamic = "force-dynamic"; @@ -23,6 +23,10 @@ interface ImportResult { ok: boolean; stats?: EmissionStats; error?: string; + /** The machine discriminator when ok:false is the BYOK refusal (owner, + * 2026-09-22) — the client can open the setup flow off it. */ + code?: "models_not_configured"; + missing?: string[]; } export async function POST(request: Request) { @@ -43,6 +47,16 @@ export async function POST(request: Request) { const stats = await extraction.importNotes({ filePath: file.name, text }); results.push({ filePath: file.name, ok: true, stats }); } catch (err) { + if (err instanceof ModelsNotConfiguredError) { + results.push({ + filePath: file.name, + ok: false, + error: err.envelope.error, + code: "models_not_configured", + missing: err.envelope.missing, + }); + continue; + } results.push({ filePath: file.name, ok: false, error: err instanceof Error ? err.message : String(err) }); } } diff --git a/config.ts b/config.ts index daa0fbb..84bacac 100644 --- a/config.ts +++ b/config.ts @@ -34,56 +34,44 @@ export const PATHS = { } as const; // ─── Models ──────────────────────────────────────────────────────────────── -// THE single place every LLM is chosen and tuned. .env stays for secrets -// (API keys) only — model choice and thinking budget are product tuning, -// not deployment environment, and this section is the one place they grow. -// `effort` is pi's thinking-level scale (off|minimal|low|medium|high|xhigh -// |max): the per-model truth lives in pi-ai's catalog (thinkingLevelMap) — -// getSupportedThinkingLevels(model) lists what a model actually accepts, -// and streamSimple clamps any other value to the nearest supported one -// (e.g. GLM-5.3-flash supports exactly low|high|max). +// BYOK ALL THE WAY DOWN (owner, 2026-09-22): there are NO factory default +// models. Every engine resolves from the user's model slots (settings.json +// models section: strong/cheap/vision) over per-role/ChatSettings overrides, +// and a chain that ends empty REFUSES honestly (503 models_not_configured). +// .env stays for secrets (API keys) only. What remains here are ENGINE +// CONSTRUCTION FACTS: the thinking-level scale and the default order. export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; -export interface ModelSpec { - /** The provider that owns the model (model slots wave, 2026-09-20): - * pi-ai provider id. Native ids are plain strings ("glm-5.3-flash" on - * zai); OpenRouter ids are vendor slugs ("z-ai/glm-5.3-flash"). The - * factory pins stay OpenRouter (the BYOK default provider). */ - provider: string; - /** The model id on that provider — must exist in pi-ai's catalog. */ - model: string; - /** Thinking budget — what the model may spend on reasoning tokens. */ - effort: ThinkingLevel; -} +/** Every agent role in the app. The conversation pair (prober/teacher) + * resolves chat.model → strong slot; background roles resolve their own + * override → tier slot (core/settings-resolution.ts ROLE_SLOT). */ +export type ModelRole = + | "prober" + | "teacher" + | "researcher" + | "demoBuilder" + | "visualMaker" + | "pdfReader" + | "extractor" + | "auditor" + | "labeler"; -export const MODELS = { - /** The prober (agent A) — edge-bracketing questions; keep it lively and adaptive. */ - prober: { provider: "openrouter", model: "z-ai/glm-5.3-flash", effort: "low" }, - /** The teacher (agent B) — planning + streaming, lesson-facing: smart, but keep it lively. */ - teacher: { provider: "openrouter", model: "z-ai/glm-5.3-flash", effort: "low" }, - /** Background sub-agents: fast and cheap beats deep thought. */ - researcher: { provider: "openrouter", model: "z-ai/glm-5.3-flash", effort: "low" }, - demoBuilder: { provider: "openrouter", model: "z-ai/glm-5.3-flash", effort: "low" }, - /** Must be vision-capable: it looks at rendered SVGs. */ - visualMaker: { provider: "openrouter", model: "qwen/qwen3.8-flash", effort: "low" }, - /** SEAM (master-approved 2026-09, revised by the user 2026-09-19): the - * pdf-reader sub-agent. Must be vision-capable — it looks at rendered - * PDF pages; its briefs are text. The user overrode the initial - * glm-5.3-flash pick (catalog says it accepts images, but qwen is - * proven for exactly this look-at-renders job — visualMaker's daily - * workload). */ - pdfReader: { provider: "openrouter", model: "qwen/qwen3.8-flash", effort: "low" }, - /** Extraction is judgment over given evidence — light thinking suffices. */ - extractor: { provider: "openrouter", model: "z-ai/glm-5.3-flash", effort: "low" }, - /** The auditor verifies evidence exists; keep it cheap. */ - auditor: { provider: "openrouter", model: "z-ai/glm-5.3-flash", effort: "low" }, - /** Names detected communities — one singular call, given member names - * + statements. Purely a labeling flourish; keep it cheap. */ - labeler: { provider: "openrouter", model: "z-ai/glm-5.3-flash", effort: "low" }, -} satisfies Record; - -export type ModelRole = keyof typeof MODELS; +/** The DEFAULT thinking level for UNSET levels only (owner, 2026-09-22): + * a slot/override/chat that names a model but no thinking level resolves + * to the FIRST entry in this order that the model supports + * (getSupportedThinkingLevels); a non-reasoning model resolves "off" + * regardless (pi reports exactly ["off"]). EXPLICITLY set levels never + * consult this list — they clamp nearest via pi's native + * clampThinkingLevel on model switches. */ +export const DEFAULT_THINKING_ORDER = [ + "medium", + "low", + "high", + "minimal", + "xhigh", + "max", +] as const satisfies readonly ThinkingLevel[]; /** * The extraction pipeline (vision §1 construction): the sole graph writer. diff --git a/src/agent/models.ts b/src/agent/models.ts index fab5257..529f342 100644 --- a/src/agent/models.ts +++ b/src/agent/models.ts @@ -1,6 +1,7 @@ import { clampThinkingLevel, createModels, + getSupportedThinkingLevels, type Api, type Model, type Models, @@ -8,19 +9,18 @@ import { } from "@earendil-works/pi-ai"; import { builtinProviders } from "@earendil-works/pi-ai/providers/all"; import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; -import { MODELS, type ModelRole } from "../../config.ts"; -import type { ChatSettings } from "../core/contracts/chat.ts"; +import { DEFAULT_THINKING_ORDER, type ModelRole } from "../../config.ts"; /** * The one engine factory every agent's model comes from (the pi-ai swap, - * docs/pi-engine-swap.md). Model choice and thinking budget live in - * config.ts (MODELS) — the single tuning table; this module only wires them - * to pi-ai's OpenRouter provider. + * docs/pi-engine-swap.md). BYOK ALL THE WAY DOWN (owner, 2026-09-22): + * there are no factory pins — every engine resolves from a RESOLVED SPEC + * (the settings chain's output) and the one builder here wires it to + * pi-ai's federated catalogs. * * An "engine" is the model + the stream function the agent loop runs on — * the two things a test fake replaces. Production engines resolve the model - * from pi-ai's static catalog (glm-5.3-flash and qwen3.8-max are both in - * 0.85.1's built-ins — no overlay fetch) and stream through the same Models + * from pi-ai's static catalogs and stream through the same Models * collection. */ @@ -34,12 +34,6 @@ export interface PiEngine { export type { ThinkingLevel }; -/** The role's config thinking level (config's scale IS pi's since the - * model-control wave — no mapping needed). */ -export function thinkingLevelFor(role: ModelRole): ThinkingLevel { - return MODELS[role].effort; -} - let collection: ReturnType | undefined; /** The process-wide pi-ai Models collection — EVERY builtin provider's @@ -63,14 +57,26 @@ export function openRouterApiKey(): string | undefined { return process.env.OPENROUTER_API_KEY || undefined; } -/** A fully-resolved model pin — the shape every engine now comes from - * (model slots wave, 2026-09-20): the settings-resolution chain's output. - * Every field concrete; the CALLER resolved the chain (role override → - * slot → factory pin, or the chat's conversation chain). */ +/** A fully-resolved model pin — the shape every engine now comes from: + * the settings-resolution chain's output (role override → slot, or the + * chat's conversation chain). thinkingLevel is OPTIONAL: unset means + * the baked default resolves here (owner 2026-09-22) — the first entry + * of DEFAULT_THINKING_ORDER the model supports. */ export interface ResolvedModelSpec { provider: string; id: string; - thinkingLevel: ThinkingLevel; + thinkingLevel?: ThinkingLevel; +} + +/** The first entry of the baked default order the model supports (owner, + * 2026-09-22). A non-reasoning model reports exactly ["off"], so the + * order walk lands there naturally. */ +function defaultThinkingLevel(model: Model): ThinkingLevel { + const supported = getSupportedThinkingLevels(model); + for (const level of DEFAULT_THINKING_ORDER) { + if (supported.includes(level)) return level; + } + return supported[0] ?? "off"; } /** The production engine for a RESOLVED spec (model slots wave, @@ -84,7 +90,12 @@ export function piEngineForSpec(spec: ResolvedModelSpec): PiEngine { if (!model) { throw new Error(`${spec.provider} catalog has no model ${spec.id}`); } - const level = clampThinkingLevel(model, spec.thinkingLevel); + // UNSET level → the baked default order (first entry the model + // supports); SET level → pi's native nearest-clamp (model swaps keep + // the user's intent and adapt honestly). + const level = spec.thinkingLevel + ? clampThinkingLevel(model, spec.thinkingLevel) + : defaultThinkingLevel(model); return { model, level, @@ -99,17 +110,14 @@ export function piEngineForSpec(spec: ResolvedModelSpec): PiEngine { }; } -/** The production engine for one agent role — the LEGACY fallback path: - * the chat's model-control override over config.ts's pin. Production - * callers (chat-handler) now resolve the full chain (chat override → - * strong slot → factory pin) and call piEngineForSpec; this stays for - * direct config-pin fallbacks (sub-agents without an injected engine — - * the job runner injects one — and tests). */ -export function piEngineFor(role: ModelRole, settings?: ChatSettings): PiEngine { - const spec = MODELS[role]; - return piEngineForSpec({ - provider: settings?.model?.provider ?? spec.provider, - id: settings?.model?.id ?? spec.model, - thinkingLevel: settings?.thinkingLevel ?? thinkingLevelFor(role), - }); +/** The LEGACY role entry point — BYOK made it an honest THROWER (owner, + * 2026-09-22): no factory pins exist to fall back to, and this module + * cannot read settings (architecture law — the agent layer never + * imports the server layer). Production resolves the chain in + * chat-handler and ALWAYS injects engines; reaching this means a + * wiring bug, so it fails loudly instead of guessing a model. */ +export function piEngineFor(role: ModelRole): PiEngine { + throw new Error( + `no configured model for role "${role}" — set your model slots (settings → models)`, + ); } diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index 468cdca..1606823 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -4,6 +4,7 @@ import type { Usage } from "@earendil-works/pi-ai"; import { z } from "zod"; import { lhTool, toPiTools } from "../lh-tool.ts"; import type { ModelRole } from "../../../config.ts"; +import { ModelsNotConfiguredError, ROLE_SLOT, type ModelSlotName } from "../../core/settings.ts"; import type { PiEngine } from "../models.ts"; import { runSubagent } from "./pi-run.ts"; import type { @@ -562,11 +563,12 @@ export interface ExtractionModels { auditor?: PiEngine; /** Test seam for the cluster namer (post-extraction garnish). */ labeler?: PiEngine; - /** Settings-resolved engine per role (model slots wave, 2026-09-20) — - * the same slot treatment the job runner got: the role-override → - * slot → factory-pin chain, read FRESH per stage call (a settings PUT - * between stages lands). Absent → config pins, unchanged. */ - engineFor?: (role: ModelRole) => PiEngine; + /** Settings-resolved engine per role — the same slot treatment the + * job runner got, read FRESH per stage call (a settings PUT between + * stages lands). BYOK (owner, 2026-09-22): the chain (role override → + * slot → NOTHING) may end EMPTY → undefined; the front doors refuse + * with ModelsNotConfiguredError before any stage runs. */ + engineFor?: (role: ModelRole) => PiEngine | undefined; } export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels { @@ -615,13 +617,24 @@ export function callUsage(usage: Usage): CallUsage { export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline { const { chatStore, graphStore, embedder: embed } = deps; - // Stage-engine resolution (model slots wave, 2026-09-20): a direct test - // value wins; otherwise the engineFor chain (role override → slot → - // factory pin, resolved FRESH per stage call server-side) — the job - // runner's exact treatment, closing the flagged gap where extraction - // stayed config-pin-bound. + // Stage-engine resolution: a direct test value wins; otherwise the + // engineFor chain (role override → slot → nothing, resolved FRESH per + // stage call server-side) — the job runner's exact treatment. undefined + // = the BYOK chain came up empty; the front-door guards turn that into + // an honest refusal. const stageEngine = (role: ModelRole, override?: PiEngine): PiEngine | undefined => override ?? deps.engineFor?.(role); + /** The per-need refusal guard: which slot tiers the pipeline's stages + * found unset (test values count as configured — they ARE engines). */ + function refuseIfStagesUnconfigured(): void { + const missing = new Set(); + if (!stageEngine("extractor", deps.extractor)) missing.add(ROLE_SLOT.extractor); + if (!stageEngine("auditor", deps.auditor)) missing.add(ROLE_SLOT.auditor); + if (missing.size > 0) { + console.info(`[extraction] refused — no model configured for: ${[...missing].join(", ")}`); + throw new ModelsNotConfiguredError([...missing]); + } + } // THE SHARED REGISTRIES (runs() above — globalThis): every pipeline // instance (route entries in dev, tests) sees the same runs, the same // single-flight guard. No module-level state left in this factory. @@ -743,6 +756,10 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti try { const record = chatStore.getChat(chatId); if (!record) throw new Error(`Chat not found: ${chatId}`); + // BYOK per-need gate (owner, 2026-09-22): the write channel needs + // extractor + auditor engines; an empty chain refuses BEFORE any + // stage runs (typed — the trigger skips, routes surface 503). + refuseIfStagesUnconfigured(); const covered = coverageOf(record.messages as UIMessage[], graphStore.readCoverage(), chatId); if (covered.uncovered.length === 0) { console.info(`[extraction ${chatId}] nothing uncovered — no-op`); @@ -811,11 +828,16 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti ); // Post-extraction garnish (DECIDED 2026-09-13): name new/changed // clusters. Fire-and-forget — naming never fails the run. - refreshClusterLabels({ - graphStore, - engine: stageEngine("labeler", deps.labeler), - signal: runningExtractions.get(chatId)?.controller.signal, - }).catch(() => {}); + const labelerEngine = stageEngine("labeler", deps.labeler); + if (labelerEngine) { + refreshClusterLabels({ + graphStore, + engine: labelerEngine, + signal: runningExtractions.get(chatId)?.controller.signal, + }).catch(() => {}); + } else { + console.info(`[extraction ${chatId}] cluster naming skipped — no labeler model configured`); + } return stats; } catch (err) { persistRunUsage(chatId, Math.round(performance.now() - t0), batchesDone > 0 ? total : undefined); @@ -908,6 +930,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti inFlight.add(id); const t0 = performance.now(); try { + // BYOK per-need gate (owner, 2026-09-22): same refusal as the chat + // front door — no extractor/auditor engines, no run. + refuseIfStagesUnconfigured(); console.info(`[extraction ${id}] notes pipeline start (${input.filePath})`); setPhase(id, "extracting", "notes", input.filePath); const findings = await notesExtract(input); @@ -933,11 +958,16 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti `[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + `(${Math.round(performance.now() - t0)}ms total)`, ); - refreshClusterLabels({ - graphStore, - engine: stageEngine("labeler", deps.labeler), - signal: runningExtractions.get(id)?.controller.signal, - }).catch(() => {}); + const labelerEngine = stageEngine("labeler", deps.labeler); + if (labelerEngine) { + refreshClusterLabels({ + graphStore, + engine: labelerEngine, + signal: runningExtractions.get(id)?.controller.signal, + }).catch(() => {}); + } else { + console.info(`[extraction ${id}] cluster naming skipped — no labeler model configured`); + } return stats; } catch (err) { persistRunUsage(id, Math.round(performance.now() - t0), undefined); diff --git a/src/agent/subagents/pdf-reader.ts b/src/agent/subagents/pdf-reader.ts index 0a2aabc..fea9349 100644 --- a/src/agent/subagents/pdf-reader.ts +++ b/src/agent/subagents/pdf-reader.ts @@ -1,7 +1,7 @@ import type { AssistantMessage, Message, ToolResultMessage, Usage } from "@earendil-works/pi-ai"; import { z } from "zod"; import { lhTool, piToolDeclarations, type LhTool } from "../lh-tool.ts"; -import { piEngineFor, thinkingLevelFor, type PiEngine } from "../models.ts"; +import { piEngineFor, type PiEngine, type ThinkingLevel } from "../models.ts"; import type { PdfResult, PdfToolInput } from "../../core/contracts.ts"; import { getHeadless, type HeadlessApi } from "./headless.ts"; import { pdfExtract, pdfInfo, pdfProbe, pdfSearch, type PdfSearchHit } from "./pdf-ops.ts"; @@ -181,7 +181,9 @@ export async function runPdfReaderAgent( const tools = createPdfTools(pdfPath); const toolDeclarations = piToolDeclarations(tools); const engine = opts.engine ?? piEngineFor("pdfReader"); - const reasoning = thinkingLevelFor("pdfReader"); + // BYOK (owner, 2026-09-22): the level rides the injected engine — no + // factory pin to consult (engine.level is the resolved/clamped value). + const reasoning = engine.level; const prompt = [ `PDF path: ${pdfPath}`, `Question (what the teacher needs from this document): ${input.question || "(no question — give a structural overview)"}`, @@ -392,11 +394,11 @@ async function streamOnce( systemPrompt: string, messages: Message[], tools: ReturnType, - reasoning: ReturnType, + reasoning: ThinkingLevel | undefined, abortSignal?: AbortSignal, ): Promise { const stream = engine.streamFn(engine.model, { systemPrompt, messages, tools }, { - reasoning: reasoning === "off" ? undefined : reasoning, + reasoning: reasoning && reasoning !== "off" ? reasoning : undefined, apiKey: process.env.OPENROUTER_API_KEY || undefined, signal: abortSignal, } as never); diff --git a/src/agent/subagents/pi-run.ts b/src/agent/subagents/pi-run.ts index 17d4c95..890fe04 100644 --- a/src/agent/subagents/pi-run.ts +++ b/src/agent/subagents/pi-run.ts @@ -1,6 +1,6 @@ import { Agent, type AgentEvent, type AgentMessage, type AgentTool } from "@earendil-works/pi-agent-core"; import type { AssistantMessage, ToolResultMessage, Usage } from "@earendil-works/pi-ai"; -import { openRouterApiKey, piEngineFor, thinkingLevelFor, type PiEngine } from "../models.ts"; +import { openRouterApiKey, piEngineFor, type PiEngine } from "../models.ts"; import type { ModelRole } from "../../../config.ts"; /** @@ -47,6 +47,9 @@ export interface SubagentRunResult { } export function subagentEngine(role: ModelRole, override?: PiEngine): PiEngine { + // BYOK (owner, 2026-09-22): no factory pins — the override is the only + // real path (the runner injects settings-resolved engines); a missed + // injection hits the honest thrower in models.ts instead of a guess. return override ?? piEngineFor(role); } @@ -57,7 +60,7 @@ export async function runSubagent(tools: AgentTool[], opts: SubagentRu initialState: { systemPrompt: opts.systemPrompt, model: engine.model, - thinkingLevel: thinkingLevelFor(opts.role), + thinkingLevel: engine.level, tools, messages: opts.messages ?? [], }, diff --git a/src/agent/subagents/runner.ts b/src/agent/subagents/runner.ts index 41d4d6d..bf0c4c5 100644 --- a/src/agent/subagents/runner.ts +++ b/src/agent/subagents/runner.ts @@ -20,7 +20,8 @@ import { type VisualToolInput, } from "../../core/contracts.ts"; import { registerJobAbort, unregisterJobAbort } from "./aborts.ts"; -import { MODELS, type ModelRole } from "../../../config.ts"; +import type { ModelRole } from "../../../config.ts"; +import { ROLE_SLOT, modelsNotConfigured } from "../../core/settings.ts"; import type { ChatStore, JobStore } from "../../core/ports.ts"; import { runDemoBuilder } from "./demo-builder.ts"; import { runPdfReaderAgent } from "./pdf-reader.ts"; @@ -76,7 +77,10 @@ export interface JobRunnerDeps { * old gap where per-role settings overrides never reached job engines. * Test overrides (below) still win; absent → config pins. */ - engineFor?: (role: ModelRole) => PiEngine; + /** Settings-resolved engine per role — BYOK (owner, 2026-09-22): the + * chain (role override → slot → NOTHING) may end empty → undefined; + * runJob then refuses the spawn honestly instead of guessing. */ + engineFor?: (role: ModelRole) => PiEngine | undefined; /** The search seam (search port wave, 2026-09-20): resolves the LIVE * provider fresh per job — server/search.ts composition injected by * the wiring; null = honestly unavailable (the researcher's web_search @@ -114,20 +118,6 @@ const KIND_ROLE: Record, ModelRole> = { pdf: "pdfReader", }; -/** The config-pin model stamp for a job kind — JobUsage.model when no - * engine was resolved (engineFor absent AND no test override). Stamps are - * provider-qualified ("provider:id") since the model slots wave, matching - * the price table's uniform keys. Context-injection jobs make NO model - * call ("" — onUsage never fires for them; the kind is deliberately - * absent from the runners below). */ -const KIND_MODEL: Record = { - research: `${MODELS.researcher.provider}:${MODELS.researcher.model}`, - demo: `${MODELS.demoBuilder.provider}:${MODELS.demoBuilder.model}`, - visual: `${MODELS.visualMaker.provider}:${MODELS.visualMaker.model}`, - pdf: `${MODELS.pdfReader.provider}:${MODELS.pdfReader.model}`, - "graph-context": "", -}; - export function createJobRunner(deps: JobRunnerDeps): JobRunner { const { chatStore, jobStore, deliver } = deps; @@ -135,10 +125,10 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner { * then writes through the job store so the poll sees progress). */ const usageTotals = new Map(); - function makeLoopContext(jobId: string, kind: JobKind, engine?: PiEngine): LoopContext { - // Provider-qualified stamp (model slots wave): the engine's actual - // model, else the legacy config-pin id (see KIND_MODEL). - const model = engine ? `${engine.model.provider}:${engine.model.id}` : KIND_MODEL[kind]; + function makeLoopContext(jobId: string, engine: PiEngine): LoopContext { + // Provider-qualified stamp: the engine's actual model — the only + // thing that ran (BYOK: no config-pin fallback stamps exist). + const model = `${engine.model.provider}:${engine.model.id}`; // Real-kill wiring (2026-09 abort semantics): the controller is what // the abort endpoint fires (agent/subagents/aborts.ts registry); the // signal rides into every model call of the loop. @@ -171,8 +161,8 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner { // are kind-correct by construction; transcript fallbacks are cast after). /** The engine one job runs on: test override first (the old seam — * tests swap scripted engines without touching deps), then the injected - * settings-resolved engineFor (model slots wave, 2026-09-20), else - * undefined → the sub-agent factories fall back to config pins. */ + * settings-resolved engineFor. undefined = the BYOK chain came up + * empty (owner, 2026-09-22) — runJob refuses below, honestly. */ function pickEngine(kind: Exclude): PiEngine | undefined { const testOverride: Record = { researcher: researchModelOverride, @@ -185,7 +175,7 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner { const KIND_RUNNERS: Record< Exclude, - (jobId: string, engine: PiEngine | undefined, input: JobToolInput | undefined, loop: LoopContext) => Promise + (jobId: string, engine: PiEngine, input: JobToolInput | undefined, loop: LoopContext) => Promise > = { research: (jobId, engine, input, loop) => runResearchJob(jobId, engine, input, loop), demo: (jobId, engine, input, loop) => runDemoJob(jobId, engine, input, loop), @@ -212,9 +202,23 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner { jobStore.updateJob(jobId, { status: "error", error: `unknown kind ${job.kind}`, activity: "failed" }); return; } + const engine = pickEngine(job.kind); + if (!engine) { + // BYOK per-need refusal (owner, 2026-09-22): this kind's tier has + // no model — the job record surfaces the honest error (dock + + // teacher read it), and no engine-less loop ever starts. + const tier = ROLE_SLOT[KIND_ROLE[job.kind]]; + const envelope = modelsNotConfigured([tier]); + console.info(`[job ${jobId}] refused — ${JSON.stringify(envelope)}`); + jobStore.updateJob(jobId, { + status: "error", + error: `${envelope.error} (missing: ${tier})`, + activity: "failed", + }); + return; + } try { - const engine = pickEngine(job.kind); - await run(jobId, engine, input, makeLoopContext(jobId, job.kind, engine)); + await run(jobId, engine, input, makeLoopContext(jobId, engine)); } finally { unregisterJobAbort(jobId); // The last usage write may have been mid-flight when the loop died; diff --git a/src/agent/subagents/visual-maker.ts b/src/agent/subagents/visual-maker.ts index b243fde..8ff6313 100644 --- a/src/agent/subagents/visual-maker.ts +++ b/src/agent/subagents/visual-maker.ts @@ -1,7 +1,7 @@ import type { AssistantMessage, Message, ToolResultMessage, Usage } from "@earendil-works/pi-ai"; import { z } from "zod"; import { lhTool, piToolDeclarations, type LhTool } from "../lh-tool.ts"; -import { piEngineFor, thinkingLevelFor, type PiEngine } from "../models.ts"; +import { piEngineFor, type PiEngine, type ThinkingLevel } from "../models.ts"; import type { VisualResult } from "../../core/contracts.ts"; import { getHeadless, type HeadlessApi } from "./headless.ts"; @@ -132,7 +132,9 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO const tools = createVisualTools(); const toolDeclarations = piToolDeclarations(tools); const engine = opts.engine ?? piEngineFor("visualMaker"); - const reasoning = thinkingLevelFor("visualMaker"); + // BYOK (owner, 2026-09-22): the level rides the injected engine — no + // factory pin to consult (engine.level is the resolved/clamped value). + const reasoning = engine.level; const prompt = [ "Brief for ONE precise figure (from the teacher, verbatim — it is everything you know):", brief, @@ -300,11 +302,11 @@ async function streamOnce( systemPrompt: string, messages: Message[], tools: ReturnType, - reasoning: ReturnType, + reasoning: ThinkingLevel | undefined, abortSignal?: AbortSignal, ): Promise { const stream = engine.streamFn(engine.model, { systemPrompt, messages, tools }, { - reasoning: reasoning === "off" ? undefined : reasoning, + reasoning: reasoning && reasoning !== "off" ? reasoning : undefined, apiKey: process.env.OPENROUTER_API_KEY || undefined, signal: abortSignal, } as never); diff --git a/src/agent/teacher-agent.ts b/src/agent/teacher-agent.ts index 8e92ae7..335e2d3 100644 --- a/src/agent/teacher-agent.ts +++ b/src/agent/teacher-agent.ts @@ -2,8 +2,7 @@ import { Agent } from "@earendil-works/pi-agent-core"; import type { AgentEvent, AgentMessage, AgentTool } from "@earendil-works/pi-agent-core"; import type { Usage } from "@earendil-works/pi-ai"; import { PATHS } from "../../config.ts"; -import { MODELS } from "../../config.ts"; -import { piEngineFor, thinkingLevelFor, openRouterApiKey, type PiEngine } from "./models.ts"; +import { piEngineFor, openRouterApiKey, type PiEngine } from "./models.ts"; import { effectivePromptSegments } from "./prompt-segments.ts"; import { askUserQuestionTool, @@ -154,7 +153,7 @@ function baseAgentOptions(kind: AgentKind, opts: AgentBuildOptions, tools: Agent initialState: { systemPrompt, model: engine.model, - thinkingLevel: thinkingLevelFor(kind), + thinkingLevel: engine.level, tools, messages: opts.messages ?? [], }, @@ -227,7 +226,3 @@ export type ProberAgent = ReturnType; export type TeacherAgent = ReturnType; export type TeachingAgent = ProberAgent | TeacherAgent; -/** The model id a role's engine resolves to (observability). */ -export function modelIdFor(kind: AgentKind): string { - return MODELS[kind].model; -} \ No newline at end of file diff --git a/src/core/contracts/models.ts b/src/core/contracts/models.ts index 4e3b4c4..b2af34b 100644 --- a/src/core/contracts/models.ts +++ b/src/core/contracts/models.ts @@ -26,38 +26,46 @@ export interface ModelCatalogEntry { maxTokens: number; } -/** The conversation agents' config pin (config.ts MODELS prober/teacher — - * one controller, so one pin): what a chat without settings runs. */ +/** The conversation agents' resolved default (BYOK, owner 2026-09-22): the + * chat's model override over the STRONG slot — one controller, so one + * default. thinkingLevel is optional: unset resolves from the baked + * DEFAULT_THINKING_ORDER at engine build. */ export interface CatalogDefaults { model: { provider: string; id: string }; - thinkingLevel: ThinkingLevel; + thinkingLevel?: ThinkingLevel; } -/** One background role's RESOLVED pin (settings interlude, 2026-09-19): - * the user's agent-models override falling back to the config.ts MODELS - * pin — what that role actually runs today. The settings UI shows each - * agent-role row's "Default: " caption from it. */ +/** One background role's RESOLVED pin (BYOK, owner 2026-09-22): what that + * role actually runs today — role override → tier slot → NOTHING. NULL + * = the role's chain is empty (the per-need gates refuse its work); the + * settings UI shows "not configured" from it. */ export type RolePins = Record< BackgroundModelRole, - { model: { provider: string; id: string }; thinkingLevel?: ThinkingLevel } + { model: { provider: string; id: string }; thinkingLevel?: ThinkingLevel } | null >; -/** One slot tier's RESOLVED pin (model slots wave, 2026-09-20): the - * strong/cheap/vision tier's concrete model — slot override falling back - * to the factory rep pin. The settings UI captions each slot row with it - * and the picker pre-fills from it. */ -export type SlotPins = Record; +/** One slot tier's RESOLVED pin (BYOK, owner 2026-09-22): NULL = the tier + * holds no model. The settings UI captions each slot row with it and the + * picker pre-fills from it. */ +export type SlotPins = Record< + ModelSlotName, + { model: { provider: string; id: string }; thinkingLevel?: ThinkingLevel } | null +>; /** GET /api/models response. */ export interface ModelCatalogResponse { models: ModelCatalogEntry[]; - defaults: CatalogDefaults; + /** The conversation agents' resolved default — NULL when the + * conversation chain is empty (no factory floor; the turn gate + * refuses with 503 models_not_configured). */ + defaults: CatalogDefaults | null; /** The six background roles' resolved pins (same resolution the agent - * factories use — src/server/stores.ts effectiveSettings). */ + * factories use — src/server/stores.ts effectiveSettings); NULL per + * role = chain empty. */ rolePins: RolePins; - /** The three slot tiers' resolved pins (model slots wave, 2026-09-20): - * same resolution chain — role override → slot → factory pin — reported - * at tier granularity; effectiveSettings().models verbatim. */ + /** The three slot tiers' resolved pins: role override → slot at tier + * granularity; effectiveSettings().models verbatim, NULL per unset + * tier. */ slots: SlotPins; /** Providers with a PRESENT key, and which env var gates each — the * model slots wave's federation surface (2026-09-20): the picker lists diff --git a/src/core/settings-resolution.ts b/src/core/settings-resolution.ts index 6069b20..c399767 100644 --- a/src/core/settings-resolution.ts +++ b/src/core/settings-resolution.ts @@ -5,17 +5,17 @@ * delta-walking of their own. * * Resolution rule (owner decision): explicit value in the record wins; - * absent key resolves to the compiled factory default. Model-role defaults - * come from config.ts's MODELS, threaded in as a parameter — this module - * must stay browser-safe/pure, so it imports config TYPES only, never the - * runtime module (config.ts pulls node:path). The composition root - * (src/server/stores.ts) owns the glue: effectiveSettings() reads the - * record through the store and passes MODELS here. + * absent key resolves to the compiled factory default — EXCEPT MODELS + * (BYOK all the way down, owner 2026-09-22): no factory pins exist, so a + * model chain that ends empty resolves to null/undefined and whatever + * needs it refuses honestly (503 models_not_configured). This module + * stays browser-safe/pure, so it imports config TYPES only, never the + * runtime module (config.ts pulls node:path). * * Deliberately NOT resolved here (see settings.ts module doc): - * - per-chat overrides: the chat record's ChatSettings applies via - * piEngineFor(role, chatRecord.settings) exactly as before — prober and - * teacher stay chat-controlled, never settings-record-controlled; + * - per-chat overrides: the chat record's ChatSettings applies via the + * conversation chain in chat-handler (chat.model → strong slot) — prober + * and teacher stay chat-controlled, never settings-record-controlled; * - tool toggles: passed through as the delta map; consumers look up by * tool id with absent → enabled (the honoring lives in the agent band); * - prompt switches: passed through; effective composition lives in @@ -42,27 +42,24 @@ import { type UserSettings, } from "./settings.ts"; -/** The factory layer for model roles — config.ts's MODELS shape (types only - * here; the runtime object is threaded in by the caller). Model slots - * (2026-09-20): the factory map spans ALL nine roles (the strong slot's - * factory fallback is the prober pin), while per-role OVERRIDES stay - * background-only (the conversation pair is chat-settings-controlled). */ -export type ModelRoleSpecs = Record; - -/** A fully-resolved model slot: the tier's concrete model (slot over factory - * rep pin) — strong's factory rep is the prober pin, cheap's the researcher - * pin, vision's the visualMaker pin (owner-settled tier mapping). */ +/** A configured model slot: the tier's concrete model, with its optional + * thinking level. UNSET level is legal — the engine build resolves it + * from config.ts's DEFAULT_THINKING_ORDER (first entry the model + * supports; owner, 2026-09-22). A slot is CONFIGURED iff its model is + * present; a level-only slot is inert (kept on disk, resolved as null). */ export interface EffectiveSlot { provider: string; id: string; - thinkingLevel: ThinkingLevel; + thinkingLevel?: ThinkingLevel; } -/** A fully-resolved model pin for one background role. */ +/** A fully-resolved model pin for one role — same shape/level semantics + * as EffectiveSlot; undefined means the role's whole chain came up + * empty and whatever needs it refuses honestly. */ export interface EffectiveModelSpec { provider: string; id: string; - thinkingLevel: ThinkingLevel; + thinkingLevel?: ThinkingLevel; } /** The concrete appearance view — same shape as the factory's. */ @@ -75,7 +72,7 @@ export interface EffectiveSettings { * ({} = no default = BYOK "no model" first boot). */ chat: ChatSettings; agent: { - models: Record; + models: Record; /** Delta map as stored — consumers resolve absent key → enabled. */ tools: Record; /** Delta map as stored — absent key → override off. */ @@ -85,84 +82,81 @@ export interface EffectiveSettings { * lives in server/search.ts, where keys are visible. */ search: { engine?: SearchEngineId }; }; - /** The resolved strong/cheap/vision slots (model slots wave, - * 2026-09-20): each tier's concrete model (slot over factory rep pin) - * — the per-role chain (override → slot → pin) is resolved into - * agent.models; these are the tier-level views the picker captions. */ - models: Record; + /** The resolved strong/cheap/vision slots: each tier's concrete model or + * NULL when the tier holds no model (no factory floor — BYOK all the + * way down, owner 2026-09-22). These are the tier-level views the + * picker captions and the per-need refusal gates read. */ + models: Record; appearance: EffectiveAppearance; graph: Required; } -/** Full-chain resolution for ANY role (model slots wave, 2026-09-20): - * per-role override (background roles only) → slot → factory pin. The - * conversation pair (prober/teacher) resolves the same chain MINUS the - * role override (their override is the chat's own settings — the - * chat-handler overlays it itself). This is the engine wiring's one call. */ -export function resolveRoleSpec( - role: ModelRole, - record: UserSettings | null, - factory: ModelRoleSpecs, -): EffectiveModelSpec { - const bg = BACKGROUND_MODEL_ROLES as readonly string[]; - const override = bg.includes(role) ? record?.agent?.models?.[role as BackgroundModelRole] : undefined; - return resolveModelSpec(role as BackgroundModelRole, override, factory, record?.models); -} - -/** The factory representative pin for a slot tier — the pin the slot - * falls back to when unset (owner-settled mapping: strong's rep is the - * prober pin — prober and teacher share the strong tier — cheap's the - * researcher pin, vision's the visualMaker pin). */ -export function factorySlotRep(slot: ModelSlotName, factory: ModelRoleSpecs): EffectiveSlot { - const rep = - slot === "strong" ? factory.prober : slot === "cheap" ? factory.researcher : factory.visualMaker; - // The factory contract spans all nine roles (stores.ts MODEL_PINS) — - // the rep lookup cannot miss; the cast is for the checker only. - const r = rep as { provider: string; model: string; effort: ThinkingLevel }; - return { provider: r.provider, id: r.model, thinkingLevel: r.effort }; -} - -/** Resolve one slot: the slot override per field, the factory rep pin per - * absent field (model slots wave, 2026-09-20). */ -export function resolveSlot( - slot: ModelSlotName, - override: ModelSlot | undefined, - factory: ModelRoleSpecs, -): EffectiveSlot { - const rep = factorySlotRep(slot, factory); +/** The conversation pair's chain (BYOK, owner 2026-09-22): the chat's own + * model override over the STRONG slot, each field independent (a bare + * chat thinking level rides the strong slot's model). null = no model + * anywhere — the turn gate refuses with the 503 models_not_configured + * envelope. */ +export function resolveConversationSpec( + chat: ChatSettings | undefined, + strong: EffectiveSlot | null, +): EffectiveModelSpec | null { + const model = chat?.model ?? (strong ? { provider: strong.provider, id: strong.id } : undefined); + if (!model) return null; return { - provider: override?.model?.provider ?? rep.provider, - id: override?.model?.id ?? rep.id, - thinkingLevel: override?.thinkingLevel ?? rep.thinkingLevel, + provider: model.provider, + id: model.id, + thinkingLevel: chat?.thinkingLevel ?? strong?.thinkingLevel, }; } -/** Resolve one background role's spec — the model slots chain (owner - * decision 2026-09-20): role override → SLOT → factory pin, each field - * independent (a role can borrow only the slot's model, or only its - * thinking budget). An absent provider coordinate anywhere resolves to - * the factory pin's provider (the BYOK default, openrouter today) — - * legacy ids without a provider ride the factory provider. */ +/** Full-chain resolution for ANY role (BYOK, owner 2026-09-22): + * per-role override (background roles only) → SLOT → NOTHING. The + * conversation pair (prober/teacher) resolves the same chain MINUS the + * role override (their override is the chat's own settings — the + * chat-handler overlays it itself). undefined = the chain came up + * empty; the per-need refusal gates fire on exactly that. This is the + * engine wiring's one call. */ +export function resolveRoleSpec(role: ModelRole, record: UserSettings | null): EffectiveModelSpec | undefined { + const bg = BACKGROUND_MODEL_ROLES as readonly string[]; + const override = bg.includes(role) ? record?.agent?.models?.[role as BackgroundModelRole] : undefined; + return resolveModelSpec(role, override, record?.models); +} + +/** Resolve one slot to its configured view, or null when the tier holds + * no model (a thinkingLevel-only slot is inert — the level has nothing + * to ride without the factory floor). */ +export function resolveSlot(slot: ModelSlot | undefined): EffectiveSlot | null { + if (!slot?.model) return null; + return { provider: slot.model.provider, id: slot.model.id, thinkingLevel: slot.thinkingLevel }; +} + +/** Resolve one role's spec — the BYOK chain (owner 2026-09-22): role + * override → SLOT → NOTHING, each field independent (an override may + * borrow only the slot's model, or only its thinking budget — a level + * with no model anywhere stays unset and the engine build resolves it + * from DEFAULT_THINKING_ORDER). undefined = no model resolved; the + * per-need refusal gates fire on exactly that. */ export function resolveModelSpec( - role: BackgroundModelRole, + role: ModelRole, override: RoleModelOverride | undefined, - factory: ModelRoleSpecs, slots?: Partial>, -): EffectiveModelSpec { - // The factory contract spans all nine roles; background roles always hit. - const spec = factory[role] as { provider: string; model: string; effort: ThinkingLevel }; +): EffectiveModelSpec | undefined { const slotName = ROLE_SLOT[role] as ModelSlotName | undefined; const slot = slotName ? slots?.[slotName] : undefined; + const model = override?.model ?? slot?.model; + if (!model) return undefined; return { - provider: override?.model?.provider ?? slot?.model?.provider ?? spec.provider, - id: override?.model?.id ?? slot?.model?.id ?? spec.model, - thinkingLevel: override?.thinkingLevel ?? slot?.thinkingLevel ?? spec.effort, + provider: model.provider, + id: model.id, + thinkingLevel: override?.thinkingLevel ?? slot?.thinkingLevel, }; } /** Resolve the whole record. `record` may be null (first boot) — every - * field then lands on its factory default. */ -export function resolveSettings(record: UserSettings | null, factory: ModelRoleSpecs): EffectiveSettings { + * factory-defaulted field lands on its default; MODEL fields resolve to + * null/undefined when unset (no factory pins exist — BYOK all the way + * down, owner 2026-09-22). */ +export function resolveSettings(record: UserSettings | null): EffectiveSettings { const agent = record?.agent ?? {}; const modelsSection = record?.models ?? {}; const appearance = record?.appearance ?? {}; @@ -170,9 +164,9 @@ export function resolveSettings(record: UserSettings | null, factory: ModelRoleS const graph = record?.graph ?? {}; const factoryAppearance = FACTORY_SETTINGS.appearance; const factoryMd = factoryAppearance.mdRendering; - const models = {} as Record; + const models = {} as Record; for (const role of BACKGROUND_MODEL_ROLES) { - models[role] = resolveModelSpec(role, agent.models?.[role], factory, modelsSection); + models[role] = resolveModelSpec(role, agent.models?.[role], modelsSection); } return { general: { @@ -186,9 +180,9 @@ export function resolveSettings(record: UserSettings | null, factory: ModelRoleS search: agent.search ?? {}, }, models: { - strong: resolveSlot("strong", modelsSection.strong, factory), - cheap: resolveSlot("cheap", modelsSection.cheap, factory), - vision: resolveSlot("vision", modelsSection.vision, factory), + strong: resolveSlot(modelsSection.strong), + cheap: resolveSlot(modelsSection.cheap), + vision: resolveSlot(modelsSection.vision), }, appearance: { mode: appearance.mode ?? factoryAppearance.mode, diff --git a/src/core/settings.ts b/src/core/settings.ts index 6e52a91..d959fdc 100644 --- a/src/core/settings.ts +++ b/src/core/settings.ts @@ -5,8 +5,10 @@ * * DELTA SEMANTICS (owner decision, 2026-09-19): the record stores EXPLICIT * user choices only — ABSENT keys resolve to the factory defaults compiled - * into this module (FACTORY_SETTINGS) and config.ts's MODELS (the compiled - * factory layer for model roles). A software update that ADDS a knob (new + * into this module (FACTORY_SETTINGS). MODELS are the exception (owner, + * 2026-09-22 — BYOK all the way down): no factory pins exist, so an absent + * model key resolves to EMPTY and the per-need gates refuse honestly. A + * software update that ADDS a knob (new * tool, new role, new appearance field) therefore needs no migration: old * records simply lack the new keys. `version` is a PRIVATE schema counter * for BREAKING shape changes only (field renamed / meaning changed) — @@ -26,6 +28,7 @@ */ import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; +import type { ModelRole } from "../../config.ts"; import type { ChatSettings } from "./contracts/chat.ts"; import type { SearchEngineId } from "./search.ts"; @@ -34,9 +37,8 @@ import type { SearchEngineId } from "./search.ts"; export const CURRENT_SETTINGS_VERSION = 1; /** The background-agent roles whose models the Agent settings expose. - * Derived from config.ts's MODELS minus the conversation pair — prober and - * teacher stay chat-settings-controlled (one controller, the model picker). - * Keep in sync with config.ts's MODELS (tests pin the set). */ + * The conversation pair (prober/teacher) stays out — chat-controlled via + * the strong slot (one controller, the model picker). Tests pin the set. */ export const BACKGROUND_MODEL_ROLES = [ "researcher", "demoBuilder", @@ -67,9 +69,44 @@ export interface RoleModelOverride { export const MODEL_SLOTS = ["strong", "cheap", "vision"] as const; export type ModelSlotName = (typeof MODEL_SLOTS)[number]; -/** One slot's override — same per-field independence as a role override: - * absent model keeps the tier's current model, absent thinkingLevel keeps - * its thinking budget. */ +/** The machine-readable NOT-CONFIGURED refusal envelope (owner, 2026-09-22): + * every per-need gate — the turn gate in chat-handler, a job spawn, a + * notes import — refuses with this exact shape. HTTP callers get 503 + + * this JSON; in-band consumers (the job record's error field) carry the + * human `error` line. `missing` names the slot TIER(S) the refused + * action needs and found unset. */ +export const MODELS_NOT_CONFIGURED_CODE = "models_not_configured" as const; +export interface ModelsNotConfigured { + /** Human-readable — safe to render as-is. */ + error: string; + /** The machine discriminator the client switches on. */ + code: typeof MODELS_NOT_CONFIGURED_CODE; + /** Which slot tiers the refused action needs ("strong" | "cheap" | "vision"). */ + missing: ModelSlotName[]; +} +export function modelsNotConfigured(missing: ModelSlotName[]): ModelsNotConfigured { + return { + error: "No model configured — set up your models first", + code: MODELS_NOT_CONFIGURED_CODE, + missing, + }; +} +/** The typed throw for in-process seams (extraction pipeline): callers + * catch it to skip honestly (trigger) or surface the 503 envelope + * (routes) instead of logging a failure. */ +export class ModelsNotConfiguredError extends Error { + readonly envelope: ModelsNotConfigured; + constructor(missing: ModelSlotName[]) { + super(modelsNotConfigured(missing).error); + this.name = "ModelsNotConfiguredError"; + this.envelope = modelsNotConfigured(missing); + } +} + +/** One slot's stored state. BYOK (owner, 2026-09-22): the WRITE side + * requires model (a level alone can never run — no factory floor); the + * sanitizer keeps whatever is on disk, but RESOLUTION treats a slot + * without a model as unset (inert level, no phantom default). */ export interface ModelSlot { /** The model swap: both coordinates or neither. */ model?: { provider: string; id: string }; @@ -99,7 +136,7 @@ export function slotsConfigured(models: ModelsSettings | undefined): boolean { /** Which slot a role draws from (model slots wave, 2026-09-20). Includes the * conversation pair and pdfReader (not BackgroundModelRoles — the slot * tier spans every role; only background roles surface per-role overrides). */ -export const ROLE_SLOT: Readonly> = { +export const ROLE_SLOT: Readonly> = { prober: "strong", teacher: "strong", researcher: "cheap", @@ -201,11 +238,11 @@ export interface UserSettings { [key: string]: unknown; } -/** Factory defaults for everything core can pin (owner decisions 2026-09-19). - * Model-role defaults live in config.ts's MODELS (the compiled factory - * layer); the chat section has NO factory model (BYOK first boot doctrine - * — "no model" until the user picks one); tools default on; prompts - * default off — hence their absence here. */ +/** Factory defaults for everything core can pin (owner decisions + * 2026-09-19). NO factory models exist anywhere (owner, 2026-09-22 — BYOK + * all the way down): the chat section and every slot stay empty until the + * user picks, and the per-need gates refuse honestly; tools default on; + * prompts default off — hence their absence here. */ export interface FactorySettings { version: number; general: Required; diff --git a/src/server/chat-handler.ts b/src/server/chat-handler.ts index 36be74f..b2508c8 100644 --- a/src/server/chat-handler.ts +++ b/src/server/chat-handler.ts @@ -1,7 +1,7 @@ import type { UIMessage } from "ai"; import type { Usage } from "@earendil-works/pi-ai"; import { createProberAgent, createTeacherAgent, type AgentBuildOptions, type TeachingAgent } from "../agent/teacher-agent.ts"; -import { piEngineForSpec } from "../agent/models.ts"; +import { piEngineForSpec, type PiEngine } from "../agent/models.ts"; import { createJobRunner } from "../agent/subagents/runner.ts"; import { createExtractionPipeline } from "../agent/subagents/extraction.ts"; import { runGraphContextJob, type GraphContextDeps } from "../agent/subagents/graph-context.ts"; @@ -10,9 +10,15 @@ import type { ModelRole } from "../../config.ts"; import { claimSteers, deliverJobResult, setContextRespawn, setJobRespawn, setTurnRunner } from "./jobs.ts"; import { recordToPiMessages } from "./pi/record-adapter.ts"; import { createPiTurnResponse } from "./pi/ui-stream.ts"; -import { effectiveSettings, MODEL_PINS, stores } from "./stores.ts"; +import { effectiveSettings, stores } from "./stores.ts"; import { searchAvailable, searchProvider } from "./search.ts"; -import { extractionEnabled, graphContextEnabled, resolveRoleSpec } from "../core/settings-resolution.ts"; +import { + extractionEnabled, + graphContextEnabled, + resolveConversationSpec, + resolveRoleSpec, +} from "../core/settings-resolution.ts"; +import { modelsNotConfigured } from "../core/settings.ts"; import { beginTurn, endTurn } from "./turn-liveness.ts"; import { persistTurn, triggerExtraction, triggerGraphContext, withMessageIds } from "./turn-service.ts"; import type { CallUsage } from "../core/contracts.ts"; @@ -40,12 +46,12 @@ const jobRunner = createJobRunner({ chatStore: stores.chatStore, jobStore: stores.jobStore, deliver: deliverJobResult, - // Model slots wave (2026-09-20): job engines resolve the full chain — - // role override → slot → factory pin — read FRESH per job (a settings - // PUT between dispatch and spawn lands). Closes the old gap where - // per-role overrides reached rolePins but never the engines. + // BYOK (owner, 2026-09-22): job engines resolve the chain — role + // override → slot → NOTHING — read FRESH per job (a settings PUT + // between dispatch and spawn lands); an empty chain flows through as + // undefined and the runner refuses the spawn honestly. engineFor: (role) => - piEngineForSpec(resolveRoleSpec(role, stores.settingsStore.readSettings(), MODEL_PINS)), + resolveEngine(role), // Search port wave (2026-09-20): the researcher's web_search runs on the // injected port — resolved FRESH per job (key rotations land immediately). // null = honestly unavailable; the dispatch-level drop (below) uses the @@ -60,13 +66,12 @@ export const extraction = createExtractionPipeline({ // The REAL embedder (local transformer model): post-merge embedding // batch — semantic clusters are computed and persisted here. embedder: embedNewNodes, - // Model slots wave (2026-09-20): the extraction stages resolve the FULL - // chain — role override → slot → factory pin — read FRESH per stage call - // (the flagged gap: ExtractionModels deps were engine VALUES, keeping - // extractor/auditor/labeler config-pin-bound while the job runner went - // slot-native). Direct test values still win inside the pipeline. + // BYOK (owner, 2026-09-22): the extraction stages resolve the chain — + // role override → slot → NOTHING — read FRESH per stage call; an empty + // chain flows through as undefined and the pipeline refuses the run + // with the typed ModelsNotConfiguredError. Direct test values still win. engineFor: (role) => - piEngineForSpec(resolveRoleSpec(role, stores.settingsStore.readSettings(), MODEL_PINS)), + resolveEngine(role), }); /** Graph context injection (vision-context-injection.md): the runner's @@ -97,6 +102,15 @@ setTurnRunner(async (chatId, messages) => { await res.arrayBuffer().catch(() => undefined); return res.status; }); +/** The settings-resolved engine for one role — BYOK (owner, 2026-09-22): + * the chain (role override → slot → NOTHING) may end empty, and undefined + * flows through to the per-need refusal gates (runner/extraction), read + * FRESH so a settings PUT between dispatch and spawn lands. */ +function resolveEngine(role: ModelRole): PiEngine | undefined { + const spec = resolveRoleSpec(role, stores.settingsStore.readSettings()); + return spec ? piEngineForSpec(spec) : undefined; +} + setJobRespawn((jobId) => void jobRunner.runJob(jobId)); setContextRespawn((jobId) => void runGraphContextJob(graphContext, jobId)); @@ -135,6 +149,25 @@ export async function handleChatRequest( return Response.json({ error: "messages must be an array" }, { status: 400 }); } + // THE UNIVERSAL TURN GATE (BYOK, owner, 2026-09-22): a turn needs the + // CONVERSATION chain — the chat's own model override over the strong + // slot — and there is no factory floor; when both are empty the turn + // refuses honestly with the machine-readable 503 envelope (new sessions + // and continuing ones alike — wake-pipeline turns ride this same path). + // Placement is deliberate: BEFORE beginTurn (no liveness effects), the + // write-ahead persist (nothing durable changes), and every downstream + // resolution (jobs/extraction gate per-need on their own tiers). + const effective = effectiveSettings(); + const conversation = resolveConversationSpec( + stores.chatStore.getChat(chatId)?.settings, + effective.models.strong, + ); + if (!conversation) { + const envelope = modelsNotConfigured(["strong"]); + console.info(`[chat ${chatId}] turn refused — ${JSON.stringify(envelope)}`); + return Response.json(envelope, { status: 503 }); + } + // One turn per chat, ever. A second request while a turn runs means a // stale/remounted client raced a live one — its message list is outdated, // and letting it through would double-run the teacher and clobber the @@ -158,10 +191,8 @@ export async function handleChatRequest( ) as UIMessage[]; // THE SETTINGS VIEW (settings-storage wave, 2026-09-19): resolved fresh - // per turn — graph gates, tool toggles, and (via the store) the record's - // chat defaults all read from ONE small file. No caching: a stale toggle - // after a settings PUT would be worse than the read. - const effective = effectiveSettings(); + // per turn (the gate above owns the read) — graph gates, tool toggles, + // and the record's chat defaults all read from ONE small file. // Write-ahead persistence: the request body IS the client's canonical // state, so it goes on disk BEFORE the stream starts. The learner's answer @@ -220,19 +251,10 @@ export async function handleChatRequest( // steers, usage, extraction) — only the agent differs. const teaching = outgoing.some((m) => m.parts.some((p) => p.type === "tool-probe_done")); const role: ModelRole = teaching ? "teacher" : "prober"; - // The chat's model-control settings (prober/teacher only — one controller - // for the conversation agents; piEngineFor falls back to config on absent - // fields and clamps the thinking level to the model's supported list). - const settings = stores.chatStore.getChat(chatId)?.settings; - // Model slots wave (2026-09-20): the conversation chain is the chat's - // own override over the STRONG slot (prober + teacher's tier) over the - // factory pin — each field independent, same discipline as the roles. - const strong = effective.models.strong; - const engine = piEngineForSpec({ - provider: settings?.model?.provider ?? strong.provider, - id: settings?.model?.id ?? strong.id, - thinkingLevel: settings?.thinkingLevel ?? strong.thinkingLevel, - }); + // BYOK (owner, 2026-09-22): the conversation chain was resolved by the + // turn gate above (chat.model override → strong slot → refused) — the + // engine build is pure from here; thinking levels clamp natively. + const engine = piEngineForSpec(conversation); // Provider-qualified stamp (model slots wave): matches the price table's // provider:id keys. const modelId = `${engine.model.provider}:${engine.model.id}`; diff --git a/src/server/model-catalog.ts b/src/server/model-catalog.ts index bd1da87..8175db4 100644 --- a/src/server/model-catalog.ts +++ b/src/server/model-catalog.ts @@ -39,63 +39,77 @@ import { BACKGROUND_MODEL_ROLES, type BackgroundModelRole, type ModelSlotName, t /** The picker's client-safe model entry (contract: core/contracts/models.ts). */ export type ModelDto = ModelCatalogEntry; -/** The conversation agents' resolved default pin (pure form — the record's - * chat section in, CatalogDefaults out; `catalogDefaults()` below is the - * fs wrapper). Absent fields (or no record — BYOK first boot) fall back - * to the config pin, exactly as before. */ -export function catalogDefaultsFor(chat: ChatSettings | undefined, strong: EffectiveSlot): CatalogDefaults { +/** The conversation agents' resolved default (pure form — the record's chat + * section + the strong slot in, CatalogDefaults out; `catalogDefaults()` + * below is the fs wrapper). BYOK (owner, 2026-09-22): there is no factory + * floor — when neither the chat section nor the strong slot names a + * model, the default is NULL and the turn gate refuses with the 503 + * models_not_configured envelope. An unset thinkingLevel resolves from + * DEFAULT_THINKING_ORDER at engine build (reported as absent here). */ +export function catalogDefaultsFor( + chat: ChatSettings | undefined, + strong: EffectiveSlot | null, +): CatalogDefaults | null { + const model = chat?.model ?? (strong ? { provider: strong.provider, id: strong.id } : undefined); + if (!model) return null; + const level = chat?.thinkingLevel ?? strong?.thinkingLevel; return { - model: chat?.model ?? { provider: strong.provider, id: strong.id }, - thinkingLevel: chat?.thinkingLevel ?? strong.thinkingLevel, + model, + ...(level !== undefined ? { thinkingLevel: level } : {}), }; } -/** The conversation agents' config pin — what a settings-less chat runs. - * (settings-storage wave, 2026-09-19): the user's picked default rides in - * the settings record's `chat` section; the picker's defaults row shows - * it once chosen. */ -export function catalogDefaults(): CatalogDefaults { +/** The conversation agents' resolved default — what a settings-less chat + * runs. (settings-storage wave, 2026-09-19): the user's picked default + * rides in the settings record's `chat` section; the picker's defaults + * row shows it once chosen. */ +export function catalogDefaults(): CatalogDefaults | null { return catalogDefaultsFor(settingsStore.readSettings()?.chat, effectiveSettings().models.strong); } /** Project the resolved per-role specs to the client DTO (pure — the * settings UI's "Default: " caption reads this). - * thinkingLevel is always concrete in the resolved view; the contract - * keeps it optional so the UI tolerates a future role without one. */ -export function rolePinsFor(models: Record): RolePins { + * BYOK (owner, 2026-09-22): an empty chain projects to NULL — the caption + * honestly shows "not configured", never a phantom default. */ +export function rolePinsFor( + models: Record, +): RolePins { const pins = {} as RolePins; for (const role of BACKGROUND_MODEL_ROLES) { const spec = models[role]; - pins[role] = { - model: { provider: spec.provider, id: spec.id }, - thinkingLevel: spec.thinkingLevel, - }; + pins[role] = spec + ? { + model: { provider: spec.provider, id: spec.id }, + ...(spec.thinkingLevel !== undefined ? { thinkingLevel: spec.thinkingLevel } : {}), + } + : null; } return pins; } -/** The six background roles' RESOLVED pins (settings interlude, - * 2026-09-19): the user's agent-models override falling back to the - * config.ts MODELS pin — the same resolution the agent factories use - * (src/server/stores.ts effectiveSettings), so the caption can never - * drift from what actually runs. */ +/** The six background roles' RESOLVED pins: the user's agent-models + * override → tier slot → nothing — the same resolution the agent wiring + * uses (src/server/stores.ts effectiveSettings), so the caption can + * never drift from what actually runs. */ export function catalogRolePins(): RolePins { return rolePinsFor(effectiveSettings().agent.models); } -/** The three slot tiers' RESOLVED pins (model slots wave, 2026-09-20) — - * effectiveSettings().models verbatim: slot over factory rep pin, the - * same resolution the engine wiring uses, so the Models settings page's - * captions cannot drift from what actually runs. */ +/** The three slot tiers' RESOLVED pins — effectiveSettings().models + * verbatim: the same resolution the engine wiring uses, so the Models + * settings page's captions cannot drift from what actually runs. NULL + * per tier that holds no model. */ export function catalogSlots(): SlotPins { const slots = effectiveSettings().models; const pins = {} as SlotPins; for (const name of ["strong", "cheap", "vision"] as const) { - const slot: EffectiveSlot = slots[name]; - pins[name] = { - model: { provider: slot.provider, id: slot.id }, - thinkingLevel: slot.thinkingLevel, - }; + const slot: EffectiveSlot | null = slots[name]; + pins[name] = slot + ? { + model: { provider: slot.provider, id: slot.id }, + ...(slot.thinkingLevel !== undefined ? { thinkingLevel: slot.thinkingLevel } : {}), + } + : null; } return pins; } @@ -183,11 +197,14 @@ export function validateChatSettings(settings: ChatSettings): SettingsValidation } } if (settings.thinkingLevel !== undefined) { - // The conversation chain's fallback is the strong slot (model slots - // wave, 2026-09-20) — the model a settings-less chat would run. + // The conversation chain's fallback is the strong slot — the model + // a settings-less chat would run. BYOK (owner, 2026-09-22): with no + // model to check against (no chat.model, no strong slot), the level + // rides unchecked — it clamps against whatever the chain later holds. const strong = effectiveSettings().models.strong; - const provider = settings.model?.provider ?? strong.provider; - const id = settings.model?.id ?? strong.id; + const provider = settings.model?.provider ?? strong?.provider; + const id = settings.model?.id ?? strong?.id; + if (!provider || !id) return { ok: true }; const model = piModels().getModel(provider, id); if (!model) return { ok: false, error: `No model "${id}" in the ${provider} catalog` }; const supported = getSupportedThinkingLevels(model); @@ -202,9 +219,10 @@ export function validateChatSettings(settings: ChatSettings): SettingsValidation return { ok: true }; } /** Validate a models-section write (PUT /api/settings/models) against the - * catalog — the same gate the chat section gets. Model swaps must name a - * real provider+model; a thinkingLevel must be supported by the model the - * slot would resolve to (the slot's own model, else the factory rep pin). + * catalog — the same gate the chat section gets. A slot must NAME a real + * provider+model (owner, 2026-09-22: no factory floor exists, so a level + * alone can never run — the parser rejects it, this gate re-checks the + * model); a thinkingLevel must be supported by the slot's own model. * keyPresent is NOT gated here (a stored pin for a provider whose key * lapsed stays valid — it fails honestly at stream time, and the picker * stops offering it). */ @@ -230,8 +248,14 @@ function validateSlotSpec(name: ModelSlotName, slot: { model?: { provider: strin } } if (slot.thinkingLevel !== undefined) { - const provider = slot.model?.provider ?? effectiveSettings().models[name].provider; - const id = slot.model?.id ?? effectiveSettings().models[name].id; + // The write-side parser requires slot.model (owner, 2026-09-22 — a + // level-only slot can never run), so this fallback is tolerance for + // future callers; with no model to check against, accept (the + // engine build clamps whatever the chain later holds). + const resolved = effectiveSettings().models[name]; + const provider = slot.model?.provider ?? resolved?.provider; + const id = slot.model?.id ?? resolved?.id; + if (!provider || !id) return { ok: true }; const model = piModels().getModel(provider, id); if (!model) return { ok: false, error: `No model "${id}" in the ${provider} catalog` }; const supported = getSupportedThinkingLevels(model); diff --git a/src/server/settings-body.ts b/src/server/settings-body.ts index f4c9fa7..9c5e91f 100644 --- a/src/server/settings-body.ts +++ b/src/server/settings-body.ts @@ -309,11 +309,14 @@ export function parseModelsSettingsBody(body: unknown): SettingsBodyResult { + // BYOK (owner, 2026-09-22): an unconfigured write channel SKIPS + // (the coverage ledger keeps the messages uncovered — the next + // trigger retries once slots are set); only real failures log. + if (err instanceof ModelsNotConfiguredError) { + console.info(`[chat ${chatId}] extraction skipped — ${err.envelope.error} (missing: ${err.envelope.missing.join(", ")})`); + return; + } console.error(`[chat ${chatId}] extraction pipeline failed:`, err); }); } catch (err) { diff --git a/tests/chat-flow.test.ts b/tests/chat-flow.test.ts index 774f56d..0407780 100644 --- a/tests/chat-flow.test.ts +++ b/tests/chat-flow.test.ts @@ -25,6 +25,7 @@ import { } from "@earendil-works/pi-ai"; import { createTeacherAgent } from "../src/agent/teacher-agent.ts"; import { handleChatRequest } from "../src/server/chat-handler.ts"; +import { effectiveSettings } from "../src/server/stores.ts"; import { isTurnRunning } from "../src/server/turn-liveness.ts"; import { recordToPiMessages } from "../src/server/pi/record-adapter.ts"; import { chatStore } from "../src/store/chats.ts"; @@ -98,6 +99,17 @@ function post(chatId: string, messages: unknown[]): Request { }); } + +/** BYOK turn gate (owner, 2026-09-22): no factory floor — a chat needs a + * conversation model before any turn streams. The scripted tests give + * each chat a per-chat pin exactly like the setup flow does in + * production (the injected faux engine still runs the loop). */ +function createConfiguredChat(title: string) { + const chat = chatStore.createChat(title); + chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } }); + return chat; +} + const createdChats: string[] = []; after(() => { for (const id of createdChats) { @@ -112,7 +124,7 @@ after(() => { describe("teacher chat flow (scripted model)", () => { it("quiz tool call persists as a pending part; the answer continues the lesson", async () => { const { engine, calls } = scriptedTeacher(defaultLesson); - const chat = chatStore.createChat("Test: solar system"); + const chat = createConfiguredChat("Test: solar system"); createdChats.push(chat.id); // ── Turn 1: learner opens with a topic; teacher probes with a quiz ── @@ -199,7 +211,7 @@ describe("teacher chat flow (scripted model)", () => { // tool-input-start chunk. If a transport change ever stopped emitting // it, the indicator would silently die — pin the protocol. const { engine } = scriptedTeacher(defaultLesson); - const chat = chatStore.createChat("Test: protocol pin"); + const chat = createConfiguredChat("Test: protocol pin"); createdChats.push(chat.id); const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] }; const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine))); @@ -212,7 +224,7 @@ describe("teacher chat flow (scripted model)", () => { it("persists even when the stream is aborted (resume keeps partial state)", async () => { const { engine } = scriptedTeacher(defaultLesson); - const chat = chatStore.createChat("Test: abort"); + const chat = createConfiguredChat("Test: abort"); createdChats.push(chat.id); const req = post(chat.id, [{ id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] }]); @@ -260,7 +272,7 @@ describe("teacher chat flow (scripted model)", () => { ]; const { engine, calls } = scriptedTeacher(steps); - const chat = chatStore.createChat("Test: bad quiz retry"); + const chat = createConfiguredChat("Test: bad quiz retry"); createdChats.push(chat.id); const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] }; const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine))); @@ -332,7 +344,7 @@ describe("teacher chat flow (scripted model)", () => { ]; const { engine } = scriptedTeacher(() => steps); - const chat = chatStore.createChat("Test: string options"); + const chat = createConfiguredChat("Test: string options"); createdChats.push(chat.id); const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] }; const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine))); @@ -374,7 +386,7 @@ describe("teacher chat flow (scripted model)", () => { ), ]); - const chat = chatStore.createChat("Test: parallel quizzes"); + const chat = createConfiguredChat("Test: parallel quizzes"); createdChats.push(chat.id); const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me twice" }] }; const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine))); @@ -441,7 +453,7 @@ describe("chat store", () => { describe("mid-turn navigation (turn liveness + write-ahead)", () => { it("one turn per chat: a concurrent request is 409'd; the next one after finish is accepted", async () => { const { engine } = scriptedTeacher(defaultLesson); - const chat = chatStore.createChat("Test: turn guard"); + const chat = createConfiguredChat("Test: turn guard"); createdChats.push(chat.id); const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] }; @@ -462,7 +474,7 @@ describe("mid-turn navigation (turn liveness + write-ahead)", () => { it("write-ahead: the learner's answer is durable BEFORE the turn finishes", async () => { const { engine } = scriptedTeacher(defaultLesson); - const chat = chatStore.createChat("Test: write-ahead"); + const chat = createConfiguredChat("Test: write-ahead"); createdChats.push(chat.id); // Turn 1: the teacher asks the quiz (pending part in the record). @@ -511,7 +523,7 @@ describe("mid-turn navigation (turn liveness + write-ahead)", () => { }); it("a turn that fails to start (malformed messages) clears the registry — no lockout", async () => { - const chat = chatStore.createChat("Test: turn failed to start"); + const chat = createConfiguredChat("Test: turn failed to start"); createdChats.push(chat.id); // A text part with no text: fails message validation before the stream // exists — the path where no onFinish will ever run. @@ -557,3 +569,46 @@ describe("renameChat (the header's click-to-edit title)", () => { assert.equal(renamed.title, "Untitled session"); }); }); + +describe("the BYOK turn gate (no factory floor — owner, 2026-09-22)", () => { + it("a chat with no model refuses the turn with the 503 envelope — nothing lingers", async () => { + const chat = chatStore.createChat("Test: unconfigured chat"); + createdChats.push(chat.id); + + // The record's strong slot belongs to settings-store.test.ts (the + // one-fs-owner policy) — READ the resolved truth, never write it. + // Refusal holds when the chain is empty; a record-carried strong + // slot legitimately opens the gate (the chain is honest both ways). + const expectRefusal = effectiveSettings().models.strong === null; + + const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] }; + const res = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(scriptedTeacher(defaultLesson).engine)); + + if (expectRefusal) { + assert.equal(res.status, 503, "the universal turn gate refuses before beginTurn"); + const body = (await res.json()) as { error: string; code: string; missing: string[] }; + assert.equal(body.code, "models_not_configured"); + assert.deepEqual(body.missing, ["strong"]); + assert.match(body.error, /No model configured/); + } else { + assert.equal(res.status, 200, "the record's strong slot satisfies the conversation chain"); + await drain(res); + } + + // Either way: no phantom liveness, nothing durable from a refused turn. + if (expectRefusal) { + assert.equal(isTurnRunning(chat.id), false, "the refusal never registers a live turn"); + const saved = chatStore.getChat(chat.id)!; + assert.equal(saved.messages.length, 0, "no write-ahead persist before the gate"); + } + }); + + it("a per-chat model alone opens the gate (the setup flow's exact write)", async () => { + const chat = createConfiguredChat("Test: per-chat pin only"); + createdChats.push(chat.id); + const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] }; + const res = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(scriptedTeacher(defaultLesson).engine)); + assert.equal(res.status, 200, "chat.settings.model satisfies the chain without any record slot"); + await drain(res); + }); +}); diff --git a/tests/demos.test.tsx b/tests/demos.test.tsx index d81a402..a7aac25 100644 --- a/tests/demos.test.tsx +++ b/tests/demos.test.tsx @@ -754,6 +754,9 @@ describe("demo/visual job runners", () => { it("dispatch through a teacher turn: the call acks mid-turn and the runner completes in the background", async () => { const chat = chatStore.createChat("Test: demo spawn dispatch"); createdChats.push(chat.id); + // BYOK turn gate (owner, 2026-09-22): the scripted chat needs a + // conversation model — the per-chat pin, exactly like the setup flow. + chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } }); // A teacher that calls demo on turn 1, then continues in the SAME turn. const teacher = scriptedModel([ diff --git a/tests/extraction.test.ts b/tests/extraction.test.ts index 1bf755f..fd0e4f2 100644 --- a/tests/extraction.test.ts +++ b/tests/extraction.test.ts @@ -41,6 +41,7 @@ import type { } from "../src/core/contracts.ts"; import type { GraphStore } from "../src/core/ports.ts"; import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts"; +import { ModelsNotConfiguredError } from "../src/core/settings.ts"; import { batchTag, coverageOf, coverageSlots } from "../src/core/coverage.ts"; import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts"; import { semanticClusters } from "../src/core/semantic-clusters.ts"; @@ -394,6 +395,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { assert.equal(covered.length, record.messages.length); assert.ok(covered.every((c) => c.chatId === chatId && c.fp.length === 64 && c.tokens > 0)); assert.equal(ledgers.readCheckpoints().length, 0, "the chat path writes coverage, not the legacy checkpoint"); + assert.equal( coverageOf(record.messages as UIMessage[], covered, chatId).uncoveredCount, 0, @@ -401,6 +403,46 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { ); }); + it("BYOK refusal: no stage engines anywhere → the run throws honestly before any stage (owner, 2026-09-22)", async () => { + const { chatId } = fixtureChat(); + const ledgers = memoryLedgers(); + // No direct engines AND no engineFor — the chain came up empty. + const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, embedder: fakeEmbedder }); + await assert.rejects(pipeline.runExtraction(chatId), (err: unknown) => { + assert.ok(err instanceof ModelsNotConfiguredError); + assert.deepEqual(err.envelope.missing, ["cheap"]); // extractor + auditor are the cheap tier + assert.equal(err.envelope.code, "models_not_configured"); + return true; + }); + // nothing ran: no phase residue, no usage, no coverage writes + assert.equal(ledgers.readCoverage().length, 0); + }); + + it("a labeler-less run merges fine and SKIPS cluster naming honestly (BYOK, 2026-09-22)", async () => { + const { chatId } = fixtureChat(); + const ledgers = memoryLedgers(); + // Identity-pass verdict: p1 becomes a new node, p2 dropped (no links). + const verdict: AuditResult = { + propositions: [{ ...FINDINGS.propositions[0]! }], + edges: [], + attribution: { quiz: { "probe-1": "p1" } }, + dropped: [{ ref: "p2", reason: "evidence not found" }], + }; + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + embedder: fakeEmbedder, + extractor: submitModel("submit_findings", FINDINGS), + auditor: submitModel("submit_review", verdict), + // NO labeler: the cluster-naming garnish skips (console line), + // the merge itself is unaffected. + }); + const stats = await pipeline.runExtraction(chatId); + assert.deepEqual(stats, { nodes: 1, edges: 0, events: 3 }); + }); + + + it("engineFor fills the stages (model slots wave): the chain replaces config pins", async () => { const { chatId } = fixtureChat(); const ledgers = memoryLedgers(); diff --git a/tests/jobs.test.ts b/tests/jobs.test.ts index ffd4587..de85f93 100644 --- a/tests/jobs.test.ts +++ b/tests/jobs.test.ts @@ -356,6 +356,9 @@ describe("dispatch (async protocol §2)", () => { it("the research call resolves immediately with an ack; the turn continues and the runner steers", async () => { const chat = chatStore.createChat("Test: research dispatch"); trackChat(chat.id); + // BYOK turn gate (owner, 2026-09-22): the scripted chat needs a + // conversation model — the per-chat pin, exactly like the setup flow. + chatStore.saveSettings(chat.id, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } }); const delivery = fakeDeliver(); const search = stubSearch(); @@ -1344,3 +1347,35 @@ describe("stale-job recovery sweep (liveness-gated)", () => { assert.equal(jobStore.getJob(jobId)!.status, "done"); }); }); + +// ─── BYOK: the unresolvable-engine refusal (owner, 2026-09-22) ────────────── + +describe("BYOK job refusal (no factory floor)", () => { + it("runJob with NO resolvable engine lands the job as error — the envelope wording, no phantom run", async () => { + const chat = chatStore.createChat("Test: unconfigured spawn"); + trackChat(chat.id); + chatStore.saveMessages(chat.id, [ + { id: "u1", role: "user", parts: [{ type: "text", text: "hi" }] }, + { + id: "a1", + role: "assistant", + parts: [{ type: "tool-research", toolCallId: "call-nc1", state: "output-available", input: RESEARCH_INPUT, output: "Dispatched." }], + }, + ] as never); + + const job = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "call-nc1" }); + trackJob(job.id); + + const delivery = fakeDeliver(); + // NO test override, NO engineFor: the chain (override → slot → floor) + // comes up empty — the runner must refuse honestly, not fall back. + const runner = createJobRunner({ chatStore, jobStore, deliver: delivery.deliver }); + await runner.runJob(job.id); + + const done = jobStore.getJob(job.id)!; + assert.equal(done.status, "error", "the job lands error, never a phantom run"); + assert.match(done.error ?? "", /No model configured/); + assert.match(done.error ?? "", /missing: cheap/); + assert.equal(delivery.steers.length, 0, "no result steer — nothing ran"); + }); +}); diff --git a/tests/model-catalog.test.ts b/tests/model-catalog.test.ts index 1276652..d900fce 100644 --- a/tests/model-catalog.test.ts +++ b/tests/model-catalog.test.ts @@ -2,23 +2,30 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; import { modelCatalog, validateChatSettings, validateModelsSettings, catalogDefaultsFor, catalogProviders, catalogSlots, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts"; import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai"; -import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; -import { MODELS } from "../config.ts"; -import { piEngineFor } from "../src/agent/models.ts"; -import { BACKGROUND_MODEL_ROLES, type BackgroundModelRole } from "../src/core/settings.ts"; +import { DEFAULT_THINKING_ORDER } from "../config.ts"; +import { piEngineForSpec, piModels } from "../src/agent/models.ts"; +import { BACKGROUND_MODEL_ROLES } from "../src/core/settings.ts"; import { resolveSettings } from "../src/core/settings-resolution.ts"; -import { MODEL_PINS } from "../src/server/stores.ts"; // The model-control wave (2026-09-12): pi-ai owns the registry — these pins // hold our projection + validation honest against the shipped catalog, and // pin pi's per-model thinking truth so a pi-ai upgrade can't silently change -// what we send. +// what we send. BYOK wave (2026-09-22): no factory pins — unset surfaces +// are NULL/undefined, never phantom defaults. // The picker federates only providers with a PRESENT key (model slots // wave, 2026-09-20): pin openrouter's env key so its full catalog // federates in tests regardless of the developer shell's exports. process.env.OPENROUTER_API_KEY ??= "test-key-openrouter-catalog"; +/** The catalog's glm entry — the old factory pin's model, still the + * best-known thinking-truth specimen (low|high|max). */ +function glmModel() { + const glm = piModels().getModel("openrouter", "z-ai/glm-5.3-flash"); + assert.ok(glm, "glm-5.3-flash in catalog"); + return glm!; +} + describe("the model catalog projection", () => { it("serves the keyed providers' catalogs with the picker's DTO shape", () => { const { models, defaults, providers, slots } = modelCatalog(); @@ -48,32 +55,41 @@ describe("the model catalog projection", () => { for (const id of keyless) { assert.ok(!models.some((m) => m.provider === id), `keyless ${id} must not federate`); } + // BYOK: nothing configured in this environment's record → defaults NULL, + // slots NULL — no phantom ids anywhere in the payload + assert.equal(defaults, null); + for (const name of ["strong", "cheap", "vision"] as const) { + assert.equal(slots[name], null, name); + } }); - it("defaults mirror the conversation agents' config pin (strong rep = prober pin)", () => { - const strong = { provider: "openrouter", id: MODELS.prober.model, thinkingLevel: MODELS.prober.effort as ThinkingLevel }; + it("defaults: the chat section over the strong slot; null when both empty (BYOK)", () => { + assert.equal(catalogDefaultsFor(undefined, null), null); + const strong = { provider: "openrouter", id: "z-ai/glm-5.3-flash", thinkingLevel: "low" as const }; const d = catalogDefaultsFor(undefined, strong); - assert.deepEqual(d, { model: { provider: "openrouter", id: MODELS.prober.model }, thinkingLevel: MODELS.prober.effort }); - assert.equal(d.model.id, MODELS.teacher.model, "one controller: prober and teacher share the pin"); - }); - - it("a picked default in the record's chat section overrides the pin (catalogDefaultsFor, settings wave)", () => { - const strong = { provider: "openrouter", id: MODELS.prober.model, thinkingLevel: MODELS.prober.effort as ThinkingLevel }; - const d = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" }, strong); - assert.deepEqual(d, { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" }); - // partial: only the model chosen → thinking falls back to the pin + assert.deepEqual(d, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "low" }); + assert.deepEqual(catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" }, strong), { + model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, + thinkingLevel: "high", + }); + // partial: only the model chosen → level falls back to the slot's const partial = catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }, strong); - assert.equal(partial.thinkingLevel, MODELS.prober.effort); - assert.equal(partial.model.id, "qwen/qwen3.8-flash"); + assert.ok(partial); + assert.equal(partial!.thinkingLevel, "low"); // graphContextTokens-only record: model untouched assert.deepEqual(catalogDefaultsFor({ graphContextTokens: 512 }, strong), { - model: { provider: "openrouter", id: MODELS.prober.model }, - thinkingLevel: MODELS.prober.effort, + model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, + thinkingLevel: "low", }); + // chat.model with NO strong slot: the conversation runs on it alone + assert.deepEqual( + catalogDefaultsFor({ model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }, null), + { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }, + ); }); it("toModelDto projects the raw catalog model verbatim", () => { - const raw = piEngineFor("prober").model; // the catalog's glm entry + const raw = glmModel(); const dto = toModelDto(raw); assert.equal(dto.id, raw.id); assert.equal(dto.name, raw.name); @@ -84,43 +100,40 @@ describe("the model catalog projection", () => { }); describe("the background roles' resolved pins (rolePins)", () => { - it("no overrides → every role pins its config MODELS spec, nested DTO shape, all six roles", () => { - const pins = rolePinsFor(resolveSettings(null, MODEL_PINS).agent.models); + it("no record → every role is NULL (BYOK: no phantom defaults)", () => { + const pins = rolePinsFor(resolveSettings(null).agent.models); assert.deepEqual(Object.keys(pins), [...BACKGROUND_MODEL_ROLES]); for (const role of BACKGROUND_MODEL_ROLES) { - assert.deepEqual(pins[role].model, { provider: "openrouter", id: MODELS[role].model }); - assert.equal(pins[role].thinkingLevel, MODELS[role].effort); + assert.equal(pins[role], null, role); } - // the vision pin rides along like any other - assert.equal(pins.visualMaker.model.id, "qwen/qwen3.8-flash"); }); - it("a model override wins for that role only; others stay pinned", () => { + it("a slot re-tiers its roles; overrides win for that role only", () => { const record = { version: 1, - agent: { models: { researcher: { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } } } }, + models: { cheap: { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } } }, + agent: { models: { researcher: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "high" as const } } }, }; - const pins = rolePinsFor(resolveSettings(record, MODEL_PINS).agent.models); - assert.equal(pins.researcher.model.id, "qwen/qwen3.8-flash"); - assert.equal(pins.researcher.thinkingLevel, MODELS.researcher.effort, "level stays the pin unless overridden too"); - assert.equal(pins.visualMaker.model.id, MODELS.visualMaker.model); - assert.equal(pins.labeler.model.id, MODELS.labeler.model); + const pins = rolePinsFor(resolveSettings(record).agent.models); + assert.deepEqual(pins.researcher, { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "high" }); + assert.deepEqual(pins.labeler, { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" } }); // level unset + assert.equal(pins.visualMaker, null, "vision tier untouched"); }); - it("a thinkingLevel-only override keeps the model, changes the level", () => { - const record = { version: 1, agent: { models: { extractor: { thinkingLevel: "high" as const } } } }; - const pins = rolePinsFor(resolveSettings(record, MODEL_PINS).agent.models); - assert.equal(pins.extractor.model.id, MODELS.extractor.model); - assert.equal(pins.extractor.thinkingLevel, "high"); + it("catalogRolePins matches the same resolution the engine wiring uses", () => { + const pins = catalogRolePins(); + for (const role of BACKGROUND_MODEL_ROLES) { + assert.ok(pins[role] === null || typeof pins[role]!.model.id === "string"); + } }); }); describe("the model slots surface (model slots wave, 2026-09-20)", () => { - it("catalogSlots reports the factory rep pins when no record exists", () => { + it("catalogSlots reports NULL when a tier holds no model", () => { const slots = catalogSlots(); - assert.deepEqual(slots.strong.model, { provider: "openrouter", id: MODELS.prober.model }); - assert.deepEqual(slots.cheap.model, { provider: "openrouter", id: MODELS.researcher.model }); - assert.deepEqual(slots.vision.model, { provider: "openrouter", id: MODELS.visualMaker.model }); + for (const name of ["strong", "cheap", "vision"] as const) { + assert.equal(slots[name], null, name); + } }); it("catalogProviders reads keys env-wins, fresh per call", () => { @@ -153,7 +166,7 @@ describe("the models-section validation gate", () => { ); }); - it("thinkingLevel resolves against the slot's model, else the tier's resolved pin", () => { + it("thinkingLevel resolves against the slot's model; level-only slots check nothing", () => { // level checked against the named model assert.equal( validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "max" } }).ok, @@ -163,8 +176,9 @@ describe("the models-section validation gate", () => { validateModelsSettings({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "minimal" } }).ok, false, ); - // bare level: against the strong slot's resolved model (config pin) - assert.equal(validateModelsSettings({ strong: { thinkingLevel: "minimal" } }).ok, false); + // (a level-only slot never passes the WRITE parser — owner, 2026-09-22; + // the validation gate tolerates it here with nothing to check against) + assert.equal(validateModelsSettings({ strong: { thinkingLevel: "minimal" } }).ok, true); }); it("accepts empty settings (the clear-slots write)", () => { @@ -184,10 +198,8 @@ describe("the settings validation gate", () => { } }); - it("resolves a bare thinking level against the config default model", () => { - assert.deepEqual(validateChatSettings({ thinkingLevel: "high" }), { ok: true }); - // "minimal" is unsupported on config's glm — rejected even with no model set - assert.equal(validateChatSettings({ thinkingLevel: "minimal" }).ok, false); + it("a bare thinking level checks against the chat model, else the strong slot, else rides unchecked", () => { + assert.deepEqual(validateChatSettings({ thinkingLevel: "minimal" }), { ok: true }, "no model to check against → accepted (clamps at run)"); }); it("rejects unknown providers and models", () => { @@ -202,7 +214,7 @@ describe("the settings validation gate", () => { describe("pi thinking truth (pinned against pi-ai upgrades)", () => { it("glm-5.3-flash: low|high|max; anything else clamps to the nearest", () => { - const glm = piEngineFor("prober").model; + const glm = glmModel(); assert.deepEqual(getSupportedThinkingLevels(glm), ["low", "high", "max"]); assert.equal(clampThinkingLevel(glm, "minimal"), "low"); assert.equal(clampThinkingLevel(glm, "off"), "low", "GLM cannot disable thinking (map: off → null)"); @@ -210,21 +222,36 @@ describe("pi thinking truth (pinned against pi-ai upgrades)", () => { assert.equal(clampThinkingLevel(glm, "high"), "high"); }); - it("the engine factory honors settings, falls back to config, clamps", () => { - // override both fields - const over = piEngineFor("teacher", { model: { provider: "openrouter", id: "qwen/qwen3.8-flash" }, thinkingLevel: "high" }); + it("the engine factory clamps explicit levels; unknown models throw honestly", () => { + const over = piEngineForSpec({ provider: "openrouter", id: "qwen/qwen3.8-flash", thinkingLevel: "high" }); assert.equal(over.model.id, "qwen/qwen3.8-flash"); assert.equal(over.level, "high"); - // no settings → config pin - const pinned = piEngineFor("prober"); - assert.equal(pinned.model.id, MODELS.prober.model); - assert.equal(pinned.level, MODELS.prober.effort); - // settings level unsupported by the settings model → clamped honestly - const clamped = piEngineFor("teacher", { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, thinkingLevel: "minimal" }); + const clamped = piEngineForSpec({ provider: "openrouter", id: "z-ai/glm-5.3-flash", thinkingLevel: "minimal" }); assert.equal(clamped.level, "low"); - // bare level resolves against the config default model - assert.equal(piEngineFor("prober", { thinkingLevel: "max" }).level, "max"); - // unknown model → honest throw, never a silent fallback - assert.throws(() => piEngineFor("teacher", { model: { provider: "openrouter", id: "nope/nope" } }), /catalog has no model/); + assert.throws(() => piEngineForSpec({ provider: "openrouter", id: "nope/nope", thinkingLevel: "low" }), /catalog has no model/); + }); + + it("UNSET levels resolve from the baked DEFAULT_THINKING_ORDER (owner, 2026-09-22)", () => { + assert.deepEqual([...DEFAULT_THINKING_ORDER], ["medium", "low", "high", "minimal", "xhigh", "max"]); + // glm supports low|high|max → first supported in the order is "low" + const glm = piEngineForSpec({ provider: "openrouter", id: "z-ai/glm-5.3-flash" }); + assert.equal(glm.level, "low"); + // explicit levels never consult the order — pi clamps nearest + assert.equal(piEngineForSpec({ provider: "openrouter", id: "z-ai/glm-5.3-flash", thinkingLevel: "max" }).level, "max"); + // a non-reasoning model reports exactly ["off"] → the order lands there + const flat = piModels() + .getProviders() + .flatMap((p) => piModels().getModels(p.id)) + .find((m) => !m.reasoning); + if (flat) { + const engine = piEngineForSpec({ provider: flat.provider, id: flat.id }); + assert.deepEqual(getSupportedThinkingLevels(flat), ["off"]); + assert.equal(engine.level, "off"); + } + }); + + it("the legacy piEngineFor role entry point throws honestly (BYOK: no pins)", async () => { + const { piEngineFor } = await import("../src/agent/models.ts"); + assert.throws(() => piEngineFor("prober"), /no configured model for role "prober"/); }); }); \ No newline at end of file diff --git a/tests/prompt-overrides.test.ts b/tests/prompt-overrides.test.ts index 6d6dac0..26569a2 100644 --- a/tests/prompt-overrides.test.ts +++ b/tests/prompt-overrides.test.ts @@ -16,6 +16,15 @@ import * as teacherAgent from "../src/agent/teacher-agent.ts"; import { allSegmentPaths, effectivePromptSegments } from "../src/agent/prompt-segments.ts"; import { isValidSegmentPath } from "../src/core/prompt-segments.ts"; import { promptOverrides } from "../src/store/prompt-overrides.ts"; +import { createFauxCore } from "@earendil-works/pi-ai"; +import type { PiEngine } from "../src/agent/models.ts"; + +/** A scripted pi engine (pi-ai's faux provider) — the agent build needs an + * injected engine since BYOK removed the factory pins (owner, 2026-09-22). */ +function fauxEngine(): PiEngine { + const faux = createFauxCore({}); + return { model: faux.getModel(), streamFn: faux.streamSimple }; +} after(() => { fs.rmSync(PROMPTS_OVERRIDE_DIR, { recursive: true, force: true }); @@ -210,9 +219,13 @@ describe("composeInstructions: override honoring + the byte-identical source pat it("promptSwitches threads through agent construction into the system prompt", () => { promptOverrides.writeOverride("system-prompt.md", "TOTALLY CUSTOM BASE"); try { - const agent = createTeacherAgent(undefined, { promptSwitches: { "system-prompt.md": true } }); + // BYOK (owner, 2026-09-22): no factory pins — the agent build needs + // an injected engine; a scripted faux one stands in (the prompt is + // what's under test here, not the model). + const engine = fauxEngine(); + const agent = createTeacherAgent(undefined, { engine, promptSwitches: { "system-prompt.md": true } }); assert.ok(agent.state.systemPrompt.includes("TOTALLY CUSTOM BASE")); - const source = createTeacherAgent(undefined, {}); + const source = createTeacherAgent(undefined, { engine }); assert.ok(!source.state.systemPrompt.includes("TOTALLY CUSTOM BASE"), "no switches → cached source"); } finally { promptOverrides.deleteOverride("system-prompt.md"); diff --git a/tests/settings-body.test.ts b/tests/settings-body.test.ts index 32d0bca..5660cea 100644 --- a/tests/settings-body.test.ts +++ b/tests/settings-body.test.ts @@ -165,18 +165,24 @@ describe("SETTINGS_SECTIONS (the route's section map)", () => { }); describe("parseModelsSettingsBody (model slots wave, 2026-09-20)", () => { - it("parses the three slot tiers, each field independent", () => { + it("parses the three slot tiers — model required, level rides (BYOK, owner 2026-09-22)", () => { const r = parseModelsSettingsBody({ strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } }, - cheap: { thinkingLevel: "high" }, + cheap: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "high" }, vision: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "low" }, }); assert.ok(r.ok); assert.deepEqual(r.settings, { strong: { model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" } }, - cheap: { thinkingLevel: "high" }, + cheap: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "high" }, vision: { model: { provider: "zai", id: "glm-5.3-flash" }, thinkingLevel: "low" }, }); + // a thinkingLevel WITHOUT a model can never run (no factory floor) + const levelOnly = parseModelsSettingsBody({ cheap: { thinkingLevel: "high" } }); + assert.equal(levelOnly.ok, false); + if (!levelOnly.ok) { + assert.match(levelOnly.error, /slot "cheap" must set a model/); + } }); it("{} is the legal clear-slots write", () => { @@ -194,7 +200,7 @@ describe("parseModelsSettingsBody (model slots wave, 2026-09-20)", () => { it("a present-but-empty slot object is rejected (absent slot = unset)", () => { const bad = parseModelsSettingsBody({ strong: {} }); assert.equal(bad.ok, false); - if (!bad.ok) assert.match(bad.error, /slot "strong" must set model or thinkingLevel/); + if (!bad.ok) assert.match(bad.error, /slot "strong" must set a model/); }); it("malformed slot shapes are 400s", () => { diff --git a/tests/settings-resolution.test.ts b/tests/settings-resolution.test.ts index 7d62b8a..6efe0d8 100644 --- a/tests/settings-resolution.test.ts +++ b/tests/settings-resolution.test.ts @@ -1,72 +1,41 @@ /** * Tests for the resolution layer (settings-storage wave, 2026-09-19; - * model-slots wave 2026-09-20 adds the slot tier): core/settings-resolution.ts - * — the delta→concrete projection. The factory layer is threaded in as a - * parameter (core stays pure; server/stores.ts threads config.ts's MODELS), - * so these tests use a small synthetic pin spanning ALL NINE roles (the - * factory contract since the slots wave — the strong slot's rep IS the - * prober pin). + * model-slots wave 2026-09-20; BYOK all-the-way-down wave 2026-09-22): + * core/settings-resolution.ts — the delta→concrete projection. There are + * NO factory pins anymore: a model chain that ends empty resolves to + * null/undefined, and the per-need refusal gates (turn, job spawn, + * extraction) fire on exactly that. */ import assert from "node:assert/strict"; import { describe, it } from "node:test"; import { extractionEnabled, - factorySlotRep, graphContextEnabled, + resolveConversationSpec, resolveModelSpec, + resolveRoleSpec, resolveSettings, resolveSlot, - type ModelRoleSpecs, } from "../src/core/settings-resolution.ts"; -import { BACKGROUND_MODEL_ROLES, type ModelSlotName, type UserSettings } from "../src/core/settings.ts"; - -const FACTORY_PINS = Object.fromEntries( - [ - "prober", - "teacher", - "researcher", - "demoBuilder", - "visualMaker", - "pdfReader", - "extractor", - "auditor", - "labeler", - ].map((role, i) => [ - role, - { provider: i === 5 ? "other" : "openrouter", model: `factory/model-${i}`, effort: i % 2 === 0 ? "low" : "off" }, - ]), -) as ModelRoleSpecs; +import { BACKGROUND_MODEL_ROLES, type UserSettings } from "../src/core/settings.ts"; describe("resolveSettings", () => { - it("null record (first boot): every knob lands on its factory default", () => { - const eff = resolveSettings(null, FACTORY_PINS); + it("null record (first boot): non-model knobs land on factory defaults; every model view is EMPTY", () => { + const eff = resolveSettings(null); assert.deepEqual(eff.general, { autoUpdate: true }); assert.deepEqual(eff.chat, {}); // BYOK: no default model, ever assert.deepEqual(eff.agent.tools, {}); assert.deepEqual(eff.agent.prompts, {}); assert.deepEqual(eff.agent.search, {}); + // BYOK (owner, 2026-09-22): no factory floor — every role undefined, + // every slot null. No phantom ids anywhere. for (const role of BACKGROUND_MODEL_ROLES) { - assert.equal(eff.agent.models[role].id, FACTORY_PINS[role]!.model); - assert.equal(eff.agent.models[role].provider, FACTORY_PINS[role]!.provider); - assert.equal(eff.agent.models[role].thinkingLevel, FACTORY_PINS[role]!.effort); + assert.equal(eff.agent.models[role], undefined, role); } - // slots with no record: every tier lands on its factory rep pin - assert.deepEqual(eff.models.strong, { - provider: "openrouter", - id: FACTORY_PINS.prober!.model, - thinkingLevel: FACTORY_PINS.prober!.effort, - }); - assert.deepEqual(eff.models.cheap, { - provider: "openrouter", - id: FACTORY_PINS.researcher!.model, - thinkingLevel: FACTORY_PINS.researcher!.effort, - }); - assert.deepEqual(eff.models.vision, { - provider: "openrouter", - id: FACTORY_PINS.visualMaker!.model, - thinkingLevel: FACTORY_PINS.visualMaker!.effort, - }); + assert.equal(eff.models.strong, null); + assert.equal(eff.models.cheap, null); + assert.equal(eff.models.vision, null); assert.equal(eff.appearance.mode, "dark"); assert.equal(eff.appearance.background, null); assert.equal(eff.appearance.fontFamily, null); @@ -90,13 +59,15 @@ describe("resolveSettings", () => { appearance: { mode: "light", proseFontSize: 20, mdRendering: { headingWeight: 600 } }, graph: { enabled: false }, }; - const eff = resolveSettings(record, FACTORY_PINS); + const eff = resolveSettings(record); assert.equal(eff.general.autoUpdate, false); assert.deepEqual(eff.chat, { thinkingLevel: "high" }); assert.deepEqual(eff.agent.models.researcher, { provider: "openrouter", id: "qwen/qwen3.8-flash", - thinkingLevel: "low", // thinking NOT overridden → factory effort + // thinking NOT overridden and no cheap slot → UNSET (the engine + // build resolves it from DEFAULT_THINKING_ORDER, not a pin). + thinkingLevel: undefined, }); assert.deepEqual(eff.agent.search, { engine: "tavily" }); assert.deepEqual(eff.agent.tools, { research: false }); @@ -109,103 +80,145 @@ describe("resolveSettings", () => { assert.deepEqual(eff.graph, { enabled: false, embedding: true, clustering: true, labeling: true }); }); - it("partial role override: thinkingLevel alone swaps the budget, model stays pinned", () => { - const eff = resolveSettings( - { version: 1, agent: { models: { extractor: { thinkingLevel: "medium" } } } }, - FACTORY_PINS, - ); - const extractor = FACTORY_PINS.extractor!; + it("partial role override: thinkingLevel alone borrows the SLOT's model (or nothing, without one)", () => { + const eff = resolveSettings({ + version: 1, + models: { cheap: { model: { provider: "zai", id: "glm-5.3-flash" } } }, + agent: { models: { extractor: { thinkingLevel: "medium" } } }, + }); assert.deepEqual(eff.agent.models.extractor, { - provider: extractor.provider, - id: extractor.model, + provider: "zai", + id: "glm-5.3-flash", thinkingLevel: "medium", }); - // sibling roles untouched - assert.equal(eff.agent.models.researcher.thinkingLevel, FACTORY_PINS.researcher!.effort); + const bare = resolveSettings({ version: 1, agent: { models: { extractor: { thinkingLevel: "medium" } } } }); + assert.equal(bare.agent.models.extractor, undefined); // no model anywhere → chain empty }); it("resolveModelSpec is field-independent per override aspect", () => { + const slots = { cheap: { model: { provider: "zai", id: "slot-model" }, thinkingLevel: "high" as const } }; assert.deepEqual( - resolveModelSpec("labeler", { model: { provider: "other", id: "x" } }, FACTORY_PINS), - { provider: "other", id: "x", thinkingLevel: FACTORY_PINS.labeler!.effort }, + resolveModelSpec("labeler", { model: { provider: "other", id: "x" } }, slots), + { provider: "other", id: "x", thinkingLevel: "high" }, // override model + slot level ); - assert.deepEqual(resolveModelSpec("labeler", {}, FACTORY_PINS), { - provider: FACTORY_PINS.labeler!.provider, - id: FACTORY_PINS.labeler!.model, - thinkingLevel: FACTORY_PINS.labeler!.effort, + assert.deepEqual(resolveModelSpec("labeler", { thinkingLevel: "max" }, slots), { + provider: "zai", + id: "slot-model", + thinkingLevel: "max", // override level + slot model }); + assert.equal(resolveModelSpec("labeler", undefined, {}), undefined); }); }); -describe("the slot tier (model slots wave, 2026-09-20)", () => { +describe("the slot tier (BYOK, owner 2026-09-22)", () => { it("slot alone re-tiers every role of its tier (no role overrides present)", () => { - const eff = resolveSettings( - { version: 1, models: { cheap: { model: { provider: "zai", id: "glm-5.3-flash" } } } }, - FACTORY_PINS, - ); + const eff = resolveSettings({ version: 1, models: { cheap: { model: { provider: "zai", id: "glm-5.3-flash" } } } }); for (const role of ["researcher", "demoBuilder", "extractor", "auditor", "labeler"] as const) { - assert.deepEqual(eff.agent.models[role as "researcher"], { - provider: "zai", - id: "glm-5.3-flash", - thinkingLevel: FACTORY_PINS[role]!.effort, // thinking not in slot → factory - }, role); + assert.deepEqual( + eff.agent.models[role], + { provider: "zai", id: "glm-5.3-flash", thinkingLevel: undefined }, // level unset → default order + role, + ); } - // other tiers untouched - assert.equal(eff.agent.models.visualMaker.id, FACTORY_PINS.visualMaker!.model); + // other tiers untouched (empty) + assert.equal(eff.agent.models.visualMaker, undefined); }); - it("role override still beats the slot; slot beats the factory pin (the chain)", () => { - const eff = resolveSettings( - { - version: 1, - agent: { models: { researcher: { model: { provider: "groq", id: "r-override" } } } }, - models: { cheap: { model: { provider: "zai", id: "r-slot" }, thinkingLevel: "high" } }, - }, - FACTORY_PINS, - ); - // researcher: override model + slot thinking (field-independent) + it("role override still beats the slot; each field independent", () => { + const eff = resolveSettings({ + version: 1, + agent: { models: { researcher: { model: { provider: "groq", id: "r-override" } } } }, + models: { cheap: { model: { provider: "zai", id: "r-slot" }, thinkingLevel: "high" } }, + }); assert.deepEqual(eff.agent.models.researcher, { provider: "groq", id: "r-override", - thinkingLevel: "high", + thinkingLevel: "high", // slot level rides the override's model }); - // labeler (same tier, no override): full slot assert.deepEqual(eff.agent.models.labeler, { provider: "zai", id: "r-slot", thinkingLevel: "high" }); }); - it("vision slot covers visualMaker AND pdfReader; strong's rep is the prober pin", () => { - const eff = resolveSettings( - { version: 1, models: { vision: { thinkingLevel: "max" } } }, - FACTORY_PINS, - ); - assert.equal(eff.agent.models.visualMaker.thinkingLevel, "max"); - assert.equal(eff.agent.models.visualMaker.id, FACTORY_PINS.visualMaker!.model); - // resolved slot views exist for every tier, set or not - assert.deepEqual(eff.models.strong, factorySlotRep("strong", FACTORY_PINS)); + it("vision slot covers visualMaker AND pdfReader; level-only slots resolve as NULL", () => { + const eff = resolveSettings({ version: 1, models: { vision: { model: { provider: "zai", id: "v-model" }, thinkingLevel: "max" } } }); + assert.deepEqual(eff.agent.models.visualMaker, { provider: "zai", id: "v-model", thinkingLevel: "max" }); + assert.deepEqual(eff.models.vision, { provider: "zai", id: "v-model", thinkingLevel: "max" }); + // a stored level WITHOUT a model is inert (kept on disk, no phantom default) + const inert = resolveSettings({ version: 1, models: { strong: { thinkingLevel: "medium" } } }); + assert.equal(inert.models.strong, null); + assert.equal(inert.agent.models.researcher, undefined); // strong's emptiness must not leak into other tiers }); - it("resolveSlot: slot over factory rep, per field", () => { - assert.deepEqual(resolveSlot("strong", { thinkingLevel: "medium" }, FACTORY_PINS), { - provider: FACTORY_PINS.prober!.provider, - id: FACTORY_PINS.prober!.model, + it("resolveSlot: configured iff the model is present", () => { + assert.deepEqual(resolveSlot({ model: { provider: "zai", id: "m" }, thinkingLevel: "medium" }), { + provider: "zai", + id: "m", thinkingLevel: "medium", }); - const rep = factorySlotRep("vision", FACTORY_PINS); - assert.equal(rep.id, FACTORY_PINS.visualMaker!.model); - const slot: ModelSlotName = "cheap"; - assert.deepEqual(resolveSlot(slot, undefined, FACTORY_PINS), { - provider: FACTORY_PINS.researcher!.provider, - id: FACTORY_PINS.researcher!.model, - thinkingLevel: FACTORY_PINS.researcher!.effort, + assert.deepEqual(resolveSlot({ model: { provider: "zai", id: "m" } }), { + provider: "zai", + id: "m", + thinkingLevel: undefined, // unset → resolved from the default order at build }); + assert.equal(resolveSlot({ thinkingLevel: "medium" }), null); + assert.equal(resolveSlot(undefined), null); }); - it("absent provider coordinate anywhere rides the factory provider (legacy ids)", () => { - // resolveModelSpec with no slots and a factory provider of "other" - // (pdfReader is index 5 in the synthetic pins): a legacy override - // with no provider keeps the FACTORY provider, not a hardcoded one. - const spec = resolveModelSpec("auditor", {}, FACTORY_PINS); - assert.equal(spec.provider, FACTORY_PINS.auditor!.provider); + it("resolveRoleSpec: background roles and the conversation pair share the chain", () => { + const record: UserSettings = { + version: 1, + models: { strong: { model: { provider: "zai", id: "strong-model" } } }, + }; + // conversation pair: slot only (their override is chat-controlled) + assert.deepEqual(resolveRoleSpec("prober", record), { + provider: "zai", + id: "strong-model", + thinkingLevel: undefined, // level unset → default order at build + }); + // background role override beats the slot + const withOverride: UserSettings = { + ...record, + agent: { models: { researcher: { model: { provider: "groq", id: "r" } } } }, + }; + assert.deepEqual(resolveRoleSpec("researcher", withOverride), { provider: "groq", id: "r", thinkingLevel: undefined }); + // pdfReader rides the vision slot (never a background override) + const visionRecord: UserSettings = { + version: 1, + models: { vision: { model: { provider: "zai", id: "v" }, thinkingLevel: "low" } }, + }; + assert.deepEqual(resolveRoleSpec("pdfReader", visionRecord), { provider: "zai", id: "v", thinkingLevel: "low" }); + assert.equal(resolveRoleSpec("teacher", null), undefined); + }); +}); + +describe("resolveConversationSpec (the turn gate's chain)", () => { + it("chat.model over the strong slot; null when both empty", () => { + assert.equal(resolveConversationSpec(undefined, null), null); + assert.deepEqual( + resolveConversationSpec(undefined, { provider: "zai", id: "s" }), + { provider: "zai", id: "s", thinkingLevel: undefined }, + ); + assert.deepEqual( + resolveConversationSpec({ model: { provider: "groq", id: "c" } }, null), + { provider: "groq", id: "c", thinkingLevel: undefined }, + ); + }); + + it("bare chat thinking level rides the strong slot's model", () => { + assert.deepEqual( + resolveConversationSpec({ thinkingLevel: "high" }, { provider: "zai", id: "s", thinkingLevel: "low" }), + { provider: "zai", id: "s", thinkingLevel: "high" }, + ); + // chat.model present, strong unset: level may still ride the chat alone + assert.deepEqual( + resolveConversationSpec({ model: { provider: "groq", id: "c" }, thinkingLevel: "max" }, null), + { provider: "groq", id: "c", thinkingLevel: "max" }, + ); + // nothing set anywhere: unset level, not a phantom one + assert.deepEqual(resolveConversationSpec({ model: { provider: "groq", id: "c" } }, null), { + provider: "groq", + id: "c", + thinkingLevel: undefined, + }); }); }); @@ -223,11 +236,11 @@ describe("the graph channel gates (the wave's seam honoring)", () => { }); it("both gates ride resolveSettings' factory defaults (all on → both open)", () => { - const eff = resolveSettings(null, FACTORY_PINS); + const eff = resolveSettings(null); assert.equal(extractionEnabled(eff.graph), true); assert.equal(graphContextEnabled(eff.graph), true); - const off = resolveSettings({ version: 1, graph: { enabled: false } }, FACTORY_PINS); + const off = resolveSettings({ version: 1, graph: { enabled: false } }); assert.equal(extractionEnabled(off.graph), false); assert.equal(graphContextEnabled(off.graph), false); }); -}); +}); \ No newline at end of file diff --git a/tests/tool-toggles-bridge.test.ts b/tests/tool-toggles-bridge.test.ts index e0fc4ce..8d20c46 100644 --- a/tests/tool-toggles-bridge.test.ts +++ b/tests/tool-toggles-bridge.test.ts @@ -18,6 +18,15 @@ import type { ProberToolContext } from "../src/agent/tools.ts"; import { jobStore } from "../src/store/jobs.ts"; import { parseAgentSettingsBody } from "../src/server/settings-body.ts"; import { chatStore } from "../src/store/chats.ts"; +import { createFauxCore } from "@earendil-works/pi-ai"; +import type { PiEngine } from "../src/agent/models.ts"; + +/** A scripted pi engine — the agent builds need an injected engine since + * BYOK removed the factory pins (owner, 2026-09-22). */ +function fauxEngine(): PiEngine { + const faux = createFauxCore({}); + return { model: faux.getModel(), streamFn: faux.streamSimple }; +} import type { AgentSettings } from "../src/core/settings.ts"; describe("AgentBuildOptions.toolToggles (the P2 bridge)", () => { @@ -27,14 +36,14 @@ describe("AgentBuildOptions.toolToggles (the P2 bridge)", () => { it("flows into the teacher's registry: a job-tool delta is a no-op on the bare set", () => { // No ctx → bare agent: quiz/ask only. A job-tool delta names tools // that don't exist there — the bridge must not crash on it. - const bare = createTeacherAgent(undefined, { toolToggles: { research: false } }); + const bare = createTeacherAgent(undefined, { engine: fauxEngine(), toolToggles: { research: false } }); assert.deepEqual(toolNames(bare), ["ask_user_question", "quiz"]); }); it("flows into the prober's registry; probe_done survives everything", () => { const chatId = chatStore.createChat("Toggle bridge test").id; const ctx: ProberToolContext = { chatId, chatStore, deliver: () => {}, jobStore, spawn: () => {} }; - const prober = createProberAgent(ctx, { toolToggles: { ask_user_question: false } }); + const prober = createProberAgent(ctx, { engine: fauxEngine(), toolToggles: { ask_user_question: false } }); const names = toolNames(prober); assert.ok(!names.includes("ask_user_question"), "the delta dropped the tool"); assert.ok(names.includes("probe_done"), "probe_done always-on"); @@ -42,6 +51,7 @@ describe("AgentBuildOptions.toolToggles (the P2 bridge)", () => { it("an explicitly injected opts.tools registry is NEVER toggle-filtered", () => { const teacher = createTeacherAgent(undefined, { + engine: fauxEngine(), tools: { quiz: { name: "quiz", description: "test", clientQuestion: true, schema: z.object({}) }, },