diff --git a/src/agent/models.ts b/src/agent/models.ts index 001f5c1..69815cf 100644 --- a/src/agent/models.ts +++ b/src/agent/models.ts @@ -43,8 +43,8 @@ export const APP_IDENTITY = Object.freeze({ export interface PiEngine { model: Model; streamFn: StreamFunction; - /** The resolved (config-or-settings, CLAMPED) thinking level — - * observability + tests; the streamFn pins it anyway. */ + /** The initially resolved (config-or-settings, CLAMPED) thinking level. + * Custom streams re-resolve the original preference per operation. */ level?: ThinkingLevel; } @@ -128,8 +128,9 @@ export function piEngineForSpec(spec: ResolvedModelSpec, background: boolean | ( // UNSET level → the baked default order (first entry the model // supports); SET level → pi's native nearest-clamp (model swaps keep // the user's intent and adapt honestly). - const level = spec.thinkingLevel - ? clampThinkingLevel(model, spec.thinkingLevel) + const thinkingPreference = spec.thinkingLevel; + const level = thinkingPreference + ? clampThinkingLevel(model, thinkingPreference) : defaultThinkingLevel(model); // A lesson's agent loop can stay alive through many tool calls. Remember // only that this is a custom route, never its configuration for that run. @@ -137,21 +138,24 @@ export function piEngineForSpec(spec: ResolvedModelSpec, background: boolean | ( return { model, level, - // bound streamSimple: the agent loop forwards state.thinkingLevel as - // `reasoning` (undefined = off on pi-ai's scale); the clamp above keeps - // the pin honest even when config or settings drift off the map. + // The agent loop forwards state.thinkingLevel as `reasoning` (undefined + // = off on pi-ai's scale). Built-ins keep the factory clamp; custom + // requests re-resolve thinking against their current metadata below. streamFn: (requestModel, context, options) => { - const requestOptions = { ...options, reasoning: level === "off" ? undefined : level }; if (custom) { // Revalidate before EVERY upstream model operation, including later - // tool-loop steps. Pi resolves auth and clamps the requested thinking - // against the current native model. Its in-flight operation retains - // its own setup; a subsequent operation uses the reloaded provider. + // tool-loop steps. Resolve thinking from the ORIGINAL preference (or + // existing default order), not a previous metadata snapshot's clamp. + // Pi retains the setup of an operation already in flight. const current = piModels(); const currentModel = current.getModel(spec.provider, spec.id); if (!currentModel) return lazyStream(requestModel, async () => { throw new Error("The selected custom model is unavailable."); }); - return current.streamSimple(currentModel, context, requestOptions); + const currentLevel = thinkingPreference + ? clampThinkingLevel(currentModel, thinkingPreference) + : defaultThinkingLevel(currentModel); + return current.streamSimple(currentModel, context, { ...options, reasoning: currentLevel === "off" ? undefined : currentLevel }); } + const requestOptions = { ...options, reasoning: level === "off" ? undefined : level }; const purpose = typeof background === "function" ? background : () => background; const stream = spec.provider === "chatgpt" ? chatGPTStream(purpose, lifecycle) : nativeId ? nativeSubscriptionStream(models.getProvider(nativeId)!, purpose, lifecycle) : models.streamSimple.bind(models); return stream(requestModel, context, requestOptions); diff --git a/tests/custom-providers.test.ts b/tests/custom-providers.test.ts index 2cdd415..71e9aa3 100644 --- a/tests/custom-providers.test.ts +++ b/tests/custom-providers.test.ts @@ -3,7 +3,8 @@ import { after, describe, it } from "node:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { createModels, normalizeContext } from "@earendil-works/pi-ai"; +import { createModels, normalizeContext, type SimpleStreamOptions } from "@earendil-works/pi-ai"; +import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; import { parseCustomProviders, resolveCustomValue } from "../src/core/custom-providers.ts"; import { createCustomProvider, customProviderApis } from "../src/agent/custom-providers.ts"; @@ -187,6 +188,46 @@ describe("custom providers in the existing application wiring", () => { assert.equal(requests[0]!.body.max_tokens, 1024); assert.equal(requests[1]!.body.max_tokens, 512, "current Pi model metadata, not the stale engine model, controls the request"); }); + const thinking = (...levels: ThinkingLevel[]) => ({ + reasoning: true, + // Pi treats omitted levels through high as supported; mark every + // unselected level explicitly unsupported for these narrow fixtures. + thinkingLevelMap: { off: null, minimal: null, low: null, medium: null, high: null, xhigh: null, max: null, ...Object.fromEntries(levels.map((level) => [level, level])) }, + }); + const thinkingCases: { name: string; preference?: ThinkingLevel; initial: object; updated: object; before: ThinkingLevel; after: ThinkingLevel }[] = [ + { name: "recovers saved max after high-only metadata expands", preference: "max", initial: thinking("high"), updated: thinking("high", "max"), before: "high", after: "max" }, + { name: "recovers saved max after reasoning-off metadata expands", preference: "max", initial: { reasoning: false }, updated: thinking("high", "max"), before: "off", after: "max" }, + { name: "selects the existing medium-first default after reasoning becomes supported", initial: { reasoning: false }, updated: thinking("low", "medium", "high"), before: "off", after: "medium" }, + { name: "reselects the existing low-before-high default against updated metadata", initial: thinking("high"), updated: thinking("low", "high"), before: "high", after: "low" }, + { name: "clamps saved max when supported thinking contracts", preference: "max", initial: thinking("high", "max"), updated: thinking("high"), before: "max", after: "high" }, + { name: "omits reasoning when the model loses thinking support", preference: "max", initial: thinking("high", "max"), updated: { reasoning: false }, before: "max", after: "off" }, + ]; + for (const scenario of thinkingCases) it(scenario.name, async () => { + const definition = (metadata: object) => config({ compat: { supportsReasoningEffort: true, thinkingFormat: "openai" } }, [model("chat", metadata)]); + save(definition(scenario.initial)); + const engine = piEngineForSpec({ provider: "gateway", id: "chat", thinkingLevel: scenario.preference }); + assert.equal(engine.level, scenario.before); + const requests: Record[] = []; + const original = globalThis.fetch; + globalThis.fetch = async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + requests.push(await request.json() as Record); + return sseText(); + }; + try { + const context = normalizeContext({ messages: [{ role: "user", content: "hello", timestamp: 1 }] }); + // A long-lived agent still forwards its initial thinking setting. + const options: SimpleStreamOptions = { reasoning: engine.level === "off" ? undefined : engine.level }; + assert.equal((await engine.streamFn(engine.model, context, options).result()).stopReason, "stop"); + save(definition(scenario.updated)); + assert.equal((await engine.streamFn(engine.model, context, options).result()).stopReason, "stop"); + } finally { globalThis.fetch = original; } + assert.equal(requests.length, 2); + for (const [index, level] of [scenario.before, scenario.after].entries()) { + assert.equal(requests[index]!.reasoning_effort, level === "off" ? undefined : level); + if (level === "off") assert.equal("reasoning_effort" in requests[index]!, false); + } + }); it("disables custom routes on malformed edits without removing built-ins or saved choices", () => { fs.writeFileSync(SETTINGS_FILE, JSON.stringify({ version: 1, models: { strong: { model: { provider: "gateway", id: "chat" }, thinkingLevel: "high" } } })); const before = fs.readFileSync(SETTINGS_FILE, "utf8");