sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Recompute custom thinking preferences for each model request

Browse Source
This commit is contained in:
amosblomqvist
2026-10-28 05:07:55 +02:00
parent fb7341d82b
commit 313a3c88a3
2 changed files with 58 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent/models.ts
+16 -12
View File
@@ -43,8 +43,8 @@ export const APP_IDENTITY = Object.freeze({
export interface PiEngine {
model: Model<Api>;
streamFn: StreamFunction;
/** The resolved (config-or-settings, CLAMPED) thinking level —
* observability + tests; the streamFn pins it anyway. */
/** The initially resolved (config-or-settings, CLAMPED) thinking level.
* Custom streams re-resolve the original preference per operation. */
level?: ThinkingLevel;
}
@@ -128,8 +128,9 @@ export function piEngineForSpec(spec: ResolvedModelSpec, background: boolean | (
// UNSET level → the baked default order (first entry the model
// supports); SET level → pi's native nearest-clamp (model swaps keep
// the user's intent and adapt honestly).
const level = spec.thinkingLevel
? clampThinkingLevel(model, spec.thinkingLevel)
const thinkingPreference = spec.thinkingLevel;
const level = thinkingPreference
? clampThinkingLevel(model, thinkingPreference)
: defaultThinkingLevel(model);
// A lesson's agent loop can stay alive through many tool calls. Remember
// only that this is a custom route, never its configuration for that run.
@@ -137,21 +138,24 @@ export function piEngineForSpec(spec: ResolvedModelSpec, background: boolean | (
return {
model,
level,
// bound streamSimple: the agent loop forwards state.thinkingLevel as
// `reasoning` (undefined = off on pi-ai's scale); the clamp above keeps
// the pin honest even when config or settings drift off the map.
// The agent loop forwards state.thinkingLevel as `reasoning` (undefined
// = off on pi-ai's scale). Built-ins keep the factory clamp; custom
// requests re-resolve thinking against their current metadata below.
streamFn: (requestModel, context, options) => {
const requestOptions = { ...options, reasoning: level === "off" ? undefined : level };
if (custom) {
// Revalidate before EVERY upstream model operation, including later
// tool-loop steps. Pi resolves auth and clamps the requested thinking
// against the current native model. Its in-flight operation retains
// its own setup; a subsequent operation uses the reloaded provider.
// tool-loop steps. Resolve thinking from the ORIGINAL preference (or
// existing default order), not a previous metadata snapshot's clamp.
// Pi retains the setup of an operation already in flight.
const current = piModels();
const currentModel = current.getModel(spec.provider, spec.id);
if (!currentModel) return lazyStream(requestModel, async () => { throw new Error("The selected custom model is unavailable."); });
return current.streamSimple(currentModel, context, requestOptions);
const currentLevel = thinkingPreference
? clampThinkingLevel(currentModel, thinkingPreference)
: defaultThinkingLevel(currentModel);
return current.streamSimple(currentModel, context, { ...options, reasoning: currentLevel === "off" ? undefined : currentLevel });
}
const requestOptions = { ...options, reasoning: level === "off" ? undefined : level };
const purpose = typeof background === "function" ? background : () => background;
const stream = spec.provider === "chatgpt" ? chatGPTStream(purpose, lifecycle) : nativeId ? nativeSubscriptionStream(models.getProvider(nativeId)!, purpose, lifecycle) : models.streamSimple.bind(models);
return stream(requestModel, context, requestOptions);
tests/custom-providers.test.ts
+42 -1
View File
@@ -3,7 +3,8 @@ import { after, describe, it } from "node:test";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { createModels, normalizeContext } from "@earendil-works/pi-ai";
import { createModels, normalizeContext, type SimpleStreamOptions } from "@earendil-works/pi-ai";
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
import { parseCustomProviders, resolveCustomValue } from "../src/core/custom-providers.ts";
import { createCustomProvider, customProviderApis } from "../src/agent/custom-providers.ts";
@@ -187,6 +188,46 @@ describe("custom providers in the existing application wiring", () => {
assert.equal(requests[0]!.body.max_tokens, 1024);
assert.equal(requests[1]!.body.max_tokens, 512, "current Pi model metadata, not the stale engine model, controls the request");
});
const thinking = (...levels: ThinkingLevel[]) => ({
reasoning: true,
// Pi treats omitted levels through high as supported; mark every
// unselected level explicitly unsupported for these narrow fixtures.
thinkingLevelMap: { off: null, minimal: null, low: null, medium: null, high: null, xhigh: null, max: null, ...Object.fromEntries(levels.map((level) => [level, level])) },
});
const thinkingCases: { name: string; preference?: ThinkingLevel; initial: object; updated: object; before: ThinkingLevel; after: ThinkingLevel }[] = [
{ name: "recovers saved max after high-only metadata expands", preference: "max", initial: thinking("high"), updated: thinking("high", "max"), before: "high", after: "max" },
{ name: "recovers saved max after reasoning-off metadata expands", preference: "max", initial: { reasoning: false }, updated: thinking("high", "max"), before: "off", after: "max" },
{ name: "selects the existing medium-first default after reasoning becomes supported", initial: { reasoning: false }, updated: thinking("low", "medium", "high"), before: "off", after: "medium" },
{ name: "reselects the existing low-before-high default against updated metadata", initial: thinking("high"), updated: thinking("low", "high"), before: "high", after: "low" },
{ name: "clamps saved max when supported thinking contracts", preference: "max", initial: thinking("high", "max"), updated: thinking("high"), before: "max", after: "high" },
{ name: "omits reasoning when the model loses thinking support", preference: "max", initial: thinking("high", "max"), updated: { reasoning: false }, before: "max", after: "off" },
];
for (const scenario of thinkingCases) it(scenario.name, async () => {
const definition = (metadata: object) => config({ compat: { supportsReasoningEffort: true, thinkingFormat: "openai" } }, [model("chat", metadata)]);
save(definition(scenario.initial));
const engine = piEngineForSpec({ provider: "gateway", id: "chat", thinkingLevel: scenario.preference });
assert.equal(engine.level, scenario.before);
const requests: Record<string, unknown>[] = [];
const original = globalThis.fetch;
globalThis.fetch = async (input, init) => {
const request = input instanceof Request ? input : new Request(input, init);
requests.push(await request.json() as Record<string, unknown>);
return sseText();
};
try {
const context = normalizeContext({ messages: [{ role: "user", content: "hello", timestamp: 1 }] });
// A long-lived agent still forwards its initial thinking setting.
const options: SimpleStreamOptions = { reasoning: engine.level === "off" ? undefined : engine.level };
assert.equal((await engine.streamFn(engine.model, context, options).result()).stopReason, "stop");
save(definition(scenario.updated));
assert.equal((await engine.streamFn(engine.model, context, options).result()).stopReason, "stop");
} finally { globalThis.fetch = original; }
assert.equal(requests.length, 2);
for (const [index, level] of [scenario.before, scenario.after].entries()) {
assert.equal(requests[index]!.reasoning_effort, level === "off" ? undefined : level);
if (level === "off") assert.equal("reasoning_effort" in requests[index]!, false);
}
});
it("disables custom routes on malformed edits without removing built-ins or saved choices", () => {
fs.writeFileSync(SETTINGS_FILE, JSON.stringify({ version: 1, models: { strong: { model: { provider: "gateway", id: "chat" }, thinkingLevel: "high" } } }));
const before = fs.readFileSync(SETTINGS_FILE, "utf8");