From 695948271f014c37c85fb68f25c0286ea33ad650 Mon Sep 17 00:00:00 2001 From: Daniel Buckner Date: Tue, 27 Oct 2026 16:25:03 -0700 Subject: [PATCH] feat: user pins for local model limits (context, max output, thinking) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A local endpoint that publishes no limits — LM Studio's /models says nothing — left every model on the 8192/4096 defaults with no way to say "this one has a 131k window". Each discovered model now gets three optional pins in the endpoint editor: context window, max output tokens, thinking (auto/on/off). A pin always wins: user pin → what the endpoint published → compiled default. Blank means no opinion, so discovery keeps ruling wherever you don't pin. Pins live in customProvider.overrides keyed by model id, stored apart from the discovery snapshot: a reconnect refreshes the models and prunes the pins of models that vanished while keeping the rest. Pins are written by PATCH /api/custom-provider, which never probes — the whole point is configuring a server that isn't answering /models correctly. Connect still probes and still fails loudly when the endpoint is down. A pin under 2048 tokens is rejected on the write side and clamped up on the read path, matching the compaction-floor discipline. --- app/api/custom-provider/route.ts | 52 ++++++-- components/settings/keys-section.tsx | 175 ++++++++++++++++++++++++++- src/core/settings-resolution.ts | 7 +- src/core/settings.ts | 54 +++++++++ src/server/custom-provider.ts | 38 ++++-- src/server/settings-body.ts | 42 ++++++- tests/custom-endpoint.test.ts | 115 +++++++++++++++++- 7 files changed, 455 insertions(+), 28 deletions(-) diff --git a/app/api/custom-provider/route.ts b/app/api/custom-provider/route.ts index 05becc5..193dd2c 100644 --- a/app/api/custom-provider/route.ts +++ b/app/api/custom-provider/route.ts @@ -1,24 +1,28 @@ /** * The custom endpoint's connect surface (one provider in the Keys list). - * GET → non-secret config { baseUrl, models } + keyPresent (masked). - * POST → connect: validate + normalize the base URL, probe /models (fail - * fast), store the key, persist { baseUrl, models }, re-sync the - * provider into piModels(). Discovery lives here because connecting - * and discovering are one user action. + * GET → non-secret config { baseUrl, models, overrides } + keyPresent (masked). + * POST → connect: validate + normalize the base URL, probe /models (fail + * fast), store the key, persist { baseUrl, models }, re-sync the + * provider into piModels(). Discovery lives here because connecting + * and discovering are one user action. Stored user pins survive a + * reconnect, pruned to the ids the endpoint still advertises. + * PATCH → store per-model pins only. No probe: pins must be editable with + * the local server switched off (the case they exist for). * DELETE → disconnect: drop the stored config + the stored key and unregister - * the provider (the mirror image of POST; idempotent). + * the provider (the mirror image of POST; idempotent). */ import { deleteKey, discoverModels, effectiveSettings, keyValue, + parseCustomModelOverridesBody, readSettingsRecord, saveKey, syncCustomProvider, writeSettingsRecord, } from "@/src/server"; -import { CUSTOM_PROVIDER, normalizeBaseUrl } from "@/src/core"; +import { CUSTOM_PROVIDER, normalizeBaseUrl, sanitizeCustomModelOverrides } from "@/src/core"; export const dynamic = "force-dynamic"; @@ -27,6 +31,7 @@ export async function GET() { return Response.json({ baseUrl: cp.baseUrl, models: cp.models, + overrides: cp.overrides, keyPresent: keyValue(CUSTOM_PROVIDER.envVar) !== undefined, }); } @@ -54,11 +59,42 @@ export async function POST(request: Request) { } const record = readSettingsRecord() ?? { version: 1 }; - const next = writeSettingsRecord({ ...record, customProvider: { baseUrl: base, models: found.models } }); + // User pins are never rewritten by discovery — only pruned to what the + // endpoint still advertises (the single prune law, core/settings.ts). + const overrides = sanitizeCustomModelOverrides( + record.customProvider?.overrides, + found.models.map((m) => m.id), + ); + const next = writeSettingsRecord({ + ...record, + customProvider: { baseUrl: base, models: found.models, overrides }, + }); syncCustomProvider(next.customProvider); return Response.json({ ok: true, models: next.customProvider?.models ?? [] }); } +/** Store the per-model pins (replace-whole). Pruned against the STORED model + * list, never a probe — an offline endpoint keeps its pins editable. */ +export async function PATCH(request: Request) { + const parsed = parseCustomModelOverridesBody(await request.json().catch(() => null)); + if (!parsed.ok) return Response.json({ error: parsed.error }, { status: 400 }); + + const cp = effectiveSettings().customProvider; + if (!cp.baseUrl) return Response.json({ error: "No custom endpoint configured" }, { status: 409 }); + + const overrides = sanitizeCustomModelOverrides( + parsed.settings, + cp.models.map((m) => m.id), + ); + const record = readSettingsRecord() ?? { version: 1 }; + const next = writeSettingsRecord({ + ...record, + customProvider: { baseUrl: cp.baseUrl, models: cp.models, overrides }, + }); + syncCustomProvider(next.customProvider); + return Response.json({ ok: true, overrides: next.customProvider?.overrides ?? {} }); +} + /** Disconnect: forget the endpoint AND its key, then unregister the provider. * Deleting the section (rather than blanking baseUrl) keeps settings.json the * same shape as a never-connected install. Idempotent by construction. */ diff --git a/components/settings/keys-section.tsx b/components/settings/keys-section.tsx index 8b28ad3..5aac78d 100644 --- a/components/settings/keys-section.tsx +++ b/components/settings/keys-section.tsx @@ -24,7 +24,15 @@ */ import { useEffect, useRef, useState } from "react"; -import { AUTH_KEYS, CUSTOM_PROVIDER, type AuthKeyStatus, type CustomModel } from "@/src/core"; +import { + AUTH_KEYS, + CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW, + CUSTOM_MODEL_DEFAULT_MAX_TOKENS, + CUSTOM_PROVIDER, + type AuthKeyStatus, + type CustomModel, + type CustomModelOverride, +} from "@/src/core"; import { SettingsGroup } from "./controls.tsx"; import { SubscriptionConnections } from "./subscription-connections.tsx"; import { ProviderMenuList } from "./model-menu.tsx"; @@ -49,6 +57,8 @@ interface KeysResponse { interface CustomStatus { baseUrl: string; models: CustomModel[]; + /** User pins by model id (context / max output / thinking). */ + overrides: Record; keyPresent: boolean; } @@ -402,6 +412,7 @@ export function KeysSection({ setCustom({ baseUrl: cp.baseUrl ?? "", models: cp.models ?? [], + overrides: cp.overrides ?? {}, keyPresent: Boolean(cp.keyPresent), }); } catch { @@ -500,11 +511,17 @@ export function KeysSection({ { setAddingCustom(false); await reload(); notifySettingsSaved("keys"); }} + onPinsSaved={async () => { + await reload(); + notifySettingsSaved("keys"); + }} onDelete={disconnectCustom} onCancel={() => setAddingCustom(false)} /> @@ -663,18 +680,68 @@ function CustomEndpointRow({ ); } +/** A pin row's raw input state — "" means "no pin": the endpoint's own value, + * then the compiled default, decides. `reasoning` is tri-state. */ +interface PinDraft { + context: string; + maxTokens: string; + reasoning: "" | "on" | "off"; +} + +const pinDraft = (pin: CustomModelOverride | undefined): PinDraft => ({ + context: pin?.context === undefined ? "" : String(pin.context), + maxTokens: pin?.maxTokens === undefined ? "" : String(pin.maxTokens), + reasoning: pin?.reasoning === undefined ? "" : pin.reasoning ? "on" : "off", +}); + +/** One draft row per discovered model, seeded from the stored pins. */ +function draftsFor( + models: readonly CustomModel[], + overrides: Record, +): Record { + return Object.fromEntries(models.map((m) => [m.id, pinDraft(overrides[m.id])])); +} + +/** Drafts → the PATCH payload (replace-whole): a blank field sends nothing and + * a fully blank model is absent — which IS the un-pin. */ +function pinsPayload(drafts: Record): Record { + const out: Record = {}; + for (const [id, d] of Object.entries(drafts)) { + const pin: CustomModelOverride = {}; + if (d.context.trim()) pin.context = Number(d.context); + if (d.maxTokens.trim()) pin.maxTokens = Number(d.maxTokens); + if (d.reasoning !== "") pin.reasoning = d.reasoning === "on"; + if (Object.keys(pin).length) out[id] = pin; + } + return out; +} + +/** testids must survive ids carrying slashes, colons, dots. */ +const pinTestid = (prefix: string, id: string) => `${prefix}-${id.replace(/[^a-z0-9-]/gi, "_")}`; + /** The custom endpoint's connect editor: base URL + key → POST /api/custom-provider - * (validate + discover + register). Image capability is detected automatically. */ + * (validate + discover + register), and the per-model limits form → PATCH (pins, + * no probe, so it works with the local server switched off). Image capability is + * detected automatically. */ function CustomEndpointEditor({ baseUrl0, keyPresent, + models, + overrides, onSaved, + onPinsSaved, onCancel, onDelete, }: { baseUrl0: string; keyPresent: boolean; + /** The discovered models — one limits row each. */ + models: readonly CustomModel[]; + /** Their stored user pins. */ + overrides: Record; onSaved: () => void | Promise; + /** Pins saved — the editor stays open (a form, not a connect). */ + onPinsSaved: () => void | Promise; onCancel: () => void; onDelete?: () => void | Promise; }) { @@ -683,11 +750,46 @@ function CustomEndpointEditor({ const [keyTouched, setKeyTouched] = useState(false); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); + // The limits form: edited drafts over the stored pins, plus the last saved + // payload as the dirty baseline (the server's echo is what gets stored). + const [drafts, setDrafts] = useState>(() => draftsFor(models, overrides)); + const [baseline, setBaseline] = useState(() => JSON.stringify(pinsPayload(draftsFor(models, overrides)))); + const [pinsBusy, setPinsBusy] = useState(false); + const [pinsError, setPinsError] = useState(null); // A key is required to enable chatting; local servers accept any value. A // stored key is kept when the field is left blank. const needsKey = !keyPresent; const canConnect = Boolean(baseUrl.trim()) && (!needsKey || Boolean(key.trim())); + const draftOf = (m: CustomModel): PinDraft => drafts[m.id] ?? pinDraft(overrides[m.id]); + const setDraft = (m: CustomModel, patch: Partial) => + setDrafts((prev) => ({ ...prev, [m.id]: { ...draftOf(m), ...patch } })); + const current = Object.fromEntries(models.map((m) => [m.id, draftOf(m)])) as Record; + const dirty = JSON.stringify(pinsPayload(current)) !== baseline; + + async function savePins() { + if (pinsBusy || !dirty) return; + setPinsBusy(true); + setPinsError(null); + const res = await fetch("/api/custom-provider", { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ overrides: pinsPayload(current) }), + }).catch(() => null); + setPinsBusy(false); + if (!res || !res.ok) { + const b = (await res?.json().catch(() => ({}))) as { error?: string } | undefined; + setPinsError(res ? (b?.error ?? `Save failed (${res.status})`) : "Save failed. Check your connection."); + return; + } + // The server's stored map is the truth (it floors + prunes) — echo it back. + const body = (await res.json().catch(() => null)) as { overrides?: Record } | null; + const next = draftsFor(models, body?.overrides ?? pinsPayload(current)); + setDrafts(next); + setBaseline(JSON.stringify(pinsPayload(next))); + await onPinsSaved(); + } + async function save() { if (busy || !baseUrl.trim()) return; setBusy(true); @@ -769,6 +871,75 @@ function CustomEndpointEditor({ )} + {models.length > 0 && ( +
+ + Model limits — blank uses the endpoint’s value, then the default + +
+ {models.map((m) => { + const d = draftOf(m); + return ( +
+ + {m.id} + + setDraft(m, { context: e.target.value })} + placeholder={String(m.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW)} + aria-label={`${m.id} context window in tokens`} + data-testid={pinTestid("custom-pin-context", m.id)} + disabled={pinsBusy} + className="h-7 w-20 shrink-0 rounded-sm border border-border bg-background px-1.5 text-right font-mono text-[11px] text-foreground outline-none placeholder:text-muted-foreground/60 focus:border-muted-foreground/50" + /> + setDraft(m, { maxTokens: e.target.value })} + placeholder={String(m.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS)} + aria-label={`${m.id} max output tokens`} + data-testid={pinTestid("custom-pin-maxtokens", m.id)} + disabled={pinsBusy} + className="h-7 w-20 shrink-0 rounded-sm border border-border bg-background px-1.5 text-right font-mono text-[11px] text-foreground outline-none placeholder:text-muted-foreground/60 focus:border-muted-foreground/50" + /> + +
+ ); + })} +
+
+ + {pinsError && ( +

+ {pinsError} +

+ )} +
+
+ )} {needsKey && (

A key is required to enable chatting — local servers (Ollama, LM Studio) accept any value. diff --git a/src/core/settings-resolution.ts b/src/core/settings-resolution.ts index a7682de..ed1974b 100644 --- a/src/core/settings-resolution.ts +++ b/src/core/settings-resolution.ts @@ -38,6 +38,7 @@ import { type DictationSettings, type CompactionSettings, type CustomModel, + type CustomModelOverride, type GraphSettings, type ModelSlot, type ModelSlotName, @@ -96,8 +97,9 @@ export interface EffectiveSettings { /** Compaction (owner round 14): resolved concrete — absent record * section resolves to the factory 150k. */ compaction: Required; - /** Custom endpoint, resolved concrete: baseUrl defaults "", models default []. */ - customProvider: { baseUrl: string; models: CustomModel[] }; + /** Custom endpoint, resolved concrete: baseUrl defaults "", models and + * user pins default to empty. */ + customProvider: { baseUrl: string; models: CustomModel[]; overrides: Record }; } /** The conversation pair's chain (BYOK, owner 2026-09-22): the chat's own @@ -227,6 +229,7 @@ export function resolveSettings(record: UserSettings | null): EffectiveSettings customProvider: { baseUrl: record?.customProvider?.baseUrl ?? "", models: record?.customProvider?.models ?? [], + overrides: record?.customProvider?.overrides ?? {}, }, }; } diff --git a/src/core/settings.ts b/src/core/settings.ts index d495e07..8e5d638 100644 --- a/src/core/settings.ts +++ b/src/core/settings.ts @@ -217,6 +217,12 @@ export interface ModelsSettings { export const CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW = 8192; export const CUSTOM_MODEL_DEFAULT_MAX_TOKENS = 4096; +/** Floor for a user-pinned token count (owner, 2026-10-27): the read side + * clamps up, the write side rejects below — the compaction-floor discipline. + * Pins exist because endpoints that publish no limits (LM Studio) otherwise + * sit on the 8192 default with no way to say better. */ +export const CUSTOM_MODEL_PIN_FLOOR = 2_048; + /** One discovered custom model. `image`/`reasoning`/`context`/`maxTokens` are * detected from the endpoint's /models metadata; anything it omits falls back * to a default. */ @@ -233,12 +239,26 @@ export interface CustomModel { maxTokens?: number; } +/** A user pin for one custom model: it beats what the endpoint publishes + * (owner ruling — explicit user intent wins; a local server's number is + * often wrong or absent). Absent field = no pin. `reasoning` is tri-state: + * true forces thinking on, false forces it off (a server that advertises + * reasoning but rejects reasoning_effort), absent = what discovery found. */ +export interface CustomModelOverride { + context?: number; + maxTokens?: number; + reasoning?: boolean; +} + /** The single custom endpoint's stored config (delta section). */ export interface CustomProviderSettings { /** Normalized base URL (see normalizeBaseUrl). Empty/absent = not configured. */ baseUrl?: string; /** Models discovered from the endpoint (refreshed on connect). */ models?: CustomModel[]; + /** User pins keyed by model id — pure user intent, never rewritten by + * discovery, pruned to the endpoint's live ids on connect. */ + overrides?: Record; } /** Canonical base-URL form (single source): trims, requires absolute http(s), @@ -562,6 +582,38 @@ function positiveInt(value: unknown): number | undefined { return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined; } +/** One pin, clamped up to the floor (clamp-never-reject on read). */ +function sanitizeCustomModelOverride(raw: unknown): CustomModelOverride | undefined { + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return undefined; + const rec = raw as Record; + const out: CustomModelOverride = {}; + const context = positiveInt(rec.context); + if (context) out.context = Math.max(CUSTOM_MODEL_PIN_FLOOR, context); + const maxTokens = positiveInt(rec.maxTokens); + if (maxTokens) out.maxTokens = Math.max(CUSTOM_MODEL_PIN_FLOOR, maxTokens); + if (typeof rec.reasoning === "boolean") out.reasoning = rec.reasoning; + return Object.keys(out).length ? out : undefined; +} + +/** THE pins map + THE prune law: well-formed entries only, ids trimmed, and + * when `liveIds` is given every entry for a model the endpoint no longer + * advertises drops. POST passes the discovered ids, PATCH the stored list. */ +export function sanitizeCustomModelOverrides( + raw: unknown, + liveIds?: Iterable, +): Record { + const out: Record = {}; + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return out; + const live = liveIds ? new Set(liveIds) : undefined; + for (const [key, value] of Object.entries(raw)) { + const id = key.trim(); + if (!id || (live && !live.has(id))) continue; + const pin = sanitizeCustomModelOverride(value); + if (pin) out[id] = pin; + } + return out; +} + /** The customProvider section: normalized base URL (invalid → dropped) and * discovered models (ids trimmed, dupes/empties dropped; image kept only when * true; token counts only as positive ints). Clamp-never-reject. */ @@ -590,6 +642,8 @@ function sanitizeCustomProvider(raw: Record): CustomProviderSet } out.models = [...byId.values()]; } + const overrides = sanitizeCustomModelOverrides(raw.overrides); + if (Object.keys(overrides).length) out.overrides = overrides; return out; } diff --git a/src/server/custom-provider.ts b/src/server/custom-provider.ts index c6015aa..b9f9b8c 100644 --- a/src/server/custom-provider.ts +++ b/src/server/custom-provider.ts @@ -15,6 +15,7 @@ import { customProviderUsable, normalizeBaseUrl, type CustomModel, + type CustomModelOverride, type CustomProviderSettings, } from "../core/settings.ts"; import { keyValue } from "./keys.ts"; @@ -36,23 +37,31 @@ const customEndpointAuth: ApiKeyAuth = { }; /** Map a discovered model to a pi-ai entry: image→input, reasoning→thinking - * levels, context/maxTokens→limits (else CUSTOM_MODEL_DEFAULT_*). baseUrl is - * stamped per-model. */ -function toPiModel(model: CustomModel, baseUrl: string): Model { + * levels, context/maxTokens→limits. Resolution ladder (owner, 2026-10-27): + * the user's pin wins, then the endpoint's own metadata, then the compiled + * default — a local server that says nothing (or says 8k) can be corrected. + * baseUrl is stamped per-model. */ +function toPiModel( + model: CustomModel, + baseUrl: string, + overrides: Record, +): Model { + const pin = overrides[model.id]; return { id: model.id, name: model.id, api: "openai-completions", provider: CUSTOM_PROVIDER.id, baseUrl, - // Detected from the endpoint's metadata (see detectReasoning). true makes - // pi-ai offer the standard levels and send reasoning_effort on the wire; - // no thinkingLevelMap → the default off/minimal/low/medium/high set. - reasoning: Boolean(model.reasoning), + // Pin wins; else detected from the endpoint's metadata (see + // detectReasoning). true makes pi-ai offer the standard levels and send + // reasoning_effort on the wire; no thinkingLevelMap → the default + // off/minimal/low/medium/high set. + reasoning: pin?.reasoning ?? Boolean(model.reasoning), input: model.image ? ["text", "image"] : ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: model.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW, - maxTokens: model.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS, + contextWindow: pin?.context ?? model.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW, + maxTokens: pin?.maxTokens ?? model.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS, type: "chat", }; } @@ -61,13 +70,16 @@ function toPiModel(model: CustomModel, baseUrl: string): Model { export function buildOpenAiCompatibleProvider(input: { baseUrl: string; models: readonly CustomModel[]; + /** User pins by model id (see CustomModelOverride) — absent = none. */ + overrides?: Record; }): Provider { + const overrides = input.overrides ?? {}; return createProvider({ id: CUSTOM_PROVIDER.id, name: CUSTOM_PROVIDER.label, baseUrl: input.baseUrl, auth: { apiKey: customEndpointAuth }, - models: input.models.map((m) => toPiModel(m, input.baseUrl)), + models: input.models.map((m) => toPiModel(m, input.baseUrl, overrides)), api: openAICompletionsApi(), }); } @@ -81,7 +93,11 @@ export function syncCustomProvider(cp: CustomProviderSettings | undefined): void return; } models.setProvider( - buildOpenAiCompatibleProvider({ baseUrl: normalizeBaseUrl(cp.baseUrl), models: cp.models ?? [] }), + buildOpenAiCompatibleProvider({ + baseUrl: normalizeBaseUrl(cp.baseUrl), + models: cp.models ?? [], + overrides: cp.overrides ?? {}, + }), ); } diff --git a/src/server/settings-body.ts b/src/server/settings-body.ts index a733003..d07cd6e 100644 --- a/src/server/settings-body.ts +++ b/src/server/settings-body.ts @@ -18,13 +18,14 @@ import type { AppearanceSettings, BackgroundModelRole, CompactionSettings, + CustomModelOverride, GeneralSettings, DictationSettings, GraphSettings, ModelsSettings, RoleModelOverride, } from "../core/settings.ts"; -import { BACKGROUND_MODEL_ROLES, COMPACTION_THRESHOLD_FLOOR, EXTRACTION_CHUNK_TOKENS_FLOOR, MODEL_SLOTS } from "../core/settings.ts"; +import { BACKGROUND_MODEL_ROLES, COMPACTION_THRESHOLD_FLOOR, CUSTOM_MODEL_PIN_FLOOR, EXTRACTION_CHUNK_TOKENS_FLOOR, MODEL_SLOTS } from "../core/settings.ts"; import { WRITABLE_TOOL_KEYS } from "../core/pending.ts"; import { isDictationLanguage } from "../core/dictation.ts"; @@ -386,6 +387,45 @@ export function parseDictationSettingsBody(body: unknown): SettingsBodyResult> { + const raw = asRecord(body); + if (!raw) return { ok: false, error: "Invalid JSON body" }; + const map = asRecord(raw.overrides); + if (!map) return { ok: false, error: "overrides must be a { modelId: { context?, maxTokens?, reasoning? } } map" }; + const out: Record = {}; + for (const [key, value] of Object.entries(map)) { + const id = key.trim(); + if (!id) return { ok: false, error: "override keys must be non-empty model ids" }; + const entry = asRecord(value); + if (!entry) return { ok: false, error: `override for ${id} must be an object` }; + const pin: CustomModelOverride = {}; + for (const field of ["context", "maxTokens"] as const) { + const v = entry[field]; + if (v === undefined) continue; + if (typeof v !== "number" || !Number.isInteger(v) || v < CUSTOM_MODEL_PIN_FLOOR) { + return { + ok: false, + error: `${field} for ${id} must be a whole number of tokens at least ${CUSTOM_MODEL_PIN_FLOOR}`, + }; + } + pin[field] = v; + } + if (entry.reasoning !== undefined) { + if (typeof entry.reasoning !== "boolean") return { ok: false, error: `reasoning for ${id} must be true or false` }; + pin.reasoning = entry.reasoning; + } + out[id] = pin; + } + return { ok: true, settings: out }; +} + /** The section→parser map for PUT /api/settings/[section]. The custom endpoint * is NOT a section — it connects via POST /api/custom-provider. */ export const SETTINGS_SECTIONS = { diff --git a/tests/custom-endpoint.test.ts b/tests/custom-endpoint.test.ts index d1d4845..8b64253 100644 --- a/tests/custom-endpoint.test.ts +++ b/tests/custom-endpoint.test.ts @@ -2,8 +2,9 @@ * Tests for the custom OpenAI-compatible endpoint (custom-endpoint wave). * Covers the pure core (normalizeBaseUrl, customProviderUsable, the sanitizer, * resolution), the deep server module (provider build + live sync + model - * discovery incl. image-modality and reasoning detection), and the catalog - * federation row. No network: the discovery probe runs against a stubbed fetch. + * discovery incl. image-modality, reasoning detection and the per-model user + * pins), the pins' strict write-side parser, and the catalog federation row. + * No network: the discovery probe runs against a stubbed fetch. */ import assert from "node:assert/strict"; @@ -11,11 +12,14 @@ import { after, describe, it } from "node:test"; import { CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW, CUSTOM_MODEL_DEFAULT_MAX_TOKENS, + CUSTOM_MODEL_PIN_FLOOR, customProviderUsable, normalizeBaseUrl, + sanitizeCustomModelOverrides, sanitizeUserSettings, } from "../src/core/settings.ts"; import { resolveSettings } from "../src/core/settings-resolution.ts"; +import { parseCustomModelOverridesBody } from "../src/server/settings-body.ts"; import { CUSTOM_PROVIDER } from "../src/core/auth-keys.ts"; import { buildOpenAiCompatibleProvider, @@ -84,10 +88,86 @@ describe("sanitizeUserSettings — customProvider", () => { }); }); -// ─── resolution ────────────────────────────────────────────────────────────── +// ─── per-model pins: the read side + THE prune law ─────────────────────────── +describe("sanitizeUserSettings / sanitizeCustomModelOverrides — pins", () => { + it("keeps well-formed pins, clamps token counts up to the floor, keeps reasoning:false", () => { + const out = sanitizeUserSettings({ + customProvider: { + baseUrl: "https://h/v1", + overrides: { + llama: { context: 131072, maxTokens: 8192, reasoning: true }, + qwen: { context: 10 }, // under the floor → clamped up (clamp-never-reject on read) + quiet: { reasoning: false }, // a real pin, not an absent one + junk: { context: "big", maxTokens: -1, reasoning: "yes" }, // every field junk + " ": { context: 4096 }, // no usable id + nope: "not-an-object", + }, + }, + }); + assert.deepEqual(out?.customProvider?.overrides, { + llama: { context: 131072, maxTokens: 8192, reasoning: true }, + qwen: { context: CUSTOM_MODEL_PIN_FLOOR }, + quiet: { reasoning: false }, + }); + }); + it("prunes pins for models the endpoint no longer advertises — the single prune law", () => { + const pins = { a: { context: 32768 }, b: { context: 32768 } }; + assert.deepEqual(Object.keys(sanitizeCustomModelOverrides(pins, ["a"])), ["a"]); + assert.deepEqual( + Object.keys(sanitizeCustomModelOverrides(pins)), + ["a", "b"], + "no liveIds → no prune (that is the routes' call, not the record sanitizer's)", + ); + assert.deepEqual(sanitizeCustomModelOverrides("nope"), {}, "a non-object map resolves empty"); + }); + it("drops an invalid overrides section from the record entirely", () => { + const out = sanitizeUserSettings({ customProvider: { baseUrl: "https://h/v1", overrides: "nope" } }); + assert.equal(out?.customProvider?.overrides, undefined); + }); +}); + +// ─── pins: the strict write side (PATCH) ────────────────────────────────── +describe("parseCustomModelOverridesBody — the strict write side", () => { + it("accepts whole counts at or above the floor and the tri-state reasoning pin", () => { + const r = parseCustomModelOverridesBody({ + overrides: { + a: { context: CUSTOM_MODEL_PIN_FLOOR, reasoning: false }, + b: { maxTokens: 8192 }, + c: {}, // an empty pin is legal — the sanitizer drops it on write + }, + }); + assert.ok(r.ok); + assert.deepEqual(r.settings, { + a: { context: CUSTOM_MODEL_PIN_FLOOR, reasoning: false }, + b: { maxTokens: 8192 }, + c: {}, + }); + }); + it("rejects a count under the floor, naming the floor", () => { + const r = parseCustomModelOverridesBody({ overrides: { a: { context: CUSTOM_MODEL_PIN_FLOOR - 1 } } }); + assert.equal(r.ok, false); + if (!r.ok) assert.ok(r.error.includes(String(CUSTOM_MODEL_PIN_FLOOR)), r.error); + }); + it("rejects a fractional count, a non-boolean reasoning, and a missing overrides map", () => { + assert.equal(parseCustomModelOverridesBody({ overrides: { a: { context: 4096.5 } } }).ok, false); + assert.equal(parseCustomModelOverridesBody({ overrides: { a: { reasoning: "on" } } }).ok, false); + assert.equal(parseCustomModelOverridesBody({ overrides: { a: "nope" } }).ok, false); + assert.equal(parseCustomModelOverridesBody({}).ok, false); + assert.equal(parseCustomModelOverridesBody(null).ok, false); + }); +}); + +// ─── resolution ────────────────────────────────────────────────────────── describe("resolveSettings — customProvider", () => { it("resolves an absent section to concrete defaults", () => { - assert.deepEqual(resolveSettings(null).customProvider, { baseUrl: "", models: [] }); + assert.deepEqual(resolveSettings(null).customProvider, { baseUrl: "", models: [], overrides: {} }); + }); + it("carries stored pins through to the concrete view", () => { + const resolved = resolveSettings({ + version: 1, + customProvider: { baseUrl: "http://h/v1", models: [{ id: "a" }], overrides: { a: { context: 32768 } } }, + }); + assert.deepEqual(resolved.customProvider.overrides, { a: { context: 32768 } }); }); }); @@ -124,6 +204,33 @@ describe("buildOpenAiCompatibleProvider / syncCustomProvider", () => { assert.equal(big!.maxTokens, 393216); }); + it("a user pin beats the endpoint, which beats the compiled default", () => { + const provider = buildOpenAiCompatibleProvider({ + baseUrl: "http://h/v1", + models: [{ id: "pinned", context: 32768, maxTokens: 4096 }, { id: "detected", context: 65536 }, { id: "bare" }], + overrides: { pinned: { context: 131072, maxTokens: 16384 } }, + }); + const [pinned, detected, bare] = provider.getModels(); + assert.equal(pinned!.contextWindow, 131072, "the pin wins over the endpoint's own number"); + assert.equal(pinned!.maxTokens, 16384); + assert.equal(detected!.contextWindow, 65536, "no pin → the endpoint's value"); + assert.equal(detected!.maxTokens, CUSTOM_MODEL_DEFAULT_MAX_TOKENS, "endpoint silent → the compiled default"); + assert.equal(bare!.contextWindow, CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW); + }); + + it("the thinking pin is tri-state: false beats a detected-true endpoint, true beats silence", () => { + const provider = buildOpenAiCompatibleProvider({ + baseUrl: "http://h/v1", + models: [{ id: "liar", reasoning: true }, { id: "silent" }], + overrides: { liar: { reasoning: false }, silent: { reasoning: true } }, + }); + const [liar, silent] = provider.getModels(); + assert.equal(liar!.reasoning, false, "a server that advertises reasoning but rejects it can be pinned off"); + assert.deepEqual(getSupportedThinkingLevels(liar!), ["off"]); + assert.equal(silent!.reasoning, true, "LM Studio publishes nothing; the user says otherwise"); + assert.deepEqual(getSupportedThinkingLevels(silent!), ["off", "minimal", "low", "medium", "high"]); + }); + it("gives a reasoning-capable model the standard thinking levels (others stay \"off\")", () => { const provider = buildOpenAiCompatibleProvider({ baseUrl: "http://h/v1",