feat: user pins for local model limits (context, max output, thinking)

A local endpoint that publishes no limits — LM Studio's /models says nothing —
left every model on the 8192/4096 defaults with no way to say "this one has a
131k window". Each discovered model now gets three optional pins in the
endpoint editor: context window, max output tokens, thinking (auto/on/off).

A pin always wins: user pin → what the endpoint published → compiled default.
Blank means no opinion, so discovery keeps ruling wherever you don't pin. Pins
live in customProvider.overrides keyed by model id, stored apart from the
discovery snapshot: a reconnect refreshes the models and prunes the pins of
models that vanished while keeping the rest.

Pins are written by PATCH /api/custom-provider, which never probes — the whole
point is configuring a server that isn't answering /models correctly. Connect
still probes and still fails loudly when the endpoint is down. A pin under 2048
tokens is rejected on the write side and clamped up on the read path, matching
the compaction-floor discipline.
This commit is contained in:
Daniel Buckner
2026-10-27 16:25:03 -07:00
parent 286a95ba04
commit 695948271f
7 changed files with 455 additions and 28 deletions
+40 -4
View File
@@ -1,10 +1,13 @@
/** /**
* The custom endpoint's connect surface (one provider in the Keys list). * The custom endpoint's connect surface (one provider in the Keys list).
* GET → non-secret config { baseUrl, models } + keyPresent (masked). * GET → non-secret config { baseUrl, models, overrides } + keyPresent (masked).
* POST → connect: validate + normalize the base URL, probe /models (fail * POST → connect: validate + normalize the base URL, probe /models (fail
* fast), store the key, persist { baseUrl, models }, re-sync the * fast), store the key, persist { baseUrl, models }, re-sync the
* provider into piModels(). Discovery lives here because connecting * provider into piModels(). Discovery lives here because connecting
* and discovering are one user action. * and discovering are one user action. Stored user pins survive a
* reconnect, pruned to the ids the endpoint still advertises.
* PATCH → store per-model pins only. No probe: pins must be editable with
* the local server switched off (the case they exist for).
* DELETE → disconnect: drop the stored config + the stored key and unregister * DELETE → disconnect: drop the stored config + the stored key and unregister
* the provider (the mirror image of POST; idempotent). * the provider (the mirror image of POST; idempotent).
*/ */
@@ -13,12 +16,13 @@ import {
discoverModels, discoverModels,
effectiveSettings, effectiveSettings,
keyValue, keyValue,
parseCustomModelOverridesBody,
readSettingsRecord, readSettingsRecord,
saveKey, saveKey,
syncCustomProvider, syncCustomProvider,
writeSettingsRecord, writeSettingsRecord,
} from "@/src/server"; } from "@/src/server";
import { CUSTOM_PROVIDER, normalizeBaseUrl } from "@/src/core"; import { CUSTOM_PROVIDER, normalizeBaseUrl, sanitizeCustomModelOverrides } from "@/src/core";
export const dynamic = "force-dynamic"; export const dynamic = "force-dynamic";
@@ -27,6 +31,7 @@ export async function GET() {
return Response.json({ return Response.json({
baseUrl: cp.baseUrl, baseUrl: cp.baseUrl,
models: cp.models, models: cp.models,
overrides: cp.overrides,
keyPresent: keyValue(CUSTOM_PROVIDER.envVar) !== undefined, keyPresent: keyValue(CUSTOM_PROVIDER.envVar) !== undefined,
}); });
} }
@@ -54,11 +59,42 @@ export async function POST(request: Request) {
} }
const record = readSettingsRecord() ?? { version: 1 }; const record = readSettingsRecord() ?? { version: 1 };
const next = writeSettingsRecord({ ...record, customProvider: { baseUrl: base, models: found.models } }); // User pins are never rewritten by discovery — only pruned to what the
// endpoint still advertises (the single prune law, core/settings.ts).
const overrides = sanitizeCustomModelOverrides(
record.customProvider?.overrides,
found.models.map((m) => m.id),
);
const next = writeSettingsRecord({
...record,
customProvider: { baseUrl: base, models: found.models, overrides },
});
syncCustomProvider(next.customProvider); syncCustomProvider(next.customProvider);
return Response.json({ ok: true, models: next.customProvider?.models ?? [] }); return Response.json({ ok: true, models: next.customProvider?.models ?? [] });
} }
/** Store the per-model pins (replace-whole). Pruned against the STORED model
* list, never a probe — an offline endpoint keeps its pins editable. */
export async function PATCH(request: Request) {
const parsed = parseCustomModelOverridesBody(await request.json().catch(() => null));
if (!parsed.ok) return Response.json({ error: parsed.error }, { status: 400 });
const cp = effectiveSettings().customProvider;
if (!cp.baseUrl) return Response.json({ error: "No custom endpoint configured" }, { status: 409 });
const overrides = sanitizeCustomModelOverrides(
parsed.settings,
cp.models.map((m) => m.id),
);
const record = readSettingsRecord() ?? { version: 1 };
const next = writeSettingsRecord({
...record,
customProvider: { baseUrl: cp.baseUrl, models: cp.models, overrides },
});
syncCustomProvider(next.customProvider);
return Response.json({ ok: true, overrides: next.customProvider?.overrides ?? {} });
}
/** Disconnect: forget the endpoint AND its key, then unregister the provider. /** Disconnect: forget the endpoint AND its key, then unregister the provider.
* Deleting the section (rather than blanking baseUrl) keeps settings.json the * Deleting the section (rather than blanking baseUrl) keeps settings.json the
* same shape as a never-connected install. Idempotent by construction. */ * same shape as a never-connected install. Idempotent by construction. */
+173 -2
View File
@@ -24,7 +24,15 @@
*/ */
import { useEffect, useRef, useState } from "react"; import { useEffect, useRef, useState } from "react";
import { AUTH_KEYS, CUSTOM_PROVIDER, type AuthKeyStatus, type CustomModel } from "@/src/core"; import {
AUTH_KEYS,
CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW,
CUSTOM_MODEL_DEFAULT_MAX_TOKENS,
CUSTOM_PROVIDER,
type AuthKeyStatus,
type CustomModel,
type CustomModelOverride,
} from "@/src/core";
import { SettingsGroup } from "./controls.tsx"; import { SettingsGroup } from "./controls.tsx";
import { SubscriptionConnections } from "./subscription-connections.tsx"; import { SubscriptionConnections } from "./subscription-connections.tsx";
import { ProviderMenuList } from "./model-menu.tsx"; import { ProviderMenuList } from "./model-menu.tsx";
@@ -49,6 +57,8 @@ interface KeysResponse {
interface CustomStatus { interface CustomStatus {
baseUrl: string; baseUrl: string;
models: CustomModel[]; models: CustomModel[];
/** User pins by model id (context / max output / thinking). */
overrides: Record<string, CustomModelOverride>;
keyPresent: boolean; keyPresent: boolean;
} }
@@ -402,6 +412,7 @@ export function KeysSection({
setCustom({ setCustom({
baseUrl: cp.baseUrl ?? "", baseUrl: cp.baseUrl ?? "",
models: cp.models ?? [], models: cp.models ?? [],
overrides: cp.overrides ?? {},
keyPresent: Boolean(cp.keyPresent), keyPresent: Boolean(cp.keyPresent),
}); });
} catch { } catch {
@@ -500,11 +511,17 @@ export function KeysSection({
<CustomEndpointEditor <CustomEndpointEditor
baseUrl0={custom?.baseUrl ?? ""} baseUrl0={custom?.baseUrl ?? ""}
keyPresent={custom?.keyPresent ?? false} keyPresent={custom?.keyPresent ?? false}
models={custom?.models ?? []}
overrides={custom?.overrides ?? {}}
onSaved={async () => { onSaved={async () => {
setAddingCustom(false); setAddingCustom(false);
await reload(); await reload();
notifySettingsSaved("keys"); notifySettingsSaved("keys");
}} }}
onPinsSaved={async () => {
await reload();
notifySettingsSaved("keys");
}}
onDelete={disconnectCustom} onDelete={disconnectCustom}
onCancel={() => setAddingCustom(false)} onCancel={() => setAddingCustom(false)}
/> />
@@ -663,18 +680,68 @@ function CustomEndpointRow({
); );
} }
/** A pin row's raw input state — "" means "no pin": the endpoint's own value,
* then the compiled default, decides. `reasoning` is tri-state. */
interface PinDraft {
context: string;
maxTokens: string;
reasoning: "" | "on" | "off";
}
const pinDraft = (pin: CustomModelOverride | undefined): PinDraft => ({
context: pin?.context === undefined ? "" : String(pin.context),
maxTokens: pin?.maxTokens === undefined ? "" : String(pin.maxTokens),
reasoning: pin?.reasoning === undefined ? "" : pin.reasoning ? "on" : "off",
});
/** One draft row per discovered model, seeded from the stored pins. */
function draftsFor(
models: readonly CustomModel[],
overrides: Record<string, CustomModelOverride>,
): Record<string, PinDraft> {
return Object.fromEntries(models.map((m) => [m.id, pinDraft(overrides[m.id])]));
}
/** Drafts → the PATCH payload (replace-whole): a blank field sends nothing and
* a fully blank model is absent — which IS the un-pin. */
function pinsPayload(drafts: Record<string, PinDraft>): Record<string, CustomModelOverride> {
const out: Record<string, CustomModelOverride> = {};
for (const [id, d] of Object.entries(drafts)) {
const pin: CustomModelOverride = {};
if (d.context.trim()) pin.context = Number(d.context);
if (d.maxTokens.trim()) pin.maxTokens = Number(d.maxTokens);
if (d.reasoning !== "") pin.reasoning = d.reasoning === "on";
if (Object.keys(pin).length) out[id] = pin;
}
return out;
}
/** testids must survive ids carrying slashes, colons, dots. */
const pinTestid = (prefix: string, id: string) => `${prefix}-${id.replace(/[^a-z0-9-]/gi, "_")}`;
/** The custom endpoint's connect editor: base URL + key → POST /api/custom-provider /** The custom endpoint's connect editor: base URL + key → POST /api/custom-provider
* (validate + discover + register). Image capability is detected automatically. */ * (validate + discover + register), and the per-model limits form → PATCH (pins,
* no probe, so it works with the local server switched off). Image capability is
* detected automatically. */
function CustomEndpointEditor({ function CustomEndpointEditor({
baseUrl0, baseUrl0,
keyPresent, keyPresent,
models,
overrides,
onSaved, onSaved,
onPinsSaved,
onCancel, onCancel,
onDelete, onDelete,
}: { }: {
baseUrl0: string; baseUrl0: string;
keyPresent: boolean; keyPresent: boolean;
/** The discovered models — one limits row each. */
models: readonly CustomModel[];
/** Their stored user pins. */
overrides: Record<string, CustomModelOverride>;
onSaved: () => void | Promise<void>; onSaved: () => void | Promise<void>;
/** Pins saved — the editor stays open (a form, not a connect). */
onPinsSaved: () => void | Promise<void>;
onCancel: () => void; onCancel: () => void;
onDelete?: () => void | Promise<void>; onDelete?: () => void | Promise<void>;
}) { }) {
@@ -683,11 +750,46 @@ function CustomEndpointEditor({
const [keyTouched, setKeyTouched] = useState(false); const [keyTouched, setKeyTouched] = useState(false);
const [busy, setBusy] = useState(false); const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
// The limits form: edited drafts over the stored pins, plus the last saved
// payload as the dirty baseline (the server's echo is what gets stored).
const [drafts, setDrafts] = useState<Record<string, PinDraft>>(() => draftsFor(models, overrides));
const [baseline, setBaseline] = useState(() => JSON.stringify(pinsPayload(draftsFor(models, overrides))));
const [pinsBusy, setPinsBusy] = useState(false);
const [pinsError, setPinsError] = useState<string | null>(null);
// A key is required to enable chatting; local servers accept any value. A // A key is required to enable chatting; local servers accept any value. A
// stored key is kept when the field is left blank. // stored key is kept when the field is left blank.
const needsKey = !keyPresent; const needsKey = !keyPresent;
const canConnect = Boolean(baseUrl.trim()) && (!needsKey || Boolean(key.trim())); const canConnect = Boolean(baseUrl.trim()) && (!needsKey || Boolean(key.trim()));
const draftOf = (m: CustomModel): PinDraft => drafts[m.id] ?? pinDraft(overrides[m.id]);
const setDraft = (m: CustomModel, patch: Partial<PinDraft>) =>
setDrafts((prev) => ({ ...prev, [m.id]: { ...draftOf(m), ...patch } }));
const current = Object.fromEntries(models.map((m) => [m.id, draftOf(m)])) as Record<string, PinDraft>;
const dirty = JSON.stringify(pinsPayload(current)) !== baseline;
async function savePins() {
if (pinsBusy || !dirty) return;
setPinsBusy(true);
setPinsError(null);
const res = await fetch("/api/custom-provider", {
method: "PATCH",
headers: { "content-type": "application/json" },
body: JSON.stringify({ overrides: pinsPayload(current) }),
}).catch(() => null);
setPinsBusy(false);
if (!res || !res.ok) {
const b = (await res?.json().catch(() => ({}))) as { error?: string } | undefined;
setPinsError(res ? (b?.error ?? `Save failed (${res.status})`) : "Save failed. Check your connection.");
return;
}
// The server's stored map is the truth (it floors + prunes) — echo it back.
const body = (await res.json().catch(() => null)) as { overrides?: Record<string, CustomModelOverride> } | null;
const next = draftsFor(models, body?.overrides ?? pinsPayload(current));
setDrafts(next);
setBaseline(JSON.stringify(pinsPayload(next)));
await onPinsSaved();
}
async function save() { async function save() {
if (busy || !baseUrl.trim()) return; if (busy || !baseUrl.trim()) return;
setBusy(true); setBusy(true);
@@ -769,6 +871,75 @@ function CustomEndpointEditor({
</button> </button>
)} )}
</div> </div>
{models.length > 0 && (
<div className="flex flex-col gap-1" data-testid="custom-pins">
<span className="text-[11px] text-muted-foreground">
Model limits — blank uses the endpoint’s value, then the default
</span>
<div className="flex max-h-56 flex-col gap-1 overflow-y-auto pr-1">
{models.map((m) => {
const d = draftOf(m);
return (
<div key={m.id} className="flex items-center gap-2" data-testid={pinTestid("custom-pin-row", m.id)}>
<span className="min-w-0 flex-1 truncate font-mono text-[11px] text-foreground" title={m.id}>
{m.id}
</span>
<input
type="text"
inputMode="numeric"
value={d.context}
onChange={(e) => setDraft(m, { context: e.target.value })}
placeholder={String(m.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW)}
aria-label={`${m.id} context window in tokens`}
data-testid={pinTestid("custom-pin-context", m.id)}
disabled={pinsBusy}
className="h-7 w-20 shrink-0 rounded-sm border border-border bg-background px-1.5 text-right font-mono text-[11px] text-foreground outline-none placeholder:text-muted-foreground/60 focus:border-muted-foreground/50"
/>
<input
type="text"
inputMode="numeric"
value={d.maxTokens}
onChange={(e) => setDraft(m, { maxTokens: e.target.value })}
placeholder={String(m.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS)}
aria-label={`${m.id} max output tokens`}
data-testid={pinTestid("custom-pin-maxtokens", m.id)}
disabled={pinsBusy}
className="h-7 w-20 shrink-0 rounded-sm border border-border bg-background px-1.5 text-right font-mono text-[11px] text-foreground outline-none placeholder:text-muted-foreground/60 focus:border-muted-foreground/50"
/>
<select
value={d.reasoning}
onChange={(e) => setDraft(m, { reasoning: e.target.value as PinDraft["reasoning"] })}
aria-label={`${m.id} thinking`}
data-testid={pinTestid("custom-pin-reasoning", m.id)}
disabled={pinsBusy}
className="h-7 shrink-0 rounded-sm border border-border bg-background px-1 font-ui text-[11px] text-foreground outline-none focus:border-muted-foreground/50"
>
<option value="">think: auto</option>
<option value="on">think: on</option>
<option value="off">think: off</option>
</select>
</div>
);
})}
</div>
<div className="flex items-center gap-2">
<button
type="button"
onClick={() => void savePins()}
disabled={pinsBusy || !dirty}
data-testid="custom-pins-save"
className="shrink-0 text-xs text-foreground underline-offset-2 hover:underline disabled:text-muted-foreground/40 disabled:no-underline"
>
{pinsBusy ? "Saving…" : "Save limits"}
</button>
{pinsError && (
<p role="alert" data-testid="custom-pins-error" className="text-[11px] text-red-500">
{pinsError}
</p>
)}
</div>
</div>
)}
{needsKey && ( {needsKey && (
<p className="text-[11px] text-muted-foreground"> <p className="text-[11px] text-muted-foreground">
A key is required to enable chatting — local servers (Ollama, LM Studio) accept any value. A key is required to enable chatting — local servers (Ollama, LM Studio) accept any value.
+5 -2
View File
@@ -38,6 +38,7 @@ import {
type DictationSettings, type DictationSettings,
type CompactionSettings, type CompactionSettings,
type CustomModel, type CustomModel,
type CustomModelOverride,
type GraphSettings, type GraphSettings,
type ModelSlot, type ModelSlot,
type ModelSlotName, type ModelSlotName,
@@ -96,8 +97,9 @@ export interface EffectiveSettings {
/** Compaction (owner round 14): resolved concrete — absent record /** Compaction (owner round 14): resolved concrete — absent record
* section resolves to the factory 150k. */ * section resolves to the factory 150k. */
compaction: Required<CompactionSettings>; compaction: Required<CompactionSettings>;
/** Custom endpoint, resolved concrete: baseUrl defaults "", models default []. */ /** Custom endpoint, resolved concrete: baseUrl defaults "", models and
customProvider: { baseUrl: string; models: CustomModel[] }; * user pins default to empty. */
customProvider: { baseUrl: string; models: CustomModel[]; overrides: Record<string, CustomModelOverride> };
} }
/** The conversation pair's chain (BYOK, owner 2026-09-22): the chat's own /** The conversation pair's chain (BYOK, owner 2026-09-22): the chat's own
@@ -227,6 +229,7 @@ export function resolveSettings(record: UserSettings | null): EffectiveSettings
customProvider: { customProvider: {
baseUrl: record?.customProvider?.baseUrl ?? "", baseUrl: record?.customProvider?.baseUrl ?? "",
models: record?.customProvider?.models ?? [], models: record?.customProvider?.models ?? [],
overrides: record?.customProvider?.overrides ?? {},
}, },
}; };
} }
+54
View File
@@ -217,6 +217,12 @@ export interface ModelsSettings {
export const CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW = 8192; export const CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW = 8192;
export const CUSTOM_MODEL_DEFAULT_MAX_TOKENS = 4096; export const CUSTOM_MODEL_DEFAULT_MAX_TOKENS = 4096;
/** Floor for a user-pinned token count (owner, 2026-10-27): the read side
* clamps up, the write side rejects below — the compaction-floor discipline.
* Pins exist because endpoints that publish no limits (LM Studio) otherwise
* sit on the 8192 default with no way to say better. */
export const CUSTOM_MODEL_PIN_FLOOR = 2_048;
/** One discovered custom model. `image`/`reasoning`/`context`/`maxTokens` are /** One discovered custom model. `image`/`reasoning`/`context`/`maxTokens` are
* detected from the endpoint's /models metadata; anything it omits falls back * detected from the endpoint's /models metadata; anything it omits falls back
* to a default. */ * to a default. */
@@ -233,12 +239,26 @@ export interface CustomModel {
maxTokens?: number; maxTokens?: number;
} }
/** A user pin for one custom model: it beats what the endpoint publishes
* (owner ruling — explicit user intent wins; a local server's number is
* often wrong or absent). Absent field = no pin. `reasoning` is tri-state:
* true forces thinking on, false forces it off (a server that advertises
* reasoning but rejects reasoning_effort), absent = what discovery found. */
export interface CustomModelOverride {
context?: number;
maxTokens?: number;
reasoning?: boolean;
}
/** The single custom endpoint's stored config (delta section). */ /** The single custom endpoint's stored config (delta section). */
export interface CustomProviderSettings { export interface CustomProviderSettings {
/** Normalized base URL (see normalizeBaseUrl). Empty/absent = not configured. */ /** Normalized base URL (see normalizeBaseUrl). Empty/absent = not configured. */
baseUrl?: string; baseUrl?: string;
/** Models discovered from the endpoint (refreshed on connect). */ /** Models discovered from the endpoint (refreshed on connect). */
models?: CustomModel[]; models?: CustomModel[];
/** User pins keyed by model id — pure user intent, never rewritten by
* discovery, pruned to the endpoint's live ids on connect. */
overrides?: Record<string, CustomModelOverride>;
} }
/** Canonical base-URL form (single source): trims, requires absolute http(s), /** Canonical base-URL form (single source): trims, requires absolute http(s),
@@ -562,6 +582,38 @@ function positiveInt(value: unknown): number | undefined {
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined; return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
} }
/** One pin, clamped up to the floor (clamp-never-reject on read). */
function sanitizeCustomModelOverride(raw: unknown): CustomModelOverride | undefined {
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return undefined;
const rec = raw as Record<string, unknown>;
const out: CustomModelOverride = {};
const context = positiveInt(rec.context);
if (context) out.context = Math.max(CUSTOM_MODEL_PIN_FLOOR, context);
const maxTokens = positiveInt(rec.maxTokens);
if (maxTokens) out.maxTokens = Math.max(CUSTOM_MODEL_PIN_FLOOR, maxTokens);
if (typeof rec.reasoning === "boolean") out.reasoning = rec.reasoning;
return Object.keys(out).length ? out : undefined;
}
/** THE pins map + THE prune law: well-formed entries only, ids trimmed, and
* when `liveIds` is given every entry for a model the endpoint no longer
* advertises drops. POST passes the discovered ids, PATCH the stored list. */
export function sanitizeCustomModelOverrides(
raw: unknown,
liveIds?: Iterable<string>,
): Record<string, CustomModelOverride> {
const out: Record<string, CustomModelOverride> = {};
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return out;
const live = liveIds ? new Set(liveIds) : undefined;
for (const [key, value] of Object.entries(raw)) {
const id = key.trim();
if (!id || (live && !live.has(id))) continue;
const pin = sanitizeCustomModelOverride(value);
if (pin) out[id] = pin;
}
return out;
}
/** The customProvider section: normalized base URL (invalid → dropped) and /** The customProvider section: normalized base URL (invalid → dropped) and
* discovered models (ids trimmed, dupes/empties dropped; image kept only when * discovered models (ids trimmed, dupes/empties dropped; image kept only when
* true; token counts only as positive ints). Clamp-never-reject. */ * true; token counts only as positive ints). Clamp-never-reject. */
@@ -590,6 +642,8 @@ function sanitizeCustomProvider(raw: Record<string, unknown>): CustomProviderSet
} }
out.models = [...byId.values()]; out.models = [...byId.values()];
} }
const overrides = sanitizeCustomModelOverrides(raw.overrides);
if (Object.keys(overrides).length) out.overrides = overrides;
return out; return out;
} }
+27 -11
View File
@@ -15,6 +15,7 @@ import {
customProviderUsable, customProviderUsable,
normalizeBaseUrl, normalizeBaseUrl,
type CustomModel, type CustomModel,
type CustomModelOverride,
type CustomProviderSettings, type CustomProviderSettings,
} from "../core/settings.ts"; } from "../core/settings.ts";
import { keyValue } from "./keys.ts"; import { keyValue } from "./keys.ts";
@@ -36,23 +37,31 @@ const customEndpointAuth: ApiKeyAuth = {
}; };
/** Map a discovered model to a pi-ai entry: image→input, reasoning→thinking /** Map a discovered model to a pi-ai entry: image→input, reasoning→thinking
* levels, context/maxTokens→limits (else CUSTOM_MODEL_DEFAULT_*). baseUrl is * levels, context/maxTokens→limits. Resolution ladder (owner, 2026-10-27):
* stamped per-model. */ * the user's pin wins, then the endpoint's own metadata, then the compiled
function toPiModel(model: CustomModel, baseUrl: string): Model<Api> { * default — a local server that says nothing (or says 8k) can be corrected.
* baseUrl is stamped per-model. */
function toPiModel(
model: CustomModel,
baseUrl: string,
overrides: Record<string, CustomModelOverride>,
): Model<Api> {
const pin = overrides[model.id];
return { return {
id: model.id, id: model.id,
name: model.id, name: model.id,
api: "openai-completions", api: "openai-completions",
provider: CUSTOM_PROVIDER.id, provider: CUSTOM_PROVIDER.id,
baseUrl, baseUrl,
// Detected from the endpoint's metadata (see detectReasoning). true makes // Pin wins; else detected from the endpoint's metadata (see
// pi-ai offer the standard levels and send reasoning_effort on the wire; // detectReasoning). true makes pi-ai offer the standard levels and send
// no thinkingLevelMap → the default off/minimal/low/medium/high set. // reasoning_effort on the wire; no thinkingLevelMap → the default
reasoning: Boolean(model.reasoning), // off/minimal/low/medium/high set.
reasoning: pin?.reasoning ?? Boolean(model.reasoning),
input: model.image ? ["text", "image"] : ["text"], input: model.image ? ["text", "image"] : ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: model.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW, contextWindow: pin?.context ?? model.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW,
maxTokens: model.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS, maxTokens: pin?.maxTokens ?? model.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS,
type: "chat", type: "chat",
}; };
} }
@@ -61,13 +70,16 @@ function toPiModel(model: CustomModel, baseUrl: string): Model<Api> {
export function buildOpenAiCompatibleProvider(input: { export function buildOpenAiCompatibleProvider(input: {
baseUrl: string; baseUrl: string;
models: readonly CustomModel[]; models: readonly CustomModel[];
/** User pins by model id (see CustomModelOverride) — absent = none. */
overrides?: Record<string, CustomModelOverride>;
}): Provider<Api> { }): Provider<Api> {
const overrides = input.overrides ?? {};
return createProvider<Api>({ return createProvider<Api>({
id: CUSTOM_PROVIDER.id, id: CUSTOM_PROVIDER.id,
name: CUSTOM_PROVIDER.label, name: CUSTOM_PROVIDER.label,
baseUrl: input.baseUrl, baseUrl: input.baseUrl,
auth: { apiKey: customEndpointAuth }, auth: { apiKey: customEndpointAuth },
models: input.models.map((m) => toPiModel(m, input.baseUrl)), models: input.models.map((m) => toPiModel(m, input.baseUrl, overrides)),
api: openAICompletionsApi(), api: openAICompletionsApi(),
}); });
} }
@@ -81,7 +93,11 @@ export function syncCustomProvider(cp: CustomProviderSettings | undefined): void
return; return;
} }
models.setProvider( models.setProvider(
buildOpenAiCompatibleProvider({ baseUrl: normalizeBaseUrl(cp.baseUrl), models: cp.models ?? [] }), buildOpenAiCompatibleProvider({
baseUrl: normalizeBaseUrl(cp.baseUrl),
models: cp.models ?? [],
overrides: cp.overrides ?? {},
}),
); );
} }
+41 -1
View File
@@ -18,13 +18,14 @@ import type {
AppearanceSettings, AppearanceSettings,
BackgroundModelRole, BackgroundModelRole,
CompactionSettings, CompactionSettings,
CustomModelOverride,
GeneralSettings, GeneralSettings,
DictationSettings, DictationSettings,
GraphSettings, GraphSettings,
ModelsSettings, ModelsSettings,
RoleModelOverride, RoleModelOverride,
} from "../core/settings.ts"; } from "../core/settings.ts";
import { BACKGROUND_MODEL_ROLES, COMPACTION_THRESHOLD_FLOOR, EXTRACTION_CHUNK_TOKENS_FLOOR, MODEL_SLOTS } from "../core/settings.ts"; import { BACKGROUND_MODEL_ROLES, COMPACTION_THRESHOLD_FLOOR, CUSTOM_MODEL_PIN_FLOOR, EXTRACTION_CHUNK_TOKENS_FLOOR, MODEL_SLOTS } from "../core/settings.ts";
import { WRITABLE_TOOL_KEYS } from "../core/pending.ts"; import { WRITABLE_TOOL_KEYS } from "../core/pending.ts";
import { isDictationLanguage } from "../core/dictation.ts"; import { isDictationLanguage } from "../core/dictation.ts";
@@ -386,6 +387,45 @@ export function parseDictationSettingsBody(body: unknown): SettingsBodyResult<Di
return { ok: true, settings: { language: raw.language } }; return { ok: true, settings: { language: raw.language } };
} }
/** The custom endpoint's per-model pins (PATCH /api/custom-provider): the
* whole desired map, replace-whole like every section. Strict write side — a
* token count must be a whole number at or above the floor (the read side
* clamps, this rejects — the compaction discipline). `reasoning` is the
* tri-state pin: true forces thinking on, false forces it off. */
export function parseCustomModelOverridesBody(
body: unknown,
): SettingsBodyResult<Record<string, CustomModelOverride>> {
const raw = asRecord(body);
if (!raw) return { ok: false, error: "Invalid JSON body" };
const map = asRecord(raw.overrides);
if (!map) return { ok: false, error: "overrides must be a { modelId: { context?, maxTokens?, reasoning? } } map" };
const out: Record<string, CustomModelOverride> = {};
for (const [key, value] of Object.entries(map)) {
const id = key.trim();
if (!id) return { ok: false, error: "override keys must be non-empty model ids" };
const entry = asRecord(value);
if (!entry) return { ok: false, error: `override for ${id} must be an object` };
const pin: CustomModelOverride = {};
for (const field of ["context", "maxTokens"] as const) {
const v = entry[field];
if (v === undefined) continue;
if (typeof v !== "number" || !Number.isInteger(v) || v < CUSTOM_MODEL_PIN_FLOOR) {
return {
ok: false,
error: `${field} for ${id} must be a whole number of tokens at least ${CUSTOM_MODEL_PIN_FLOOR}`,
};
}
pin[field] = v;
}
if (entry.reasoning !== undefined) {
if (typeof entry.reasoning !== "boolean") return { ok: false, error: `reasoning for ${id} must be true or false` };
pin.reasoning = entry.reasoning;
}
out[id] = pin;
}
return { ok: true, settings: out };
}
/** The section→parser map for PUT /api/settings/[section]. The custom endpoint /** The section→parser map for PUT /api/settings/[section]. The custom endpoint
* is NOT a section — it connects via POST /api/custom-provider. */ * is NOT a section — it connects via POST /api/custom-provider. */
export const SETTINGS_SECTIONS = { export const SETTINGS_SECTIONS = {
+111 -4
View File
@@ -2,8 +2,9 @@
* Tests for the custom OpenAI-compatible endpoint (custom-endpoint wave). * Tests for the custom OpenAI-compatible endpoint (custom-endpoint wave).
* Covers the pure core (normalizeBaseUrl, customProviderUsable, the sanitizer, * Covers the pure core (normalizeBaseUrl, customProviderUsable, the sanitizer,
* resolution), the deep server module (provider build + live sync + model * resolution), the deep server module (provider build + live sync + model
* discovery incl. image-modality and reasoning detection), and the catalog * discovery incl. image-modality, reasoning detection and the per-model user
* federation row. No network: the discovery probe runs against a stubbed fetch. * pins), the pins' strict write-side parser, and the catalog federation row.
* No network: the discovery probe runs against a stubbed fetch.
*/ */
import assert from "node:assert/strict"; import assert from "node:assert/strict";
@@ -11,11 +12,14 @@ import { after, describe, it } from "node:test";
import { import {
CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW, CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW,
CUSTOM_MODEL_DEFAULT_MAX_TOKENS, CUSTOM_MODEL_DEFAULT_MAX_TOKENS,
CUSTOM_MODEL_PIN_FLOOR,
customProviderUsable, customProviderUsable,
normalizeBaseUrl, normalizeBaseUrl,
sanitizeCustomModelOverrides,
sanitizeUserSettings, sanitizeUserSettings,
} from "../src/core/settings.ts"; } from "../src/core/settings.ts";
import { resolveSettings } from "../src/core/settings-resolution.ts"; import { resolveSettings } from "../src/core/settings-resolution.ts";
import { parseCustomModelOverridesBody } from "../src/server/settings-body.ts";
import { CUSTOM_PROVIDER } from "../src/core/auth-keys.ts"; import { CUSTOM_PROVIDER } from "../src/core/auth-keys.ts";
import { import {
buildOpenAiCompatibleProvider, buildOpenAiCompatibleProvider,
@@ -84,10 +88,86 @@ describe("sanitizeUserSettings — customProvider", () => {
}); });
}); });
// ─── resolution ────────────────────────────────────────────────────────────── // ─── per-model pins: the read side + THE prune law ───────────────────────────
describe("sanitizeUserSettings / sanitizeCustomModelOverrides — pins", () => {
it("keeps well-formed pins, clamps token counts up to the floor, keeps reasoning:false", () => {
const out = sanitizeUserSettings({
customProvider: {
baseUrl: "https://h/v1",
overrides: {
llama: { context: 131072, maxTokens: 8192, reasoning: true },
qwen: { context: 10 }, // under the floor → clamped up (clamp-never-reject on read)
quiet: { reasoning: false }, // a real pin, not an absent one
junk: { context: "big", maxTokens: -1, reasoning: "yes" }, // every field junk
" ": { context: 4096 }, // no usable id
nope: "not-an-object",
},
},
});
assert.deepEqual(out?.customProvider?.overrides, {
llama: { context: 131072, maxTokens: 8192, reasoning: true },
qwen: { context: CUSTOM_MODEL_PIN_FLOOR },
quiet: { reasoning: false },
});
});
it("prunes pins for models the endpoint no longer advertises — the single prune law", () => {
const pins = { a: { context: 32768 }, b: { context: 32768 } };
assert.deepEqual(Object.keys(sanitizeCustomModelOverrides(pins, ["a"])), ["a"]);
assert.deepEqual(
Object.keys(sanitizeCustomModelOverrides(pins)),
["a", "b"],
"no liveIds → no prune (that is the routes' call, not the record sanitizer's)",
);
assert.deepEqual(sanitizeCustomModelOverrides("nope"), {}, "a non-object map resolves empty");
});
it("drops an invalid overrides section from the record entirely", () => {
const out = sanitizeUserSettings({ customProvider: { baseUrl: "https://h/v1", overrides: "nope" } });
assert.equal(out?.customProvider?.overrides, undefined);
});
});
// ─── pins: the strict write side (PATCH) ──────────────────────────────────
describe("parseCustomModelOverridesBody — the strict write side", () => {
it("accepts whole counts at or above the floor and the tri-state reasoning pin", () => {
const r = parseCustomModelOverridesBody({
overrides: {
a: { context: CUSTOM_MODEL_PIN_FLOOR, reasoning: false },
b: { maxTokens: 8192 },
c: {}, // an empty pin is legal — the sanitizer drops it on write
},
});
assert.ok(r.ok);
assert.deepEqual(r.settings, {
a: { context: CUSTOM_MODEL_PIN_FLOOR, reasoning: false },
b: { maxTokens: 8192 },
c: {},
});
});
it("rejects a count under the floor, naming the floor", () => {
const r = parseCustomModelOverridesBody({ overrides: { a: { context: CUSTOM_MODEL_PIN_FLOOR - 1 } } });
assert.equal(r.ok, false);
if (!r.ok) assert.ok(r.error.includes(String(CUSTOM_MODEL_PIN_FLOOR)), r.error);
});
it("rejects a fractional count, a non-boolean reasoning, and a missing overrides map", () => {
assert.equal(parseCustomModelOverridesBody({ overrides: { a: { context: 4096.5 } } }).ok, false);
assert.equal(parseCustomModelOverridesBody({ overrides: { a: { reasoning: "on" } } }).ok, false);
assert.equal(parseCustomModelOverridesBody({ overrides: { a: "nope" } }).ok, false);
assert.equal(parseCustomModelOverridesBody({}).ok, false);
assert.equal(parseCustomModelOverridesBody(null).ok, false);
});
});
// ─── resolution ──────────────────────────────────────────────────────────
describe("resolveSettings — customProvider", () => { describe("resolveSettings — customProvider", () => {
it("resolves an absent section to concrete defaults", () => { it("resolves an absent section to concrete defaults", () => {
assert.deepEqual(resolveSettings(null).customProvider, { baseUrl: "", models: [] }); assert.deepEqual(resolveSettings(null).customProvider, { baseUrl: "", models: [], overrides: {} });
});
it("carries stored pins through to the concrete view", () => {
const resolved = resolveSettings({
version: 1,
customProvider: { baseUrl: "http://h/v1", models: [{ id: "a" }], overrides: { a: { context: 32768 } } },
});
assert.deepEqual(resolved.customProvider.overrides, { a: { context: 32768 } });
}); });
}); });
@@ -124,6 +204,33 @@ describe("buildOpenAiCompatibleProvider / syncCustomProvider", () => {
assert.equal(big!.maxTokens, 393216); assert.equal(big!.maxTokens, 393216);
}); });
it("a user pin beats the endpoint, which beats the compiled default", () => {
const provider = buildOpenAiCompatibleProvider({
baseUrl: "http://h/v1",
models: [{ id: "pinned", context: 32768, maxTokens: 4096 }, { id: "detected", context: 65536 }, { id: "bare" }],
overrides: { pinned: { context: 131072, maxTokens: 16384 } },
});
const [pinned, detected, bare] = provider.getModels();
assert.equal(pinned!.contextWindow, 131072, "the pin wins over the endpoint's own number");
assert.equal(pinned!.maxTokens, 16384);
assert.equal(detected!.contextWindow, 65536, "no pin → the endpoint's value");
assert.equal(detected!.maxTokens, CUSTOM_MODEL_DEFAULT_MAX_TOKENS, "endpoint silent → the compiled default");
assert.equal(bare!.contextWindow, CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW);
});
it("the thinking pin is tri-state: false beats a detected-true endpoint, true beats silence", () => {
const provider = buildOpenAiCompatibleProvider({
baseUrl: "http://h/v1",
models: [{ id: "liar", reasoning: true }, { id: "silent" }],
overrides: { liar: { reasoning: false }, silent: { reasoning: true } },
});
const [liar, silent] = provider.getModels();
assert.equal(liar!.reasoning, false, "a server that advertises reasoning but rejects it can be pinned off");
assert.deepEqual(getSupportedThinkingLevels(liar!), ["off"]);
assert.equal(silent!.reasoning, true, "LM Studio publishes nothing; the user says otherwise");
assert.deepEqual(getSupportedThinkingLevels(silent!), ["off", "minimal", "low", "medium", "high"]);
});
it("gives a reasoning-capable model the standard thinking levels (others stay \"off\")", () => { it("gives a reasoning-capable model the standard thinking levels (others stay \"off\")", () => {
const provider = buildOpenAiCompatibleProvider({ const provider = buildOpenAiCompatibleProvider({
baseUrl: "http://h/v1", baseUrl: "http://h/v1",