feat: user pins for local model limits (context, max output, thinking)
A local endpoint that publishes no limits — LM Studio's /models says nothing — left every model on the 8192/4096 defaults with no way to say "this one has a 131k window". Each discovered model now gets three optional pins in the endpoint editor: context window, max output tokens, thinking (auto/on/off). A pin always wins: user pin → what the endpoint published → compiled default. Blank means no opinion, so discovery keeps ruling wherever you don't pin. Pins live in customProvider.overrides keyed by model id, stored apart from the discovery snapshot: a reconnect refreshes the models and prunes the pins of models that vanished while keeping the rest. Pins are written by PATCH /api/custom-provider, which never probes — the whole point is configuring a server that isn't answering /models correctly. Connect still probes and still fails loudly when the endpoint is down. A pin under 2048 tokens is rejected on the write side and clamped up on the read path, matching the compaction-floor discipline.
This commit is contained in:
Daniel Buckner
@@ -15,6 +15,7 @@ import {
|
||||
customProviderUsable,
|
||||
normalizeBaseUrl,
|
||||
type CustomModel,
|
||||
type CustomModelOverride,
|
||||
type CustomProviderSettings,
|
||||
} from "../core/settings.ts";
|
||||
import { keyValue } from "./keys.ts";
|
||||
@@ -36,23 +37,31 @@ const customEndpointAuth: ApiKeyAuth = {
|
||||
};
|
||||
|
||||
/** Map a discovered model to a pi-ai entry: image→input, reasoning→thinking
|
||||
* levels, context/maxTokens→limits (else CUSTOM_MODEL_DEFAULT_*). baseUrl is
|
||||
* stamped per-model. */
|
||||
function toPiModel(model: CustomModel, baseUrl: string): Model<Api> {
|
||||
* levels, context/maxTokens→limits. Resolution ladder (owner, 2026-10-27):
|
||||
* the user's pin wins, then the endpoint's own metadata, then the compiled
|
||||
* default — a local server that says nothing (or says 8k) can be corrected.
|
||||
* baseUrl is stamped per-model. */
|
||||
function toPiModel(
|
||||
model: CustomModel,
|
||||
baseUrl: string,
|
||||
overrides: Record<string, CustomModelOverride>,
|
||||
): Model<Api> {
|
||||
const pin = overrides[model.id];
|
||||
return {
|
||||
id: model.id,
|
||||
name: model.id,
|
||||
api: "openai-completions",
|
||||
provider: CUSTOM_PROVIDER.id,
|
||||
baseUrl,
|
||||
// Detected from the endpoint's metadata (see detectReasoning). true makes
|
||||
// pi-ai offer the standard levels and send reasoning_effort on the wire;
|
||||
// no thinkingLevelMap → the default off/minimal/low/medium/high set.
|
||||
reasoning: Boolean(model.reasoning),
|
||||
// Pin wins; else detected from the endpoint's metadata (see
|
||||
// detectReasoning). true makes pi-ai offer the standard levels and send
|
||||
// reasoning_effort on the wire; no thinkingLevelMap → the default
|
||||
// off/minimal/low/medium/high set.
|
||||
reasoning: pin?.reasoning ?? Boolean(model.reasoning),
|
||||
input: model.image ? ["text", "image"] : ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: model.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW,
|
||||
maxTokens: model.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS,
|
||||
contextWindow: pin?.context ?? model.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW,
|
||||
maxTokens: pin?.maxTokens ?? model.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS,
|
||||
type: "chat",
|
||||
};
|
||||
}
|
||||
@@ -61,13 +70,16 @@ function toPiModel(model: CustomModel, baseUrl: string): Model<Api> {
|
||||
export function buildOpenAiCompatibleProvider(input: {
|
||||
baseUrl: string;
|
||||
models: readonly CustomModel[];
|
||||
/** User pins by model id (see CustomModelOverride) — absent = none. */
|
||||
overrides?: Record<string, CustomModelOverride>;
|
||||
}): Provider<Api> {
|
||||
const overrides = input.overrides ?? {};
|
||||
return createProvider<Api>({
|
||||
id: CUSTOM_PROVIDER.id,
|
||||
name: CUSTOM_PROVIDER.label,
|
||||
baseUrl: input.baseUrl,
|
||||
auth: { apiKey: customEndpointAuth },
|
||||
models: input.models.map((m) => toPiModel(m, input.baseUrl)),
|
||||
models: input.models.map((m) => toPiModel(m, input.baseUrl, overrides)),
|
||||
api: openAICompletionsApi(),
|
||||
});
|
||||
}
|
||||
@@ -81,7 +93,11 @@ export function syncCustomProvider(cp: CustomProviderSettings | undefined): void
|
||||
return;
|
||||
}
|
||||
models.setProvider(
|
||||
buildOpenAiCompatibleProvider({ baseUrl: normalizeBaseUrl(cp.baseUrl), models: cp.models ?? [] }),
|
||||
buildOpenAiCompatibleProvider({
|
||||
baseUrl: normalizeBaseUrl(cp.baseUrl),
|
||||
models: cp.models ?? [],
|
||||
overrides: cp.overrides ?? {},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -18,13 +18,14 @@ import type {
|
||||
AppearanceSettings,
|
||||
BackgroundModelRole,
|
||||
CompactionSettings,
|
||||
CustomModelOverride,
|
||||
GeneralSettings,
|
||||
DictationSettings,
|
||||
GraphSettings,
|
||||
ModelsSettings,
|
||||
RoleModelOverride,
|
||||
} from "../core/settings.ts";
|
||||
import { BACKGROUND_MODEL_ROLES, COMPACTION_THRESHOLD_FLOOR, EXTRACTION_CHUNK_TOKENS_FLOOR, MODEL_SLOTS } from "../core/settings.ts";
|
||||
import { BACKGROUND_MODEL_ROLES, COMPACTION_THRESHOLD_FLOOR, CUSTOM_MODEL_PIN_FLOOR, EXTRACTION_CHUNK_TOKENS_FLOOR, MODEL_SLOTS } from "../core/settings.ts";
|
||||
import { WRITABLE_TOOL_KEYS } from "../core/pending.ts";
|
||||
import { isDictationLanguage } from "../core/dictation.ts";
|
||||
|
||||
@@ -386,6 +387,45 @@ export function parseDictationSettingsBody(body: unknown): SettingsBodyResult<Di
|
||||
return { ok: true, settings: { language: raw.language } };
|
||||
}
|
||||
|
||||
/** The custom endpoint's per-model pins (PATCH /api/custom-provider): the
|
||||
* whole desired map, replace-whole like every section. Strict write side — a
|
||||
* token count must be a whole number at or above the floor (the read side
|
||||
* clamps, this rejects — the compaction discipline). `reasoning` is the
|
||||
* tri-state pin: true forces thinking on, false forces it off. */
|
||||
export function parseCustomModelOverridesBody(
|
||||
body: unknown,
|
||||
): SettingsBodyResult<Record<string, CustomModelOverride>> {
|
||||
const raw = asRecord(body);
|
||||
if (!raw) return { ok: false, error: "Invalid JSON body" };
|
||||
const map = asRecord(raw.overrides);
|
||||
if (!map) return { ok: false, error: "overrides must be a { modelId: { context?, maxTokens?, reasoning? } } map" };
|
||||
const out: Record<string, CustomModelOverride> = {};
|
||||
for (const [key, value] of Object.entries(map)) {
|
||||
const id = key.trim();
|
||||
if (!id) return { ok: false, error: "override keys must be non-empty model ids" };
|
||||
const entry = asRecord(value);
|
||||
if (!entry) return { ok: false, error: `override for ${id} must be an object` };
|
||||
const pin: CustomModelOverride = {};
|
||||
for (const field of ["context", "maxTokens"] as const) {
|
||||
const v = entry[field];
|
||||
if (v === undefined) continue;
|
||||
if (typeof v !== "number" || !Number.isInteger(v) || v < CUSTOM_MODEL_PIN_FLOOR) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `${field} for ${id} must be a whole number of tokens at least ${CUSTOM_MODEL_PIN_FLOOR}`,
|
||||
};
|
||||
}
|
||||
pin[field] = v;
|
||||
}
|
||||
if (entry.reasoning !== undefined) {
|
||||
if (typeof entry.reasoning !== "boolean") return { ok: false, error: `reasoning for ${id} must be true or false` };
|
||||
pin.reasoning = entry.reasoning;
|
||||
}
|
||||
out[id] = pin;
|
||||
}
|
||||
return { ok: true, settings: out };
|
||||
}
|
||||
|
||||
/** The section→parser map for PUT /api/settings/[section]. The custom endpoint
|
||||
* is NOT a section — it connects via POST /api/custom-provider. */
|
||||
export const SETTINGS_SECTIONS = {
|
||||
|
||||
Reference in New Issue
Block a user