sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

feat: user pins for local model limits (context, max output, thinking)

Browse Source 
A local endpoint that publishes no limits — LM Studio's /models says nothing —
left every model on the 8192/4096 defaults with no way to say "this one has a
131k window". Each discovered model now gets three optional pins in the
endpoint editor: context window, max output tokens, thinking (auto/on/off).

A pin always wins: user pin → what the endpoint published → compiled default.
Blank means no opinion, so discovery keeps ruling wherever you don't pin. Pins
live in customProvider.overrides keyed by model id, stored apart from the
discovery snapshot: a reconnect refreshes the models and prunes the pins of
models that vanished while keeping the rest.

Pins are written by PATCH /api/custom-provider, which never probes — the whole
point is configuring a server that isn't answering /models correctly. Connect
still probes and still fails loudly when the endpoint is down. A pin under 2048
tokens is rejected on the write side and clamped up on the read path, matching
the compaction-floor discipline.
This commit is contained in:
Daniel Buckner
2026-10-27 16:25:03 -07:00
parent 286a95ba04
commit 695948271f
7 changed files with 455 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/custom-provider.ts
+27 -11
View File
@@ -15,6 +15,7 @@ import {
customProviderUsable,
normalizeBaseUrl,
type CustomModel,
type CustomModelOverride,
type CustomProviderSettings,
} from "../core/settings.ts";
import { keyValue } from "./keys.ts";
@@ -36,23 +37,31 @@ const customEndpointAuth: ApiKeyAuth = {
};
/** Map a discovered model to a pi-ai entry: image→input, reasoning→thinking
* levels, context/maxTokens→limits (else CUSTOM_MODEL_DEFAULT_*). baseUrl is
* stamped per-model. */
function toPiModel(model: CustomModel, baseUrl: string): Model<Api> {
* levels, context/maxTokens→limits. Resolution ladder (owner, 2026-10-27):
* the user's pin wins, then the endpoint's own metadata, then the compiled
* default — a local server that says nothing (or says 8k) can be corrected.
* baseUrl is stamped per-model. */
function toPiModel(
model: CustomModel,
baseUrl: string,
overrides: Record<string, CustomModelOverride>,
): Model<Api> {
const pin = overrides[model.id];
return {
id: model.id,
name: model.id,
api: "openai-completions",
provider: CUSTOM_PROVIDER.id,
baseUrl,
// Detected from the endpoint's metadata (see detectReasoning). true makes
// pi-ai offer the standard levels and send reasoning_effort on the wire;
// no thinkingLevelMap → the default off/minimal/low/medium/high set.
reasoning: Boolean(model.reasoning),
// Pin wins; else detected from the endpoint's metadata (see
// detectReasoning). true makes pi-ai offer the standard levels and send
// reasoning_effort on the wire; no thinkingLevelMap → the default
// off/minimal/low/medium/high set.
reasoning: pin?.reasoning ?? Boolean(model.reasoning),
input: model.image ? ["text", "image"] : ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: model.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW,
maxTokens: model.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS,
contextWindow: pin?.context ?? model.context ?? CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW,
maxTokens: pin?.maxTokens ?? model.maxTokens ?? CUSTOM_MODEL_DEFAULT_MAX_TOKENS,
type: "chat",
};
}
@@ -61,13 +70,16 @@ function toPiModel(model: CustomModel, baseUrl: string): Model<Api> {
export function buildOpenAiCompatibleProvider(input: {
baseUrl: string;
models: readonly CustomModel[];
/** User pins by model id (see CustomModelOverride) — absent = none. */
overrides?: Record<string, CustomModelOverride>;
}): Provider<Api> {
const overrides = input.overrides ?? {};
return createProvider<Api>({
id: CUSTOM_PROVIDER.id,
name: CUSTOM_PROVIDER.label,
baseUrl: input.baseUrl,
auth: { apiKey: customEndpointAuth },
models: input.models.map((m) => toPiModel(m, input.baseUrl)),
models: input.models.map((m) => toPiModel(m, input.baseUrl, overrides)),
api: openAICompletionsApi(),
});
}
@@ -81,7 +93,11 @@ export function syncCustomProvider(cp: CustomProviderSettings | undefined): void
return;
}
models.setProvider(
buildOpenAiCompatibleProvider({ baseUrl: normalizeBaseUrl(cp.baseUrl), models: cp.models ?? [] }),
buildOpenAiCompatibleProvider({
baseUrl: normalizeBaseUrl(cp.baseUrl),
models: cp.models ?? [],
overrides: cp.overrides ?? {},
}),
);
}