feat: user pins for local model limits (context, max output, thinking)
A local endpoint that publishes no limits — LM Studio's /models says nothing — left every model on the 8192/4096 defaults with no way to say "this one has a 131k window". Each discovered model now gets three optional pins in the endpoint editor: context window, max output tokens, thinking (auto/on/off). A pin always wins: user pin → what the endpoint published → compiled default. Blank means no opinion, so discovery keeps ruling wherever you don't pin. Pins live in customProvider.overrides keyed by model id, stored apart from the discovery snapshot: a reconnect refreshes the models and prunes the pins of models that vanished while keeping the rest. Pins are written by PATCH /api/custom-provider, which never probes — the whole point is configuring a server that isn't answering /models correctly. Connect still probes and still fails loudly when the endpoint is down. A pin under 2048 tokens is rejected on the write side and clamped up on the read path, matching the compaction-floor discipline.
This commit is contained in:
@@ -1,24 +1,28 @@
|
||||
/**
|
||||
* The custom endpoint's connect surface (one provider in the Keys list).
|
||||
* GET → non-secret config { baseUrl, models } + keyPresent (masked).
|
||||
* POST → connect: validate + normalize the base URL, probe /models (fail
|
||||
* fast), store the key, persist { baseUrl, models }, re-sync the
|
||||
* provider into piModels(). Discovery lives here because connecting
|
||||
* and discovering are one user action.
|
||||
* GET → non-secret config { baseUrl, models, overrides } + keyPresent (masked).
|
||||
* POST → connect: validate + normalize the base URL, probe /models (fail
|
||||
* fast), store the key, persist { baseUrl, models }, re-sync the
|
||||
* provider into piModels(). Discovery lives here because connecting
|
||||
* and discovering are one user action. Stored user pins survive a
|
||||
* reconnect, pruned to the ids the endpoint still advertises.
|
||||
* PATCH → store per-model pins only. No probe: pins must be editable with
|
||||
* the local server switched off (the case they exist for).
|
||||
* DELETE → disconnect: drop the stored config + the stored key and unregister
|
||||
* the provider (the mirror image of POST; idempotent).
|
||||
* the provider (the mirror image of POST; idempotent).
|
||||
*/
|
||||
import {
|
||||
deleteKey,
|
||||
discoverModels,
|
||||
effectiveSettings,
|
||||
keyValue,
|
||||
parseCustomModelOverridesBody,
|
||||
readSettingsRecord,
|
||||
saveKey,
|
||||
syncCustomProvider,
|
||||
writeSettingsRecord,
|
||||
} from "@/src/server";
|
||||
import { CUSTOM_PROVIDER, normalizeBaseUrl } from "@/src/core";
|
||||
import { CUSTOM_PROVIDER, normalizeBaseUrl, sanitizeCustomModelOverrides } from "@/src/core";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -27,6 +31,7 @@ export async function GET() {
|
||||
return Response.json({
|
||||
baseUrl: cp.baseUrl,
|
||||
models: cp.models,
|
||||
overrides: cp.overrides,
|
||||
keyPresent: keyValue(CUSTOM_PROVIDER.envVar) !== undefined,
|
||||
});
|
||||
}
|
||||
@@ -54,11 +59,42 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
const record = readSettingsRecord() ?? { version: 1 };
|
||||
const next = writeSettingsRecord({ ...record, customProvider: { baseUrl: base, models: found.models } });
|
||||
// User pins are never rewritten by discovery — only pruned to what the
|
||||
// endpoint still advertises (the single prune law, core/settings.ts).
|
||||
const overrides = sanitizeCustomModelOverrides(
|
||||
record.customProvider?.overrides,
|
||||
found.models.map((m) => m.id),
|
||||
);
|
||||
const next = writeSettingsRecord({
|
||||
...record,
|
||||
customProvider: { baseUrl: base, models: found.models, overrides },
|
||||
});
|
||||
syncCustomProvider(next.customProvider);
|
||||
return Response.json({ ok: true, models: next.customProvider?.models ?? [] });
|
||||
}
|
||||
|
||||
/** Store the per-model pins (replace-whole). Pruned against the STORED model
|
||||
* list, never a probe — an offline endpoint keeps its pins editable. */
|
||||
export async function PATCH(request: Request) {
|
||||
const parsed = parseCustomModelOverridesBody(await request.json().catch(() => null));
|
||||
if (!parsed.ok) return Response.json({ error: parsed.error }, { status: 400 });
|
||||
|
||||
const cp = effectiveSettings().customProvider;
|
||||
if (!cp.baseUrl) return Response.json({ error: "No custom endpoint configured" }, { status: 409 });
|
||||
|
||||
const overrides = sanitizeCustomModelOverrides(
|
||||
parsed.settings,
|
||||
cp.models.map((m) => m.id),
|
||||
);
|
||||
const record = readSettingsRecord() ?? { version: 1 };
|
||||
const next = writeSettingsRecord({
|
||||
...record,
|
||||
customProvider: { baseUrl: cp.baseUrl, models: cp.models, overrides },
|
||||
});
|
||||
syncCustomProvider(next.customProvider);
|
||||
return Response.json({ ok: true, overrides: next.customProvider?.overrides ?? {} });
|
||||
}
|
||||
|
||||
/** Disconnect: forget the endpoint AND its key, then unregister the provider.
|
||||
* Deleting the section (rather than blanking baseUrl) keeps settings.json the
|
||||
* same shape as a never-connected install. Idempotent by construction. */
|
||||
|
||||
Reference in New Issue
Block a user