Daniel Buckner
695948271f
A local endpoint that publishes no limits — LM Studio's /models says nothing — left every model on the 8192/4096 defaults with no way to say "this one has a 131k window". Each discovered model now gets three optional pins in the endpoint editor: context window, max output tokens, thinking (auto/on/off). A pin always wins: user pin → what the endpoint published → compiled default. Blank means no opinion, so discovery keeps ruling wherever you don't pin. Pins live in customProvider.overrides keyed by model id, stored apart from the discovery snapshot: a reconnect refreshes the models and prunes the pins of models that vanished while keeping the rest. Pins are written by PATCH /api/custom-provider, which never probes — the whole point is configuring a server that isn't answering /models correctly. Connect still probes and still fails loudly when the endpoint is down. A pin under 2048 tokens is rejected on the write side and clamped up on the read path, matching the compaction-floor discipline.
357 lines
16 KiB
TypeScript
357 lines
16 KiB
TypeScript
/**
|
|
* Tests for the custom OpenAI-compatible endpoint (custom-endpoint wave).
|
|
* Covers the pure core (normalizeBaseUrl, customProviderUsable, the sanitizer,
|
|
* resolution), the deep server module (provider build + live sync + model
|
|
* discovery incl. image-modality, reasoning detection and the per-model user
|
|
* pins), the pins' strict write-side parser, and the catalog federation row.
|
|
* No network: the discovery probe runs against a stubbed fetch.
|
|
*/
|
|
|
|
import assert from "node:assert/strict";
|
|
import { after, describe, it } from "node:test";
|
|
import {
|
|
CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW,
|
|
CUSTOM_MODEL_DEFAULT_MAX_TOKENS,
|
|
CUSTOM_MODEL_PIN_FLOOR,
|
|
customProviderUsable,
|
|
normalizeBaseUrl,
|
|
sanitizeCustomModelOverrides,
|
|
sanitizeUserSettings,
|
|
} from "../src/core/settings.ts";
|
|
import { resolveSettings } from "../src/core/settings-resolution.ts";
|
|
import { parseCustomModelOverridesBody } from "../src/server/settings-body.ts";
|
|
import { CUSTOM_PROVIDER } from "../src/core/auth-keys.ts";
|
|
import {
|
|
buildOpenAiCompatibleProvider,
|
|
discoverModels,
|
|
syncCustomProvider,
|
|
} from "../src/server/custom-provider.ts";
|
|
import { piModels } from "../src/agent/models.ts";
|
|
import { getSupportedThinkingLevels } from "@earendil-works/pi-ai";
|
|
|
|
// ─── normalizeBaseUrl ────────────────────────────────────────────────────────
|
|
describe("normalizeBaseUrl", () => {
|
|
it("drops trailing slashes and keeps the path", () => {
|
|
assert.equal(normalizeBaseUrl("https://host:8000/v1/"), "https://host:8000/v1");
|
|
assert.equal(normalizeBaseUrl(" https://host/v1 "), "https://host/v1");
|
|
assert.equal(normalizeBaseUrl("http://localhost:11434"), "http://localhost:11434");
|
|
});
|
|
it("rejects non-http(s) schemes and junk with empty string", () => {
|
|
assert.equal(normalizeBaseUrl("file:///etc/passwd"), "");
|
|
assert.equal(normalizeBaseUrl("not a url"), "");
|
|
assert.equal(normalizeBaseUrl(""), "");
|
|
});
|
|
});
|
|
|
|
// ─── customProviderUsable ────────────────────────────────────────────────────
|
|
describe("customProviderUsable", () => {
|
|
it("is true exactly when a base URL is set (the key is optional)", () => {
|
|
assert.equal(customProviderUsable({ baseUrl: "https://h/v1" }), true);
|
|
assert.equal(customProviderUsable({ baseUrl: "" }), false);
|
|
assert.equal(customProviderUsable({}), false);
|
|
assert.equal(customProviderUsable(undefined), false);
|
|
});
|
|
});
|
|
|
|
// ─── sanitizer ───────────────────────────────────────────────────────────────
|
|
describe("sanitizeUserSettings — customProvider", () => {
|
|
it("normalizes the base URL and cleans the model list (trim, dedupe, image flag)", () => {
|
|
const out = sanitizeUserSettings({
|
|
customProvider: {
|
|
baseUrl: "https://host:8000/v1/",
|
|
models: [
|
|
{ id: " llama-3 " }, // trims id; image absent
|
|
{ id: "gpt", image: true },
|
|
{ id: "gpt", image: false }, // dup id wins (last) → image dropped
|
|
{ id: "" }, // dropped: no usable id
|
|
"not-an-object", // dropped
|
|
],
|
|
},
|
|
});
|
|
const cp = out?.customProvider;
|
|
assert.equal(cp?.baseUrl, "https://host:8000/v1");
|
|
assert.deepEqual(cp?.models, [{ id: "llama-3" }, { id: "gpt" }]);
|
|
});
|
|
it("keeps image:true and drops a non-array models list / invalid base URL", () => {
|
|
const out = sanitizeUserSettings({ customProvider: { baseUrl: "ftp://x", models: "nope" } });
|
|
assert.equal(out?.customProvider?.baseUrl, undefined);
|
|
assert.equal(out?.customProvider?.models, undefined);
|
|
});
|
|
it("keeps reasoning:true and drops a non-true reasoning flag", () => {
|
|
const out = sanitizeUserSettings({
|
|
customProvider: {
|
|
baseUrl: "https://h/v1",
|
|
models: [{ id: "thinker", reasoning: true }, { id: "plain", reasoning: "yes" }],
|
|
},
|
|
});
|
|
assert.deepEqual(out?.customProvider?.models, [{ id: "thinker", reasoning: true }, { id: "plain" }]);
|
|
});
|
|
});
|
|
|
|
// ─── per-model pins: the read side + THE prune law ───────────────────────────
|
|
describe("sanitizeUserSettings / sanitizeCustomModelOverrides — pins", () => {
|
|
it("keeps well-formed pins, clamps token counts up to the floor, keeps reasoning:false", () => {
|
|
const out = sanitizeUserSettings({
|
|
customProvider: {
|
|
baseUrl: "https://h/v1",
|
|
overrides: {
|
|
llama: { context: 131072, maxTokens: 8192, reasoning: true },
|
|
qwen: { context: 10 }, // under the floor → clamped up (clamp-never-reject on read)
|
|
quiet: { reasoning: false }, // a real pin, not an absent one
|
|
junk: { context: "big", maxTokens: -1, reasoning: "yes" }, // every field junk
|
|
" ": { context: 4096 }, // no usable id
|
|
nope: "not-an-object",
|
|
},
|
|
},
|
|
});
|
|
assert.deepEqual(out?.customProvider?.overrides, {
|
|
llama: { context: 131072, maxTokens: 8192, reasoning: true },
|
|
qwen: { context: CUSTOM_MODEL_PIN_FLOOR },
|
|
quiet: { reasoning: false },
|
|
});
|
|
});
|
|
it("prunes pins for models the endpoint no longer advertises — the single prune law", () => {
|
|
const pins = { a: { context: 32768 }, b: { context: 32768 } };
|
|
assert.deepEqual(Object.keys(sanitizeCustomModelOverrides(pins, ["a"])), ["a"]);
|
|
assert.deepEqual(
|
|
Object.keys(sanitizeCustomModelOverrides(pins)),
|
|
["a", "b"],
|
|
"no liveIds → no prune (that is the routes' call, not the record sanitizer's)",
|
|
);
|
|
assert.deepEqual(sanitizeCustomModelOverrides("nope"), {}, "a non-object map resolves empty");
|
|
});
|
|
it("drops an invalid overrides section from the record entirely", () => {
|
|
const out = sanitizeUserSettings({ customProvider: { baseUrl: "https://h/v1", overrides: "nope" } });
|
|
assert.equal(out?.customProvider?.overrides, undefined);
|
|
});
|
|
});
|
|
|
|
// ─── pins: the strict write side (PATCH) ──────────────────────────────────
|
|
describe("parseCustomModelOverridesBody — the strict write side", () => {
|
|
it("accepts whole counts at or above the floor and the tri-state reasoning pin", () => {
|
|
const r = parseCustomModelOverridesBody({
|
|
overrides: {
|
|
a: { context: CUSTOM_MODEL_PIN_FLOOR, reasoning: false },
|
|
b: { maxTokens: 8192 },
|
|
c: {}, // an empty pin is legal — the sanitizer drops it on write
|
|
},
|
|
});
|
|
assert.ok(r.ok);
|
|
assert.deepEqual(r.settings, {
|
|
a: { context: CUSTOM_MODEL_PIN_FLOOR, reasoning: false },
|
|
b: { maxTokens: 8192 },
|
|
c: {},
|
|
});
|
|
});
|
|
it("rejects a count under the floor, naming the floor", () => {
|
|
const r = parseCustomModelOverridesBody({ overrides: { a: { context: CUSTOM_MODEL_PIN_FLOOR - 1 } } });
|
|
assert.equal(r.ok, false);
|
|
if (!r.ok) assert.ok(r.error.includes(String(CUSTOM_MODEL_PIN_FLOOR)), r.error);
|
|
});
|
|
it("rejects a fractional count, a non-boolean reasoning, and a missing overrides map", () => {
|
|
assert.equal(parseCustomModelOverridesBody({ overrides: { a: { context: 4096.5 } } }).ok, false);
|
|
assert.equal(parseCustomModelOverridesBody({ overrides: { a: { reasoning: "on" } } }).ok, false);
|
|
assert.equal(parseCustomModelOverridesBody({ overrides: { a: "nope" } }).ok, false);
|
|
assert.equal(parseCustomModelOverridesBody({}).ok, false);
|
|
assert.equal(parseCustomModelOverridesBody(null).ok, false);
|
|
});
|
|
});
|
|
|
|
// ─── resolution ──────────────────────────────────────────────────────────
|
|
describe("resolveSettings — customProvider", () => {
|
|
it("resolves an absent section to concrete defaults", () => {
|
|
assert.deepEqual(resolveSettings(null).customProvider, { baseUrl: "", models: [], overrides: {} });
|
|
});
|
|
it("carries stored pins through to the concrete view", () => {
|
|
const resolved = resolveSettings({
|
|
version: 1,
|
|
customProvider: { baseUrl: "http://h/v1", models: [{ id: "a" }], overrides: { a: { context: 32768 } } },
|
|
});
|
|
assert.deepEqual(resolved.customProvider.overrides, { a: { context: 32768 } });
|
|
});
|
|
});
|
|
|
|
// ─── provider build + live sync ──────────────────────────────────────────────
|
|
describe("buildOpenAiCompatibleProvider / syncCustomProvider", () => {
|
|
after(() => piModels().deleteProvider(CUSTOM_PROVIDER.id));
|
|
|
|
it("maps ids to defaults and marks image-capable models image-input", () => {
|
|
const provider = buildOpenAiCompatibleProvider({
|
|
baseUrl: "http://h/v1",
|
|
models: [{ id: "m1" }, { id: "m2", image: true }],
|
|
});
|
|
assert.equal(provider.id, CUSTOM_PROVIDER.id);
|
|
const [m1, m2] = provider.getModels();
|
|
assert.equal(m1!.name, "m1");
|
|
assert.equal(m1!.api, "openai-completions");
|
|
assert.equal(m1!.provider, CUSTOM_PROVIDER.id);
|
|
assert.equal(m1!.baseUrl, "http://h/v1");
|
|
assert.equal(m1!.contextWindow, CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW);
|
|
assert.equal(m1!.maxTokens, CUSTOM_MODEL_DEFAULT_MAX_TOKENS);
|
|
assert.deepEqual(m1!.input, ["text"]); // text-only by default
|
|
assert.deepEqual(m2!.input, ["text", "image"]); // image-capable → Vision-eligible
|
|
assert.equal(m1!.contextWindow, CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW); // no endpoint value → default
|
|
assert.equal(m1!.maxTokens, CUSTOM_MODEL_DEFAULT_MAX_TOKENS);
|
|
});
|
|
|
|
it("carries endpoint token limits when present", () => {
|
|
const provider = buildOpenAiCompatibleProvider({
|
|
baseUrl: "http://h/v1",
|
|
models: [{ id: "big", context: 196608, maxTokens: 393216 }],
|
|
});
|
|
const [big] = provider.getModels();
|
|
assert.equal(big!.contextWindow, 196608); // 192k from the endpoint
|
|
assert.equal(big!.maxTokens, 393216);
|
|
});
|
|
|
|
it("a user pin beats the endpoint, which beats the compiled default", () => {
|
|
const provider = buildOpenAiCompatibleProvider({
|
|
baseUrl: "http://h/v1",
|
|
models: [{ id: "pinned", context: 32768, maxTokens: 4096 }, { id: "detected", context: 65536 }, { id: "bare" }],
|
|
overrides: { pinned: { context: 131072, maxTokens: 16384 } },
|
|
});
|
|
const [pinned, detected, bare] = provider.getModels();
|
|
assert.equal(pinned!.contextWindow, 131072, "the pin wins over the endpoint's own number");
|
|
assert.equal(pinned!.maxTokens, 16384);
|
|
assert.equal(detected!.contextWindow, 65536, "no pin → the endpoint's value");
|
|
assert.equal(detected!.maxTokens, CUSTOM_MODEL_DEFAULT_MAX_TOKENS, "endpoint silent → the compiled default");
|
|
assert.equal(bare!.contextWindow, CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW);
|
|
});
|
|
|
|
it("the thinking pin is tri-state: false beats a detected-true endpoint, true beats silence", () => {
|
|
const provider = buildOpenAiCompatibleProvider({
|
|
baseUrl: "http://h/v1",
|
|
models: [{ id: "liar", reasoning: true }, { id: "silent" }],
|
|
overrides: { liar: { reasoning: false }, silent: { reasoning: true } },
|
|
});
|
|
const [liar, silent] = provider.getModels();
|
|
assert.equal(liar!.reasoning, false, "a server that advertises reasoning but rejects it can be pinned off");
|
|
assert.deepEqual(getSupportedThinkingLevels(liar!), ["off"]);
|
|
assert.equal(silent!.reasoning, true, "LM Studio publishes nothing; the user says otherwise");
|
|
assert.deepEqual(getSupportedThinkingLevels(silent!), ["off", "minimal", "low", "medium", "high"]);
|
|
});
|
|
|
|
it("gives a reasoning-capable model the standard thinking levels (others stay \"off\")", () => {
|
|
const provider = buildOpenAiCompatibleProvider({
|
|
baseUrl: "http://h/v1",
|
|
models: [{ id: "thinker", reasoning: true }, { id: "plain" }],
|
|
});
|
|
const [thinker, plain] = provider.getModels();
|
|
assert.equal(thinker!.reasoning, true);
|
|
// No thinkingLevelMap → pi-ai's default set, same as any reasoning model.
|
|
assert.deepEqual(getSupportedThinkingLevels(thinker!), ["off", "minimal", "low", "medium", "high"]);
|
|
assert.equal(plain!.reasoning, false);
|
|
assert.deepEqual(getSupportedThinkingLevels(plain!), ["off"]);
|
|
});
|
|
|
|
it("registers on a usable sync and removes when the base URL clears", () => {
|
|
syncCustomProvider({ baseUrl: "http://h/v1", models: [{ id: "live" }] });
|
|
assert.ok(piModels().getProvider(CUSTOM_PROVIDER.id), "provider registered");
|
|
assert.ok(piModels().getModel(CUSTOM_PROVIDER.id, "live"), "model resolvable");
|
|
syncCustomProvider({ baseUrl: "", models: [{ id: "live" }] });
|
|
assert.equal(piModels().getProvider(CUSTOM_PROVIDER.id), undefined, "provider removed");
|
|
});
|
|
});
|
|
|
|
// ─── discovery (stubbed fetch) ───────────────────────────────────────────────
|
|
describe("discoverModels", () => {
|
|
const realFetch = globalThis.fetch;
|
|
after(() => {
|
|
globalThis.fetch = realFetch;
|
|
});
|
|
|
|
function stubFetch(handler: (url: string) => { status: number; body: unknown }) {
|
|
globalThis.fetch = (async (input: unknown) => {
|
|
const res = handler(String(input));
|
|
return new Response(JSON.stringify(res.body), { status: res.status });
|
|
}) as unknown as typeof fetch;
|
|
}
|
|
|
|
it("reads OpenAI {data:[{id}]} and Ollama {models:[{id}]} shapes", async () => {
|
|
stubFetch(() => ({ status: 200, body: { data: [{ id: "b" }, { id: "a" }] } }));
|
|
const openai = await discoverModels("http://h/v1");
|
|
assert.ok(openai.ok);
|
|
assert.deepEqual(openai.models, [{ id: "a" }, { id: "b" }]);
|
|
stubFetch(() => ({ status: 200, body: { models: [{ id: "x" }] } }));
|
|
const r = await discoverModels("http://h/v1");
|
|
assert.ok(r.ok && r.models.length === 1 && r.models[0]!.id === "x");
|
|
});
|
|
it("detects image capability from common modality fields", async () => {
|
|
stubFetch(() => ({
|
|
status: 200,
|
|
body: {
|
|
data: [
|
|
{ id: "vision-model", architecture: { input_modalities: ["text", "image"] } },
|
|
{ id: "cap-model", capabilities: ["completion", "vision"] },
|
|
{ id: "text-model", architecture: { input_modalities: ["text"] } },
|
|
],
|
|
},
|
|
}));
|
|
const r = await discoverModels("http://h/v1");
|
|
assert.ok(r.ok);
|
|
const byId = Object.fromEntries(r.models.map((m) => [m.id, Boolean(m.image)]));
|
|
assert.equal(byId["vision-model"], true);
|
|
assert.equal(byId["cap-model"], true);
|
|
assert.equal(byId["text-model"], false);
|
|
});
|
|
it("detects token limits from common metadata fields", async () => {
|
|
stubFetch(() => ({
|
|
status: 200,
|
|
body: {
|
|
data: [
|
|
{ id: "big", max_input_tokens: 196608, max_output_tokens: 393216 },
|
|
{ id: "ctx", context_length: 32768 },
|
|
{ id: "plain" },
|
|
],
|
|
},
|
|
}));
|
|
const r = await discoverModels("http://h/v1");
|
|
assert.ok(r.ok);
|
|
const byId = Object.fromEntries(r.models.map((m) => [m.id, m]));
|
|
assert.equal(byId["big"]!.context, 196608);
|
|
assert.equal(byId["big"]!.maxTokens, 393216);
|
|
assert.equal(byId["ctx"]!.context, 32768);
|
|
assert.equal(byId["ctx"]!.maxTokens, undefined);
|
|
assert.equal(byId["plain"]!.context, undefined);
|
|
});
|
|
it("detects reasoning from endpoint metadata only — never from the model name", async () => {
|
|
stubFetch(() => ({
|
|
status: 200,
|
|
body: {
|
|
data: [
|
|
{ id: "vllm-reasoner", supported_parameters: ["temperature", "reasoning", "reasoning_effort"] },
|
|
{ id: "bool-reasoner", reasoning: true },
|
|
{ id: "thinking-flag", thinking: true },
|
|
{ id: "cap-reasoner", capabilities: ["completion", "reasoning"] },
|
|
{ id: "deepseek-r1-distill" }, // a reasoning-sounding NAME, no metadata → stays off
|
|
],
|
|
},
|
|
}));
|
|
const r = await discoverModels("http://h/v1");
|
|
assert.ok(r.ok);
|
|
const byId = Object.fromEntries(r.models.map((m) => [m.id, Boolean(m.reasoning)]));
|
|
assert.equal(byId["vllm-reasoner"], true);
|
|
assert.equal(byId["bool-reasoner"], true);
|
|
assert.equal(byId["thinking-flag"], true);
|
|
assert.equal(byId["cap-reasoner"], true);
|
|
assert.equal(byId["deepseek-r1-distill"], false, "name guessing would risk sending reasoning_effort to a server that rejects it");
|
|
});
|
|
|
|
it("guards the scheme before any fetch", async () => {
|
|
let called = false;
|
|
globalThis.fetch = (async () => {
|
|
called = true;
|
|
return new Response("{}", { status: 200 });
|
|
}) as unknown as typeof fetch;
|
|
const r = await discoverModels("file:///etc/passwd");
|
|
assert.equal(r.ok, false);
|
|
assert.equal(called, false, "no fetch for a non-http scheme");
|
|
});
|
|
it("reports a non-2xx and an unparseable shape as honest errors", async () => {
|
|
stubFetch(() => ({ status: 500, body: {} }));
|
|
assert.equal((await discoverModels("http://h/v1")).ok, false);
|
|
stubFetch(() => ({ status: 200, body: { unexpected: true } }));
|
|
assert.equal((await discoverModels("http://h/v1")).ok, false);
|
|
});
|
|
});
|