From b48821309b7b63e98fc5425cf000fce94d73be1b Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Thu, 29 Oct 2026 04:43:53 +0200 Subject: [PATCH] feat: finish donor-led custom endpoint integration --- app/api/custom-provider/route.ts | 137 +-------- components/settings/keys-section.tsx | 148 ++++++--- components/setup-dialog.tsx | 43 +-- config.ts | 3 - docs/custom-providers.md | 252 +++++++--------- scripts/smoke-provider-runtime.mjs | 8 + src/agent/custom-providers.ts | 71 ----- src/agent/models.ts | 21 +- src/core/contracts/models.ts | 2 - src/core/custom-provider-auth.ts | 26 ++ src/core/custom-providers.ts | 146 --------- src/core/settings.ts | 22 +- src/server/custom-provider-auth.ts | 43 +++ src/server/custom-provider-body.ts | 31 ++ src/server/custom-provider-http.ts | 149 ++++++++++ src/server/custom-provider.ts | 84 ++++-- src/server/custom-providers.ts | 91 ------ src/server/index.ts | 1 + src/server/keys.ts | 32 +- src/server/model-catalog.ts | 8 +- src/server/pricing.ts | 3 +- src/server/settings-body.ts | 4 +- src/server/stores.ts | 2 +- src/store/auth.ts | 47 ++- src/store/custom-providers.ts | 12 - tests/custom-endpoint.test.ts | 13 +- tests/custom-provider-boundaries.test.ts | 191 ++++++++++++ tests/custom-provider-lifecycle.test.ts | 129 ++++++++ tests/custom-provider-restart.test.ts | 81 +++++ tests/custom-provider-route.test.ts | 5 +- tests/custom-providers.test.ts | 362 ----------------------- tests/helpers/custom-provider-auth.ts | 10 + tests/helpers/custom-provider-crash.ts | 56 ++++ 33 files changed, 1116 insertions(+), 1117 deletions(-) delete mode 100644 src/agent/custom-providers.ts create mode 100644 src/core/custom-provider-auth.ts delete mode 100644 src/core/custom-providers.ts create mode 100644 src/server/custom-provider-auth.ts create mode 100644 src/server/custom-provider-body.ts create mode 100644 src/server/custom-provider-http.ts delete mode 100644 src/server/custom-providers.ts delete mode 100644 src/store/custom-providers.ts create mode 100644 tests/custom-provider-boundaries.test.ts create mode 100644 tests/custom-provider-lifecycle.test.ts create mode 100644 tests/custom-provider-restart.test.ts delete mode 100644 tests/custom-providers.test.ts create mode 100644 tests/helpers/custom-provider-auth.ts create mode 100644 tests/helpers/custom-provider-crash.ts diff --git a/app/api/custom-provider/route.ts b/app/api/custom-provider/route.ts index f77ad78..54b267c 100644 --- a/app/api/custom-provider/route.ts +++ b/app/api/custom-provider/route.ts @@ -1,132 +1,9 @@ -/** - * The custom endpoint's connect surface (one provider in the Keys list). - * GET → non-secret config { baseUrl, models, overrides, manualModels } + - * keyPresent (masked). - * POST → connect: validate + normalize the base URL, probe /models, store - * the key, persist { baseUrl, models, overrides, manualModels }, - * re-sync the provider into piModels(). Discovery lives here because - * connecting and discovering are one user action. A failed probe is - * fatal UNLESS the user typed model IDs: an endpoint that serves chat - * but no model list is configured from those ids alone. A failed - * probe for the endpoint already stored keeps its last snapshot, so - * a blip cannot prune the stored pins. A 401/403 always fails: a - * rejected key is not a missing model list. Stored user pins survive - * a reconnect, pruned to the ids the connect ends up offering. - * PATCH → store per-model pins only. No probe: pins must be editable with - * the local server switched off (the case they exist for). - * DELETE → disconnect: drop the stored config + the stored key and unregister - * the provider (the mirror image of POST; idempotent). - */ -import { - deleteKey, - discoverModels, - effectiveSettings, - keyValue, - mergeManualModels, - parseCustomModelOverridesBody, - readSettingsRecord, - saveKey, - syncCustomProvider, - writeSettingsRecord, -} from "@/src/server"; -import { CUSTOM_PROVIDER, normalizeBaseUrl, parseModelIds, sanitizeCustomModelOverrides } from "@/src/core"; +/** PR #62's single custom endpoint: discover/manual connect, offline pins, + * and disconnect. Host boundaries and mutations live behind the server barrel. */ +import { customProviderGet, customProviderMutation } from "@/src/server"; export const dynamic = "force-dynamic"; - -export async function GET() { - const cp = effectiveSettings().customProvider; - return Response.json({ - baseUrl: cp.baseUrl, - models: cp.models, - overrides: cp.overrides, - manualModels: cp.manualModels, - keyPresent: keyValue(CUSTOM_PROVIDER.envVar) !== undefined, - }); -} - -export async function POST(request: Request) { - const body = (await request.json().catch(() => null)) as { baseUrl?: unknown; key?: unknown; manualModels?: unknown } | null; - const base = normalizeBaseUrl(typeof body?.baseUrl === "string" ? body.baseUrl : ""); - if (!base) return Response.json({ error: "Base URL must be a valid http(s) URL" }, { status: 400 }); - - // Empty key string clears it, absent leaves it stored; only a real key rides the probe. - const keyProvided = typeof body?.key === "string"; - const key = keyProvided ? (body!.key as string).trim() : undefined; - - const manualModels = parseModelIds(body?.manualModels); - const record = readSettingsRecord() ?? { version: 1 }; - const found = await discoverModels(base, key ? key : undefined); - // A rejected key is not a missing model list: never let typed ids turn an - // auth failure into a stored "connected" endpoint. - if (!found.ok && (found.status === 401 || found.status === 403)) { - return Response.json({ error: found.error }, { status: 502 }); - } - if (!found.ok && !manualModels.length) { - return Response.json({ error: `${found.error}. If this endpoint has no /models list, add model IDs manually.` }, { status: 502 }); - } - // A failed probe says nothing about what the endpoint offers: keep the last - // snapshot of the SAME endpoint, so a transient failure cannot shrink the - // model list and prune its pins. A different base URL starts clean. - const discovered = found.ok - ? found.models - : record.customProvider?.baseUrl === base - ? (record.customProvider?.models ?? []) - : []; - const models = mergeManualModels(discovered, manualModels); - - if (keyProvided) { - if (key) { - const saved = saveKey(CUSTOM_PROVIDER.envVar, key); - if (!saved.ok) return Response.json({ error: saved.error }, { status: 400 }); - } else { - deleteKey(CUSTOM_PROVIDER.envVar); - } - } - - // User pins are never rewritten by discovery — only pruned to the ids this - // connect offers (discovered ∪ typed). - const overrides = sanitizeCustomModelOverrides( - record.customProvider?.overrides, - models.map((m) => m.id), - ); - const next = writeSettingsRecord({ - ...record, - customProvider: { baseUrl: base, models, overrides, ...(manualModels.length ? { manualModels } : {}) }, - }); - syncCustomProvider(next.customProvider); - return Response.json({ ok: true, models: next.customProvider?.models ?? [] }); -} - -/** Store the per-model pins (replace-whole). Pruned against the STORED model - * list, never a probe — an offline endpoint keeps its pins editable. */ -export async function PATCH(request: Request) { - const parsed = parseCustomModelOverridesBody(await request.json().catch(() => null)); - if (!parsed.ok) return Response.json({ error: parsed.error }, { status: 400 }); - - const cp = effectiveSettings().customProvider; - if (!cp.baseUrl) return Response.json({ error: "No custom endpoint configured" }, { status: 409 }); - - const overrides = sanitizeCustomModelOverrides( - parsed.settings, - cp.models.map((m) => m.id), - ); - const record = readSettingsRecord() ?? { version: 1 }; - const next = writeSettingsRecord({ - ...record, - customProvider: { baseUrl: cp.baseUrl, models: cp.models, overrides, ...(cp.manualModels.length ? { manualModels: cp.manualModels } : {}) }, - }); - syncCustomProvider(next.customProvider); - return Response.json({ ok: true, overrides: next.customProvider?.overrides ?? {} }); -} - -/** Disconnect: forget the endpoint AND its key, then unregister the provider. - * Deleting the section (rather than blanking baseUrl) keeps settings.json the - * same shape as a never-connected install. Idempotent by construction. */ -export async function DELETE() { - const record = readSettingsRecord() ?? { version: 1 }; - const { customProvider: _dropped, ...rest } = record; - const next = writeSettingsRecord(rest); - deleteKey(CUSTOM_PROVIDER.envVar); - syncCustomProvider(next.customProvider); - return Response.json({ ok: true }); -} +export const GET = customProviderGet; +export const POST = customProviderMutation; +export const PATCH = customProviderMutation; +export const DELETE = customProviderMutation; diff --git a/components/settings/keys-section.tsx b/components/settings/keys-section.tsx index a3ddbee..219c3c5 100644 --- a/components/settings/keys-section.tsx +++ b/components/settings/keys-section.tsx @@ -29,6 +29,7 @@ import { CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW, CUSTOM_MODEL_DEFAULT_MAX_TOKENS, CUSTOM_PROVIDER, + normalizeBaseUrl, type AuthKeyStatus, type CustomModel, type CustomModelOverride, @@ -56,6 +57,10 @@ interface KeysResponse { /** The custom endpoint's non-secret status (GET /api/custom-provider). */ interface CustomStatus { + keyReuseRequired: boolean; + revision: string; + authMode: "keyless" | "apiKey"; + keySource: "env" | "file" | "none"; baseUrl: string; models: CustomModel[]; /** User pins by model id (context / max output / thinking). */ @@ -64,6 +69,13 @@ interface CustomStatus { keyPresent: boolean; } +async function readCustomStatus(inspectKey = false): Promise { + const response = await fetch(`/api/custom-provider${inspectKey ? "?authMode=apiKey" : ""}`); + if (!response.ok) throw new Error("Unable to load custom configuration."); + const cp = await response.json() as Partial; + return { keyReuseRequired: Boolean(cp.keyReuseRequired), revision: response.headers.get("etag") ?? "", authMode: cp.authMode ?? "keyless", keySource: cp.keySource ?? "none", baseUrl: cp.baseUrl ?? "", models: cp.models ?? [], overrides: cp.overrides ?? {}, manualModels: cp.manualModels ?? [], keyPresent: Boolean(cp.keyPresent) }; +} + /** The curated tools group, pinned by id (the catalog owns the copy; * these ids are the contract). Derived from the catalog (search wave, * 2026-09-20: the Google CSE pair retired, Tavily + Brave joined) — the @@ -410,14 +422,7 @@ export function KeysSection({ // Best-effort and independent: the custom endpoint's status must never // blank the provider-key list above if its fetch fails. try { - const cp = (await fetch("/api/custom-provider").then((r) => r.json())) as Partial; - setCustom({ - baseUrl: cp.baseUrl ?? "", - models: cp.models ?? [], - overrides: cp.overrides ?? {}, - manualModels: cp.manualModels ?? [], - keyPresent: Boolean(cp.keyPresent), - }); + setCustom(await readCustomStatus()); } catch { /* custom status unavailable — leave it unset */ } @@ -460,11 +465,13 @@ export function KeysSection({ /** Forget the custom endpoint (config + key server-side), same rhythm as * deleteKey: fire, re-read, broadcast so the catalog narrows everywhere. */ - async function disconnectCustom(): Promise { - await fetch("/api/custom-provider", { method: "DELETE" }).catch(() => undefined); + async function disconnectCustom(revision: string): Promise { + const res = await fetch("/api/custom-provider", { method: "DELETE", headers: { "content-type": "application/json", "if-match": revision }, body: JSON.stringify({}) }).catch(() => null); + if (!res?.ok) return res ? ((await res.json().catch(() => ({}))).error ?? "Disconnect failed.") : "Disconnect failed. Check your connection."; setAddingCustom(false); await reload(); notifySettingsSaved("keys"); + return null; } if (loading || statuses === null) { @@ -512,6 +519,11 @@ export function KeysSection({ )} {addingCustom && ( { await reload(); setAddingCustom(false); }} baseUrl0={custom?.baseUrl ?? ""} keyPresent={custom?.keyPresent ?? false} models={custom?.models ?? []} @@ -658,6 +670,20 @@ function AddProviderEditor({ ); } +/** One donor editor for first setup and Settings; setup omits offline pin and + * disconnect controls, while authentication/revisions/manual IDs stay identical. */ +export function CustomEndpointSetup({ onSaved }: { onSaved: () => void | Promise }) { + const [status, setStatus] = useState(null); + const [error, setError] = useState(false); + async function reload() { + try { setStatus(await readCustomStatus()); setError(false); } + catch { setError(true); } + } + useEffect(() => { void reload(); }, []); + if (!status || error) return
{error ? "Unable to load custom configuration." : "Loading custom configuration…"}{error && }
; + return ; +} + /** The custom endpoint's status row (shown once configured); clicking re-opens the editor. */ function CustomEndpointRow({ status, @@ -669,7 +695,7 @@ function CustomEndpointRow({ onToggle: () => void; }) { const count = status.models.length; - const line = `${status.baseUrl} · ${count} model${count === 1 ? "" : "s"}${status.keyPresent ? "" : " · keyless"}`; + const line = `${status.baseUrl} · ${count} model${count === 1 ? "" : "s"} · ${status.authMode === "keyless" ? "keyless" : status.keyPresent ? "API key" : "API key missing"}`; return (
)}
- {models.length > 0 && ( + {!setup && models.length > 0 && (
- Model limits — blank uses the endpoint’s value, then the default + Budget pins do not resize backend context. Blank uses reported limits, then fallback (8192 context / 4096 output). Custom pricing is unknown.
{models.map((m) => { @@ -904,6 +968,7 @@ function CustomEndpointEditor({
{m.id} + Context: {d.context.trim() ? "user pin" : m.context ? `reported ${m.context}` : `fallback ${CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW}`} · Output: {d.maxTokens.trim() ? "user pin" : m.maxTokens ? `reported ${m.maxTokens}` : `fallback ${CUSTOM_MODEL_DEFAULT_MAX_TOKENS}`} setDraft(m, { reasoning: e.target.value as PinDraft["reasoning"] })} - aria-label={`${m.id} thinking`} + aria-label={`${m.id} reasoning support`} data-testid={pinTestid("custom-pin-reasoning", m.id)} disabled={pinsBusy} className="h-7 shrink-0 rounded-sm border border-border bg-background px-1 font-ui text-[11px] text-foreground outline-none focus:border-muted-foreground/50" > - - - + + +
); @@ -961,11 +1026,8 @@ function CustomEndpointEditor({
)} - {needsKey && ( -

- A key is required to enable chatting — local servers (Ollama, LM Studio) accept any value. -

- )} +

Reasoning support is a capability declaration. The effort picker’s Off may not disable a custom backend’s default thinking.

+ {(error || pinsError) && } {error && (

{error} diff --git a/components/setup-dialog.tsx b/components/setup-dialog.tsx index 98b6ad7..20e389b 100644 --- a/components/setup-dialog.tsx +++ b/components/setup-dialog.tsx @@ -48,7 +48,7 @@ import { notifySettingsSaved, useSettingsSaved } from "@/components/settings-sig import { useModelCatalog } from "@/components/model-catalog"; import { savedSlotsConfigured } from "@/components/setup-cta"; import { Hint } from "@/components/ui/hint"; -import { type ProviderKeyInfo, CUSTOM_PROVIDER_INFO } from "@/components/settings/keys-section"; +import { type ProviderKeyInfo, CUSTOM_PROVIDER_INFO, CustomEndpointSetup } from "@/components/settings/keys-section"; import { ProviderMenuList } from "@/components/settings/model-menu"; import { ChatGPTBackgroundAccess } from "@/components/settings/chatgpt-connection"; import { NativeSubscriptionBackgroundAccess, SubscriptionConnections } from "@/components/settings/subscription-connections"; @@ -135,8 +135,7 @@ export function SetupKeysBody({ onSave, menuAnchor, isCustom, - baseUrl, - onBaseUrl, + onCustomSaved, }: { /** The full provider roster (GET /api/keys providers) — undefined * while it loads; the picker renders an honest empty state then. */ @@ -157,8 +156,7 @@ export function SetupKeysBody({ /** Chosen provider is the custom endpoint: a base URL field joins the key * field. All three optional so plain-key callers (and tests) stay unchanged. */ isCustom?: boolean; - baseUrl?: string; - onBaseUrl?: (v: string) => void; + onCustomSaved?: () => void; }) { return (

@@ -192,20 +190,8 @@ export function SetupKeysBody({ )}
- {provider && ( + {provider && (isCustom ? : (
- {isCustom && ( - onBaseUrl?.(e.target.value)} - placeholder="Base URL, e.g. http://localhost:11434/v1" - aria-label="Custom endpoint base URL" - data-testid="setup-custom-baseurl" - disabled={saving} - className="h-7 w-full rounded-sm border border-border bg-background px-1.5 font-mono text-xs text-foreground outline-none placeholder:text-muted-foreground/60 focus:border-muted-foreground/50" - /> - )}
{ if (e.key === "Enter" && value.trim()) onSave(); }} - placeholder={isCustom ? "API key — required (any value works for local servers)" : "Paste key…"} + placeholder="Paste key…" aria-label={`${provider.label} key`} data-testid="setup-key-input" disabled={saving} @@ -228,11 +214,11 @@ export function SetupKeysBody({ data-testid="setup-key-save" className="shrink-0 rounded-sm border border-border bg-background px-3 py-1.5 font-ui text-xs text-foreground transition-colors hover:border-muted-foreground/50 disabled:cursor-not-allowed disabled:opacity-40" > - {saving ? "Saving…" : isCustom ? "Connect" : "Save key"} + {saving ? "Saving…" : "Save key"}
- )} + ))} {error && (

{error} @@ -286,7 +272,6 @@ function OpenSetupDialog({ onClose, onDone }: SetupDialogProps) { const [keyProviders, setKeyProviders] = useState(undefined); const [keyProvider, setKeyProvider] = useState(null); const [keyValue, setKeyValue] = useState(""); - const [customBaseUrl, setCustomBaseUrl] = useState(""); const [keySaving, setKeySaving] = useState(false); const [keyError, setKeyError] = useState(null); const [keyMenuOpen, setKeyMenuOpen] = useState(false); @@ -398,18 +383,10 @@ function OpenSetupDialog({ onClose, onDone }: SetupDialogProps) { * 400s keep their user-facing line inline. */ async function saveKey() { if (!keyProvider || keySaving) return; - const isCustom = keyProvider.id === CUSTOM_PROVIDER.id; if (!keyValue.trim()) return; setKeySaving(true); setKeyError(null); - // Custom endpoint connects through its own route; every other provider is a plain key PUT. - const res = isCustom - ? await fetch("/api/custom-provider", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ baseUrl: customBaseUrl, key: keyValue }), - }).catch(() => null) - : await fetch(`/api/keys/${encodeURIComponent(keyProvider.envVar)}`, { + const res = await fetch(`/api/keys/${encodeURIComponent(keyProvider.envVar)}`, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ value: keyValue }), @@ -422,7 +399,6 @@ function OpenSetupDialog({ onClose, onDone }: SetupDialogProps) { } setKeyProvider(null); setKeyValue(""); - setCustomBaseUrl(""); notifySettingsSaved("keys"); } @@ -461,8 +437,7 @@ function OpenSetupDialog({ onClose, onDone }: SetupDialogProps) { menuOpen={keyMenuOpen} menuAnchor={keyTriggerRef} isCustom={keyProvider?.id === CUSTOM_PROVIDER.id} - baseUrl={customBaseUrl} - onBaseUrl={setCustomBaseUrl} + onCustomSaved={() => { setKeyProvider(null); notifySettingsSaved("keys"); }} onToggleMenu={() => setKeyMenuOpen((v) => !v)} onPick={(p) => { setKeyProvider(p); setKeyMenuOpen(false); setKeyError(null); }} onValue={setKeyValue} diff --git a/config.ts b/config.ts index daf0b4e..860b571 100644 --- a/config.ts +++ b/config.ts @@ -111,9 +111,6 @@ export const GRAPH_FORMAT_FILE = path.join(PATHS.graph, "format.json"); */ export const SETTINGS_FILE = path.join(DATA_ROOT, "settings.json"); -/** Explicit custom-provider/model definitions; never read the user's Pi profile. */ -export const CUSTOM_MODELS_FILE = path.join(DATA_ROOT, "models.json"); - /** * User prompt-override files (settings-storage wave, 2026-09-19): the * per-segment system-prompt overrides mirroring the SOURCE layout diff --git a/docs/custom-providers.md b/docs/custom-providers.md index bd6901d..eae4abd 100644 --- a/docs/custom-providers.md +++ b/docs/custom-providers.md @@ -1,159 +1,133 @@ -# Custom model providers +# Custom and local models -LearnHarness can register several independently named custom providers using its -existing Pi AI runtime. This is configuration wiring, not another agent runtime: -Pi owns the model protocols, request conversion, streaming, tools, thinking and -usage accounting. There is no additional dependency on Pi coding-agent. +LearnHarness supports **one custom OpenAI-compatible endpoint with multiple models**. +The setup and Settings UI are based on PR #62 by Daniel Buckner and Abhishek Thakur. +Pi owns inference, message conversion, tools, streaming, and reasoning-effort handling. -## Configuration location +## Configure an endpoint -Create `models.json` in the LearnHarness app-data directory (the directory -containing `settings.json`; `LH_DATA_DIR` relocates it). LearnHarness does not read -`~/.pi/agent/models.json`, install a model server, or edit this file for you. -Custom-provider setup UI is not included yet. Once configured, models appear in -the existing model selectors when their credential/header references resolve. +1. Open **Settings → Providers → + Add a provider → Custom endpoint**. The same + editor is available from the first-run provider picker. +2. Enter the server's complete API base URL, for example + `http://localhost:11434/v1`. LearnHarness does **not** append `/v1`. +3. Choose **Keyless** or **API key**: + - **Keyless** never uses a saved or environment custom-provider key for + authentication. Pi's OpenAI client receives an internal dummy credential. + - **API key** uses the effective `CUSTOM_OPENAI_API_KEY`. A real environment + value wins over a file-saved value; the editor identifies environment ownership. + Any nonempty trimmed custom key is accepted—there is no arbitrary length floor. + - Changing the URL requires a new effective key or explicit consent to reuse + the existing effective key. A key is never silently forwarded to a new URL. + Clearing authentication selects Keyless. +4. Enter optional **model IDs**, separated by commas or whitespace. This supports + servers whose chat endpoint works but whose `/models` endpoint is absent. +5. Click **Connect**, then choose models for Strong, Cheap, and Vision. -The file uses Pi's `providers` / `models` structure. For example: +HTTP and HTTPS, including localhost/private-network servers, are supported. URLs +with embedded credentials, query strings, or fragments are rejected. Put a key in +the authentication field, not the URL. Discovery redirects are rejected. -```json -{ - "providers": { - "my-local-server": { - "baseUrl": "http://127.0.0.1:11434/v1", - "api": "openai-completions", - "apiKey": "unused-local-key", - "compat": { - "supportsDeveloperRole": false, - "supportsReasoningEffort": false, - "maxTokensField": "max_tokens" - }, - "models": [ - { - "id": "your-installed-model", - "name": "My local model", - "contextWindow": 8192, - "maxTokens": 1024, - "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, - "cacheWrite": 0 - } - } - ] - }, - "my-gateway": { - "baseUrl": "https://your-gateway.example/v1", - "api": "openai-completions", - "apiKey": "$MY_GATEWAY_API_KEY", - "headers": { "X-Tenant": "${MY_GATEWAY_TENANT}" }, - "models": [ - { - "id": "your-gateway-model-id", - "contextWindow": 32768, - "maxTokens": 4096, - "cost": { - "input": 1, - "output": 3, - "cacheRead": 0, - "cacheWrite": 0 - } - } - ] - } - } -} -``` +Connect requests `GET /models`. Discovery metadata wins over duplicate manually +entered IDs. A failed probe can fall back to entered IDs; a failed probe for the +same URL preserves its previous model snapshot and pins. A different URL starts +with its own models. **401/403 always fails**, even with entered IDs. Response-size, +model-count, and redirect violations also fail explicitly rather than silently +truncating or bypassing a discovery limit. -**Replace all example IDs, limits, prices and compatibility flags with the actual -server/model configuration.** These numbers are examples, not recommendations or -automatically detected capabilities. Rates are USD per million tokens. Explicit -zero rates are allowed; LearnHarness does not assume that custom gateways are -free. Verify the endpoint's billing independently. +Configuration is not a continuous health check. An endpoint may be configured while +offline, and model capability or output quality is not proven by listing its ID. +No backend/model installation, downloads, or OAuth flows are performed. -Use the exact API base URL your server documents. LearnHarness does not append -`/v1`, discover models, infer capabilities from names, or download models. Ollama, -vLLM, LM Studio, 9router and other gateways work only to the extent that their -configured endpoint implements the selected Pi protocol and compatibility flags. -For a keyless OpenAI-compatible local server, supply an explicit nonempty dummy -`apiKey` as above: the native Pi OpenAI client requires a key. Omitting `apiKey` -leaves a provider unavailable rather than implying unauthenticated access. +## Model budgets and reasoning support -## Supported fields +Settings exposes offline-editable pins per model: -Provider fields: `name`, `baseUrl`, `api`, `apiKey`, `headers`, `authHeader`, -`compat`, and the explicit `models` array. Each model requires `id`, -`contextWindow`, `maxTokens`, and `cost` with all four base rates. Unlike Pi -coding-agent's defaults, LearnHarness never invents limits or prices. +- **Context budget**: user pin → reported context → **8192 fallback**. +- **Maximum output**: user pin → reported output limit → **4096 fallback**. +- **Reasoning support**: user declaration → reported support. -Optional Pi model fields are `name`, `api`, `baseUrl`, `reasoning`, `input`, -`thinkingLevelMap`, `inputLimits`, `promptCache`, `samplingParams`, -`samplingParamsByThinkingLevel`, `headers`, and `compat`. Cost tiers and Pi's -exported compatibility schemas are supported. A model's `api` and `baseUrl` -override its provider's values; model headers override provider headers. Names -default to model IDs, `reasoning` defaults to false, and `input` defaults to -`["text"]`. Declare image input only when the server/model supports it. Existing -onboarding and vision-slot requirements are unchanged. +Every positive integer token pin is accepted, including a 512-token output limit. +Blank fields remove that pin. The editor identifies reported, fallback, and user-pin +values. **These are request/compaction budgets; changing them does not resize the +backend's actual context.** Configure the server's active context separately and +enter budgets that match it. Reconnecting prunes pins only to the resulting model +IDs; pin saves do not contact the server. -The native chat transports available in the pinned Pi AI 1.4.0 dependency are: -`openai-completions`, `openai-responses`, `anthropic-messages`, -`google-generative-ai`, `azure-openai-responses`, `google-vertex`, -`bedrock-converse-stream`, `mistral-conversations`, `openai-codex-responses`, and -`pi-messages`. Protocol availability does **not** establish account access, -OAuth support, cloud credentials, subscription permission, or vendor approval. -This feature adds no login flows. In particular, an Anthropic Messages endpoint -is not a Claude Agent SDK or Claude Code runtime. +Reasoning support is a **capability declaration, not an execution toggle**. The +existing Pi-backed effort picker remains available. In particular, **Off may not +disable a custom backend's default thinking**: an absent effort parameter is not a +universal disable command. There is no compatibility/thinking-map editor, Ollama +preset, or model-name inference. Backends requiring additional nonstandard headers +or compatibility configuration are outside this release's custom-endpoint UI. -Use new provider IDs; built-in and subscription-route IDs cannot be replaced. -Provider IDs must be nonempty and cannot contain `:` (the provider/model -coordinate separator). Model IDs must be unique within a provider. Built-in -provider overrides, `modelOverrides`, OAuth definitions, executable plugins, and -automatic discovery are outside this phase. +## Images and Vision -## Credentials and trust +All custom models can be selected for Vision; models that report image support +appear first. The endpoint owner is responsible for choosing an appropriate model. +Built-in model filtering is unchanged. -API keys and header values accept literals, `$NAME`, and `${NAME}`. References -use LearnHarness's existing key lookup: environment first, then the existing -flat `auth.json` file, respecting the fresh-user environment-ignore setting. -There is no auth migration or new secret file format. Prefer references over -putting secrets into `models.json`; protect any file containing literal secrets. -Unknown variable references keep the provider unavailable. All required header -references, including model-specific ones, must resolve before a provider becomes -available. An empty key does not configure a provider. +Missing discovery metadata does not silently remove images: **every custom-engine +image-bearing request attempts delivery**, including teaching `read` results, +rendered PDF pages, and visual-maker screenshots. A request-local model clone permits +Pi's native image encoding; registered/catalog metadata is not promoted to claim +that every model understands images. A text-only backend may reject the request; +that error is surfaced, with no silent text-only fallback. This does not establish +real-model image-understanding quality. -Pi's `$$` and `$!` literal escapes are supported. `!command` credential/header -values are rejected, never executed; resolved environment/file values are data, -not commands. Optional `authHeader: true` adds `Authorization: Bearer ` in -addition to the selected native transport's normal authentication behavior. +## Credentials, reloads, and concurrent editing -Only trusted people should edit this file: its endpoints receive model prompts, -attachments, tool results and configured credentials. Only HTTP(S) URLs are -accepted, without embedded URL credentials or fragments. This is not an -arbitrary-endpoint discovery or config-editing API and does not add a network -sandbox, endpoint reachability checks, or vendor policy exemptions. +Nonsecret configuration is stored in the existing `settings.json.customProvider`; +file-saved credentials use the existing `auth.json` key seam. A reserved internal +string entry, `__learnharness_custom_provider_binding`, binds authorization to the +endpoint, mode, and a random nonsecret `credentialBindingId` in custom settings. +The file key and its binding are published in one existing atomic auth-map write. +The internal entry is not an environment variable or an editable/displayed key. +Neither keys nor raw upstream errors are returned by the custom-provider API. -## Reloading and failures +Every startup and upstream operation requires matching binding IDs, URL, and mode +before using credentials. A crash between auth/settings writes, or a failed +settings write whose auth rollback also fails, leaves the custom provider disabled +rather than sending a new key to an old endpoint. Reload and explicitly reconnect +to repair authorization. A shadowed file key retains its own authorized URL: if an +environment key is reused for another URL and later disappears, the old file key +is not silently forwarded there. Keyless also requires its nonsecret binding, but +never resolves a stored/environment key. Disconnect leaves an internal disabled +marker so a latent environment key cannot revive an interrupted disconnect. -Changes take effect on the next catalog, settings-validation, or inference -operation without restarting. Credential/header reference changes are also -picked up through the existing key lookup. Every upstream model operation -revalidates the same saved provider/model ID against the current Pi catalog, -including later requests inside a long-running agent/tool loop. It uses the -current endpoint, model metadata and resolved credentials. A removed model, -invalid configuration or missing credential blocks the next operation, without -falling back. An HTTP operation already in flight retains its own native Pi -setup and is not implicitly cancelled by a file edit. +Use the custom endpoint editor to change custom credentials; generic +`/api/keys` PUT/DELETE cannot bypass binding. No credential-derived fingerprint is +stored in settings or a public revision. These guards cover process crashes and +restart using the existing atomic rename; they are not a power-loss/fsync +transaction system. Missing auth storage can initialize normally; unreadable or +malformed private storage fails closed without overwriting its contents. -Deleting the file means no custom providers. Invalid JSON, invalid metadata, -reserved IDs, unsupported configuration, or an unreadable file disables the -custom routes, **not** built-in providers. The file accepts Pi's line comments -and trailing commas. `GET /api/models` includes an optional sanitized -`customProvidersError` diagnostic; rendering it in setup/settings UI is deferred. -Correcting the file restores valid providers on the next operation. +Each upstream model operation resolves the current custom model, endpoint, effective +credential, limits, and original saved effort preference against Pi's current model +metadata. Already-started requests keep their setup. A removed model or disconnected +provider fails rather than falling back; saved model selections and history are not +rewritten. -Saved selections and thinking preferences are not erased when a provider/model -is removed or becomes unavailable. New work using an unavailable selection fails -rather than falling back to another model. Select another configured model to -repair the slot. Custom-provider removal does not rewrite historical records; -like existing API-key models, historical cost estimates use the current catalog -and become unknown when the relevant model definition is unavailable. +Edits use an opaque nonsecret ETag/If-Match revision. A stale edit returns **409** with +reload/retry guidance. A connect also checks configuration and effective credentials +after discovery, and merges only the custom section into the latest settings record, +preserving intervening unrelated edits. Failed mutations never claim success. + +Desktop requests use the existing desktop capability. Browser mutations must be +same-origin JSON requests. Limits are **15 seconds** for discovery, **8 MiB** for its +response, **10,000** listed models, and **64 KiB** for configuration-write bodies. +These are protections for the new endpoint, not a multiuser authentication system. + +## Pricing and previous development configuration + +Custom model pricing is **unknown/unpriced**, not $0—even for an endpoint that may +be locally hosted. Pi-required internal zero-rate placeholders are not presented as +free inference. There is no rate editor in this release. + +The earlier, unshipped app-data `models.json` feature is retired and is **not read**. +No user file is deleted or renamed, and there is no importer, coexistence mode, +profile reset, history rewrite, or silent model remapping. Existing development +profiles need manual endpoint configuration and model reselection. Unbound early +GUI development configurations also require explicit reconnect/new-key entry or +the existing reuse-consent checkbox, even at the same URL; they are never silently +trusted or migrated. This release +supports one OpenAI-compatible custom endpoint, not multiple arbitrary protocols. diff --git a/scripts/smoke-provider-runtime.mjs b/scripts/smoke-provider-runtime.mjs index 7cafc1d..eb63e4f 100644 --- a/scripts/smoke-provider-runtime.mjs +++ b/scripts/smoke-provider-runtime.mjs @@ -80,6 +80,14 @@ async function probe(standalone) { // factories, never the checkout's module or a direct/fake Google adapter. const entry = pathToFileURL(path.join(root, "node_modules/@earendil-works/pi-ai/dist/index.js")); const { createModels } = await import(entry.href); + // Keep every native chat implementation evaluable in the standalone + // payload even though the custom-provider UI intentionally exposes one API. + for (const api of ["anthropic-messages", "openai-completions", "openai-responses", "openai-codex-responses", "azure-openai-responses", "google-generative-ai", "google-vertex", "bedrock-converse-stream", "mistral-conversations", "pi-messages"]) { + const implementation = await import(new URL(`api/${api}.js`, entry).href); + assert.equal(typeof implementation.streamSimple, "function", `${api}: native implementation missing`); + } + noNetwork(); + console.log("[smoke-provider-runtime] all 10 native chat implementations evaluated offline"); const models = createModels(); for (const [name, factory, api, label] of [ ["google", "googleProvider", "google-generative-ai", "Google"], diff --git a/src/agent/custom-providers.ts b/src/agent/custom-providers.ts deleted file mode 100644 index 5c82773..0000000 --- a/src/agent/custom-providers.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** Native Pi providers assembled from validated configuration. No IO or registry - * here: the server supplies key lookup, and Pi owns model dispatch/streaming. */ -import { createProvider, type Provider, type ProviderStreams } from "@earendil-works/pi-ai"; -import { anthropicMessagesApi } from "@earendil-works/pi-ai/api/anthropic-messages.lazy"; -import { openAICompletionsApi } from "@earendil-works/pi-ai/api/openai-completions.lazy"; -import { openAIResponsesApi } from "@earendil-works/pi-ai/api/openai-responses.lazy"; -import { openAICodexResponsesApi } from "@earendil-works/pi-ai/api/openai-codex-responses.lazy"; -import { azureOpenAIResponsesApi } from "@earendil-works/pi-ai/api/azure-openai-responses.lazy"; -import { googleGenerativeAIApi } from "@earendil-works/pi-ai/api/google-generative-ai.lazy"; -import { googleVertexApi } from "@earendil-works/pi-ai/api/google-vertex.lazy"; -import { lazyApi } from "@earendil-works/pi-ai/api/lazy"; -import { mistralConversationsApi } from "@earendil-works/pi-ai/api/mistral-conversations.lazy"; -import { piMessagesApi } from "@earendil-works/pi-ai/api/pi-messages.lazy"; -import { resolveCustomValue, type CustomProviderConfig } from "../core/custom-providers.ts"; - -const apis: Record = { - "anthropic-messages": anthropicMessagesApi(), - "openai-completions": openAICompletionsApi(), - "openai-responses": openAIResponsesApi(), - "openai-codex-responses": openAICodexResponsesApi(), - "azure-openai-responses": azureOpenAIResponsesApi(), - "google-generative-ai": googleGenerativeAIApi(), - "google-vertex": googleVertexApi(), - // Pi's Bedrock wrapper intentionally hides its Node-only import from - // bundlers. Give this server's standalone tracer a literal public import; - // Pi still owns both lazy forwarding and the unchanged native transport. - "bedrock-converse-stream": lazyApi(() => import("@earendil-works/pi-ai/api/bedrock-converse-stream")), - "mistral-conversations": mistralConversationsApi(), - "pi-messages": piMessagesApi(), -}; -export const customProviderApis: ReadonlySet = new Set(Object.keys(apis)); - -function resolveHeaders(headers: Record | undefined, lookup: (name: string) => string | undefined): Record | undefined { - const result: Record = {}; - for (const [name, value] of Object.entries(headers ?? {})) { - const resolved = resolveCustomValue(value, lookup); - if (resolved === undefined) return undefined; - result[name] = resolved; - } - return result; -} -export function customProviderAuth(config: CustomProviderConfig, lookup: (name: string) => string | undefined) { - const key = config.definition.apiKey === undefined ? undefined : resolveCustomValue(config.definition.apiKey, lookup); - const headers = resolveHeaders(config.definition.headers, lookup); - if (!key || !headers) return undefined; - // A model header is part of that model's request configuration. Unresolved - // references must not silently disappear from a request. - for (const model of config.models) if (!resolveHeaders(model.headers, lookup)) return undefined; - if (config.definition.authHeader) { - for (const name of Object.keys(headers)) if (name.toLowerCase() === "authorization") delete headers[name]; - headers.Authorization = `Bearer ${key}`; - } - return { apiKey: key, headers }; -} - -export function createCustomProvider(config: CustomProviderConfig, lookup: (name: string) => string | undefined): Provider { - const auth = customProviderAuth(config, lookup); - return createProvider({ - id: config.id, - name: config.definition.name ?? config.id, - baseUrl: config.definition.baseUrl, - models: config.models.map((model) => ({ ...model, headers: resolveHeaders(model.headers, lookup) })), - auth: { apiKey: { - name: "API key", - check: async () => auth ? { type: "api_key", source: "custom provider configuration" } : undefined, - resolve: async () => auth ? { auth, source: "custom provider configuration" } : undefined, - } }, - // Direct native implementations: no copied stream or request conversion. - api: apis, - }); -} diff --git a/src/agent/models.ts b/src/agent/models.ts index 8edd8d5..37a98d2 100644 --- a/src/agent/models.ts +++ b/src/agent/models.ts @@ -9,7 +9,7 @@ import { type StreamFunction, } from "@earendil-works/pi-ai"; import { builtinProviders } from "@earendil-works/pi-ai/providers/all"; -import { lazyStream } from "@earendil-works/pi-ai/api/lazy"; +import { lazyApi, lazyStream } from "@earendil-works/pi-ai/api/lazy"; import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; import { type ModelRole } from "../../config.ts"; import { DEFAULT_THINKING_ORDER } from "../core/settings.ts"; @@ -82,7 +82,14 @@ export function configureProviderSource(source: ProviderSource): void { export function piModels() { if (!collection) { collection = createModels({ identity: APP_IDENTITY }); - for (const provider of builtinProviders()) collection.setProvider(provider); + for (const provider of builtinProviders()) { + // Keep the native Bedrock implementation reachable by standalone tracing. + // Pi's default Node-only loader hides this import from bundlers. Only + // replace its lazy forwarding; preserve the native factory's auth/catalog. + collection.setProvider(provider.id === "amazon-bedrock" + ? { ...provider, ...lazyApi(() => import("@earendil-works/pi-ai/api/bedrock-converse-stream")) } + : provider); + } collection.setProvider(createClaudeCodeProvider(streamClaudeCode)); } providerSource?.sync(collection); @@ -124,7 +131,12 @@ function defaultThinkingLevel(model: Model): ThinkingLevel { export function piEngineForSpec(spec: ResolvedModelSpec, background: boolean | (() => boolean) = true, lifecycle?: ChatGPTDispatchLifecycle): PiEngine { const models: Models = piModels(); const nativeId = nativeSubscriptionId(spec.provider); - const model = spec.provider === "chatgpt" ? chatGPTModel(spec.id) : nativeId ? nativeSubscriptionModel(spec.provider, spec.id) : models.getModel(spec.provider, spec.id); + const registeredModel = spec.provider === "chatgpt" ? chatGPTModel(spec.id) : nativeId ? nativeSubscriptionModel(spec.provider, spec.id) : models.getModel(spec.provider, spec.id); + const custom = providerSource?.isCustom(spec.provider) ?? false; + // Custom endpoint owners authorize image attempts independently of sparse + // discovery metadata. Never promote the registered/catalog capabilities. + const permitImages = (value: Model): Model => ({ ...value, input: ["text", "image"] }); + const model = custom && registeredModel ? permitImages(registeredModel) : registeredModel; if (!model) { throw new Error(`${spec.provider} catalog has no model ${spec.id}`); } @@ -137,7 +149,6 @@ export function piEngineForSpec(spec: ResolvedModelSpec, background: boolean | ( : defaultThinkingLevel(model); // A lesson's agent loop can stay alive through many tool calls. Remember // only that this is a custom route, never its configuration for that run. - const custom = providerSource?.isCustom(spec.provider) ?? false; return { model, level, @@ -156,7 +167,7 @@ export function piEngineForSpec(spec: ResolvedModelSpec, background: boolean | ( const currentLevel = thinkingPreference ? clampThinkingLevel(currentModel, thinkingPreference) : defaultThinkingLevel(currentModel); - return current.streamSimple(currentModel, context, { ...options, reasoning: currentLevel === "off" ? undefined : currentLevel }); + return current.streamSimple(permitImages(currentModel), context, { ...options, reasoning: currentLevel === "off" ? undefined : currentLevel }); } const requestOptions = { ...options, reasoning: level === "off" ? undefined : level }; const purpose = typeof background === "function" ? background : () => background; diff --git a/src/core/contracts/models.ts b/src/core/contracts/models.ts index e0225a2..22000ec 100644 --- a/src/core/contracts/models.ts +++ b/src/core/contracts/models.ts @@ -75,8 +75,6 @@ export interface ModelCatalogResponse { * are simply absent (the Keys page is the discovery surface, same * discipline as the search engines). */ providers: CatalogProvider[]; - /** Sanitized configuration diagnostic; built-in catalogs remain available. */ - customProvidersError?: string; } /** One row of the providers surface. */ diff --git a/src/core/custom-provider-auth.ts b/src/core/custom-provider-auth.ts new file mode 100644 index 0000000..8f161ee --- /dev/null +++ b/src/core/custom-provider-auth.ts @@ -0,0 +1,26 @@ +/** Private custom credential authorization; no credential-derived public value. */ +import { normalizeBaseUrl, type CustomProviderSettings } from "./settings.ts"; + +export const CUSTOM_AUTH_BINDING = "__learnharness_custom_provider_binding"; +export interface CustomAuthBinding { + version: 1; + id: string; + baseUrl?: string; + authMode: "apiKey" | "keyless" | "disabled"; + /** A shadowed file key keeps its own destination when an env key moves. */ + fileKeyBaseUrl?: string; +} +export function parseCustomAuthBinding(value: unknown): CustomAuthBinding | undefined { + try { + const b = typeof value === "string" ? JSON.parse(value) : undefined; + if (!b || b.version !== 1 || typeof b.id !== "string" || !/^[a-f0-9]{32}$/.test(b.id)) return; + if (!["apiKey", "keyless", "disabled"].includes(b.authMode)) return; + if (b.authMode !== "disabled" && b.baseUrl === undefined) return; + if (b.baseUrl !== undefined && (typeof b.baseUrl !== "string" || !b.baseUrl || normalizeBaseUrl(b.baseUrl) !== b.baseUrl)) return; + if (b.fileKeyBaseUrl !== undefined && (typeof b.fileKeyBaseUrl !== "string" || !b.fileKeyBaseUrl || normalizeBaseUrl(b.fileKeyBaseUrl) !== b.fileKeyBaseUrl)) return; + return { version: 1, id: b.id, authMode: b.authMode, ...(b.baseUrl !== undefined ? { baseUrl: b.baseUrl } : {}), ...(b.fileKeyBaseUrl !== undefined ? { fileKeyBaseUrl: b.fileKeyBaseUrl } : {}) }; + } catch { return; } +} +export function customBindingMatches(cp: CustomProviderSettings | undefined, binding: CustomAuthBinding | undefined): boolean { + return Boolean(cp?.baseUrl && binding && binding.authMode !== "disabled" && cp.credentialBindingId === binding.id && cp.baseUrl === binding.baseUrl && cp.authMode === binding.authMode); +} diff --git a/src/core/custom-providers.ts b/src/core/custom-providers.ts deleted file mode 100644 index 9aaed27..0000000 --- a/src/core/custom-providers.ts +++ /dev/null @@ -1,146 +0,0 @@ -/** Declarative custom-provider configuration. The envelope/normalization follows - * Pi coding-agent 1.4.0's model-config/provider-composer (MIT, Mario Zechner). - * Pi AI owns the shared model/compatibility schemas and the runtime models. - * Deliberate differences: new provider IDs only, explicit limits/prices, - * and no OAuth, extensions, overrides, or executable credential commands. - * - * Adapted portions: Copyright (c) Mario Zechner. Permission is hereby granted, - * free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, modify, - * merge, publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, subject to - * the following conditions: The above copyright notice and this permission - * notice shall be included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ -import { Type, type Static } from "typebox"; -import { Compile } from "typebox/compile"; -import type { Api, Model } from "@earendil-works/pi-ai"; -import { ProviderCompatSchema } from "@earendil-works/pi-ai/providers/compat-schema"; -import { ModelCostSchema, ModelInputLimitsSchema, ModelInputModalitySchema, ModelPromptCacheSchema, ThinkingLevelMapSchema } from "@earendil-works/pi-ai/providers/model-schema"; - -const Headers = Type.Record(Type.String(), Type.String()); -const Sampling = Type.Record(Type.String(), Type.Unknown()); -const ModelSchema = Type.Object({ - id: Type.String({ minLength: 1 }), - name: Type.Optional(Type.String({ minLength: 1 })), - api: Type.Optional(Type.String({ minLength: 1 })), - baseUrl: Type.Optional(Type.String({ minLength: 1 })), - reasoning: Type.Optional(Type.Boolean()), - thinkingLevelMap: Type.Optional(ThinkingLevelMapSchema), - input: Type.Optional(Type.Array(ModelInputModalitySchema)), - inputLimits: Type.Optional(ModelInputLimitsSchema), - cost: ModelCostSchema, - promptCache: Type.Optional(ModelPromptCacheSchema), - contextWindow: Type.Number({ exclusiveMinimum: 0 }), - maxTokens: Type.Number({ exclusiveMinimum: 0 }), - samplingParams: Type.Optional(Sampling), - samplingParamsByThinkingLevel: Type.Optional(Type.Object(Object.fromEntries( - ["off", "minimal", "low", "medium", "high", "xhigh", "max"].map((level) => [level, Type.Optional(Sampling)]), - ))), - headers: Type.Optional(Headers), - compat: Type.Optional(ProviderCompatSchema), -}, { additionalProperties: false }); -const ProviderSchema = Type.Object({ - name: Type.Optional(Type.String({ minLength: 1 })), - baseUrl: Type.Optional(Type.String({ minLength: 1 })), - api: Type.Optional(Type.String({ minLength: 1 })), - apiKey: Type.Optional(Type.String({ minLength: 1 })), - headers: Type.Optional(Headers), - compat: Type.Optional(ProviderCompatSchema), - authHeader: Type.Optional(Type.Boolean()), - models: Type.Array(ModelSchema), -}, { additionalProperties: false }); -const ConfigSchema = Type.Object({ - $schema: Type.Optional(Type.String()), - providers: Type.Record(Type.String(), ProviderSchema), -}, { additionalProperties: false }); -const validator = Compile(ConfigSchema); -export type CustomProviderDefinition = Static; -export interface CustomProviderConfig { - id: string; - definition: CustomProviderDefinition; - models: Model[]; -} - -/** Error messages describe fields only: never include file contents or values. */ -export class CustomProviderConfigError extends Error {} -function invalid(message: string): never { throw new CustomProviderConfigError(message); } - -/** Pi-style values, but never execute commands. Escapes are processed once; - * a literal/variable's resolved value beginning with ! is still just data. */ -export function resolveCustomValue(value: string, lookup: (name: string) => string | undefined): string | undefined { - if (value.startsWith("!")) invalid("Credential commands are not supported in models.json."); - let missing = false; - const resolved = value.replace(/\$\$|\$!|\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)/g, (match, braced: string | undefined, plain: string | undefined) => { - if (match === "$$") return "$"; - if (match === "$!") return "!"; - const found = lookup(braced ?? plain!); - if (found === undefined) missing = true; - return found ?? ""; - }); - return missing ? undefined : resolved; -} - -function endpoint(value: string | undefined): string { - if (!value) invalid("Each custom model needs an api and baseUrl, at provider or model level."); - try { - const url = new URL(value); - if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.hash) throw new Error(); - } catch { invalid("Custom baseUrl must be an HTTP(S) URL without embedded credentials or a fragment."); } - return value; -} -function compat(base: Model["compat"], override: Model["compat"]): Model["compat"] { - if (!base && !override) return undefined; - const merged = { ...base, ...override } as Record; - for (const name of ["openRouterRouting", "vercelGatewayRouting", "chatTemplateKwargs", "chatTemplateArgs"]) { - const a = (base as Record | undefined)?.[name]; - const b = (override as Record | undefined)?.[name]; - if (a || b) merged[name] = { ...(a as object | undefined), ...(b as object | undefined) }; - } - return merged as Model["compat"]; -} - -export function parseCustomProviders(text: string, reserved: ReadonlySet, supportedApis: ReadonlySet): CustomProviderConfig[] { - let raw: unknown; - // Match Pi's line comments/trailing commas without touching JSON strings. - const json = text.replace(/^\uFEFF/, "") - .replace(/"(?:\\.|[^"\\])*"|\/\/[^\n]*/g, (match) => match[0] === '"' ? match : "") - .replace(/"(?:\\.|[^"\\])*"|,(\s*[}\]])/g, (match, tail: string | undefined) => tail ?? (match[0] === '"' ? match : "")); - try { raw = JSON.parse(json); } - catch { return invalid("models.json is not valid JSON."); } - if (!validator.Check(raw)) invalid("Invalid models.json structure: check provider/model fields and explicit contextWindow, maxTokens and cost."); - return Object.entries(raw.providers).map(([id, definition]) => { - if (!id.trim() || id.includes(":")) invalid("Custom provider IDs must be nonempty and must not contain a colon."); - if (reserved.has(id)) invalid("Custom provider IDs must not replace built-in or subscription providers."); - for (const value of [definition.apiKey, ...Object.values(definition.headers ?? {}), ...definition.models.flatMap((m) => Object.values(m.headers ?? {}))]) { - if (value !== undefined) resolveCustomValue(value, () => "validation"); - } - const ids = new Set(); - const models = definition.models.map((model): Model => { - if (ids.has(model.id)) invalid("Duplicate model IDs are not supported within a custom provider."); - ids.add(model.id); - const api = model.api ?? definition.api; - if (!api || !supportedApis.has(api)) invalid("A custom model specifies a missing or unsupported Pi API."); - const rates = [model.cost, ...(model.cost.tiers ?? [])]; - if (![model.contextWindow, model.maxTokens].every(Number.isFinite) || rates.some((rate) => Object.values(rate).some((n) => typeof n === "number" && (!Number.isFinite(n) || n < 0)))) invalid("Custom model limits and costs must be finite; limits must be positive and costs nonnegative."); - return { - ...model, - provider: id, - api, - baseUrl: endpoint(model.baseUrl ?? definition.baseUrl), - name: model.name ?? model.id, - reasoning: model.reasoning ?? false, - input: model.input ?? ["text"], - compat: compat(definition.compat, model.compat), - }; - }); - return { id, definition, models }; - }); -} diff --git a/src/core/settings.ts b/src/core/settings.ts index 8dc6a37..3d6ded3 100644 --- a/src/core/settings.ts +++ b/src/core/settings.ts @@ -215,15 +215,12 @@ export interface ModelsSettings { // secret in the auth store, never here. Registered into pi-ai as a normal // provider (server/custom-provider.ts). -/** Defaults for a custom model — self-hosted endpoints are free. */ +/** Budget fallbacks only; custom endpoint pricing is unknown. */ export const CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW = 8192; export const CUSTOM_MODEL_DEFAULT_MAX_TOKENS = 4096; -/** Floor for a user-pinned token count (owner, 2026-10-27): the read side - * clamps up, the write side rejects below — the compaction-floor discipline. - * Pins exist because endpoints that publish no limits (LM Studio) otherwise - * sit on the 8192 default with no way to say better. */ -export const CUSTOM_MODEL_PIN_FLOOR = 2_048; +/** Custom budgets accept every positive integer, including small output caps. */ +export const CUSTOM_MODEL_PIN_FLOOR = 1; /** One discovered custom model. `image`/`reasoning`/`context`/`maxTokens` are * detected from the endpoint's /models metadata; anything it omits falls back @@ -244,8 +241,8 @@ export interface CustomModel { /** A user pin for one custom model: it beats what the endpoint publishes * (owner ruling — explicit user intent wins; a local server's number is * often wrong or absent). Absent field = no pin. `reasoning` is tri-state: - * true forces thinking on, false forces it off (a server that advertises - * reasoning but rejects reasoning_effort), absent = what discovery found. */ + * true declares reasoning support, false removes that declaration, absent + * uses discovery. This is NOT a backend execution/disable switch. */ export interface CustomModelOverride { context?: number; maxTokens?: number; @@ -254,6 +251,10 @@ export interface CustomModelOverride { /** The single custom endpoint's stored config (delta section). */ export interface CustomProviderSettings { + /** Server-owned random authorization nonce; never a credential hash. */ + credentialBindingId?: string; + /** Explicit authentication intent; never inferred from a saved key. */ + authMode?: "keyless" | "apiKey"; /** Normalized base URL (see normalizeBaseUrl). Empty/absent = not configured. */ baseUrl?: string; /** Models this endpoint offers: advertised ids merged with the typed @@ -287,6 +288,7 @@ export function normalizeBaseUrl(raw: string): string { return ""; } if (url.protocol !== "http:" && url.protocol !== "https:") return ""; + if (url.username || url.password || url.search || url.hash || value.includes("?") || value.includes("#")) return ""; return url.toString().replace(/\/+$/, ""); } @@ -625,7 +627,7 @@ export function sanitizeCustomModelOverrides( const id = key.trim(); if (!id || (live && !live.has(id))) continue; const pin = sanitizeCustomModelOverride(value); - if (pin) out[id] = pin; + if (pin) Object.defineProperty(out, id, { value: pin, enumerable: true, configurable: true, writable: true }); } return out; } @@ -635,6 +637,8 @@ export function sanitizeCustomModelOverrides( * true; token counts only as positive ints). Clamp-never-reject. */ function sanitizeCustomProvider(raw: Record): CustomProviderSettings { const out: CustomProviderSettings = {}; + if (typeof raw.credentialBindingId === "string" && /^[a-f0-9]{32}$/.test(raw.credentialBindingId)) out.credentialBindingId = raw.credentialBindingId; + if (raw.authMode === "keyless" || raw.authMode === "apiKey") out.authMode = raw.authMode; if (typeof raw.baseUrl === "string") { const base = normalizeBaseUrl(raw.baseUrl); if (base) out.baseUrl = base; diff --git a/src/server/custom-provider-auth.ts b/src/server/custom-provider-auth.ts new file mode 100644 index 0000000..aa04025 --- /dev/null +++ b/src/server/custom-provider-auth.ts @@ -0,0 +1,43 @@ +/** Custom-only credential composition. File credentials and their authorization + * are read from the same atomic record, never a stale fill-gapped env copy. */ +import { CUSTOM_PROVIDER } from "../core/auth-keys.ts"; +import { authEnvRecord } from "../core/auth-env.ts"; +import { customBindingMatches, type CustomAuthBinding } from "../core/custom-provider-auth.ts"; +import type { CustomProviderSettings } from "../core/settings.ts"; +import { readCustomAuth, writeCustomAuth } from "../store/auth.ts"; + +export interface CustomAuthState { + readable: boolean; + binding?: CustomAuthBinding; + key?: string; + fileKey?: string; + source: "env" | "file" | "none"; +} +/** requiredConfig prevents even credential resolution until startup/runtime + * authorization matches. Explicit UI inspection/probe authorization omits it. */ +export function customAuthState(inspectKey = false, requiredConfig?: CustomProviderSettings): CustomAuthState { + try { + const snapshot = readCustomAuth(); + const result: CustomAuthState = { readable: true, binding: snapshot.binding, source: "none" }; + if (!inspectKey || (requiredConfig && !customBindingMatches(requiredConfig, snapshot.binding))) return result; + result.fileKey = snapshot.fileKey(); + const record = authEnvRecord(); + const envKey = process.env.LH_IGNORE_ENV_KEYS !== "1" && record.realEnv.has(CUSTOM_PROVIDER.envVar) ? process.env[CUSTOM_PROVIDER.envVar] : undefined; + result.source = envKey !== undefined ? "env" : result.fileKey !== undefined ? "file" : "none"; + result.key = (envKey ?? result.fileKey)?.trim() || undefined; + return result; + } catch { return { readable: false, source: "none" }; } +} +export function customKeyAuthorized(cp: CustomProviderSettings | undefined, state: CustomAuthState): boolean { + return cp?.authMode === "apiKey" && state.readable && customBindingMatches(cp, state.binding) && Boolean(state.key) && (state.source === "env" || state.binding?.fileKeyBaseUrl === cp?.baseUrl); +} +/** Reflect only after the atomic pair is committed. Runtime file resolution + * does not rely on this compatibility mirror, including after a failed write. */ +export function persistCustomAuth(binding: CustomAuthBinding | undefined, fileKey: string | undefined): void { + writeCustomAuth(binding, fileKey); + const record = authEnvRecord(); + if (!record.realEnv.has(CUSTOM_PROVIDER.envVar)) { + if (fileKey === undefined) { delete process.env[CUSTOM_PROVIDER.envVar]; record.fillGap.delete(CUSTOM_PROVIDER.envVar); } + else { process.env[CUSTOM_PROVIDER.envVar] = fileKey; record.fillGap.add(CUSTOM_PROVIDER.envVar); } + } +} diff --git a/src/server/custom-provider-body.ts b/src/server/custom-provider-body.ts new file mode 100644 index 0000000..4abac5e --- /dev/null +++ b/src/server/custom-provider-body.ts @@ -0,0 +1,31 @@ +/** Byte-bounded JSON for the custom endpoint's request/discovery surfaces. */ +export class CustomBodyLimitError extends Error {} + +export async function readBoundedJson(message: Request | Response, limit: number, allowEmptyObject = false): Promise { + if (Number(message.headers.get("content-length")) > limit) { + await message.body?.cancel(); + throw new CustomBodyLimitError("JSON body exceeds the allowed byte limit"); + } + const reader = message.body?.getReader(); + if (!reader) { + if (allowEmptyObject) return {}; + throw new SyntaxError("Missing JSON body"); + } + const decoder = new TextDecoder(); + let text = ""; + let bytes = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > limit) throw new CustomBodyLimitError("JSON body exceeds the allowed byte limit"); + text += decoder.decode(value, { stream: true }); + } + if (bytes === 0 && allowEmptyObject) return {}; + return JSON.parse(text + decoder.decode()); + } finally { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } +} diff --git a/src/server/custom-provider-http.ts b/src/server/custom-provider-http.ts new file mode 100644 index 0000000..ad240cb --- /dev/null +++ b/src/server/custom-provider-http.ts @@ -0,0 +1,149 @@ +/** PR #62's connect/manual-model/offline-pin workflow, with host request, + * credential-binding and optimistic-write boundaries. No secret reaches GET. */ +import { createHmac, randomBytes } from "node:crypto"; +import { normalizeBaseUrl, parseModelIds, sanitizeCustomModelOverrides, type CustomProviderSettings } from "../core/settings.ts"; +import { readSettings, writeSettings } from "../store/settings.ts"; +import { readCustomAuth } from "../store/auth.ts"; +import { customAuthState, customKeyAuthorized, persistCustomAuth } from "./custom-provider-auth.ts"; +import type { CustomAuthBinding } from "../core/custom-provider-auth.ts"; +import { desktopAPIAuthorized } from "./desktop-access.ts"; +import { subscriptionRequestOrigin } from "./web-subscription-http.ts"; +import { discoverModels, mergeManualModels, syncCustomProvider } from "./custom-provider.ts"; +import { parseCustomModelOverridesBody } from "./settings-body.ts"; +import { CustomBodyLimitError, readBoundedJson } from "./custom-provider-body.ts"; + +const revisionKey = Symbol.for("learnharness.custom-provider-revision"); +const globals = globalThis as typeof globalThis & { [revisionKey]?: Buffer }; +const secret = globals[revisionKey] ??= randomBytes(32); +const failure = (error: string, status = 400) => Response.json({ error }, { status }); +const stale = () => failure("Custom configuration or credentials changed. Reload configuration and retry.", 409); + +function snapshot(inspectKey = false) { + const record = readSettings() ?? { version: 1 }; + const cp = record.customProvider; + // Keyless never looks up a key for authentication or revision generation. + // An explicit API-key inspection is used only after the user selects that mode. + const auth = customAuthState(inspectKey); + const key = auth.key; + // Public revisions contain configuration only, never credential material. + // Effective credentials are compared privately across the awaited probe. + const etag = `"${createHmac("sha256", secret).update(JSON.stringify([cp ?? null, auth.binding ?? null, auth.readable])).digest("base64url")}"`; + return { record, cp, auth, key, etag }; +} + +function response(body: object) { + return Response.json(body, { headers: { etag: snapshot().etag, "cache-control": "no-store" } }); +} + +export function customProviderGet(request: Request): Response { + const inspect = new URL(request.url).searchParams.get("authMode") === "apiKey"; + const current = snapshot(); + const s = snapshot(inspect || current.cp?.authMode === "apiKey"); + return response({ + baseUrl: s.cp?.baseUrl ?? "", authMode: s.cp?.authMode ?? "keyless", + models: s.cp?.models ?? [], overrides: s.cp?.overrides ?? {}, manualModels: s.cp?.manualModels ?? [], + keyPresent: Boolean(s.key), keySource: s.auth.source, + keyReuseRequired: Boolean(s.key) && !customKeyAuthorized(s.cp, s.auth), + }); +} + +function authorized(request: Request): boolean { + const desktop = process.env.LH_DESKTOP_SERVER === "1" || Boolean(process.env.LH_DESKTOP_API_SECRET); + if (desktop) return desktopAPIAuthorized(request.headers); + const origin = request.headers.get("origin"); + if (!origin || origin === "null") return false; + try { + // Next's standalone Request URL can use its internal hostname. Match + // the actual addressed Host through the app's existing origin seam; + // never trust forwarded-origin headers. + return origin === subscriptionRequestOrigin(request) + && (!request.headers.has("sec-fetch-site") || request.headers.get("sec-fetch-site") === "same-origin"); + } catch { return false; } +} + +/** Auth pair FIRST, matching config nonce SECOND. Any interrupted/failed + * cross-file publication is durably unusable, including with a real env key. + * Best-effort rollback restores the entire old pair, never just its secret. */ +function commit(cp: CustomProviderSettings | undefined, binding?: CustomAuthBinding, fileKey?: string): void { + const latest = readSettings() ?? { version: 1 }; + const oldAuth = binding ? readCustomAuth() : undefined; + let published = false; + try { + if (binding) { persistCustomAuth(binding, fileKey); published = true; } + const { customProvider: _old, ...rest } = latest; + writeSettings(cp ? { ...rest, customProvider: cp } : rest); + } catch { + if (published && oldAuth) { + try { persistCustomAuth(oldAuth.binding, oldAuth.fileKey()); } catch { /* durable nonce mismatch fails closed after restart */ } + } + throw new Error("Unable to save custom configuration. Reload configuration and retry."); + } + syncCustomProvider(); +} + +export async function customProviderMutation(request: Request): Promise { + if (!authorized(request)) return failure("Custom configuration requires a trusted same-origin request.", 403); + if (request.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() !== "application/json") return failure("JSON content-type is required.", 415); + try { + // Next can represent an empty DELETE as a zero-byte stream rather + // than null. Bound it either way; only DELETE permits an empty body. + const body = await readBoundedJson(request, 64 * 1024, request.method === "DELETE"); + if (!body || typeof body !== "object" || Array.isArray(body)) return failure("Invalid JSON body"); + const raw = body as Record; + const before = snapshot(raw.authMode === "apiKey"); + if (!before.auth.readable) return failure("Custom authorization storage is unavailable. No changes were saved.", 500); + if (request.headers.get("if-match") !== before.etag) return stale(); + if (request.method === "DELETE") { + commit(undefined, { version: 1, id: randomBytes(16).toString("hex"), authMode: "disabled" }); + return response({ ok: true }); + } + if (request.method === "PATCH") { + const parsed = parseCustomModelOverridesBody(raw); + if (!parsed.ok) return failure(parsed.error); + if (!before.cp?.baseUrl) return failure("No custom endpoint configured", 409); + const overrides = sanitizeCustomModelOverrides(parsed.settings, before.cp.models?.map((m) => m.id)); + commit({ ...before.cp, overrides }); + return response({ ok: true, overrides }); + } + if (request.method !== "POST") return failure("Unsupported method", 405); + const base = normalizeBaseUrl(typeof raw.baseUrl === "string" ? raw.baseUrl : ""); + if (!base) return failure("Base URL must be http(s), with no credentials, query or fragment."); + if (raw.authMode !== "keyless" && raw.authMode !== "apiKey") return failure("Choose Keyless or API key authentication."); + const authMode = raw.authMode; + if (raw.key !== undefined && typeof raw.key !== "string") return failure("API key must be a string."); + const supplied = typeof raw.key === "string" ? raw.key.trim() : undefined; + if (authMode === "apiKey" && supplied === "") return failure("Choose Keyless to clear authentication."); + const envOwned = before.auth.source === "env"; + if (envOwned && supplied) return failure("Environment owns the effective key. Change the environment or explicitly reuse that key."); + // A shadowed file key is not the key that will run. Require consent for + // the actual environment key when authorizing a different endpoint. + const hasNewEffectiveKey = Boolean(supplied) && !envOwned; + const changedTarget = before.cp?.baseUrl !== base; + if (authMode === "apiKey" && (changedTarget || !customKeyAuthorized(before.cp, before.auth)) && !hasNewEffectiveKey && raw.reuseKey !== true) { + return failure("Enter a new key or explicitly allow reuse of the effective existing key for this URL."); + } + const effectiveKey = authMode === "apiKey" ? (envOwned ? before.key : supplied || before.key) : undefined; + if (authMode === "apiKey" && !effectiveKey) return failure("API key mode requires an effective key."); + const manualModels = parseModelIds(raw.manualModels); + if (manualModels.length > 10_000) return failure("Model list exceeds 10000 entries"); + const found = await discoverModels(base, effectiveKey); + if (!found.ok && (found.fatal || found.status === 401 || found.status === 403)) return failure(found.error, 502); + if (!found.ok && !manualModels.length) return failure(`${found.error}. If this endpoint has no /models list, add model IDs manually.`, 502); + // Discovery yields; both the custom record and the credential must still + // match. Unrelated settings edits are preserved by commit's fresh merge. + const now = snapshot(authMode === "apiKey"); + if (before.etag !== now.etag || JSON.stringify(before.auth) !== JSON.stringify(now.auth)) return stale(); + const discovered = found.ok ? found.models : !changedTarget ? before.cp?.models ?? [] : []; + const models = mergeManualModels(discovered, manualModels); + if (models.length > 10_000) return failure("Model list exceeds 10000 entries"); + const overrides = sanitizeCustomModelOverrides(!changedTarget ? before.cp?.overrides : undefined, models.map((m) => m.id)); + const id = randomBytes(16).toString("hex"); + const fileKey = authMode === "keyless" ? undefined : supplied && !envOwned ? supplied : now.auth.fileKey; + const fileKeyBaseUrl = authMode === "apiKey" ? envOwned ? now.auth.binding?.fileKeyBaseUrl : base : undefined; + const binding: CustomAuthBinding = { version: 1, id, baseUrl: base, authMode, ...(fileKey && fileKeyBaseUrl ? { fileKeyBaseUrl } : {}) }; + commit({ baseUrl: base, authMode, credentialBindingId: id, models, overrides, ...(manualModels.length ? { manualModels } : {}) }, binding, fileKey); + return response({ ok: true, models }); + } catch (error) { + return failure(error instanceof CustomBodyLimitError ? "Configuration body exceeds 64 KiB." : error instanceof SyntaxError ? "Invalid JSON body." : "Unable to save custom configuration. Reload configuration and retry.", error instanceof CustomBodyLimitError ? 413 : error instanceof SyntaxError ? 400 : 500); + } +} diff --git a/src/server/custom-provider.ts b/src/server/custom-provider.ts index 2d947e6..b717022 100644 --- a/src/server/custom-provider.ts +++ b/src/server/custom-provider.ts @@ -5,10 +5,14 @@ * local servers work via a placeholder key resolved at request time. */ -import { createProvider, type ApiKeyAuth, type Api, type Model, type Provider } from "@earendil-works/pi-ai"; +import { createProvider, type Api, type Model, type Provider } from "@earendil-works/pi-ai"; import { openAICompletionsApi } from "@earendil-works/pi-ai/api/openai-completions.lazy"; -import { piModels } from "../agent/models.ts"; +import { configureProviderSource, piModels } from "../agent/models.ts"; +import { readSettings } from "../store/settings.ts"; +import { CustomBodyLimitError, readBoundedJson } from "./custom-provider-body.ts"; import { CUSTOM_PROVIDER } from "../core/auth-keys.ts"; +import { customBindingMatches } from "../core/custom-provider-auth.ts"; +import { customAuthState, customKeyAuthorized } from "./custom-provider-auth.ts"; import { CUSTOM_MODEL_DEFAULT_CONTEXT_WINDOW, CUSTOM_MODEL_DEFAULT_MAX_TOKENS, @@ -16,9 +20,8 @@ import { normalizeBaseUrl, type CustomModel, type CustomModelOverride, - type CustomProviderSettings, } from "../core/settings.ts"; -import { keyValue } from "./keys.ts"; + /** Placeholder bearer for keyless endpoints (the OpenAI client needs *a* key). */ const KEYLESS_PLACEHOLDER = "not-needed"; @@ -27,14 +30,32 @@ const KEYLESS_PLACEHOLDER = "not-needed"; const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); -/** Auth: the reserved key via the keys seam, else the keyless placeholder. */ -const customEndpointAuth: ApiKeyAuth = { - name: `${CUSTOM_PROVIDER.label} API key`, - resolve: async () => { - const key = keyValue(CUSTOM_PROVIDER.envVar)?.trim(); - return { auth: { apiKey: key || KEYLESS_PLACEHOLDER }, source: key ? CUSTOM_PROVIDER.envVar : "keyless" }; - }, -}; +/** Install the sole custom source: donor settings + explicit authentication. + * Publication captures a consistent snapshot; an already-started operation + * cannot acquire a later endpoint's key. No file configuration source exists. */ +export function installCustomProviderSource(readAuth = customAuthState): void { + let fingerprint: string | undefined; + configureProviderSource({ + isCustom: (id) => id === CUSTOM_PROVIDER.id, + sync(models) { + const cp = readSettings()?.customProvider; + // Validate the durable nonce/URL/mode before resolving ANY credential. + // Keyless reads only the nonsecret marker, never a file/env key. + const auth = readAuth(cp?.authMode === "apiKey", cp); + const key = auth.key; + const ready = customProviderUsable(cp) && auth.readable && customBindingMatches(cp, auth.binding) && (cp.authMode === "keyless" || customKeyAuthorized(cp, auth)); + const next = JSON.stringify([cp, auth]); // Private in-memory fingerprint, never a public revision. + if (next === fingerprint) return; + if (ready) models.setProvider(buildOpenAiCompatibleProvider({ ...cp, baseUrl: cp.baseUrl, models: cp.models ?? [], apiKey: key })); + else models.deleteProvider(CUSTOM_PROVIDER.id); + fingerprint = next; + }, + }); +} + +export function customProviderAvailable(): boolean { + return piModels().getProvider(CUSTOM_PROVIDER.id) !== undefined; +} /** Map a discovered model to a pi-ai entry: image→input, reasoning→thinking * levels, context/maxTokens→limits. Resolution ladder (owner, 2026-10-27): @@ -72,13 +93,18 @@ export function buildOpenAiCompatibleProvider(input: { models: readonly CustomModel[]; /** User pins by model id (see CustomModelOverride) — absent = none. */ overrides?: Record; + /** Already resolved for this publication; absent means explicit keyless. */ + apiKey?: string; }): Provider { const overrides = input.overrides ?? {}; return createProvider({ id: CUSTOM_PROVIDER.id, name: CUSTOM_PROVIDER.label, baseUrl: input.baseUrl, - auth: { apiKey: customEndpointAuth }, + auth: { apiKey: { + name: `${CUSTOM_PROVIDER.label} API key`, + resolve: async () => ({ auth: { apiKey: input.apiKey ?? KEYLESS_PLACEHOLDER }, source: input.apiKey ? CUSTOM_PROVIDER.envVar : "keyless" }), + } }, models: input.models.map((m) => toPiModel(m, input.baseUrl, overrides)), api: openAICompletionsApi(), }); @@ -86,19 +112,8 @@ export function buildOpenAiCompatibleProvider(input: { /** Re-sync the provider into piModels() (idempotent): usable → rebuild; * not usable → remove. Never throws. */ -export function syncCustomProvider(cp: CustomProviderSettings | undefined): void { - const models = piModels(); - if (!customProviderUsable(cp)) { - models.deleteProvider(CUSTOM_PROVIDER.id); - return; - } - models.setProvider( - buildOpenAiCompatibleProvider({ - baseUrl: normalizeBaseUrl(cp.baseUrl), - models: cp.models ?? [], - overrides: cp.overrides ?? {}, - }), - ); +export function syncCustomProvider(): void { + piModels(); } /** Outcome of a model-discovery probe. `status` is the endpoint's HTTP status @@ -106,7 +121,7 @@ export function syncCustomProvider(cp: CustomProviderSettings | undefined): void * model list. */ export type DiscoverResult = | { ok: true; models: CustomModel[] } - | { ok: false; error: string; status?: number }; + | { ok: false; error: string; status?: number; fatal?: boolean }; /** Probe GET {baseUrl}/models for model ids. Scheme validated (http/https * only); keyless endpoints answer without a bearer. Non-2xx or unparseable @@ -114,22 +129,26 @@ export type DiscoverResult = export async function discoverModels(baseUrl: string, apiKey?: string): Promise { const base = normalizeBaseUrl(baseUrl); if (!base) return { ok: false, error: "Base URL must be a valid http(s) URL" }; - const key = apiKey?.trim() || keyValue(CUSTOM_PROVIDER.envVar)?.trim(); + const key = apiKey?.trim(); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 15_000); try { const res = await fetch(`${base}/models`, { headers: key ? { authorization: `Bearer ${key}` } : {}, signal: controller.signal, + redirect: "manual", }); - if (!res.ok) return { ok: false, error: `Endpoint returned ${res.status} ${res.statusText}`, status: res.status }; - const body = (await res.json().catch(() => null)) as unknown; + if (res.status >= 300 && res.status < 400) { + await res.body?.cancel(); + return { ok: false, error: "Discovery redirects are not allowed", fatal: true }; + } + if (!res.ok) { await res.body?.cancel(); return { ok: false, error: `Endpoint returned HTTP ${res.status}`, status: res.status }; } + const body = await readBoundedJson(res, 8 * 1024 * 1024); const list = extractModels(body); if (list === null) return { ok: false, error: "Endpoint response was not a model list" }; return { ok: true, models: list }; } catch (err) { - const msg = err instanceof Error ? (err.name === "AbortError" ? "Endpoint timed out" : err.message) : "Request failed"; - return { ok: false, error: msg }; + return { ok: false, error: err instanceof CustomBodyLimitError ? err.message : controller.signal.aborted ? "Endpoint timed out" : "Discovery request failed", fatal: err instanceof CustomBodyLimitError }; } finally { clearTimeout(timer); } @@ -210,6 +229,7 @@ const MAX_TOKEN_KEYS = ["max_output_tokens"] as const; function extractModels(body: unknown): CustomModel[] | null { const rows = modelRows(body); if (rows === null) return null; + if (rows.length > 10_000) throw new CustomBodyLimitError("Model list exceeds 10000 entries"); const byId = new Map(); for (const row of rows) { if (typeof row === "string") { diff --git a/src/server/custom-providers.ts b/src/server/custom-providers.ts deleted file mode 100644 index d78342e..0000000 --- a/src/server/custom-providers.ts +++ /dev/null @@ -1,91 +0,0 @@ -/** Configuration composition only. Pi's Models collection is the one runtime - * registry. A failed reload removes custom routes, never alters built-ins. */ -import type { MutableModels } from "@earendil-works/pi-ai"; -import { builtinProviders } from "@earendil-works/pi-ai/providers/all"; -import { createCustomProvider, customProviderApis, customProviderAuth } from "../agent/custom-providers.ts"; -import { configureProviderSource } from "../agent/models.ts"; -import { parseCustomProviders, resolveCustomValue, CustomProviderConfigError, type CustomProviderConfig } from "../core/custom-providers.ts"; -import { NATIVE_SUBSCRIPTIONS } from "../core/contracts/subscriptions.ts"; -import { CLAUDE_CODE_PROVIDER } from "../core/contracts/claude-code.ts"; -import { readCustomProvidersFile } from "../store/custom-providers.ts"; - -const reserved = new Set([...builtinProviders().map((p) => p.id), "chatgpt", CLAUDE_CODE_PROVIDER, ...NATIVE_SUBSCRIPTIONS.map((p) => p.logicalId)]); - -/** Instance-local so every Next server module graph registers into its own Pi - * collection. No process-global registry or personal Pi config is consulted. */ -export class CustomProviderSource { - private text?: string; - private fingerprint?: string; - private configs: CustomProviderConfig[] = []; - private registered = new Set(); - private ready = new Map(); - private error?: string; - - constructor( - private readonly read: typeof readCustomProvidersFile, - private readonly key: (name: string) => string | undefined, - ) {} - - sync(models: MutableModels): void { - try { - const record = this.read(); - if (record.error) throw new CustomProviderConfigError(record.error); - if (record.text !== this.text || this.error) { - this.configs = parseCustomProviders(record.text!, reserved, customProviderApis); - this.text = record.text; - this.fingerprint = undefined; - } - // Resolve only referenced variables through the app's existing key seam. - // Changes in credentials/headers trigger a new immutable provider snapshot - // even when the configuration file itself has not changed. - const values = new Map(); - for (const config of this.configs) { - for (const template of [config.definition.apiKey, ...Object.values(config.definition.headers ?? {}), ...config.models.flatMap((m) => Object.values(m.headers ?? {}))]) { - if (template !== undefined) resolveCustomValue(template, (name) => { - const value = this.key(name); - values.set(name, value); - return value; - }); - } - } - const fingerprint = JSON.stringify([...values]); - if (this.fingerprint === fingerprint) return; - const lookup = (name: string) => values.get(name); - const providers = this.configs.map((config) => createCustomProvider(config, lookup)); - const ready = new Map(this.configs.map((config) => [config.id, customProviderAuth(config, lookup) !== undefined])); - // Everything validates/builds before the synchronous publication begins. - for (const id of this.registered) models.deleteProvider(id); - this.registered = new Set(providers.map((p) => p.id)); - for (const provider of providers) models.setProvider(provider); - this.ready = ready; - this.fingerprint = fingerprint; - this.error = undefined; - } catch (error) { - for (const id of this.registered) models.deleteProvider(id); - this.registered.clear(); - this.ready.clear(); - this.fingerprint = undefined; - this.error = error instanceof CustomProviderConfigError ? error.message : "Unable to configure custom providers from models.json."; - } - } - - configured(provider: string): boolean | undefined { return this.ready.get(provider); } - providers(): { id: string; envVar: string; keyPresent: boolean }[] { - return [...this.ready].map(([id, keyPresent]) => ({ id, envVar: "", keyPresent })); - } - getError(): string | undefined { return this.error; } -} - -let source: CustomProviderSource | undefined; -export function installCustomProviderSource(key: (name: string) => string | undefined): void { - const next = new CustomProviderSource(readCustomProvidersFile, key); - configureProviderSource({ - sync: (models) => next.sync(models), - isCustom: (id) => next.configured(id) !== undefined, - }); - source = next; -} -/** Call piModels() before these reads to synchronize the snapshot. */ -export function customProviderRows() { return source?.providers() ?? []; } -export function customProviderConfigured(provider: string): boolean | undefined { return source?.configured(provider); } -export function customProvidersError(): string | undefined { return source?.getError(); } diff --git a/src/server/index.ts b/src/server/index.ts index 9f24424..bf93d54 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -49,3 +49,4 @@ export { claudeCodeInstallationResponse } from "./claude-code.ts"; // The custom endpoint module (registers into piModels() + probes for models). // No import side effects; the boot sync lives in stores.ts. export * from "./custom-provider.ts"; +export { customProviderGet, customProviderMutation } from "./custom-provider-http.ts"; diff --git a/src/server/keys.ts b/src/server/keys.ts index 46b2be3..5bb2b6a 100644 --- a/src/server/keys.ts +++ b/src/server/keys.ts @@ -54,19 +54,19 @@ import { } from "../core/auth-keys.ts"; import { authStore } from "../store/auth.ts"; import { isSubscriptionProvider, subscriptionConnected } from "../agent/subscription-connections.ts"; -import { piModels } from "../agent/models.ts"; -import { customProviderConfigured, installCustomProviderSource } from "./custom-providers.ts"; +import { customProviderAvailable, installCustomProviderSource } from "./custom-provider.ts"; import { claudeCodeAvailable } from "./claude-code-config.ts"; import { CLAUDE_CODE_PROVIDER } from "../agent/claude-code/constants.ts"; +import { CUSTOM_AUTH_BINDING } from "../core/custom-provider-auth.ts"; // Shared HMR-stable provenance lets external CLIs exclude file-only injections // without importing this store-backed module into the agent layer. import { authEnvRecord as boot } from "../core/auth-env.ts"; -/** Is `id` a WRITABLE key name — curated catalog, model-provider env var, or - * the custom endpoint's reserved key (writable through the same seam). */ +/** Ordinary writable key names. Custom auth writes must use the atomic + * endpoint-binding seam, never the generic single-key API. */ export function isWritableAuthKey(id: string): boolean { - return isKnownAuthKey(id) || isProviderEnvVar(id) || id === CUSTOM_PROVIDER.envVar; + return id !== CUSTOM_AUTH_BINDING && id !== CUSTOM_PROVIDER.envVar && (isKnownAuthKey(id) || isProviderEnvVar(id)); } /** The settings page's provider dropdown: every pi-ai provider that takes @@ -92,7 +92,7 @@ export function envKeysIgnored(): boolean { */ export function initAuthEnv(): void { const record = boot(); - const fileVars = authStore.authVars(); + const fileVars = Object.fromEntries(Object.entries(authStore.authVars()).filter(([name]) => name !== CUSTOM_AUTH_BINDING)); const managed = new Set([ ...Object.keys(fileVars), ...AUTH_KEYS.map((k) => k.id), @@ -100,6 +100,7 @@ export function initAuthEnv(): void { // an env-exported OPENAI_API_KEY with a file copy gets honest shadow // semantics, exactly like the curated vars. ...AUTH_PROVIDER_KEYS.map((p) => p.envVar), + CUSTOM_PROVIDER.envVar, ]); if (envKeysIgnored()) { // THE FRESH-USER SWITCH: delete every managed env var (re-run on @@ -127,7 +128,7 @@ export function initAuthEnv(): void { /** Is this env-var name one the keys system manages (curated, provider, * or held in the file)? */ function isManagedName(name: string): boolean { - return isWritableAuthKey(name) || name in authStore.authVars(); + return name !== CUSTOM_AUTH_BINDING && (isWritableAuthKey(name) || name in authStore.authVars()); } /** @@ -149,6 +150,7 @@ export function resolveKeyStatuses( } const rows: AuthKeyStatus[] = []; for (const id of ids) { + if (id === CUSTOM_AUTH_BINDING) continue; const envValue = env[id]; const fileValue = fileVars[id]; const isRealEnv = realEnv.has(id); @@ -170,7 +172,7 @@ export function resolveKeyStatuses( /** The GET view: every curated key (present or not) + unknown file vars. */ export function keyStatuses(): AuthKeyStatus[] { - return resolveKeyStatuses(authStore.authVars(), process.env, boot().realEnv, AUTH_PROVIDER_ENV_VARS); + return resolveKeyStatuses(authStore.authVars(), process.env, boot().realEnv, new Set([...AUTH_PROVIDER_ENV_VARS, CUSTOM_PROVIDER.envVar])); } /** The live value for one var, env-first (the env-wins rule) — the canonical @@ -178,6 +180,7 @@ export function keyStatuses(): AuthKeyStatus[] { * STT route was the one cold-boot hole: a first-ever request to it would * otherwise read env before any fill-gap ran). */ export function keyValue(id: string): string | undefined { + if (id === CUSTOM_AUTH_BINDING) return undefined; // The fresh-user switch: ONLY the file store counts (belt-and- // suspenders — the boot deletion already emptied the env; this guards // any env key reintroduced after boot). @@ -192,9 +195,7 @@ export function keyValue(id: string): string | undefined { * (gate by what would run). Providers absent from AUTH_PROVIDER_KEYS take * no key at all: nothing to gate, present by definition. */ export function providerKeyPresent(provider: string): boolean { - piModels(); // Synchronize hand-edited custom configuration before admission. - const custom = customProviderConfigured(provider); - if (custom !== undefined) return custom; + if (provider === CUSTOM_PROVIDER.id) return customProviderAvailable(); // Explicit opt-in + installation only, never authentication readiness. // Consulted at admission, not inside already-running agent loops. if (provider === CLAUDE_CODE_PROVIDER) return claudeCodeAvailable(); @@ -210,10 +211,8 @@ export function providerKeyPresent(provider: string): boolean { * reports shadowedByEnv so the page can explain it). */ export function saveKey(id: string, value: string): { ok: true; status: AuthKeyStatus } | { ok: false; error: string } { - const shape = - isProviderEnvVar(id) || id === CUSTOM_PROVIDER.envVar - ? genericProviderKeyGuard(id, value) - : validateKeyShape(id, value); + if (id === CUSTOM_PROVIDER.envVar || id === CUSTOM_AUTH_BINDING) return { ok: false, error: "Use the custom endpoint editor" }; + const shape = isProviderEnvVar(id) ? genericProviderKeyGuard(id, value) : validateKeyShape(id, value); if (!shape.ok) return { ok: false, error: shape.error }; const trimmed = value.trim(); authStore.setAuthVar(id, trimmed); @@ -232,6 +231,7 @@ export function saveKey(id: string, value: string): { ok: true; status: AuthKeyS * wins) and the status shows the key still present from env. */ export function deleteKey(id: string): AuthKeyStatus { + if (id === CUSTOM_PROVIDER.envVar || id === CUSTOM_AUTH_BINDING) throw new Error("Use the custom endpoint editor"); authStore.deleteAuthVar(id); const record = boot(); if (record.fillGap.has(id) && !record.realEnv.has(id)) { @@ -244,4 +244,4 @@ export function deleteKey(id: string): AuthKeyStatus { // Reflect the file into the env at first import (idempotent; the module is // imported by every route/handler path that touches keys). initAuthEnv(); -installCustomProviderSource(keyValue); +installCustomProviderSource(); diff --git a/src/server/model-catalog.ts b/src/server/model-catalog.ts index fa24801..44883dd 100644 --- a/src/server/model-catalog.ts +++ b/src/server/model-catalog.ts @@ -20,7 +20,7 @@ import { clampThinkingLevel, getSupportedThinkingLevels, type Api, type Model } from "@earendil-works/pi-ai"; import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; import { piModels } from "../agent/models.ts"; -import { customProviderRows, customProvidersError } from "./custom-providers.ts"; +import { customProviderAvailable } from "./custom-provider.ts"; import { chatGPTConnected, chatGPTModel, chatGPTModels, refreshChatGPTModels } from "../agent/chatgpt-bridge.ts"; import { NATIVE_SUBSCRIPTIONS } from "../core/contracts/subscriptions.ts"; import { nativeSubscriptionId, nativeSubscriptionConnected, nativeSubscriptionModel, nativeSubscriptionModels, refreshSubscriptionModels } from "../agent/subscription-bridge.ts"; @@ -155,11 +155,10 @@ export function catalogProviders(): CatalogProvider[] { { id: "chatgpt", envVar: "", keyPresent: chatGPTConnected() }, { id: CLAUDE_CODE_PROVIDER, envVar: "", keyPresent: claudeCodeAvailable() }, ...NATIVE_SUBSCRIPTIONS.map((p) => ({ id: p.logicalId, envVar: "", keyPresent: nativeSubscriptionConnected(p.providerId) })), - ...customProviderRows(), // The custom endpoint: a row only when usable (base URL set). keyPresent // reflects usability, not key presence (keyless local servers are allowed). ...(customProviderUsable(effectiveSettings().customProvider) - ? [{ id: CUSTOM_PROVIDER.id, envVar: CUSTOM_PROVIDER.envVar, keyPresent: true }] + ? [{ id: CUSTOM_PROVIDER.id, envVar: CUSTOM_PROVIDER.envVar, keyPresent: customProviderAvailable() }] : []), ]).sort((a, b) => a.id.localeCompare(b.id)); } @@ -180,7 +179,7 @@ export function toModelDto(model: Model): ModelDto { reasoning: model.reasoning, input: [...model.input], supportedLevels: getSupportedThinkingLevels(model), - cost: model.provider === CLAUDE_CODE_PROVIDER ? null : { + cost: model.provider === CLAUDE_CODE_PROVIDER || model.provider === CUSTOM_PROVIDER.id ? null : { input: model.cost.input, output: model.cost.output, cacheRead: model.cost.cacheRead, @@ -231,7 +230,6 @@ export function modelCatalog(): ModelCatalogResponse { rolePins: catalogRolePins(), slots: catalogSlots(), providers: catalogProviders(), - ...(customProvidersError() ? { customProvidersError: customProvidersError() } : {}), }; } diff --git a/src/server/pricing.ts b/src/server/pricing.ts index 1454c11..097669c 100644 --- a/src/server/pricing.ts +++ b/src/server/pricing.ts @@ -4,6 +4,7 @@ */ import { calculateCost, type Usage } from "@earendil-works/pi-ai"; import { piModels } from "../agent/models.ts"; +import { CUSTOM_PROVIDER } from "../core/auth-keys.ts"; import { CLAUDE_CODE_PROVIDER } from "../core/contracts/claude-code.ts"; import type { CallUsage, ChatRecord, JobRecord, TurnUsage } from "../core/contracts.ts"; import { isSubscriptionModel, sessionCost as foldSessionCost, turnCost as foldTurnCost, jobCost as foldJobCost } from "../core/prices.ts"; @@ -12,7 +13,7 @@ function catalogModel(id: string) { const colon = id.indexOf(":"); // Native CLI routing determines billing. Adapter zero-rate placeholders // must never become an apparent $0 estimate or subscription classification. - if (colon < 0 || isSubscriptionModel(id) || id.slice(0, colon) === CLAUDE_CODE_PROVIDER) return undefined; + if (colon < 0 || isSubscriptionModel(id) || [CLAUDE_CODE_PROVIDER, CUSTOM_PROVIDER.id].includes(id.slice(0, colon))) return undefined; return piModels().getModel(id.slice(0, colon), id.slice(colon + 1)); } diff --git a/src/server/settings-body.ts b/src/server/settings-body.ts index faad497..e84a7c9 100644 --- a/src/server/settings-body.ts +++ b/src/server/settings-body.ts @@ -400,7 +400,7 @@ export function parseDictationSettingsBody(body: unknown): SettingsBodyResult> { @@ -430,7 +430,7 @@ export function parseCustomModelOverridesBody( if (typeof entry.reasoning !== "boolean") return { ok: false, error: `reasoning for ${id} must be true or false` }; pin.reasoning = entry.reasoning; } - out[id] = pin; + Object.defineProperty(out, id, { value: pin, enumerable: true, configurable: true, writable: true }); } return { ok: true, settings: out }; } diff --git a/src/server/stores.ts b/src/server/stores.ts index 27e6c3e..2e543b1 100644 --- a/src/server/stores.ts +++ b/src/server/stores.ts @@ -36,4 +36,4 @@ export function effectiveSettings(): EffectiveSettings { // Boot: register the custom endpoint (when configured) into piModels(); the // POST /api/custom-provider route re-syncs after every connect. -syncCustomProvider(effectiveSettings().customProvider); +syncCustomProvider(); diff --git a/src/store/auth.ts b/src/store/auth.ts index 2fe6edd..4985cb4 100644 --- a/src/store/auth.ts +++ b/src/store/auth.ts @@ -7,9 +7,10 @@ * * The server (src/server/keys.ts) is the only intended writer: it applies * shape validation before setAuthVar and env precedence on top of - * authVars(). This adapter persists what it is given, never throws - * (absent/corrupt file = {} = "no keys yet", the documented first-boot - * state, matching settings doctrine), and preserves unknown vars verbatim + * authVars(). Ordinary reads never throw (absent/corrupt file = {} = + * "no keys yet"); custom-only pair reads/writes below are deliberately + * strict so an unreadable private record is never overwritten as {}. + * Ordinary writes preserve unknown string vars verbatim * (forward-compat: future entries — e.g. a plans-login token — ride along * untouched until a catalog entry adopts them). */ @@ -18,6 +19,8 @@ import fs from "node:fs"; import path from "node:path"; import { AUTH_FILE } from "../../config.ts"; import type { AuthStore } from "../core/ports.ts"; +import { CUSTOM_PROVIDER } from "../core/auth-keys.ts"; +import { CUSTOM_AUTH_BINDING, parseCustomAuthBinding, type CustomAuthBinding } from "../core/custom-provider-auth.ts"; /** The flat string→string map, or {} for absent/corrupt/non-object. */ function readMap(): Record { @@ -36,7 +39,7 @@ function readMap(): Record { * leave a 0644 file (secrets) or a truncated JSON (looks like "no keys"). * tmp-in-same-dir + rename keeps the swap atomic; chmod BEFORE the * rename so the mode never exists relaxed on the final path. */ -function writeMap(map: Record): void { +function writeMap(map: Record): void { const dir = path.dirname(AUTH_FILE); fs.mkdirSync(dir, { recursive: true }); const tmp = path.join(dir, `.auth.json.tmp-${process.pid}`); @@ -50,14 +53,50 @@ export const authStore: AuthStore = { return readMap(); }, setAuthVar(name, value) { + assertOrdinaryVar(name); const map = readMap(); map[name] = value; writeMap(map); }, deleteAuthVar(name) { + assertOrdinaryVar(name); const map = readMap(); if (!(name in map)) return; // absent = silent success, idempotent delete map[name]; writeMap(map); }, }; + +/** Custom writes must never turn an unreadable/corrupt private file into {}. + * Other providers retain their existing adapter semantics. Unknown JSON values + * survive custom-only updates, not just the ordinary string projection. */ +function readCustomMap(): Record { + let text: string; + try { text = fs.readFileSync(AUTH_FILE, "utf8"); } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return {}; throw new Error("Custom authorization storage unavailable"); } + let raw: unknown; + try { raw = JSON.parse(text); } catch { throw new Error("Custom authorization storage unavailable"); } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error("Custom authorization storage unavailable"); + const map = raw as Record; + if (Object.hasOwn(map, CUSTOM_AUTH_BINDING) && !parseCustomAuthBinding(map[CUSTOM_AUTH_BINDING])) throw new Error("Invalid custom authorization record"); + if (Object.hasOwn(map, CUSTOM_PROVIDER.envVar) && typeof map[CUSTOM_PROVIDER.envVar] !== "string") throw new Error("Invalid custom credential record"); + return map; +} +function assertOrdinaryVar(name: string): void { + if (name === CUSTOM_AUTH_BINDING || name === CUSTOM_PROVIDER.envVar) throw new Error("Use the custom endpoint editor"); +} +/** One immutable file snapshot. Keyless callers never invoke fileKey(). */ +export function readCustomAuth() { + const map = readCustomMap(); + return { binding: parseCustomAuthBinding(map[CUSTOM_AUTH_BINDING]), fileKey: () => map[CUSTOM_PROVIDER.envVar] as string | undefined }; +} +/** Replace just the custom pair in ONE existing atomic record write. */ +export function writeCustomAuth(binding: CustomAuthBinding | undefined, fileKey: string | undefined): void { + const map = readCustomMap(); + if (binding && !parseCustomAuthBinding(JSON.stringify(binding))) throw new Error("Invalid custom authorization record"); + if (fileKey === undefined) delete map[CUSTOM_PROVIDER.envVar]; + else Object.defineProperty(map, CUSTOM_PROVIDER.envVar, { value: fileKey, enumerable: true, configurable: true, writable: true }); + if (binding === undefined) delete map[CUSTOM_AUTH_BINDING]; + else Object.defineProperty(map, CUSTOM_AUTH_BINDING, { value: JSON.stringify(binding), enumerable: true, configurable: true, writable: true }); + writeMap(map); +} diff --git a/src/store/custom-providers.ts b/src/store/custom-providers.ts deleted file mode 100644 index 2245823..0000000 --- a/src/store/custom-providers.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** Read-only adapter. Configuration editing/UI is deliberately a later phase. */ -import fs from "node:fs"; -import { CUSTOM_MODELS_FILE } from "../../config.ts"; - -export function readCustomProvidersFile(): { text?: string; error?: string } { - try { return { text: fs.readFileSync(CUSTOM_MODELS_FILE, "utf8") }; } - catch (error) { - return (error as NodeJS.ErrnoException).code === "ENOENT" - ? { text: '{"providers":{}}' } - : { error: "Unable to read models.json." }; - } -} diff --git a/tests/custom-endpoint.test.ts b/tests/custom-endpoint.test.ts index d477b79..5aea909 100644 --- a/tests/custom-endpoint.test.ts +++ b/tests/custom-endpoint.test.ts @@ -26,7 +26,6 @@ import { buildOpenAiCompatibleProvider, discoverModels, mergeManualModels, - syncCustomProvider, } from "../src/server/custom-provider.ts"; import { piModels } from "../src/agent/models.ts"; import { getSupportedThinkingLevels } from "@earendil-works/pi-ai"; @@ -92,13 +91,13 @@ describe("sanitizeUserSettings — customProvider", () => { // ─── per-model pins: the read side + THE prune law ─────────────────────────── describe("sanitizeUserSettings / sanitizeCustomModelOverrides — pins", () => { - it("keeps well-formed pins, clamps token counts up to the floor, keeps reasoning:false", () => { + it("keeps all positive integer pins and reasoning:false", () => { const out = sanitizeUserSettings({ customProvider: { baseUrl: "https://h/v1", overrides: { llama: { context: 131072, maxTokens: 8192, reasoning: true }, - qwen: { context: 10 }, // under the floor → clamped up (clamp-never-reject on read) + qwen: { context: 10 }, // Small positive budgets are preserved. quiet: { reasoning: false }, // a real pin, not an absent one junk: { context: "big", maxTokens: -1, reasoning: "yes" }, // every field junk " ": { context: 4096 }, // no usable id @@ -108,7 +107,7 @@ describe("sanitizeUserSettings / sanitizeCustomModelOverrides — pins", () => { }); assert.deepEqual(out?.customProvider?.overrides, { llama: { context: 131072, maxTokens: 8192, reasoning: true }, - qwen: { context: CUSTOM_MODEL_PIN_FLOOR }, + qwen: { context: 10 }, quiet: { reasoning: false }, }); }); @@ -281,11 +280,11 @@ describe("buildOpenAiCompatibleProvider / syncCustomProvider", () => { assert.deepEqual(getSupportedThinkingLevels(plain!), ["off"]); }); - it("registers on a usable sync and removes when the base URL clears", () => { - syncCustomProvider({ baseUrl: "http://h/v1", models: [{ id: "live" }] }); + it("registers the native provider in the shared collection", () => { + piModels().setProvider(buildOpenAiCompatibleProvider({ baseUrl: "http://h/v1", models: [{ id: "live" }] })); assert.ok(piModels().getProvider(CUSTOM_PROVIDER.id), "provider registered"); assert.ok(piModels().getModel(CUSTOM_PROVIDER.id, "live"), "model resolvable"); - syncCustomProvider({ baseUrl: "", models: [{ id: "live" }] }); + piModels().deleteProvider(CUSTOM_PROVIDER.id); assert.equal(piModels().getProvider(CUSTOM_PROVIDER.id), undefined, "provider removed"); }); }); diff --git a/tests/custom-provider-boundaries.test.ts b/tests/custom-provider-boundaries.test.ts new file mode 100644 index 0000000..2ea1006 --- /dev/null +++ b/tests/custom-provider-boundaries.test.ts @@ -0,0 +1,191 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { after, beforeEach, test, mock } from "node:test"; + +const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-custom-boundaries-")); +const originalEnv = process.env; +const originalFetch = globalThis.fetch; +process.env = { ...(originalEnv.NODE_TEST_CONTEXT ? { NODE_TEST_CONTEXT: originalEnv.NODE_TEST_CONTEXT } : {}), ...(originalEnv.NODE_OPTIONS ? { NODE_OPTIONS: originalEnv.NODE_OPTIONS } : {}), NODE_ENV: "test", HOME: scratch, LH_DATA_DIR: scratch, LH_AUTH_FILE: path.join(scratch, "auth.json"), LH_IGNORE_ENV_KEYS: "1" }; +const route = await import("../app/api/custom-provider/route.ts"); +const keys = await import("../src/server/keys.ts"); +const { authEnvRecord } = await import("../src/core/auth-env.ts"); +const { CUSTOM_PROVIDER } = await import("../src/core/auth-keys.ts"); +const { authorizeFixture } = await import("./helpers/custom-provider-auth.ts"); +const { readCustomAuth } = await import("../src/store/auth.ts"); +const { persistCustomAuth } = await import("../src/server/custom-provider-auth.ts"); +const settingsFile = path.join(scratch, "settings.json"); +const seed = (cp: import("../src/core/settings.ts").CustomProviderSettings = { baseUrl: "http://old/v1", authMode: "keyless", models: [{ id: "m" }] }, key?: string) => fs.writeFileSync(settingsFile, JSON.stringify({ version: 1, customProvider: authorizeFixture(cp, key) })); +const setFileKey = (key: string) => { const auth = readCustomAuth(); persistCustomAuth({ ...auth.binding!, fileKeyBaseUrl: auth.binding!.baseUrl }, key); }; +const current = () => JSON.parse(fs.readFileSync(settingsFile, "utf8")); +const get = (inspect = false) => route.GET(new Request(`http://localhost/api/custom-provider${inspect ? "?authMode=apiKey" : ""}`)); +function mutate(body: unknown, method = "POST", headers: Record = {}) { + return route.POST(new Request("http://localhost/api/custom-provider", { method, headers: { origin: "http://localhost", "content-type": "application/json", "if-match": get().headers.get("etag")!, ...headers }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) })); +} +let requests: { url: string; authorization: string | null }[]; +beforeEach(() => { + mock.restoreAll(); + process.env.LH_IGNORE_ENV_KEYS = "1"; + delete process.env.LH_DESKTOP_SERVER; delete process.env.LH_DESKTOP_API_SECRET; + fs.writeFileSync(path.join(scratch, "auth.json"), "{}"); seed(); + requests = []; + globalThis.fetch = async (url, options) => { + requests.push({ url: String(url), authorization: new Headers(options?.headers).get("authorization") }); + assert.equal(options?.redirect, "manual"); + assert.ok(options?.signal instanceof AbortSignal); + return Response.json({ data: [{ id: "m" }] }); + }; +}); +after(() => { mock.restoreAll(); process.env = originalEnv; globalThis.fetch = originalFetch; fs.rmSync(scratch, { recursive: true, force: true }); }); + +test("keyless probe never forwards a stored key, and clearing removes authentication", async () => { + setFileKey("private-old"); + assert.equal((await mutate({ baseUrl: "http://new/v1", authMode: "keyless", key: "do-not-use-this" })).status, 200); + assert.equal(requests[0]?.authorization, null); + assert.equal(keys.keyValue(CUSTOM_PROVIDER.envVar), undefined); + assert.equal(current().customProvider.authMode, "keyless"); +}); +test("new keys have no arbitrary minimum; the exact probed key is stored", async () => { + assert.equal((await mutate({ baseUrl: "http://new/v1", authMode: "apiKey", key: " k " })).status, 200); + assert.equal(requests[0]?.authorization, "Bearer k"); + assert.equal(keys.keyValue(CUSTOM_PROVIDER.envVar), "k"); + const body = await get().text(); assert.ok(!body.includes("Bearer")); +}); +test("changing destination cannot silently reuse an existing credential", async () => { + setFileKey("private-old"); + const payload = { baseUrl: "http://new/v1", authMode: "apiKey" }; + assert.equal((await mutate(payload)).status, 400); assert.equal(requests.length, 0); + assert.equal((await mutate({ ...payload, reuseKey: true })).status, 200); + assert.equal(requests[0]?.authorization, "Bearer private-old"); +}); +test("same-URL unbound development key needs existing explicit reuse consent", async () => { + seed({ baseUrl: "http://old/v1", authMode: "apiKey", models: [{ id: "m" }] }, "legacy-key"); + const { CUSTOM_AUTH_BINDING } = await import("../src/core/custom-provider-auth.ts"); + const file = path.join(scratch, "auth.json"); const map = JSON.parse(fs.readFileSync(file, "utf8")); delete map[CUSTOM_AUTH_BINDING]; fs.writeFileSync(file, JSON.stringify(map)); + assert.equal((await get().json()).keyReuseRequired, true); + assert.equal((await mutate({ baseUrl: "http://old/v1", authMode: "apiKey" })).status, 400); assert.equal(requests.length, 0); + assert.equal((await mutate({ baseUrl: "http://old/v1", authMode: "apiKey", reuseKey: true })).status, 200); + assert.equal(requests[0]?.authorization, "Bearer legacy-key"); assert.equal((await get().json()).keyReuseRequired, false); +}); +test("generic custom/reserved key writes cannot bypass the atomic binding seam", async () => { + const ordinary = await import("../app/api/keys/[id]/route.ts"); + const { CUSTOM_AUTH_BINDING } = await import("../src/core/custom-provider-auth.ts"); + const { authStore } = await import("../src/store/auth.ts"); + const before = fs.readFileSync(path.join(scratch, "auth.json"), "utf8"); + for (const id of [CUSTOM_PROVIDER.envVar, CUSTOM_AUTH_BINDING]) { + const context = { params: Promise.resolve({ id }) }; + assert.equal((await ordinary.PUT(new Request("http://localhost", { method: "PUT", body: JSON.stringify({ value: "bypass" }) }), context)).status, 404); + assert.equal((await ordinary.DELETE(new Request("http://localhost", { method: "DELETE" }), context)).status, 404); + assert.equal(keys.saveKey(id, "bypass").ok, false); assert.throws(() => keys.deleteKey(id)); + assert.throws(() => authStore.setAuthVar(id, "bypass")); assert.throws(() => authStore.deleteAuthVar(id)); + } + assert.equal(fs.readFileSync(path.join(scratch, "auth.json"), "utf8"), before); + assert.ok(!keys.keyStatuses().some(row => row.id === CUSTOM_AUTH_BINDING)); assert.equal(process.env[CUSTOM_AUTH_BINDING], undefined); +}); +test("same target can retain its effective key", async () => { + seed({ baseUrl: "http://old/v1", authMode: "apiKey", models: [{ id: "m" }] }, "retained"); + assert.equal((await mutate({ baseUrl: "http://old/v1", authMode: "apiKey" })).status, 200); + assert.equal(requests[0]?.authorization, "Bearer retained"); +}); +test("environment ownership is shown; consent uses the environment key in probe and inference", async () => { + process.env.LH_IGNORE_ENV_KEYS = "0"; + process.env[CUSTOM_PROVIDER.envVar] = "environment-secret"; + authEnvRecord().realEnv.add(CUSTOM_PROVIDER.envVar); + try { + assert.equal((await get(true).json()).keySource, "env"); + assert.equal((await mutate({ baseUrl: "http://new/v1", authMode: "apiKey", key: "shadowed" })).status, 400); + assert.equal((await mutate({ baseUrl: "http://new/v1", authMode: "apiKey", reuseKey: true })).status, 200); + assert.equal(requests[0]?.authorization, "Bearer environment-secret"); + assert.equal(keys.keyValue(CUSTOM_PROVIDER.envVar), "environment-secret"); + assert.ok(!(await get().text()).includes("environment-secret")); + assert.equal((await mutate({ baseUrl: "http://other/v1", authMode: "keyless" })).status, 200); + assert.equal(requests.at(-1)?.authorization, null); + } finally { authEnvRecord().realEnv.delete(CUSTOM_PROVIDER.envVar); delete process.env[CUSTOM_PROVIDER.envVar]; } +}); +test("cross-origin/opaque/missing-origin and non-JSON writes cannot probe or mutate", async () => { + for (const origin of ["http://evil", "null", ""]) assert.equal((await mutate({ baseUrl: "http://new", authMode: "keyless" }, "POST", { origin })).status, 403); + assert.equal((await mutate({}, "POST", { "content-type": "text/plain" })).status, 415); + assert.equal((await mutate({}, "POST", { "sec-fetch-site": "cross-site" })).status, 403); + assert.equal(requests.length, 0); +}); +test("standalone internal URL uses the addressed Host origin, never forwarded origins", async () => { + const headers = { "content-type": "application/json", "if-match": get().headers.get("etag")!, host: "127.0.0.1:3317", origin: "http://127.0.0.1:3317" }; + assert.equal((await route.DELETE(new Request("http://localhost:3317/api/custom-provider", { method: "DELETE", headers }))).status, 200); + assert.equal((await mutate({}, "DELETE", { "x-forwarded-host": "evil", origin: "http://evil" })).status, 403); +}); +test("empty DELETE streams normalize consistently without accepting malformed JSON", async () => { + const headers = { "content-type": "application/json", "if-match": get().headers.get("etag")!, origin: "http://localhost" }; + const init: RequestInit & { duplex: "half" } = { method: "DELETE", headers, body: new ReadableStream({ start(controller) { controller.close(); } }), duplex: "half" }; + assert.equal((await route.DELETE(new Request("http://localhost/api/custom-provider", init))).status, 200); + assert.equal((await route.DELETE(new Request("http://localhost/api/custom-provider", { method: "DELETE", headers: { ...headers, "if-match": get().headers.get("etag")! }, body: "{" }))).status, 400); + assert.equal((await route.POST(new Request("http://localhost/api/custom-provider", { method: "POST", headers, body: "" }))).status, 400); +}); +test("desktop capability remains mandatory in desktop mode", async () => { + process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "x".repeat(64); + assert.equal((await mutate({}, "DELETE")).status, 403); + assert.equal((await mutate(undefined, "DELETE", { origin: "null", "x-learnharness-desktop": "x".repeat(64) })).status, 200); +}); +test("forbidden URL components are rejected before probing", async () => { + for (const baseUrl of ["ftp://h", "https://user:pass@h/v1", "http://h/v1?q=secret", "http://h/v1#x", "http://h/v1?", "http://h/v1#"]) assert.equal((await mutate({ baseUrl, authMode: "keyless" })).status, 400); + assert.equal(requests.length, 0); +}); +test("redirects/auth failures/model and byte limits cannot be bypassed with typed IDs", async () => { + for (const answer of [ + () => new Response(null, { status: 302, headers: { location: "http://other" } }), + () => new Response("private response", { status: 401 }), + () => new Response("private response", { status: 403 }), + () => Response.json({ data: Array(10001).fill({ id: "m" }) }), + () => new Response("x".repeat(8 * 1024 * 1024 + 1)), + () => new Response("{}", { headers: { "content-length": String(8 * 1024 * 1024 + 1) } }), + ]) { + globalThis.fetch = async () => answer(); + const before = current(); + const res = await mutate({ baseUrl: "http://h/v1", authMode: "keyless", manualModels: "m" }); + assert.equal(res.status, 502); assert.deepEqual(current(), before); + assert.ok(!(await res.text()).includes("private response")); + } +}); +test("write size bounds are explicit for streamed bodies", async () => { + assert.equal((await mutate({ baseUrl: "x".repeat(65536), authMode: "keyless" })).status, 413); + assert.equal((await mutate({ padding: "x".repeat(65536) }, "DELETE")).status, 413); + assert.equal(requests.length, 0); +}); +test("stale save, pin edit and disconnect all return reload guidance without writing", async () => { + const etag = get().headers.get("etag")!; + seed({ baseUrl: "http://other/v1", authMode: "keyless", models: [{ id: "different" }] }); + const before = current(); + for (const method of ["POST", "PATCH", "DELETE"]) { + const res = await mutate({ baseUrl: "http://h", authMode: "keyless", overrides: {} }, method, { "if-match": etag }); + assert.equal(res.status, 409); assert.match(await res.text(), /Reload/); + } + assert.equal(requests.length, 0); assert.deepEqual(current(), before); +}); +test("after discovery preserves unrelated edits and rejects intervening custom edits", async () => { + globalThis.fetch = async () => { fs.writeFileSync(settingsFile, JSON.stringify({ ...current(), appearance: { mode: "dark" }, claudeCode: { enabled: true } })); return Response.json({ data: [{ id: "m" }] }); }; + assert.equal((await mutate({ baseUrl: "http://h", authMode: "keyless" })).status, 200); + assert.deepEqual(current().appearance, { mode: "dark" }); assert.equal(current().claudeCode.enabled, true); + globalThis.fetch = async () => { seed({ baseUrl: "http://changed", authMode: "keyless", models: [] }); return Response.json({ data: [{ id: "m" }] }); }; + assert.equal((await mutate({ baseUrl: "http://h", authMode: "keyless" })).status, 409); + assert.equal(current().customProvider.baseUrl, "http://changed"); +}); +test("after discovery rejects a changed effective key", async () => { + seed({ baseUrl: "http://old/v1", authMode: "apiKey", models: [{ id: "m" }] }, "before"); + globalThis.fetch = async () => { setFileKey("rotated"); return Response.json({ data: [{ id: "m" }] }); }; + assert.equal((await mutate({ baseUrl: "http://old/v1", authMode: "apiKey" })).status, 409); + assert.equal(keys.keyValue(CUSTOM_PROVIDER.envVar), "rotated"); +}); +test("offline pins accept 512/1, retain support declaration, and do not probe", async () => { + const res = await mutate({ overrides: { m: { context: 1, maxTokens: 512, reasoning: false } } }, "PATCH"); + assert.equal(res.status, 200); assert.equal(requests.length, 0); + assert.deepEqual(current().customProvider.overrides.m, { context: 1, maxTokens: 512, reasoning: false }); +}); +test("settings-write failure restores previous key/config and reports failure", async () => { + setFileKey("old-key"); + const before = current(); + const write = fs.writeFileSync; + mock.method(fs, "writeFileSync", (...args: Parameters) => { if (args[0] === settingsFile) throw new Error("private-io-details"); return write(...args); }); + const res = await mutate({ baseUrl: "http://new", authMode: "apiKey", key: "new-key" }); + assert.equal(res.status, 500); assert.ok(!(await res.text()).includes("private-io-details")); + assert.equal(keys.keyValue(CUSTOM_PROVIDER.envVar), "old-key"); assert.deepEqual(current(), before); +}); diff --git a/tests/custom-provider-lifecycle.test.ts b/tests/custom-provider-lifecycle.test.ts new file mode 100644 index 0000000..d4b259c --- /dev/null +++ b/tests/custom-provider-lifecycle.test.ts @@ -0,0 +1,129 @@ +/** Native Pi payloads with synthetic transport and owned scratch files only. */ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { after, beforeEach, test } from "node:test"; +import { createCanvas } from "@napi-rs/canvas"; +import { normalizeContext, fauxAssistantMessage, fauxToolCall, type Message } from "@earendil-works/pi-ai"; + +const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-custom-lifecycle-")); +const originalEnv = process.env; +const originalFetch = globalThis.fetch; +process.env = { ...(originalEnv.NODE_TEST_CONTEXT ? { NODE_TEST_CONTEXT: originalEnv.NODE_TEST_CONTEXT } : {}), ...(originalEnv.NODE_OPTIONS ? { NODE_OPTIONS: originalEnv.NODE_OPTIONS } : {}), NODE_ENV: "test", HOME: scratch, LH_DATA_DIR: scratch, LH_AUTH_FILE: path.join(scratch, "auth.json"), LH_IGNORE_ENV_KEYS: "1" }; +const { writeSettings, readSettings } = await import("../src/store/settings.ts"); +const { piEngineForSpec, piModels } = await import("../src/agent/models.ts"); +await import("../src/server/keys.ts"); +const { readCustomAuth } = await import("../src/store/auth.ts"); +const { persistCustomAuth, customAuthState } = await import("../src/server/custom-provider-auth.ts"); +const { authorizeFixture } = await import("./helpers/custom-provider-auth.ts"); +const { modelCatalog, catalogContextLimits } = await import("../src/server/model-catalog.ts"); +const { callCost } = await import("../src/server/pricing.ts"); +const { createReadTool } = await import("../src/agent/tools/read.ts"); +const { installCustomProviderSource } = await import("../src/server/custom-provider.ts"); +let requests: { url: string; auth: string | null; body: Record }[]; +function configure(baseUrl = "http://first/v1", reasoning = false, maxTokens = 512, authMode: "keyless" | "apiKey" = "apiKey") { + const key = authMode === "apiKey" ? readCustomAuth().fileKey() ?? "first-key" : undefined; + writeSettings({ version: 1, customProvider: authorizeFixture({ baseUrl, authMode, models: [{ id: "m", context: 16384, maxTokens, reasoning }] }, key) }); +} +function answer() { + return new Response(`data: ${JSON.stringify({ id: "fake", choices: [{ index: 0, delta: { role: "assistant", content: "ok" }, finish_reason: null }] })}\n\ndata: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 1 } })}\n\ndata: [DONE]\n\n`, { headers: { "content-type": "text/event-stream" } }); +} +function capture(url: unknown, init?: RequestInit) { + requests.push({ url: String(url), auth: new Headers(init?.headers).get("authorization"), body: JSON.parse(String(init?.body)) }); +} +const context = (messages: Message[] = [{ role: "user", content: "synthetic", timestamp: 0 }]) => normalizeContext({ messages }); +const run = (engine: ReturnType, messages?: Message[]) => engine.streamFn(engine.model, context(messages), {}).result(); +beforeEach(() => { + requests = []; fs.writeFileSync(path.join(scratch, "auth.json"), "{}"); configure(); + globalThis.fetch = async (url, init) => { capture(url, init); return answer(); }; +}); +after(() => { process.env = originalEnv; globalThis.fetch = originalFetch; fs.rmSync(scratch, { recursive: true, force: true }); }); + +test("native requests revalidate endpoint/model/key/limits with no fallback and retain in-flight setup", async () => { + const engine = piEngineForSpec({ provider: "custom", id: "m" }); + let started!: () => void; const ready = new Promise((r) => { started = r; }); + let release!: () => void; + globalThis.fetch = async (url, init) => { capture(url, init); started(); return new Promise((resolve) => { release = () => resolve(answer()); }); }; + const inFlight = run(engine); await ready; + configure("http://second/v1", false, 1024); persistCustomAuth(readCustomAuth().binding, "second-key"); + globalThis.fetch = async (url, init) => { capture(url, init); return answer(); }; + assert.equal((await run(engine)).stopReason, "stop"); + release(); assert.equal((await inFlight).stopReason, "stop"); + assert.match(requests[0]!.url, /^http:\/\/first\/v1\/chat\/completions$/); + assert.equal(requests[0]!.auth, "Bearer first-key"); assert.equal(requests[0]!.body.max_completion_tokens, 512); + assert.match(requests[1]!.url, /^http:\/\/second\/v1\/chat\/completions$/); + assert.equal(requests[1]!.auth, "Bearer second-key"); assert.equal(requests[1]!.body.max_completion_tokens, 1024); + writeSettings({ version: 1, customProvider: { ...readSettings()!.customProvider, models: [{ id: "other" }] } }); + assert.ok(piModels().getProvider("custom"), "provider stays authorized while this model is removed"); + assert.equal((await run(engine)).stopReason, "error"); + writeSettings({ version: 1 }); assert.equal((await run(engine)).stopReason, "error"); + assert.equal(requests.length, 2); + assert.ok(piModels().getProvider("openai"), "native route survives custom removal"); +}); +test("current key changes without configuration edits are reflected on the next operation", async () => { + const engine = piEngineForSpec({ provider: "custom", id: "m" }); + await run(engine); persistCustomAuth(readCustomAuth().binding, "rotated-key"); await run(engine); + assert.deepEqual(requests.map((r) => r.auth), ["Bearer first-key", "Bearer rotated-key"]); +}); +test("original saved thinking preference recovers after capability expansion and contracts again", async () => { + const engine = piEngineForSpec({ provider: "custom", id: "m", thinkingLevel: "high" }); + assert.equal(engine.level, "off"); + await run(engine); configure("http://first/v1", true); await run(engine); configure(); await run(engine); + assert.deepEqual(requests.map((r) => r.body.reasoning_effort), [undefined, "high", undefined]); +}); +test("unset preference reselects the existing default against current reasoning metadata", async () => { + const engine = piEngineForSpec({ provider: "custom", id: "m" }); + configure("http://first/v1", true); await run(engine); + assert.equal(requests[0]!.body.reasoning_effort, "medium"); + configure(); await run(engine); assert.equal(requests[1]!.body.reasoning_effort, undefined); +}); +test("catalog capabilities remain reported and prices unknown despite runtime image permission", () => { + const engine = piEngineForSpec({ provider: "custom", id: "m" }); + assert.deepEqual(engine.model.input, ["text", "image"]); + const dto = modelCatalog().models.find((m) => m.provider === "custom")!; + assert.deepEqual(dto.input, ["text"]); assert.equal(dto.cost, null); + assert.equal(callCost({ inputTokens: 100, outputTokens: 100 }, "custom:m"), null); + assert.equal(catalogContextLimits().get("custom:m"), 16384); +}); +const image = { type: "image" as const, data: createCanvas(1, 1).toBuffer("image/png").toString("base64"), mimeType: "image/png" }; +test("actual native encoder preserves USER and tool-result images under missing metadata, including after reload", async () => { + const engine = piEngineForSpec({ provider: "custom", id: "m" }); + const messages: Message[] = [ + { role: "user", content: [{ type: "text", text: "look" }, image], timestamp: 0 }, + fauxAssistantMessage([fauxToolCall("read", {}, { id: "read-1" })], { stopReason: "toolUse" }), + { role: "toolResult", toolCallId: "read-1", toolName: "read", content: [image], isError: false, timestamp: 1 }, + ]; + await run(engine, messages); configure("http://second/v1"); await run(engine, messages); + for (const request of requests) { + const body = JSON.stringify(request.body.messages); + assert.equal((body.match(/"type":"image_url"/g) ?? []).length, 2); + assert.equal((body.match(/data:image\/png;base64/g) ?? []).length, 2); + assert.ok(!body.includes("image omitted")); + } + assert.deepEqual(piModels().getModel("custom", "m")!.input, ["text"]); +}); +test("teacher-bound read returns image without contradictory omission note", async () => { + const file = path.join(scratch, "synthetic.png"); fs.writeFileSync(file, Buffer.from(image.data, "base64")); + const engine = piEngineForSpec({ provider: "custom", id: "m" }); + const result = await createReadTool(engine.model).execute("read-1", { path: file }); + assert.ok(result.content.some((block) => block.type === "image")); + assert.ok(!JSON.stringify(result).includes("image will be omitted")); +}); +test("backend image rejection surfaces once, never retries without images", async () => { + globalThis.fetch = async (url, init) => { capture(url, init); return Response.json({ error: { message: "images rejected by backend" } }, { status: 400 }); }; + const result = await run(piEngineForSpec({ provider: "custom", id: "m" }), [{ role: "user", content: [image], timestamp: 0 }]); + assert.equal(result.stopReason, "error"); assert.match(result.errorMessage ?? "", /images rejected/); assert.equal(requests.length, 1); +}); +test("existing engine refuses nonce mismatch on EVERY operation, without falling back to env", async () => { + const engine = piEngineForSpec({ provider: "custom", id: "m" }); await run(engine); + persistCustomAuth({ ...readCustomAuth().binding!, id: "b".repeat(32) }, "different-key"); + assert.equal((await run(engine)).stopReason, "error"); assert.equal(requests.length, 1); +}); +test("keyless publication never reads the credential lookup", async () => { + configure("http://first/v1", false, 512, "keyless"); + let credentialLookups = 0; + installCustomProviderSource((inspect, cp) => { if (inspect) credentialLookups++; return customAuthState(inspect, cp); }); + try { assert.equal((await run(piEngineForSpec({ provider: "custom", id: "m" }))).stopReason, "stop"); assert.equal(credentialLookups, 0); assert.equal(requests[0]!.auth, "Bearer not-needed"); } + finally { installCustomProviderSource(); } +}); diff --git a/tests/custom-provider-restart.test.ts b/tests/custom-provider-restart.test.ts new file mode 100644 index 0000000..1361c78 --- /dev/null +++ b/tests/custom-provider-restart.test.ts @@ -0,0 +1,81 @@ +/** No real endpoints/accounts. Actual process death and cold startup, not HMR. */ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { after, test } from "node:test"; +import { CUSTOM_AUTH_BINDING } from "../src/core/custom-provider-auth.ts"; +const root = fs.mkdtempSync(path.join(os.tmpdir(), "lh-custom-restart-")); +after(() => fs.rmSync(root, { recursive: true, force: true })); +const base = "http://first.invalid/v1"; +function profile() { + const home = fs.mkdtempSync(path.join(root, "case-")); + const id = "a".repeat(32); + fs.writeFileSync(path.join(home, "settings.json"), JSON.stringify({ version: 1, appearance: { mode: "dark" }, customProvider: { baseUrl: base, authMode: "apiKey", credentialBindingId: id, models: [{ id: "m" }] } })); + fs.writeFileSync(path.join(home, "auth.json"), JSON.stringify({ CUSTOM_OPENAI_API_KEY: "fixture-old-key", OTHER_PROVIDER_KEY: "untouched", future: { preserve: true }, [CUSTOM_AUTH_BINDING]: JSON.stringify({ version: 1, id, baseUrl: base, authMode: "apiKey", fileKeyBaseUrl: base }) }), { mode: 0o600 }); + return home; +} +function child(home: string, mode: "mutate" | "inspect", action = "key", fault = "none", envKey?: string) { + const result = spawnSync(process.execPath, ["--import", "tsx", "tests/helpers/custom-provider-crash.ts", mode, action, fault], { cwd: process.cwd(), env: { PATH: process.env.PATH, HOME: home, USERPROFILE: home, NODE_ENV: "test", LH_DATA_DIR: home, LH_AUTH_FILE: path.join(home, "auth.json"), LH_IGNORE_ENV_KEYS: "0", ...(envKey ? { CUSTOM_OPENAI_API_KEY: envKey } : {}) }, encoding: "utf8", timeout: 15000 }); + assert.ifError(result.error); + assert.ok(result.status === 0 || result.status === 77, `${result.status}: ${result.stderr}`); + return { exit: result.status, data: result.stdout.trim() ? JSON.parse(result.stdout.trim().split("\n").at(-1)!) : undefined }; +} +function closed(home: string, envKey?: string) { + const { data } = child(home, "inspect", "key", "none", envKey); + assert.equal(data.enabled, false); assert.deepEqual(data.requests, []); assert.equal(data.metadataInEnv, false); + assert.ok(!data.statusIds.includes(CUSTOM_AUTH_BINDING)); +} +test("cold restart refuses the new auth/old URL gap and double-I/O failure, with or without env", () => { + for (const fault of ["after-auth", "double-io"]) { + const home = profile(); const result = child(home, "mutate", "key", fault); + if (fault === "after-auth") assert.equal(result.exit, 77); else assert.equal(result.data.status, 500); + closed(home); closed(home, "fixture-real-env"); + const auth = JSON.parse(fs.readFileSync(path.join(home, "auth.json"), "utf8")); + assert.equal(auth.OTHER_PROVIDER_KEY, "untouched"); assert.deepEqual(auth.future, { preserve: true }); + } +}); +test("failed auth publication retains old matching setup; landed settings plus restored auth fails closed", () => { + const home = profile(); assert.equal(child(home, "mutate", "key", "auth-before-rename").data.status, 500); + const old = child(home, "inspect").data; assert.equal(old.enabled, true); assert.equal(old.requests[0].url, base + "/chat/completions"); assert.equal(old.requests[0].authorization, "Bearer fixture-old-key"); + const other = profile(); assert.equal(child(other, "mutate", "key", "settings-landed").data.status, 500); closed(other); +}); +test("clear/disconnect half commits stay closed across restart even with genuine env keys", () => { + for (const action of ["keyless", "disconnect"]) { + for (const fault of ["after-auth", "double-io"]) { + const home = profile(); child(home, "mutate", action, fault, "fixture-real-env"); closed(home, "fixture-real-env"); + } + } +}); +test("successful keyless checks only binding; disconnect tombstone blocks latent env after restart", () => { + const home = profile(); assert.equal(child(home, "mutate", "keyless", "none", "fixture-real-env").data.status, 200); + const state = child(home, "inspect", "key", "none", "fixture-real-env").data; assert.equal(state.enabled, true); assert.equal(state.requests[0].authorization, "Bearer not-needed"); + assert.equal(child(home, "mutate", "disconnect", "none", "fixture-real-env").data.status, 200); closed(home, "fixture-real-env"); +}); +test("env reuse never reauthorizes a shadowed file key for another destination", () => { + const home = profile(); assert.equal(child(home, "mutate", "envMove", "none", "fixture-real-env").data.status, 200); + const live = child(home, "inspect", "key", "none", "fixture-real-env").data; assert.equal(live.enabled, true); assert.equal(live.requests[0].url, "http://second.invalid/v1/chat/completions"); assert.equal(live.requests[0].authorization, "Bearer fixture-real-env"); + closed(home); + const auth = JSON.parse(fs.readFileSync(path.join(home, "auth.json"), "utf8")); assert.equal(JSON.parse(auth[CUSTOM_AUTH_BINDING]).fileKeyBaseUrl, base); assert.equal(auth.CUSTOM_OPENAI_API_KEY, "fixture-old-key"); + assert.equal(child(home, "mutate", "reuse").data.status, 200); assert.equal(child(home, "inspect").data.requests[0].authorization, "Bearer fixture-old-key"); +}); +test("unbound development records require explicit manual reauthorization, never automatic migration", () => { + const home = profile(); const file = path.join(home, "auth.json"); const auth = JSON.parse(fs.readFileSync(file, "utf8")); delete auth[CUSTOM_AUTH_BINDING]; fs.writeFileSync(file, JSON.stringify(auth)); + closed(home); closed(home, "fixture-real-env"); + assert.equal(child(home, "mutate", "reuse").data.status, 200); assert.equal(child(home, "inspect").data.enabled, true); +}); +test("malformed/unreadable private snapshots are never rewritten or used, unrelated bytes survive", () => { + for (const raw of ['{"OTHER_PROVIDER_KEY":"do-not-erase",', '[]', JSON.stringify({ OTHER_PROVIDER_KEY: "do-not-erase", [CUSTOM_AUTH_BINDING]: "not-json" })]) { + const home = profile(); const file = path.join(home, "auth.json"); fs.writeFileSync(file, raw); + closed(home, "fixture-real-env"); assert.equal(child(home, "mutate").data.status, 500); assert.equal(fs.readFileSync(file, "utf8"), raw); + } + const home = profile(); const file = path.join(home, "auth.json"); const before = fs.readFileSync(file, "utf8"); + assert.equal(child(home, "mutate", "key", "unreadable").data.status, 500); assert.equal(fs.readFileSync(file, "utf8"), before); + assert.equal(child(home, "inspect", "key", "unreadable", "fixture-real-env").data.enabled, false); +}); +test("missing auth file initializes normally with explicit new key, preserving other settings", () => { + const home = profile(); fs.unlinkSync(path.join(home, "auth.json")); closed(home); + assert.equal(child(home, "mutate").data.status, 200); const live = child(home, "inspect").data; assert.equal(live.enabled, true); assert.equal(live.requests[0].authorization, "Bearer fixture-new-key"); + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(home, "settings.json"), "utf8")).appearance, { mode: "dark" }); +}); diff --git a/tests/custom-provider-route.test.ts b/tests/custom-provider-route.test.ts index 6315ded..26c9d61 100644 --- a/tests/custom-provider-route.test.ts +++ b/tests/custom-provider-route.test.ts @@ -37,6 +37,7 @@ const CONNECTED = { version: 1, customProvider: { baseUrl: BASE, + authMode: "keyless", models: [{ id: "disc-1", context: 131072 }, { id: "disc-2" }], overrides: { "disc-1": { context: 200000 }, "disc-2": { maxTokens: 8192 } }, manualModels: ["m1"], @@ -59,8 +60,8 @@ function post(payload: unknown): Promise { return route.POST( new Request("http://localhost/api/custom-provider", { method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(payload), + headers: { "content-type": "application/json", origin: "http://localhost", "if-match": route.GET(new Request("http://localhost/api/custom-provider")).headers.get("etag")! }, + body: JSON.stringify({ authMode: payload && typeof payload === "object" && "key" in payload ? "apiKey" : "keyless", ...payload as object }), }), ); } diff --git a/tests/custom-providers.test.ts b/tests/custom-providers.test.ts deleted file mode 100644 index baf2080..0000000 --- a/tests/custom-providers.test.ts +++ /dev/null @@ -1,362 +0,0 @@ -import assert from "node:assert/strict"; -import { after, describe, it } from "node:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { createModels, normalizeContext, type SimpleStreamOptions } from "@earendil-works/pi-ai"; -import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; -import { parseCustomProviders, resolveCustomValue } from "../src/core/custom-providers.ts"; -import { createCustomProvider, customProviderApis } from "../src/agent/custom-providers.ts"; -import { CLAUDE_CODE_PROVIDER } from "../src/core/contracts/claude-code.ts"; - -const zero = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }; -function model(id = "chat", extra = {}) { - return { id, contextWindow: 8192, maxTokens: 1024, cost: zero, ...extra }; -} -function config(extra = {}, models = [model()]) { - return { providers: { gateway: { baseUrl: "http://example.invalid/custom/v1", api: "openai-completions", apiKey: "dummy", models, ...extra } } }; -} -function parse(value: unknown) { return parseCustomProviders(JSON.stringify(value), new Set(["openai", "chatgpt"]), customProviderApis); } - -const temp = fs.mkdtempSync(path.join(os.tmpdir(), "lh-custom-providers-")); -process.env.LH_DATA_DIR = temp; -process.env.LH_AUTH_FILE = path.join(temp, "auth.json"); -process.env.LH_IGNORE_ENV_KEYS = "1"; -fs.writeFileSync(process.env.LH_AUTH_FILE, JSON.stringify({ GW_KEY: "first-key", GW_HEADER: "first-header" })); -const { CUSTOM_MODELS_FILE, SETTINGS_FILE } = await import("../config.ts"); -const { modelCatalog, validateModelsSettings, clampModelsLevels, catalogContextLimits } = await import("../src/server/model-catalog.ts"); -const { piModels, piEngineForSpec } = await import("../src/agent/models.ts"); -const { providerKeyPresent, keyValue } = await import("../src/server/keys.ts"); -const { authStore } = await import("../src/store/auth.ts"); -const { CustomProviderSource, customProviderConfigured, installCustomProviderSource } = await import("../src/server/custom-providers.ts"); -const { callCost } = await import("../src/server/pricing.ts"); -after(() => fs.rmSync(temp, { recursive: true, force: true })); - -function save(value: unknown) { fs.writeFileSync(CUSTOM_MODELS_FILE, JSON.stringify(value)); } -function sseText(text = "ok") { - const chunks = [ - { choices: [{ index: 0, delta: { role: "assistant", content: text }, finish_reason: null }] }, - { choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 } }, - ]; - return new Response(chunks.map((chunk) => `data: ${JSON.stringify({ id: "test", object: "chat.completion.chunk", created: 1, model: "chat", ...chunk })}\n\n`).join("") + "data: [DONE]\n\n", { headers: { "content-type": "text/event-stream" } }); -} - -describe("declarative custom-provider configuration", () => { - it("uses Pi metadata and provider/model inheritance without rewriting base URLs", () => { - const [parsed] = parse(config({ compat: { supportsStore: false, chatTemplateKwargs: { a: true } } }, [model("named", { - name: "Named model", reasoning: true, input: ["text", "image"], thinkingLevelMap: { high: "high", xhigh: null }, - compat: { maxTokensField: "max_tokens", chatTemplateKwargs: { b: false } }, - samplingParams: { temperature: 0.5 }, samplingParamsByThinkingLevel: { high: { top_p: 0.9 } }, - })])); - assert.equal(parsed!.models[0]!.baseUrl, "http://example.invalid/custom/v1"); - assert.equal(parsed!.models[0]!.provider, "gateway"); - assert.equal(parsed!.models[0]!.name, "Named model"); - assert.deepEqual(parsed!.models[0]!.input, ["text", "image"]); - assert.deepEqual(parsed!.models[0]!.compat, { supportsStore: false, maxTokensField: "max_tokens", chatTemplateKwargs: { a: true, b: false } }); - assert.deepEqual(parsed!.models[0]!.samplingParamsByThinkingLevel, { high: { top_p: 0.9 } }); - }); - it("accepts Pi line comments, trailing commas and BOM without corrupting strings", () => { - const text = `\uFEFF${JSON.stringify(config()).replace('"providers":', '// comment\n"providers":').replace(/}$/, ',}')}`; - assert.equal(parseCustomProviders(text, new Set(), customProviderApis)[0]!.models[0]!.baseUrl, "http://example.invalid/custom/v1"); - }); - it("registers multiple providers and mixed native APIs without fixed backend IDs", () => { - const value = config(); - const raw = { providers: { ...value.providers, other: { ...value.providers.gateway, models: [model("chat"), model("anthropic", { api: "anthropic-messages", baseUrl: "https://example.invalid/anthropic" })] } } }; - const models = createModels(); - for (const entry of parse(raw)) models.setProvider(createCustomProvider(entry, () => undefined)); - assert.equal(models.getProviders().length, 2); - assert.equal(models.getModel("other", "anthropic")!.api, "anthropic-messages"); - assert.equal(models.getModel("other", "anthropic")!.baseUrl, "https://example.invalid/anthropic"); - }); - it("requires explicit finite limits and nonnegative costs, instead of guessing", () => { - for (const field of ["contextWindow", "maxTokens", "cost"]) { - const m: Record = model(); delete m[field]; - assert.throws(() => parse(config({}, [m as ReturnType]))); - } - assert.throws(() => parse(config({}, [model("chat", { contextWindow: -1 })]))); - assert.throws(() => parse(config({}, [model("chat", { cost: { ...zero, output: -1 } })]))); - assert.throws(() => parseCustomProviders(JSON.stringify(config()).replace('"contextWindow":8192', '"contextWindow":1e999'), new Set(), customProviderApis)); - }); - it("rejects collisions, duplicate IDs, unsupported protocols, executable fields and invalid URLs", () => { - for (const id of ["openai", "chatgpt", "", "has:colon"]) assert.throws(() => parse({ providers: { [id]: config().providers.gateway } })); - assert.throws(() => parse(config({}, [model(), model()]))); - assert.throws(() => parse(config({ api: "unknown-api" }))); - assert.throws(() => parse(config({ oauth: "radius" }))); - assert.throws(() => parse(config({ modelOverrides: {} }))); - assert.throws(() => parse(config({ apiKey: "!touch /never-execute" }))); - assert.throws(() => parse(config({ headers: { "x-key": "!secret-command" } }))); - assert.throws(() => parse(config({}, [model("chat", { headers: { Authorization: "!command" } })]))); - for (const baseUrl of ["file:///tmp/model", "https://secret@example.invalid", "http://example.invalid/#fragment"]) assert.throws(() => parse(config({ baseUrl }))); - }); - it("matches Pi value interpolation and escapes, without recursive expansion or commands", () => { - const lookup = (key: string) => ({ KEY: "secret", DATA: "!never-executed" })[key as "KEY"]; - assert.equal(resolveCustomValue("Bearer ${KEY}/$KEY", lookup), "Bearer secret/secret"); - assert.equal(resolveCustomValue("$$KEY $!literal", lookup), "$KEY !literal"); - assert.equal(resolveCustomValue("$MISSING", lookup), undefined); - assert.equal(resolveCustomValue("$DATA", lookup), "!never-executed"); - }); -}); - -describe("custom providers in the existing application wiring", () => { - it("is empty before configuration and never creates a configuration file", () => { - assert.equal(fs.existsSync(CUSTOM_MODELS_FILE), false); - assert.equal(modelCatalog().customProvidersError, undefined); - assert.equal(fs.existsSync(CUSTOM_MODELS_FILE), false); - }); - it("projects into existing pickers, validation, context limits and Pi pricing", async () => { - save(config({ apiKey: "$GW_KEY", headers: { "x-gateway": "$GW_HEADER" } }, [model("chat", { cost: { ...zero, input: 2, output: 4 } })])); - const catalog = modelCatalog(); - assert.ok(catalog.providers.find((p) => p.id === "gateway")?.keyPresent); - assert.ok(catalog.models.some((m) => m.provider === "gateway" && m.id === "chat")); - assert.equal(JSON.stringify(catalog).includes("first-key"), false); - assert.equal(JSON.stringify(catalog).includes("first-header"), false); - assert.equal(validateModelsSettings({ strong: { model: { provider: "gateway", id: "chat" } } }).ok, true); - assert.equal(catalogContextLimits().get("gateway:chat"), 8192); - assert.ok(Math.abs(callCost({ inputTokens: 100, outputTokens: 50 }, "gateway:chat")! - 0.0004) < 1e-12); - assert.equal(providerKeyPresent("gateway"), true); - assert.equal((await piModels().getAuth("gateway"))!.auth.apiKey, "first-key"); - }); - it("uses current app key precedence and marks unresolved credentials/headers unavailable", () => { - process.env.LH_IGNORE_ENV_KEYS = "0"; - process.env.GW_KEY = "real-env-key"; - try { assert.equal(keyValue("GW_KEY"), "real-env-key"); } - finally { process.env.LH_IGNORE_ENV_KEYS = "1"; delete process.env.GW_KEY; } - save(config({ apiKey: "$MISSING_KEY" })); - assert.equal(providerKeyPresent("gateway"), false); - assert.equal(modelCatalog().models.some((m) => m.provider === "gateway"), false); - save(config({ headers: { "x-required": "$MISSING_HEADER" } })); - assert.equal(providerKeyPresent("gateway"), false); - save(config({ apiKey: undefined })); - assert.equal(providerKeyPresent("gateway"), false, "Pi-style keyless endpoints need an explicit dummy key"); - }); - it("revalidates every request of one engine while leaving an in-flight HTTP operation alone", async () => { - save(config({ apiKey: "$GW_KEY", headers: { "x-gateway": "$GW_HEADER" }, compat: { supportsStore: false, supportsDeveloperRole: false, supportsReasoningEffort: false, maxTokensField: "max_tokens" } })); - const engine = piEngineForSpec({ provider: "gateway", id: "chat" }); - const context = normalizeContext({ messages: [{ role: "user", content: "hello", timestamp: 1 }] }); - let started!: () => void, release!: (response: Response) => void; - const requestStarted = new Promise((resolve) => { started = resolve; }); - const firstResponse = new Promise((resolve) => { release = resolve; }); - const requests: { url: string; headers: Headers; body: Record }[] = []; - const original = globalThis.fetch; - globalThis.fetch = async (input, init) => { - const request = input instanceof Request ? input : new Request(input, init); - requests.push({ url: request.url, headers: request.headers, body: await request.json() as Record }); - if (requests.length === 1) { started(); return firstResponse; } - return sseText("héllo 🌍"); - }; - const pending = engine.streamFn(engine.model, context); - try { - await requestStarted; - authStore.setAuthVar("GW_KEY", "second-key"); - authStore.setAuthVar("GW_HEADER", "second-header"); - save(config({ baseUrl: "https://new.example.invalid/inference/v1", apiKey: "$GW_KEY", headers: { "x-gateway": "$GW_HEADER" }, compat: { supportsStore: false, maxTokensField: "max_tokens" } }, [model("chat", { maxTokens: 512 })])); - const second = await engine.streamFn(engine.model, context).result(); - assert.equal(second.stopReason, "stop"); - assert.deepEqual(second.content, [{ type: "text", text: "héllo 🌍" }]); - assert.equal(second.usage.input, 10); - assert.equal(second.usage.output, 2); - // Credential-only edits must also take effect on this same engine. - authStore.setAuthVar("GW_KEY", "third-key"); - assert.equal((await engine.streamFn(engine.model, context).result()).stopReason, "stop"); - // None of these cases may dispatch using the engine's old model/key. - save(config({ apiKey: "$MISSING_KEY" })); - assert.equal((await engine.streamFn(engine.model, context).result()).stopReason, "error"); - save(config({}, [model("other-model")])); - assert.equal((await engine.streamFn(engine.model, context).result()).stopReason, "error"); - fs.writeFileSync(CUSTOM_MODELS_FILE, "invalid configuration"); - assert.equal((await engine.streamFn(engine.model, context).result()).stopReason, "error"); - fs.rmSync(CUSTOM_MODELS_FILE); - assert.equal((await engine.streamFn(engine.model, context).result()).stopReason, "error"); - release(sseText("original request finished")); - const first = await pending.result(); - assert.equal(first.stopReason, "stop"); - assert.deepEqual(first.content, [{ type: "text", text: "original request finished" }]); - } finally { - release(sseText()); - await pending.result(); - globalThis.fetch = original; - } - assert.equal(requests.length, 3, "unavailable configurations must not cause additional transport calls"); - assert.equal(requests[0]!.url, "http://example.invalid/custom/v1/chat/completions"); - assert.equal(requests[0]!.headers.get("authorization"), "Bearer first-key"); - assert.equal(requests[0]!.headers.get("x-gateway"), "first-header"); - assert.equal(requests[1]!.url, "https://new.example.invalid/inference/v1/chat/completions"); - assert.equal(requests[1]!.headers.get("authorization"), "Bearer second-key"); - assert.equal(requests[1]!.headers.get("x-gateway"), "second-header"); - assert.equal(requests[2]!.headers.get("authorization"), "Bearer third-key"); - assert.equal(requests[0]!.body.model, "chat"); - assert.equal("store" in requests[0]!.body, false); - assert.equal(requests[0]!.body.max_tokens, 1024); - assert.equal(requests[1]!.body.max_tokens, 512, "current Pi model metadata, not the stale engine model, controls the request"); - }); - const thinking = (...levels: ThinkingLevel[]) => ({ - reasoning: true, - // Pi treats omitted levels through high as supported; mark every - // unselected level explicitly unsupported for these narrow fixtures. - thinkingLevelMap: { off: null, minimal: null, low: null, medium: null, high: null, xhigh: null, max: null, ...Object.fromEntries(levels.map((level) => [level, level])) }, - }); - const thinkingCases: { name: string; preference?: ThinkingLevel; initial: object; updated: object; before: ThinkingLevel; after: ThinkingLevel }[] = [ - { name: "recovers saved max after high-only metadata expands", preference: "max", initial: thinking("high"), updated: thinking("high", "max"), before: "high", after: "max" }, - { name: "recovers saved max after reasoning-off metadata expands", preference: "max", initial: { reasoning: false }, updated: thinking("high", "max"), before: "off", after: "max" }, - { name: "selects the existing medium-first default after reasoning becomes supported", initial: { reasoning: false }, updated: thinking("low", "medium", "high"), before: "off", after: "medium" }, - { name: "reselects the existing low-before-high default against updated metadata", initial: thinking("high"), updated: thinking("low", "high"), before: "high", after: "low" }, - { name: "clamps saved max when supported thinking contracts", preference: "max", initial: thinking("high", "max"), updated: thinking("high"), before: "max", after: "high" }, - { name: "omits reasoning when the model loses thinking support", preference: "max", initial: thinking("high", "max"), updated: { reasoning: false }, before: "max", after: "off" }, - ]; - for (const scenario of thinkingCases) it(scenario.name, async () => { - const definition = (metadata: object) => config({ compat: { supportsReasoningEffort: true, thinkingFormat: "openai" } }, [model("chat", metadata)]); - save(definition(scenario.initial)); - const engine = piEngineForSpec({ provider: "gateway", id: "chat", thinkingLevel: scenario.preference }); - assert.equal(engine.level, scenario.before); - const requests: Record[] = []; - const original = globalThis.fetch; - globalThis.fetch = async (input, init) => { - const request = input instanceof Request ? input : new Request(input, init); - requests.push(await request.json() as Record); - return sseText(); - }; - try { - const context = normalizeContext({ messages: [{ role: "user", content: "hello", timestamp: 1 }] }); - // A long-lived agent still forwards its initial thinking setting. - const options: SimpleStreamOptions = { reasoning: engine.level === "off" ? undefined : engine.level }; - assert.equal((await engine.streamFn(engine.model, context, options).result()).stopReason, "stop"); - save(definition(scenario.updated)); - assert.equal((await engine.streamFn(engine.model, context, options).result()).stopReason, "stop"); - } finally { globalThis.fetch = original; } - assert.equal(requests.length, 2); - for (const [index, level] of [scenario.before, scenario.after].entries()) { - assert.equal(requests[index]!.reasoning_effort, level === "off" ? undefined : level); - if (level === "off") assert.equal("reasoning_effort" in requests[index]!, false); - } - }); - it("disables custom routes on malformed edits without removing built-ins or saved choices", () => { - fs.writeFileSync(SETTINGS_FILE, JSON.stringify({ version: 1, models: { strong: { model: { provider: "gateway", id: "chat" }, thinkingLevel: "high" } } })); - const before = fs.readFileSync(SETTINGS_FILE, "utf8"); - fs.writeFileSync(CUSTOM_MODELS_FILE, '{ secret-token-do-not-log'); - const catalog = modelCatalog(); - assert.ok(catalog.customProvidersError); - assert.equal(catalog.customProvidersError!.includes("secret-token"), false); - assert.equal(piModels().getProvider("gateway"), undefined); - assert.ok(piModels().getProvider("openai")); - assert.deepEqual(catalog.slots.strong!.model, { provider: "gateway", id: "chat" }); - const stored = { strong: { model: { provider: "gateway", id: "chat" }, thinkingLevel: "high" as const } }; - const repaired = { strong: { ...stored.strong, thinkingLevel: "max" as const }, cheap: { model: { provider: "openai", id: piModels().getModels("openai")[0]!.id } } }; - assert.equal(validateModelsSettings(repaired, stored).ok, true, "an unavailable custom pin must not prevent independent slot repair"); - assert.equal(clampModelsLevels(repaired).strong!.thinkingLevel, "max", "preserve thinking while custom metadata is unavailable"); - assert.equal(validateModelsSettings({ cheap: stored.strong }, stored).ok, false, "unchanged identity preservation is same-slot only"); - assert.equal(fs.readFileSync(SETTINGS_FILE, "utf8"), before); - assert.throws(() => piEngineForSpec({ provider: "gateway", id: "chat" }), /catalog has no model/); - save(config()); - assert.equal(modelCatalog().customProvidersError, undefined); - assert.ok(piModels().getProvider("gateway")); - fs.rmSync(CUSTOM_MODELS_FILE); - assert.equal(modelCatalog().models.some((m) => m.provider === "gateway"), false); - assert.equal(modelCatalog().customProvidersError, undefined); - assert.equal(fs.readFileSync(SETTINGS_FILE, "utf8"), before); - }); - it("refreshes credentials without a file edit, preserving native header precedence and snapshots", async () => { - const values = new Map([["KEY", "one"], ["HEADER", "first"]]); - const source = new CustomProviderSource(() => ({ text: JSON.stringify(config({ apiKey: "$KEY", authHeader: true, headers: { "x-shared": "$HEADER" } }, [model("chat", { headers: { "x-shared": "model-${HEADER}" } })])) }), (name) => values.get(name)); - const models = createModels(); - source.sync(models); - const captured = createModels(); - captured.setProvider(models.getProvider("gateway")!); - values.set("KEY", "two"); values.set("HEADER", "second"); - source.sync(models); - const current = (await models.getAuth(models.getModel("gateway", "chat")!))!.auth; - const previous = (await captured.getAuth(captured.getModel("gateway", "chat")!))!.auth; - assert.equal(current.apiKey, "two"); - assert.equal(current.headers!["x-shared"], "model-second"); - assert.equal(current.headers!.Authorization, "Bearer two"); - assert.equal(previous.apiKey, "one"); - assert.equal(previous.headers!["x-shared"], "model-first"); - }); - it("rejects built-in, subscription and native CLI collisions without replacing their providers", () => { - for (const id of ["openai", "github-copilot", "chatgpt", "github-copilot-subscription", "xai-subscription", CLAUDE_CODE_PROVIDER]) { - const existing = piModels().getProvider("openai")!; - const nativeCli = piModels().getProvider(CLAUDE_CODE_PROVIDER)!; - const models = createModels(); models.setProvider(existing); models.setProvider(nativeCli); - const source = new CustomProviderSource(() => ({ text: JSON.stringify({ providers: { [id]: config().providers.gateway } }) }), () => undefined); - source.sync(models); - assert.ok(source.getError()); - assert.equal(models.getProvider("openai"), existing); - assert.equal(models.getProvider(CLAUDE_CODE_PROVIDER), nativeCli); - assert.equal(models.getProviders().length, 2); - } - }); - it("cannot bypass CLI opt-in or replace/remove the native provider across custom configuration reloads", (t) => { - const nativeCli = piModels().getProvider(CLAUDE_CODE_PROVIDER)!; - assert.ok(nativeCli); - const aliases = piModels().getModels(CLAUDE_CODE_PROVIDER); - const savedConfig = fs.existsSync(CUSTOM_MODELS_FILE) ? fs.readFileSync(CUSTOM_MODELS_FILE, "utf8") : undefined; - const savedSettings = fs.existsSync(SETTINGS_FILE) ? fs.readFileSync(SETTINGS_FILE, "utf8") : undefined; - t.after(() => { - if (savedConfig === undefined) fs.rmSync(CUSTOM_MODELS_FILE, { force: true }); else fs.writeFileSync(CUSTOM_MODELS_FILE, savedConfig); - if (savedSettings === undefined) fs.rmSync(SETTINGS_FILE, { force: true }); else fs.writeFileSync(SETTINGS_FILE, savedSettings); - piModels(); - }); - fs.writeFileSync(SETTINGS_FILE, JSON.stringify({ version: 1, claudeCode: { enabled: false } })); - const collision = { providers: { [CLAUDE_CODE_PROVIDER]: config().providers.gateway } }; - for (const [value, error] of [ - [collision, true], // Initial collision must never enter registered/ready. - [config(), false], // A normal custom route still works. - [collision, true], // Invalid reload retires custom routes, not native ones. - ["malformed configuration", true], - [undefined, false], // Removing the file must not delete the native CLI. - [config(), false], - [undefined, false], - ] as const) { - if (value === undefined) fs.rmSync(CUSTOM_MODELS_FILE, { force: true }); - else if (typeof value === "string") fs.writeFileSync(CUSTOM_MODELS_FILE, value); - else save(value); - const catalog = modelCatalog(); - assert.equal(Boolean(catalog.customProvidersError), error); - assert.equal(customProviderConfigured(CLAUDE_CODE_PROVIDER), undefined); - assert.equal(providerKeyPresent(CLAUDE_CODE_PROVIDER), false, "custom auth must never bypass disabled CLI admission"); - assert.equal(piModels().getProvider(CLAUDE_CODE_PROVIDER), nativeCli, "preserve the native provider identity, not a reconstructed substitute"); - assert.deepEqual(piModels().getModels(CLAUDE_CODE_PROVIDER), aliases); - assert.equal(catalog.models.some((m) => m.provider === CLAUDE_CODE_PROVIDER), false); - } - }); - it("fails closed across partial publication and recovers without leaking internal errors", () => { - const text = JSON.stringify({ providers: { ...config().providers, second: config().providers.gateway } }); - const source = new CustomProviderSource(() => ({ text }), () => undefined); - const models = createModels(); - const builtin = piModels().getProvider("openai")!; - models.setProvider(builtin); - const publish = models.setProvider.bind(models); - models.setProvider = (provider) => { - if (provider.id === "second") throw new Error("private-value-do-not-log"); - publish(provider); - }; - source.sync(models); - assert.equal(models.getProvider("gateway"), undefined); - assert.equal(models.getProvider("second"), undefined); - assert.equal(models.getProvider("openai"), builtin); - assert.equal(source.getError()!.includes("private-value"), false); - models.setProvider = publish; - source.sync(models); - assert.ok(models.getProvider("gateway")); - assert.ok(models.getProvider("second")); - assert.equal(source.getError(), undefined); - }); - it("retires old custom routes when server configuration bootstrap is reinstalled", () => { - save(config()); - assert.ok(piModels().getProvider("gateway")); - fs.rmSync(CUSTOM_MODELS_FILE); - installCustomProviderSource(keyValue); - assert.equal(piModels().getProvider("gateway"), undefined); - assert.ok(piModels().getProvider("openai")); - }); - it("treats unreadable config as unavailable and recovers without a restart", () => { - let unreadable = true; - const source = new CustomProviderSource(() => unreadable ? { error: "Unable to read models.json." } : { text: JSON.stringify(config()) }, () => undefined); - const models = createModels(); - source.sync(models); - assert.equal(source.getError(), "Unable to read models.json."); - unreadable = false; - source.sync(models); - assert.ok(models.getProvider("gateway")); - assert.equal(source.getError(), undefined); - }); -}); diff --git a/tests/helpers/custom-provider-auth.ts b/tests/helpers/custom-provider-auth.ts new file mode 100644 index 0000000..acdeafe --- /dev/null +++ b/tests/helpers/custom-provider-auth.ts @@ -0,0 +1,10 @@ +/** Synthetic authorized records, imported only after each test's isolated env. */ +import { randomBytes } from "node:crypto"; +import type { CustomProviderSettings } from "../../src/core/settings.ts"; +import { persistCustomAuth } from "../../src/server/custom-provider-auth.ts"; + +export function authorizeFixture(cp: CustomProviderSettings, fileKey?: string): CustomProviderSettings { + const id = randomBytes(16).toString("hex"); + persistCustomAuth({ version: 1, id, baseUrl: cp.baseUrl, authMode: cp.authMode ?? "keyless", ...(fileKey ? { fileKeyBaseUrl: cp.baseUrl } : {}) }, fileKey); + return { ...cp, credentialBindingId: id }; +} diff --git a/tests/helpers/custom-provider-crash.ts b/tests/helpers/custom-provider-crash.ts new file mode 100644 index 0000000..62f2e5a --- /dev/null +++ b/tests/helpers/custom-provider-crash.ts @@ -0,0 +1,56 @@ +/** Child-process fixture: owned profile only; every upstream fetch is synthetic. */ +import fs from "node:fs"; +import { AUTH_FILE, SETTINGS_FILE } from "../../config.ts"; +const [mode, action = "key", fault = "none"] = process.argv.slice(2); +const requests: { url: string; authorization: string | null }[] = []; +globalThis.fetch = async (url, init) => { + requests.push({ url: String(url), authorization: new Headers(init?.headers).get("authorization") }); + if (String(url).endsWith("/models")) return Response.json({ data: [{ id: "m" }] }); + return new Response('data: {"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":null}]}\n\ndata: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', { headers: { "content-type": "text/event-stream" } }); +}; +const rename = fs.renameSync; +const write = fs.writeFileSync; +const read = fs.readFileSync; +let authRenames = 0; +fs.renameSync = (...args: Parameters) => { + if (args[1] === AUTH_FILE) { + authRenames++; + if (fault === "auth-before-rename" || (fault === "double-io" && authRenames > 1)) throw new Error("fixture auth I/O failure"); + rename(...args); + if (fault === "after-auth") process.exit(77); + return; + } + return rename(...args); +}; +fs.writeFileSync = (...args: Parameters) => { + if (args[0] === SETTINGS_FILE && fault === "double-io") throw new Error("fixture settings I/O failure"); + if (args[0] === SETTINGS_FILE && fault === "settings-landed") { write(...args); throw new Error("fixture ambiguous settings completion"); } + return write(...args); +}; +fs.readFileSync = ((...args: Parameters) => { + if (args[0] === AUTH_FILE && fault === "unreadable") throw Object.assign(new Error("fixture unreadable auth"), { code: "EACCES" }); + return read(...args); +}) as typeof read; +await import("../../src/server/keys.ts"); +if (mode === "inspect") { + const { piModels, piEngineForSpec } = await import("../../src/agent/models.ts"); + const { normalizeContext } = await import("@earendil-works/pi-ai"); + const enabled = Boolean(piModels().getProvider("custom")); + let reason: string | undefined; + try { + const engine = piEngineForSpec({ provider: "custom", id: "m" }); + reason = (await engine.streamFn(engine.model, normalizeContext({ messages: [{ role: "user", content: "fixture", timestamp: 0 }] }), {}).result()).stopReason; + } catch { reason = "refused"; } + const { keyStatuses } = await import("../../src/server/keys.ts"); + console.log(JSON.stringify({ enabled, reason, requests, statusIds: keyStatuses().map(row => row.id), metadataInEnv: process.env.__learnharness_custom_provider_binding !== undefined })); +} else { + const route = await import("../../app/api/custom-provider/route.ts"); + const get = route.GET(new Request("http://localhost/api/custom-provider")); + const payload = action === "keyless" ? { baseUrl: "http://second.invalid/v1", authMode: "keyless" } + : action === "envMove" ? { baseUrl: "http://second.invalid/v1", authMode: "apiKey", reuseKey: true } + : action === "reuse" ? { baseUrl: "http://first.invalid/v1", authMode: "apiKey", reuseKey: true } + : { baseUrl: "http://second.invalid/v1", authMode: "apiKey", key: "fixture-new-key" }; + const request = new Request("http://localhost/api/custom-provider", { method: action === "disconnect" ? "DELETE" : "POST", headers: { origin: "http://localhost", "content-type": "application/json", "if-match": get.headers.get("etag")! }, body: JSON.stringify(payload) }); + const result = await route.POST(request); + console.log(JSON.stringify({ status: result.status, body: await result.json(), requests })); +}