BYOK universal gate rendered honestly: mid-chat setup flow, refusal-aware cards
- existing sessions: a send that lands the 503 models_not_configured envelope renders as the SETUP FLOW, never the error banner — the record refetch drops the refused orphan (the gate refuses before the write-ahead persist), the draft goes home to the composer, and Save auto-replays it (owner ruling); parseModelsRefusal reads the envelope from the transport's verbatim error text (exported pure, tested) - persistent 'Set up your models' CTA above the chat composer while slotsConfigured is false (server-read seed via app/chat/[id]/page.tsx) — legacy per-chat overrides may still send with the pill present; the server gate stays the truth - SetupDialog prefill + 'needed' marks (owner ruling): pinned tiers open with their pick (matched in the federated catalog — lapsed keys don't prefill), unpinned tiers carry the mark; Save still requires all three; prefillFromSlots exported pure, tested - refused jobs read as INFO: dispatch card shows a quiet 'needs model setup' line + 'Set up models' deep-link (ChatActionsContext.openSetup); dock blobs' activity reads 'needs model setup' instead of 'failed'; real failures keep the red tone; isModelsRefusal added to the envelope's home module - agent-section ModelsBody.slots type honest to the nullable contract
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { parseModelsRefusal } from "../components/chat.tsx";
|
||||
import { prefillFromSlots, setupStage } from "../components/setup-dialog.tsx";
|
||||
import { isModelsRefusal, modelsNotConfigured } from "../src/core/settings.ts";
|
||||
import type { ModelCatalogEntry, ModelSlotName } from "../src/core/index.ts";
|
||||
|
||||
/**
|
||||
* The BYOK universal gate rendered honestly (owner, 2026-09-22): the server
|
||||
* refuses ALL turns with 503 {error, code: "models_not_configured", missing}
|
||||
* when a slot tier is unset. These pin the client's pure seams: the refusal
|
||||
* detector (the transport throws the 503 body verbatim as the error text),
|
||||
* the job-record refusal matcher, and the SetupDialog's prefill.
|
||||
*/
|
||||
|
||||
const REFUSAL = JSON.stringify(modelsNotConfigured(["strong"]));
|
||||
|
||||
describe("parseModelsRefusal — the send refusal detector", () => {
|
||||
it("reads the missing tiers from the envelope text", () => {
|
||||
assert.deepEqual(parseModelsRefusal(REFUSAL), ["strong"]);
|
||||
assert.deepEqual(parseModelsRefusal(JSON.stringify(modelsNotConfigured(["strong", "vision"]))), [
|
||||
"strong",
|
||||
"vision",
|
||||
]);
|
||||
});
|
||||
|
||||
it("anything else is a real error — null, the honest banner stays", () => {
|
||||
assert.equal(parseModelsRefusal(undefined), null);
|
||||
assert.equal(parseModelsRefusal("Something went wrong"), null); // not JSON
|
||||
assert.equal(parseModelsRefusal('{"error":"A turn is already running"}'), null); // JSON, wrong code
|
||||
assert.equal(parseModelsRefusal('{"code":"models_not_configured","missing":"strong"}'), null); // missing not an array
|
||||
assert.equal(parseModelsRefusal('{"code":"models_not_configured","missing":[42]}'), null); // tiers are strings
|
||||
});
|
||||
});
|
||||
|
||||
describe("isModelsRefusal — the job-record matcher", () => {
|
||||
it("matches the runner's composed wording, not real failures", () => {
|
||||
assert.equal(isModelsRefusal(`${modelsNotConfigured(["vision"]).error} (missing: vision)`), true);
|
||||
assert.equal(isModelsRefusal(modelsNotConfigured(["cheap"]).error), true);
|
||||
assert.equal(isModelsRefusal(undefined), false);
|
||||
assert.equal(isModelsRefusal("FAILED: headless crashed"), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prefillFromSlots — the SetupDialog prefill (BYOK wave)", () => {
|
||||
const entry = (id: string, provider: string): ModelCatalogEntry => ({
|
||||
id,
|
||||
name: id,
|
||||
provider,
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
supportedLevels: ["low"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 1024,
|
||||
});
|
||||
const catalog = [entry("glm-5.3-flash", "z-ai"), entry("qwen3.8-flash", "qwen")];
|
||||
const pins = (strong: boolean, vision: boolean) => ({
|
||||
strong: strong ? { model: { provider: "z-ai", id: "glm-5.3-flash" } } : null,
|
||||
cheap: null,
|
||||
vision: vision ? { model: { provider: "qwen", id: "qwen3.8-flash" } } : null,
|
||||
});
|
||||
|
||||
it("prefills exactly the pinned tiers, matched by provider AND id", () => {
|
||||
const pre = prefillFromSlots(catalog, pins(true, true));
|
||||
assert.equal(pre.strong?.id, "glm-5.3-flash");
|
||||
assert.equal(pre.vision?.id, "qwen3.8-flash");
|
||||
assert.equal(pre.cheap, undefined);
|
||||
});
|
||||
|
||||
it("nothing pinned (first-run) → nothing prefilled — all three marked needed", () => {
|
||||
const pre = prefillFromSlots(catalog, pins(false, false));
|
||||
assert.deepEqual(pre, {});
|
||||
});
|
||||
|
||||
it("a pin whose model left the federated catalog does not prefill (lapsed key)", () => {
|
||||
const pre = prefillFromSlots(catalog, { strong: { model: { provider: "gone", id: "vanished" } }, cheap: null, vision: null });
|
||||
assert.deepEqual(pre, {});
|
||||
});
|
||||
|
||||
it("provider collisions never cross-pollute (same id, different provider)", () => {
|
||||
const both = [entry("shared-id", "openai"), entry("shared-id", "openrouter")];
|
||||
const pre = prefillFromSlots(both, { strong: { model: { provider: "openrouter", id: "shared-id" } }, cheap: null, vision: null });
|
||||
assert.equal(pre.strong?.provider, "openrouter");
|
||||
});
|
||||
|
||||
it("absent slots payload (older shape) prefills nothing", () => {
|
||||
const pre = prefillFromSlots(catalog, undefined);
|
||||
assert.deepEqual(pre, {});
|
||||
});
|
||||
});
|
||||
|
||||
describe("setupStage — unchanged by the BYOK wave", () => {
|
||||
it("keys first (owner ordering), pickers when a provider is keyed", () => {
|
||||
assert.equal(setupStage(null), "keys");
|
||||
assert.equal(setupStage([{ id: "x", keyPresent: false }]), "keys");
|
||||
assert.equal(setupStage([{ id: "x", keyPresent: true }]), "pickers");
|
||||
});
|
||||
});
|
||||
@@ -65,6 +65,20 @@ describe("activity dock pure layer", () => {
|
||||
assert.equal(absent.cost, null);
|
||||
});
|
||||
|
||||
it("a models-refused job reads as info — 'needs model setup', not 'failed' (BYOK, 2026-09-22)", () => {
|
||||
const refused = job({
|
||||
status: "error",
|
||||
activity: "failed",
|
||||
error: "No model configured — set up your models first (missing: vision)",
|
||||
});
|
||||
const blob = toBlobs({ jobs: [refused], extractions: [] })[0]!;
|
||||
assert.equal(blob.activity, "needs model setup");
|
||||
// A real failure keeps the runner's honest wording.
|
||||
const crashed = job({ status: "error", activity: "failed", error: "FAILED: headless crashed" });
|
||||
const real = toBlobs({ jobs: [crashed], extractions: [] })[0]!;
|
||||
assert.equal(real.activity, "failed");
|
||||
});
|
||||
|
||||
it("context line = LAST step's tokens vs the model limit, never a cumulative sum", () => {
|
||||
const base = {
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
|
||||
@@ -18,6 +18,10 @@ import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { MessageResponse } from "../components/ai-elements/message.tsx";
|
||||
import { QuestionDock } from "../components/question-dock.tsx";
|
||||
import { ProbeDoneCard } from "../components/artifacts/probe-done-card.tsx";
|
||||
import { DispatchCard } from "../components/artifacts/dispatch-card.tsx";
|
||||
import { JobsContext } from "../components/jobs-context.ts";
|
||||
import { ChatActionsContext } from "../components/chat.tsx";
|
||||
import type { JobRecord } from "../src/core/contracts.ts";
|
||||
import { AnsweredCard } from "../components/answered-card.tsx";
|
||||
import { buildAnswer, coerceCorrectAnswer, normalizeOptions, resolveCorrect, type QuestionData } from "../src/core/question-core.ts";
|
||||
|
||||
@@ -63,6 +67,54 @@ function renderDock(props: { toolName: "quiz" | "ask_user_question"; toolCallId:
|
||||
);
|
||||
}
|
||||
|
||||
describe("DispatchCard — the BYOK refusal tone (owner, 2026-09-22)", () => {
|
||||
const job = (over: Partial<JobRecord>): JobRecord => ({
|
||||
id: "j1",
|
||||
kind: "research",
|
||||
chatId: "c1",
|
||||
toolCallId: "t1",
|
||||
status: "error",
|
||||
activity: "",
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...over,
|
||||
});
|
||||
const inJobs = (jobs: JobRecord[]) => (
|
||||
<JobsContext.Provider value={jobs}>
|
||||
<ChatActionsContext.Provider
|
||||
value={{ chatId: "c1", syncFromServer: async () => {}, sendText: () => {}, openSetup: () => {} }}
|
||||
>
|
||||
<DispatchCard toolCallId="t1" kind="research" />
|
||||
</ChatActionsContext.Provider>
|
||||
</JobsContext.Provider>
|
||||
);
|
||||
|
||||
it("a models-refused job is INFO: quiet card, 'needs model setup', deep-link to the dialog", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
inJobs([job({ error: "No model configured — set up your models first (missing: cheap)" })]),
|
||||
);
|
||||
assert.ok(html.includes('data-testid="dispatch-refused"'));
|
||||
assert.ok(html.includes("needs model setup"));
|
||||
assert.ok(html.includes('data-testid="dispatch-setup-link"'));
|
||||
assert.ok(html.includes("Set up models"));
|
||||
assert.ok(!html.includes("failed"));
|
||||
assert.ok(!html.includes("text-red-500"));
|
||||
});
|
||||
|
||||
it("a real failure keeps the honest red failure", () => {
|
||||
const html = renderToStaticMarkup(inJobs([job({ error: "FAILED: headless crashed" })]));
|
||||
assert.ok(html.includes('data-testid="dispatch-settled"'));
|
||||
assert.ok(html.includes("failed"));
|
||||
assert.ok(html.includes("text-red-500"));
|
||||
assert.ok(!html.includes("dispatch-refused"));
|
||||
});
|
||||
|
||||
it("a delivered job reads delivered, untouched by the wave", () => {
|
||||
const html = renderToStaticMarkup(inJobs([job({ status: "done", error: undefined })]));
|
||||
assert.ok(html.includes("delivered"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("render smoke tests", () => {
|
||||
it("QuizCard renders question, options, and the automatic 'I don't know'", () => {
|
||||
const html = renderDock({ toolName: "quiz", toolCallId: "q1", input: quizInput as never });
|
||||
|
||||
Reference in New Issue
Block a user