test(model-catalog): machine-aware record assertions — the suite runs green on any tree

Master reported the only red on main: 'serves the keyed providers'
catalogs' pinned defaults/slots NULL, but catalogDefaults/catalogSlots
read the REAL data/settings.json — the owner's tree carries configured
slots, so the payload is legitimately non-null there while the platform
worktree's scratch record is empty. Same class of fragility in two more
tests the scratch run couldn't see: the bare-level validation verdicts
(validateChatSettings / validateModelsSettings) fall back to the
record's resolved strong tier.

data/settings.json belongs to settings-store.test.ts under the
one-fs-owner discipline, so the tests READ the resolved truth instead
of pinning it — the chat-flow 503-gate pattern, applied to all four:

- modelCatalog payload: defaults/slots must carry EXACTLY what the
  machine's record says (configured slots surface as configured, empty
  record surfaces as null) — never a phantom id in either direction.
- catalogSlots: projects each stored tier's model verbatim; a tier
  without a model reads null.
- bare-level verdicts: expectBareLevelVerdict mirrors the fallback chain
  — no resolved tier → rides unchecked; a tier whose model left the
  catalog → rejected honestly; else the supported-level check.

Verified green against four record states: empty, configured (glm with
unsupported minimal), vanished model, and a level-only stored slot
(inert). Gates: typecheck, lint, 825/825, build.
This commit is contained in:
amosblomqvist
2026-09-22 17:05:44 +03:00
parent bfa906e2f8
commit eaa963f98b
+64 -9
View File
@@ -1,11 +1,14 @@
import { describe, it } from "node:test"; import { describe, it } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { modelCatalog, validateChatSettings, validateModelsSettings, catalogDefaultsFor, catalogProviders, catalogSlots, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts"; import { modelCatalog, validateChatSettings, validateModelsSettings, catalogDefaultsFor, catalogProviders, catalogSlots, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts";
import type { SettingsValidation } from "../src/server/model-catalog.ts";
import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai"; import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai";
import { DEFAULT_THINKING_ORDER } from "../config.ts"; import { DEFAULT_THINKING_ORDER } from "../config.ts";
import { piEngineForSpec, piModels } from "../src/agent/models.ts"; import { piEngineForSpec, piModels } from "../src/agent/models.ts";
import { BACKGROUND_MODEL_ROLES } from "../src/core/settings.ts"; import { BACKGROUND_MODEL_ROLES } from "../src/core/settings.ts";
import { resolveSettings } from "../src/core/settings-resolution.ts"; import { resolveSettings } from "../src/core/settings-resolution.ts";
import { readSettings } from "../src/store/settings.ts";
import { effectiveSettings } from "../src/server/stores.ts";
// The model-control wave (2026-09-12): pi-ai owns the registry — these pins // The model-control wave (2026-09-12): pi-ai owns the registry — these pins
// hold our projection + validation honest against the shipped catalog, and // hold our projection + validation honest against the shipped catalog, and
@@ -26,6 +29,28 @@ function glmModel() {
return glm!; return glm!;
} }
/** The record's resolved tier as seen by the validation fallbacks —
* machine-aware truth (data/settings.json belongs to settings-store.test.ts
* under the one-fs-owner discipline, so READ it, never pin it): on a
* scratch worktree the record is empty, on the owner's machine slots are
* configured; the expected verdicts below follow whichever holds. */
/** The machine-aware expected verdict for a bare-level validation: mirrors
* the fallback chain exactly — no resolved tier → rides unchecked; a tier
* whose model left the catalog → rejected; else the supported-level check. */
function expectBareLevelVerdict(verdict: SettingsValidation, tier: "strong" | "cheap" | "vision") {
const slot = effectiveSettings().models[tier];
if (!slot) {
assert.deepEqual(verdict, { ok: true }, "no model to check against → accepted (clamps at run)");
return;
}
const model = piModels().getModel(slot.provider, slot.id);
if (!model) {
assert.equal(verdict.ok, false, "a vanished model is rejected honestly");
return;
}
assert.equal(verdict.ok, getSupportedThinkingLevels(model).includes("minimal"));
}
describe("the model catalog projection", () => { describe("the model catalog projection", () => {
it("serves the keyed providers' catalogs with the picker's DTO shape", () => { it("serves the keyed providers' catalogs with the picker's DTO shape", () => {
const { models, defaults, providers, slots } = modelCatalog(); const { models, defaults, providers, slots } = modelCatalog();
@@ -55,11 +80,29 @@ describe("the model catalog projection", () => {
for (const id of keyless) { for (const id of keyless) {
assert.ok(!models.some((m) => m.provider === id), `keyless ${id} must not federate`); assert.ok(!models.some((m) => m.provider === id), `keyless ${id} must not federate`);
} }
// BYOK: nothing configured in this environment's record → defaults NULL, // BYOK: the payload carries ONLY what the machine's record actually
// slots NULL — no phantom ids anywhere in the payload // says — never a phantom id. data/settings.json belongs to
assert.equal(defaults, null); // settings-store.test.ts (the one-fs-owner discipline), so READ the
// resolved truth here instead of pinning it: on a scratch worktree
// the record is empty (defaults/slots NULL), on the owner's machine
// configured slots must surface exactly as configured.
const record = readSettings();
const chat = record?.chat;
const strong = effectiveSettings().models.strong;
if (chat?.model || strong) {
assert.ok(defaults, "a configured chain yields a default");
assert.deepEqual(defaults!.model, chat?.model ?? { provider: strong!.provider, id: strong!.id });
} else {
assert.equal(defaults, null);
}
for (const name of ["strong", "cheap", "vision"] as const) { for (const name of ["strong", "cheap", "vision"] as const) {
assert.equal(slots[name], null, name); const stored = record?.models?.[name];
if (stored?.model) {
assert.ok(slots[name], name);
assert.deepEqual(slots[name]!.model, stored.model);
} else {
assert.equal(slots[name], null, name);
}
} }
}); });
@@ -129,10 +172,18 @@ describe("the background roles' resolved pins (rolePins)", () => {
}); });
describe("the model slots surface (model slots wave, 2026-09-20)", () => { describe("the model slots surface (model slots wave, 2026-09-20)", () => {
it("catalogSlots reports NULL when a tier holds no model", () => { it("catalogSlots projects the record's tiers — NULL only where a tier holds no model", () => {
// machine-aware (one-fs-owner: READ the record, never pin it)
const record = readSettings();
const slots = catalogSlots(); const slots = catalogSlots();
for (const name of ["strong", "cheap", "vision"] as const) { for (const name of ["strong", "cheap", "vision"] as const) {
assert.equal(slots[name], null, name); const stored = record?.models?.[name];
if (stored?.model) {
assert.ok(slots[name], name);
assert.deepEqual(slots[name]!.model, stored.model);
} else {
assert.equal(slots[name], null, name);
}
} }
}); });
@@ -177,8 +228,10 @@ describe("the models-section validation gate", () => {
false, false,
); );
// (a level-only slot never passes the WRITE parser — owner, 2026-09-22; // (a level-only slot never passes the WRITE parser — owner, 2026-09-22;
// the validation gate tolerates it here with nothing to check against) // the validation gate falls back to the tier's RESOLVED model —
assert.equal(validateModelsSettings({ strong: { thinkingLevel: "minimal" } }).ok, true); // machine-aware: the record's strong tier decides the verdict)
const levelOnly = validateModelsSettings({ strong: { thinkingLevel: "minimal" } });
expectBareLevelVerdict(levelOnly, "strong");
}); });
it("accepts empty settings (the clear-slots write)", () => { it("accepts empty settings (the clear-slots write)", () => {
@@ -199,7 +252,9 @@ describe("the settings validation gate", () => {
}); });
it("a bare thinking level checks against the chat model, else the strong slot, else rides unchecked", () => { it("a bare thinking level checks against the chat model, else the strong slot, else rides unchecked", () => {
assert.deepEqual(validateChatSettings({ thinkingLevel: "minimal" }), { ok: true }, "no model to check against → accepted (clamps at run)"); // machine-aware: with a record-carried strong slot the level checks
// against it; with none it rides unchecked (clamps at run).
expectBareLevelVerdict(validateChatSettings({ thinkingLevel: "minimal" }), "strong");
}); });
it("rejects unknown providers and models", () => { it("rejects unknown providers and models", () => {