Merge compaction and native usage accounting fixes

This commit is contained in:
amosblomqvist
2026-10-24 19:53:18 +03:00
53 changed files with 930 additions and 2337 deletions
+3
View File
@@ -88,10 +88,13 @@ test("activityFeed: contextLimit is the server's live-catalog join (owner amendm
// pi 1.0.2's catalog: glm-5.3-flash contextWindow 1_048_575 (was
// 1_048_576 at 0.87.1 — upstream models.dev refresh).
assert.equal(knownJob.contextLimit, 1_048_575, "pi's catalog, joined server-side onto the feed");
assert.ok(knownJob.costUsd !== null && Math.abs(knownJob.costUsd - 0.0002) < 1e-12, "Pi's price is projected on the server");
const unknownJob = feed.jobs.find((j) => j.id === unknown.id)!;
assert.equal(unknownJob.contextLimit, null, "missing from the live catalog → null, never a guessed window");
assert.equal(unknownJob.costUsd, null);
const bareJob = feed.jobs.find((j) => j.id === bare.id)!;
assert.equal(bareJob.contextLimit, null, "no usage yet → no context line");
assert.equal(bareJob.costUsd, null);
} finally {
fs.unlinkSync(`${PATHS.chats}/${chat.id}.json`);
const jobsDir = path.join(PATHS.data, "jobs");
+15 -2
View File
@@ -1,4 +1,4 @@
import { test, after } from "node:test";
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
@@ -21,7 +21,20 @@ let revision = 1;
function connection(background: boolean) {
ipc({ channel: "lh:chatgpt", kind: "state", stamp: { epoch: 1, authorityRevision: 1, revision: revision++ }, invalidate: background ? "none" : "background", status: { available: true, pending: false, allowBackgroundUse: background, activeId: "scratch-account", accounts: [{ id: "scratch-account", connected: true, planEnabled: true, background: !background }], error: null } });
}
after(() => { fs.rmSync(dir, { recursive: true, force: true }); });
const originalSend = Object.getOwnPropertyDescriptor(process, "send"), originalConnected = Object.getOwnPropertyDescriptor(process, "connected");
const desktopMarker = process.env.LH_DESKTOP_SERVER, desktopSecret = process.env.LH_DESKTOP_API_SECRET;
before(() => {
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
Object.defineProperty(process, "connected", { configurable: true, value: true });
Object.defineProperty(process, "send", { configurable: true, value: (_message: unknown, callback?: (error: Error) => void) => { callback?.(new Error("unexpected fixture desktop request")); return false; } });
});
after(() => {
fs.rmSync(dir, { recursive: true, force: true });
if (originalSend) Object.defineProperty(process, "send", originalSend); else delete process.send;
if (originalConnected) Object.defineProperty(process, "connected", originalConnected); else delete (process as { connected?: boolean }).connected;
if (desktopMarker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = desktopMarker;
if (desktopSecret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = desktopSecret;
});
for (const revokedAfterQueue of [false, true]) {
test(`mixed learner+automatic batch: foreground runs, automatic result stays owned/deferred without consent (${revokedAfterQueue ? "revoked after handoff" : "initially off"})`, { timeout: 5000 }, async () => {
+4
View File
@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
import { piModels } from "../src/agent/models.ts";
import { refreshChatGPTModels, chatGPTModels, chatGPTStatus, chatGPTScope, chatGPTScopeCurrent, chatGPTLease, chatGPTAbortController } from "../src/agent/chatgpt-bridge.ts";
const send = Object.getOwnPropertyDescriptor(process, "send"), connected = Object.getOwnPropertyDescriptor(process, "connected");
const desktopMarker = process.env.LH_DESKTOP_SERVER, desktopSecret = process.env.LH_DESKTOP_API_SECRET;
const pending: any[] = [];
const model = piModels().getModels("openai").find((m) => m.api === "openai-responses")!;
const ipc = (m: unknown) => (process.emit as (name: string, value: unknown) => boolean)("message", m);
@@ -10,10 +11,13 @@ const status = (account: string, background = true) => ({ available: true, pendi
const value = (name: string, revision: number, account = "A", authorityRevision = 1) => ({ epoch: account === "A" ? 1 : 2, authorityRevision, revision, status: status(account), accountId: account, models: [{ id: model.id, name }] });
function reply(request: any, v: unknown) { ipc({ channel: "lh:chatgpt", id: request.id, op: request.op, ok: true, value: v }); }
before(() => {
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
Object.defineProperty(process, "connected", { configurable: true, value: true });
Object.defineProperty(process, "send", { configurable: true, value: (m: unknown, cb?: () => void) => { pending.push(m); cb?.(); return true; } });
});
after(() => {
if (desktopMarker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = desktopMarker;
if (desktopSecret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = desktopSecret;
if (send) Object.defineProperty(process, "send", send); else delete process.send;
if (connected) Object.defineProperty(process, "connected", connected); else delete (process as { connected?: boolean }).connected;
});
+53
View File
@@ -0,0 +1,53 @@
import assert from "node:assert/strict";
import test, { after } from "node:test";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-dev-ipc-"));
process.env.LH_DATA_DIR = scratch;
process.env.LH_AUTH_FILE = path.join(scratch, "auth.json");
fs.writeFileSync(process.env.LH_AUTH_FILE, "{}", { mode: 0o600 });
process.env.OPENAI_API_KEY = "sk-fixture-not-used-for-generation";
const { GET } = await import("../app/api/models/route.ts");
const { chatGPTRequest, chatGPTConnected, chatGPTModels } = await import("../src/agent/chatgpt-bridge.ts");
after(() => fs.rmSync(scratch, { recursive: true, force: true }));
test("Next-style connected IPC is not desktop transport and never delays keyed API catalogs", async () => {
const send = Object.getOwnPropertyDescriptor(process, "send");
const connected = Object.getOwnPropertyDescriptor(process, "connected");
const marker = process.env.LH_DESKTOP_SERVER, secret = process.env.LH_DESKTOP_API_SECRET;
const originalFetch = globalThis.fetch;
let sends = 0;
try {
globalThis.fetch = async () => { throw new Error("provider/network calls forbidden in catalog fixture"); };
Object.defineProperty(process, "send", { configurable: true, value: (_message: unknown, callback?: (error: Error) => void) => { sends++; callback?.(new Error("not a desktop parent")); return false; } });
Object.defineProperty(process, "connected", { configurable: true, value: true });
for (const [mode, capability] of [[undefined, undefined], ["1", undefined], [undefined, "d".repeat(64)], ["1", "short"]]) {
if (mode === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = mode;
if (capability === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = capability;
// A framework parent cannot establish ChatGPT state by sharing only IPC.
(process.emit as (name: string, value: unknown) => boolean)("message", { channel: "lh:chatgpt", kind: "state", stamp: { epoch: 1, authorityRevision: 1, revision: 1 }, status: { available: true, activeId: "fixture", accounts: [{ id: "fixture", connected: true, planEnabled: true }] } });
const response = await GET();
assert.equal(response.status, 200);
const catalog = await response.json();
assert.ok(catalog.models.some((m: { provider: string }) => m.provider === "openai"));
assert.equal(catalog.providers.find((p: { id: string }) => p.id === "openai").keyPresent, true);
assert.equal(catalog.providers.find((p: { id: string }) => p.id === "chatgpt").keyPresent, false);
assert.equal(chatGPTConnected(), false);
assert.deepEqual(chatGPTModels(), []);
await assert.rejects(chatGPTRequest("status"), /available only in the desktop app/);
assert.equal(sends, 0, "no desktop request/timer may be started over framework IPC");
}
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
Object.defineProperty(process, "connected", { configurable: true, value: false });
await GET();
await assert.rejects(chatGPTRequest("status"), /available only in the desktop app/);
assert.equal(sends, 0, "desktop markers without connected IPC do not suffice");
} finally {
globalThis.fetch = originalFetch;
if (send) Object.defineProperty(process, "send", send); else delete process.send;
if (connected) Object.defineProperty(process, "connected", connected); else delete (process as { connected?: boolean }).connected;
if (marker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = marker;
if (secret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = secret;
}
});
+4
View File
@@ -20,6 +20,7 @@ import { newUserMessage, newNextMessage, newThinkingMessage, newAnswerResult, re
const originalSend = Object.getOwnPropertyDescriptor(process, "send");
const originalConnected = Object.getOwnPropertyDescriptor(process, "connected");
const realFetch = globalThis.fetch;
const desktopMarker = process.env.LH_DESKTOP_SERVER, desktopSecret = process.env.LH_DESKTOP_API_SECRET;
const model = piModels().getModel("openai", "gpt-5") as Model<"openai-responses">;
let epoch = 1, revision = 1, background = true;
let cut: "lease" | "payload" | "full" | "afterFetch" | "none" = "none";
@@ -31,6 +32,7 @@ function revoke(full = false) {
ipc({ channel: "lh:chatgpt", kind: "state", stamp: { epoch, authorityRevision: epoch, revision: ++revision }, status: status(), invalidate: full ? "all" : "background" });
}
before(() => {
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
Object.defineProperty(process, "connected", { configurable: true, value: true });
Object.defineProperty(process, "send", { configurable: true, value: (m: any, callback?: () => void) => {
if (m.op === "lease") {
@@ -59,6 +61,8 @@ before(() => {
};
});
after(() => {
if (desktopMarker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = desktopMarker;
if (desktopSecret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = desktopSecret;
globalThis.fetch = realFetch;
fs.rmSync(scratch, { recursive: true, force: true });
if (originalSend) Object.defineProperty(process, "send", originalSend); else delete process.send;
+4
View File
@@ -9,6 +9,7 @@ import { chatGPTHistory, chatGPTPayload } from "../src/agent/chatgpt-stream.ts";
const originalSend = Object.getOwnPropertyDescriptor(process, "send");
const originalConnected = Object.getOwnPropertyDescriptor(process, "connected");
const realFetch = globalThis.fetch;
const desktopMarker = process.env.LH_DESKTOP_SERVER, desktopSecret = process.env.LH_DESKTOP_API_SECRET;
const originalKey = process.env.OPENAI_API_KEY;
let leaseError: string | null = null;
let requests: { authorization: string | null; body: Record<string, any>; url: string }[] = [];
@@ -17,6 +18,7 @@ const ipc = (value: unknown) => (process.emit as (name: string, value: unknown)
const model = piModels().getModel("openai", "gpt-5") as Model<"openai-responses">;
const status = { available: true, pending: false, allowBackgroundUse: false, activeId: "scratch-account", accounts: [{ id: "scratch-account", email: "dummy@example.invalid", registration: "oaiapp_scratch", connected: true, planEnabled: true, background: false }], error: null };
before(async () => {
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
assert(model);
process.env.OPENAI_API_KEY = "sk-scratch-must-never-be-used";
Object.defineProperty(process, "connected", { configurable: true, value: true });
@@ -36,6 +38,8 @@ before(async () => {
};
});
after(() => {
if (desktopMarker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = desktopMarker;
if (desktopSecret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = desktopSecret;
globalThis.fetch = realFetch;
if (originalSend) Object.defineProperty(process, "send", originalSend); else delete process.send;
if (originalConnected) Object.defineProperty(process, "connected", originalConnected); else delete (process as { connected?: boolean }).connected;
+2 -1
View File
@@ -10,7 +10,8 @@ import { notifySettingsSaved } from "../components/settings-signal.ts";
import { SidebarProvider } from "../components/sidebar.tsx";
import { ChatGPTConnection, ChatGPTUIProvider } from "../components/settings/chatgpt-connection.tsx";
import { GraphCostControl } from "../components/graph-view.tsx";
import { graphCostSummary } from "../src/core/cost-summary.ts";
import { graphCostSummary as foldGraphCostSummary, type CostSummaryInput } from "../src/core/cost-summary.ts";
const graphCostSummary = (input: CostSummaryInput) => foldGraphCostSummary(input, () => null);
import type { ModelCatalogResponse, ChatGPTStatus } from "../src/core/index.ts";
const dom = parseHTML("<html><body></body></html>");
Object.assign(globalThis, { window: dom.window, document: dom.document, Element: dom.window.Element, HTMLElement: dom.window.HTMLElement, CustomEvent: dom.window.CustomEvent, IS_REACT_ACT_ENVIRONMENT: true, localStorage: { getItem: () => null, setItem() {} } });
+4 -3
View File
@@ -1,7 +1,8 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { callCost, sessionCost, turnCost, jobCost } from "../src/core/prices.ts";
import { graphCostSummary } from "../src/core/cost-summary.ts";
import { callCost, sessionCost, turnCost, jobCost } from "../src/server/pricing.ts";
import { graphCostSummary as foldGraphCostSummary, type CostSummaryInput } from "../src/core/cost-summary.ts";
const graphCostSummary = (input: CostSummaryInput) => foldGraphCostSummary(input, callCost);
import type { TurnUsage } from "../src/core/contracts.ts";
import { planCostLabel } from "../components/usage-format.ts";
const step = { inputTokens: 1000, outputTokens: 100, cacheReadTokens: undefined, cacheWriteTokens: undefined, reasoningTokens: undefined };
@@ -12,7 +13,7 @@ test("subscription usage is neither API dollars nor an unpriced-model mistake, i
assert.equal(callCost(step, plan.model), null);
assert.equal(turnCost(turn(plan.model, [])), null);
assert.equal(jobCost({ usage: { model: plan.model, ...step, steps: 1 } }), null);
assert.deepEqual(sessionCost({ turns: [plan, turn(plan.model, [])] }), { cost: 0, unpriced: 0, subscription: 2 });
assert.deepEqual(sessionCost({ turns: [plan, turn(plan.model, [])] }), { cost: 0, unpriced: 0, subscription: 2, turnCosts: { "scratch-turn": null } });
const api = turn("openai:gpt-5");
const mixed = sessionCost({ turns: [plan, api, turn("unknown:missing")] });
assert.equal(mixed.cost, turnCost(api)); assert.equal(mixed.unpriced, 1); assert.equal(mixed.subscription, 1);
+55
View File
@@ -0,0 +1,55 @@
/** Golden outputs from the verified published pi-coding-agent 1.4.0
* findCutPoint + estimateTokens helpers, executed in isolation on synthetic
* message/metadata entries. No coding-agent dependency or private data.
* Source compaction.js SHA256:
* f51e78ef1da66b5803fc91e185f07b968f8f989ab9326b7744958416aad3f6f6
* Update deliberately alongside the documented local port, not automatically.
*/
import assert from "node:assert/strict";
import test from "node:test";
import { compactionCutIndex, estimateMessageTokens, newCompactionMarker, KEEP_RECENT_TOKENS, type LhMessage } from "../src/core/index.ts";
let serial = 0;
const user = (chars: number): LhMessage => ({ id: `u${serial++}`, role: "user", content: "u".repeat(chars), timestamp: 1 });
const assistant = (chars: number): LhMessage => ({ id: `a${serial++}`, role: "assistant", content: [{ type: "text", text: "a".repeat(chars) }], timestamp: 1 }) as LhMessage;
const call = (chars: number): LhMessage => ({ ...assistant(chars), content: [{ type: "text", text: "a".repeat(chars) }, { type: "toolCall", name: "quiz", arguments: {}, id: "c" }] }) as LhMessage;
const result = (chars: number): LhMessage => ({ id: `r${serial++}`, role: "toolResult", content: [{ type: "text", text: "r".repeat(chars) }], toolCallId: "c", toolName: "quiz", isError: false, timestamp: 1 });
const image = (): LhMessage => ({ id: `i${serial++}`, role: "user", content: [{ type: "image", data: "fixture", mimeType: "image/png" }], timestamp: 1 });
const metadata = (): LhMessage => ({ id: `m${serial++}`, role: "custom", customType: "plan-block", content: "invisible".repeat(1000), display: false, details: {}, timestamp: 1 });
test("local cutoff matches published native golden boundaries and estimator values", () => {
const cases: [string, LhMessage[], number, number, number[]][] = [
["exact threshold", [user(40), user(40), user(40)], 20, 1, [10, 10, 10]],
["under budget", [user(4), assistant(4)], 20, 0, [1, 1]],
["split turn", [user(400), assistant(40), assistant(40)], 15, 1, [100, 10, 10]],
["completed result boundary", [user(400), call(4), result(400), user(40)], 30, 3, [100, 3, 100, 10]],
["oversized trailing result", [user(400), user(40), call(4), result(400)], 30, 2, [100, 10, 3, 100]],
["multiple trailing results", [user(400), call(4), result(200), result(200)], 30, 1, [100, 3, 50, 50]],
["image allowance", [user(400), image(), user(40)], 1000, 1, [100, 1200, 10]],
["empty trailing text", [user(400), assistant(0), user(0)], 30, 0, [100, 0, 0]],
["pending call", [user(400), user(400), call(4)], 10, 1, [100, 100, 3]],
];
for (const [name, messages, budget, expected, tokens] of cases) {
assert.equal(compactionCutIndex(messages, budget), expected, name);
assert.deepEqual(messages.map(estimateMessageTokens), tokens, name);
}
assert.equal(compactionCutIndex([user(400), metadata(), assistant(40), metadata(), user(40)], 15), 1, "adjacent invisible metadata stays with the cut");
assert.equal(KEEP_RECENT_TOKENS, 20_000);
});
test("chained cutoff operates on active retained history, never the previous summary or discarded prefix", () => {
const old = user(400_000);
const kept = user(40);
const marker = newCompactionMarker("summary".repeat(100_000), { firstKeptMessageId: kept.id, tokensBefore: 100_000, at: "fixture" });
const messages = [old, kept, assistant(40), marker, user(40)];
assert.equal(compactionCutIndex(messages, 100), 1, "retained tail fits; no progress");
assert.equal(compactionCutIndex(messages, 15), 2, "old summary does not consume the kept-tail budget");
});
test("opaque signatures remain unmeasured and preserved; system/image allowances match native", () => {
const signed = { ...assistant(4), content: [{ type: "thinking", thinking: "", thinkingSignature: "opaque".repeat(100_000) }] } as LhMessage;
assert.equal(estimateMessageTokens(signed), 0);
assert.equal((signed as { content: { thinkingSignature: string }[] }).content[0]!.thinkingSignature.length, 600_000);
const system = { role: "system", content: [{ type: "image", data: "fixture", mimeType: "image/png" }], sections: { a: "abcd" }, toolsAdded: [{ name: "fixture" }] };
assert.equal(estimateMessageTokens(system as never), Math.ceil((4800 + 4 + JSON.stringify(system.toolsAdded).length) / 4));
});
+32 -1
View File
@@ -32,7 +32,7 @@ const {
} = await import("../src/server/compaction.ts");
import type { CompactionDeps } from "../src/server/compaction.ts";
const { subscribe } = await import("../src/core/events.ts");
const { activeCompactionOf, newUserMessage, COMPACTED_HISTORY_HEADER } = await import("../src/core/index.ts");
const { activeCompactionOf, newUserMessage, measuredContextTokens, COMPACTED_HISTORY_HEADER } = await import("../src/core/index.ts");
import type { LhMessage } from "../src/core/index.ts";
after(() => {
@@ -204,6 +204,37 @@ describe("the compaction service", () => {
kickCompaction(() => null, chatId);
});
it("a usage-heavy zero cut does not spend or append a marker", async () => {
const chat = chatStore.createChat("no-progress fixture");
const assistant = { id: "measured", role: "assistant", content: [{ type: "text", text: "ok" }], stopReason: "stop", usage: { input: 120_000, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 120_001 }, timestamp: 1 } as LhMessage;
chatStore.saveMessages(chat.id, [newUserMessage("tiny visible history"), assistant]);
const before = summarizeCalls;
await ensureCompacted(depsOf(chat.id, 40_000), chat.id);
assert.equal(summarizeCalls, before);
assert.equal(activeCompactionOf(chatStore.getChat(chat.id)!.messages), null);
assert.equal(measuredContextTokens(chatStore.getChat(chat.id)!.messages), 120_001);
});
it("starting/failing compaction preserves the meter; committing its marker clears it, regardless of summary usage", async () => {
const id = overThresholdChat("meter validity fixture");
const assistant = { id: "measured", role: "assistant", content: [{ type: "text", text: "ok" }], stopReason: "stop", usage: { input: 120_000, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 120_001 }, timestamp: 1 } as LhMessage;
chatStore.saveMessages(id, [...chatStore.getChat(id)!.messages, assistant]);
let release!: () => void;
const gate = new Promise<void>((done) => { release = done; });
const deps = depsOf(id, 40_000);
const running = ensureCompacted({ ...deps, summarize: async () => {
await gate;
throw new Error("fixture failure");
} }, id);
assert.equal(measuredContextTokens(chatStore.getChat(id)!.messages), 120_001);
release(); await running;
assert.equal(measuredContextTokens(chatStore.getChat(id)!.messages), 120_001);
await ensureCompacted(deps, id);
const record = chatStore.getChat(id)!;
assert.ok(record.turns!.at(-1)!.steps[0]!.inputTokens! > 0);
assert.equal(measuredContextTokens(record.messages), null);
});
it("a summarizer failure leaves the record untouched and the gate re-arms", async () => {
const chatId = overThresholdChat("compact: fail");
const before = summarizeCalls;
+8 -8
View File
@@ -156,7 +156,7 @@ describe("compactionCutIndex", () => {
assert.equal((messages[cut] as { content: string }).content.startsWith("m2"), true, "the oldest kept message sits at the cut");
});
it("NEVER separates a tool result from its call — the cut extends to the call's assistant message", () => {
it("native cut after an oversized result drops both call and result, never keeps an orphan", () => {
const messages = [
big("m1"),
big("m2"),
@@ -165,13 +165,13 @@ describe("compactionCutIndex", () => {
big("m3"),
big("m4"),
];
// Budget lands such that the naive stop is ON the toolResult (m2 kept,
// result next): the walk must extend to a1's assistant message.
// Budget lands on the result. Pi chooses the next valid cut AT OR
// AFTER it, dropping the complete exchange rather than extending.
const cut = compactionCutIndex(messages, estimateTokens([big("m3"), big("m4")]) + 1);
assert.ok(cut <= 2, `the cut must not sit between the call (idx 2) and its result (idx 3) — got ${cut}`);
const kept = messages.slice(cut);
const keptIds = new Set(kept.map((m) => m.id));
assert.ok(keptIds.has("a1") && keptIds.has("r_call_1"), "call and result stay together");
assert.equal(cut, 4);
const keptIds = new Set(messages.slice(cut).map((m) => m.id));
assert.equal(keptIds.has("a1"), false);
assert.equal(keptIds.has("r_call_1"), false);
});
it("a PENDING question call (no result — the pose→answer window) always survives", () => {
@@ -181,7 +181,7 @@ describe("compactionCutIndex", () => {
assert.ok(kept.some((m) => m.id === "a_pending"), "the pending pose is never dropped");
});
it("zero budget summarizes everything; whole-history-in-budget returns 0 only when nothing precedes", () => {
it("whole history within budget keeps everything (zero cut)", () => {
assert.equal(compactionCutIndex([newUserMessage("hi")], 100), 0);
});
});
+54
View File
@@ -0,0 +1,54 @@
import assert from "node:assert/strict";
import test from "node:test";
import type { AssistantMessage } from "@earendil-works/pi-ai";
import { measuredContextTokens, newCompactionMarker, type LhMessage } from "../src/core/index.ts";
import { SessionModel } from "../components/session-core.ts";
function assistant(id: string, stopReason: AssistantMessage["stopReason"] = "stop", total = 999_999): LhMessage {
return { id, role: "assistant", content: [{ type: "text", text: "fixture" }], api: "openai-completions", provider: "openrouter", model: "fixture", stopReason, timestamp: 1,
usage: { input: 10_000, output: 1_000, cacheRead: 70_000, cacheWrite: 2_000, reasoning: 500, totalTokens: total, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } } as LhMessage;
}
const marker = () => newCompactionMarker("educational summary", { firstKeptMessageId: "old", tokensBefore: 110_001, at: "fixture" });
test("meter sums four disjoint fields, ignoring reasoning subset and totalTokens overrides", () => {
assert.equal(measuredContextTokens([assistant("old")]), 83_000);
assert.equal(measuredContextTokens([]), null);
for (const stop of ["stop", "length", "toolUse"] as const) assert.equal(measuredContextTokens([assistant("a", stop)]), 83_000);
});
test("newest marker invalidates retained pre-cut usage; summarizer money log cannot supply a measurement", () => {
const old = assistant("old");
assert.equal(measuredContextTokens([old, marker()]), null);
assert.equal(measuredContextTokens([old, marker(), assistant("fresh")]), 83_000);
assert.equal(measuredContextTokens([old, marker(), assistant("fresh"), marker()]), null);
// The summarizer is logged as a money-channel turn, never an assistant
// in the native conversation list. No money-channel input exists here.
});
test("errors, aborts, all-zero/invalid usage and unfinished slots preserve the previous valid reading", () => {
const old = assistant("old");
for (const stop of ["error", "aborted"] as const) assert.equal(measuredContextTokens([old, assistant("bad", stop)]), 83_000);
const zero = assistant("zero") as AssistantMessage & { id: string };
zero.usage = { ...zero.usage, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
assert.equal(measuredContextTokens([old, zero]), 83_000);
assert.equal(measuredContextTokens([zero]), null);
assert.equal(measuredContextTokens([old, assistant("partial")], new Set(["partial"])), 83_000);
assert.equal(measuredContextTokens([marker(), assistant("bad", "error")]), null);
});
test("live terminal-looking message_update is not completion; each message_end updates immediately", () => {
const model = new SessionModel({ chatId: "fixture", initialMessages: [assistant("old")] });
const fresh = assistant("live") as AssistantMessage & { id: string };
fresh.usage = { ...fresh.usage, input: 20_000 };
model.applyWireEvent({ type: "message_start", message: fresh });
model.applyWireEvent({ type: "message_update", message: fresh });
assert.equal(model.getState().contextTokens, 83_000);
model.applyWireEvent({ type: "message_end", message: fresh });
assert.equal(model.getState().contextTokens, 93_000);
model.applyWireEvent({ type: "message_start", message: assistant("next") });
assert.equal(model.getState().contextTokens, 93_000);
model.applyWireEvent({ type: "message_end", message: assistant("next", "toolUse") });
assert.equal(model.getState().contextTokens, 83_000);
model.setMessages([assistant("old"), marker()]);
assert.equal(model.getState().contextTokens, null);
});
+4 -3
View File
@@ -8,16 +8,17 @@
import assert from "node:assert/strict";
import { describe, it as test } from "node:test";
import {
graphCostSummary,
graphCostSummary as foldGraphCostSummary,
type CostSummaryInput,
type GraphCostSummary,
} from "../src/core/cost-summary.ts";
import type { CallUsage, ExtractionRunUsage } from "../src/core/contracts.ts";
import { PRICES, callCost } from "../src/core/prices.ts";
import { callCost } from "../src/server/pricing.ts";
const graphCostSummary = (input: CostSummaryInput) => foldGraphCostSummary(input, callCost);
/** A model with full published pricing (from the generated registry). */
const PRICED = "openrouter:z-ai/glm-5.3-flash";
assert.ok(PRICES[PRICED], "fixture model must exist in PRICES");
assert.notEqual(callCost({ inputTokens: 1, outputTokens: 1 }, PRICED), null);
const step = (input: number, output: number): CallUsage => ({
inputTokens: input,
+60
View File
@@ -0,0 +1,60 @@
import React from "react";
(globalThis as Record<string, unknown>).React = React;
import assert from "node:assert/strict";
import test from "node:test";
import { renderToStaticMarkup } from "react-dom/server";
import { GraphCostControl } from "../components/graph-view.tsx";
import { graphCostSummary } from "../src/core/cost-summary.ts";
import { callCost } from "../src/server/pricing.ts";
import type { CallUsage, ExtractionRunUsage } from "../src/core/contracts.ts";
const usage = (over: Partial<CallUsage> = {}): CallUsage => ({ inputTokens: 1000, outputTokens: 100, cacheReadTokens: 0, cacheWriteTokens: 20_000, reasoningTokens: 0, ...over });
const run = (steps: CallUsage[], model = "anthropic:claude-haiku-4-5", at = 1): ExtractionRunUsage => ({ startedAt: at, at: "fixture", durationMs: 1, stages: { extractor: { model, steps }, auditor: { model, steps } } });
const summary = (runs: ExtractionRunUsage[]) => graphCostSummary({ chats: [{ title: "fixture", extractions: runs }], sources: [] }, callCost);
const headline = (runs: ExtractionRunUsage[]) => renderToStaticMarkup(<GraphCostControl costs={summary(runs)} />);
test("wholly unpriced historical graph costs render unknown, not known-zero dollars", () => {
const historical = run([usage()]); // missing long-cache split
const costs = summary([historical]);
assert.equal(costs.totalDollars, 0);
assert.equal(costs.unpricedRuns, 1);
assert.equal(costs.stages.extraction.unpricedRuns, 1);
const html = headline([historical]);
assert.match(html, />—<\/button>/);
assert.doesNotMatch(html, /\$0\.00/);
});
test("known zero, mixed priced/unknown, and subscription headline semantics stay distinct", () => {
const zero = run([usage({ inputTokens: 0, outputTokens: 0, cacheWriteTokens: 0 })]);
assert.match(headline([zero]), />\$0\.00<\/button>/, "known zero is not unknown");
const priced = run([usage({ cacheWrite1hTokens: 0 })]);
const historical = run([usage()], undefined, 2);
const mixed = summary([priced, historical]);
assert.equal(mixed.unpricedRuns, 1);
assert.ok(mixed.totalDollars > 0);
assert.match(headline([priced, historical]), new RegExp(`>\\$${mixed.totalDollars.toFixed(2).replace(".", "\\.")}<\\/button>`));
const plan = run([usage()], "chatgpt:fixture");
assert.match(headline([plan]), />ChatGPT plan<\/button>/);
assert.match(headline([plan, zero]), />\$0\.00 API \+ ChatGPT plan<\/button>/);
});
test("graph stage and recent-run aggregates preserve known one-hour cache writes", () => {
const costs = summary([run([usage({ cacheWrite1hTokens: 20_000 })]), run([usage({ cacheWrite1hTokens: 5000 })], undefined, 2)]);
assert.equal(costs.stages.extraction.tokens.cacheWrite1hTokens, 25_000);
assert.equal(costs.stages.audit.tokens.cacheWrite1hTokens, 25_000);
assert.deepEqual(costs.runs.map((r) => r.tokens.cacheWrite1hTokens), [10_000, 40_000]);
});
test("unknown historical long-cache splits stay unknown regardless of merge order", () => {
const known = run([usage({ cacheWrite1hTokens: 20_000 })]);
const historical = run([usage()], undefined, 2);
for (const runs of [[known, historical], [historical, known]]) {
const costs = summary(runs);
assert.equal(costs.stages.extraction.tokens.cacheWrite1hTokens, undefined);
assert.equal(costs.stages.audit.tokens.cacheWrite1hTokens, undefined);
}
const mixedWithinRun = summary([run([usage({ cacheWrite1hTokens: 20_000 }), usage()])]);
assert.equal(mixedWithinRun.stages.extraction.tokens.cacheWrite1hTokens, undefined);
const absentNoWrites = run([usage({ cacheWriteTokens: 0 })], undefined, 2);
assert.equal(summary([absentNoWrites, known]).stages.extraction.tokens.cacheWrite1hTokens, 20_000, "absence without writes must not poison known write detail");
});
+5
View File
@@ -346,8 +346,11 @@ describe("steer helpers (contracts.ts)", () => {
cacheReadTokens: 4,
cacheWriteTokens: undefined,
reasoningTokens: undefined,
cacheWrite1hTokens: undefined,
calls: [{ inputTokens: 10, outputTokens: 5, cacheReadTokens: 4, cacheWriteTokens: undefined, cacheWrite1hTokens: undefined, reasoningTokens: undefined }],
steps: 1,
lastStep: {
cacheWrite1hTokens: undefined,
inputTokens: 10,
outputTokens: 5,
cacheReadTokens: 4,
@@ -364,7 +367,9 @@ describe("steer helpers (contracts.ts)", () => {
assert.equal(total.cacheWriteTokens, undefined, "never reported stays absent — never zero");
assert.equal(total.reasoningTokens, undefined);
assert.equal(total.steps, 2, "one step per model call, reported or not");
assert.equal(total.calls?.length, 2);
assert.deepEqual(total.lastStep, {
cacheWrite1hTokens: undefined,
inputTokens: 20,
outputTokens: 0,
cacheReadTokens: undefined,
+1 -1
View File
@@ -4,7 +4,7 @@ import { normalizeContext } from "@earendil-works/pi-ai";
import { Type } from "typebox";
import { piEngineForSpec, piModels } from "../src/agent/models.ts";
import { modelCatalog, validateChatSettings, validateModelsSettings } from "../src/server/model-catalog.ts";
import { callCost } from "../src/core/prices.ts";
import { callCost } from "../src/server/pricing.ts";
const specimens = [
{ id: "recursive/flint-v1.0:free", input: ["text"] },
-56
View File
@@ -1,56 +0,0 @@
/**
* THE PRICE-TABLE DRIFT GUARD (owner, 2026-09-29).
*
* Failure mode this kills: bumping @earendil-works/pi-ai adds new models
* (and price corrections) to the shipped catalogs, but nobody re-runs
* scripts/sync-pricing.ts — src/core/prices.ts stays a stale generation
* and every run on a new model renders cost as "—" (deepseek-v4.1-flash,
* 2026-09-29, commit df654fb). The table must stay a generated static
* file (core is client-safe: activity-dock and stats-bar fold costs in
* the renderer, and reading the catalog JSON needs fs), so staleness is
* guarded instead of eliminated.
*
* The test asserts PRICES equals EXACTLY what the sync script would
* generate right now — it imports the script's own builder, so the guard
* and the generator can never disagree:
* - a catalog model with pricing missing from PRICES → "run the sync"
* - a price field that drifted in the catalog → "run the sync"
* - a PRICES key whose model left the catalog → "run the sync"
*
* On failure: npx tsx scripts/sync-pricing.ts, commit the regenerated
* file, re-derive any pinned-cost tests.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { PRICES } from "../src/core/prices.ts";
import { buildPrices } from "../scripts/sync-pricing.ts";
test("PRICES matches pi-ai's shipped catalogs exactly — the sync was run after the last bump", () => {
const catalog = buildPrices();
const missing = [...catalog.keys()].filter((k) => !(k in PRICES));
const stale = Object.keys(PRICES).filter((k) => !catalog.has(k));
const drifted: string[] = [];
for (const [key, cost] of catalog) {
const price = PRICES[key];
if (!price) continue;
if (
price.input !== cost.input ||
price.output !== cost.output ||
price.cacheRead !== cost.cacheRead ||
price.cacheWrite !== cost.cacheWrite
) {
drifted.push(key);
}
}
assert.equal(
`${missing.length} / ${stale.length} / ${drifted.length}`,
"0 / 0 / 0",
`price table is stale: ${missing.length} unpriced catalog models (e.g. ${missing[0] ?? "—"}), ` +
`${stale.length} keys gone from the catalog (e.g. ${stale[0] ?? "—"}), ` +
`${drifted.length} drifted prices (e.g. ${drifted[0] ?? "—"}). ` +
"Run: npx tsx scripts/sync-pricing.ts, commit the regenerated file, re-derive pinned-cost tests.",
);
});
+45
View File
@@ -0,0 +1,45 @@
/** Catalog/calculator parity replaces the old generated-table drift test. */
import assert from "node:assert/strict";
import test from "node:test";
import { calculateCost, type Usage } from "@earendil-works/pi-ai";
import { piModels } from "../src/agent/models.ts";
import { callCost, jobCost } from "../src/server/pricing.ts";
import { accumulateJobUsage, piUsageToStep } from "../src/core/contracts/jobs.ts";
const models = piModels().getProviders().flatMap((p) => piModels().getModels(p.id));
const nativeUsage = (input: number): Usage => ({ input, output: 1_000, cacheRead: 10_000, cacheWrite: 2_000, cacheWrite1h: 500, totalTokens: input + 13_000, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } });
const saved = (u: Usage) => ({ inputTokens: u.input, outputTokens: u.output, cacheReadTokens: u.cacheRead, cacheWriteTokens: u.cacheWrite, cacheWrite1hTokens: u.cacheWrite1h, reasoningTokens: 500 });
test("all catalog models price through Pi, including every tier boundary; saved usage is never mutated", () => {
assert.ok(models.length > 100);
for (const model of models) {
const sizes = [1_000, ...(model.cost.tiers ?? []).flatMap((t) => [Math.max(0, t.inputTokensAbove - 12_000), Math.max(0, t.inputTokensAbove - 12_000 + 1)])];
for (const input of sizes) {
const usage = nativeUsage(input);
const call = Object.freeze(saved(usage));
const before = structuredClone(call);
const expected = calculateCost(model, usage).total;
assert.equal(callCost(call, `${model.provider}:${model.id}`), expected, `${model.provider}:${model.id} input=${input}`);
assert.deepEqual(call, before);
}
}
});
test("job capture preserves individual tier sizes and one-hour writes; old tier aggregates are unknown", () => {
const model = models.find((m) => m.cost.tiers?.length)!;
assert.ok(model);
const id = `${model.provider}:${model.id}`;
const threshold = model.cost.tiers![0]!.inputTokensAbove;
const first = nativeUsage(Math.ceil(threshold / 2));
const second = nativeUsage(Math.ceil(threshold / 2));
const total = accumulateJobUsage(accumulateJobUsage(undefined, id, piUsageToStep(first)), id, piUsageToStep(second));
assert.equal(total.calls?.length, 2);
assert.equal(total.cacheWrite1hTokens, 1000);
assert.equal(total.lastStep?.cacheWrite1hTokens, 500);
assert.equal(jobCost({ usage: total }), calculateCost(model, first).total + calculateCost(model, second).total);
assert.equal(jobCost({ usage: { ...total, calls: undefined } }), null);
const legacyOne = { ...total, ...saved(first), calls: undefined, steps: 1 };
assert.equal(jobCost({ usage: legacyOne }), calculateCost(model, first).total);
const resumedLegacy = accumulateJobUsage({ ...total, calls: undefined }, id, piUsageToStep(first));
assert.equal(resumedLegacy.calls, undefined, "never pretend the missing old calls were reconstructed");
});
+34
View File
@@ -0,0 +1,34 @@
import assert from "node:assert/strict";
import test, { after } from "node:test";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-pricing-transport-"));
process.env.LH_DATA_DIR = scratch;
process.env.LH_AUTH_FILE = path.join(scratch, "empty-auth.json");
fs.writeFileSync(process.env.LH_AUTH_FILE, "{}", { mode: 0o600 });
const { chatStore } = await import("../src/store/chats.ts");
const { GET } = await import("../app/api/chats/[id]/route.ts");
const { sessionCost } = await import("../src/server/pricing.ts");
after(() => fs.rmSync(scratch, { recursive: true, force: true }));
test("chat GET supplies derived Pi costs without persisting dollars or changing native history", async () => {
const record = chatStore.createChat("price projection fixture");
chatStore.saveMessages(record.id, [{ id: "fixture-input", role: "user", content: "synthetic", timestamp: 1 }]);
const step = { inputTokens: 10_000, outputTokens: 1_000, cacheReadTokens: 70_000, cacheWriteTokens: 0, cacheWrite1hTokens: 0, reasoningTokens: 500 };
for (const [id, model] of [["api", "openrouter:openai/gpt-6.1-sol"], ["plan", "chatgpt:fixture"], ["unknown", "missing:fixture"]]) {
chatStore.appendTurnUsage(record.id, { id: id!, model: model!, messageId: "fixture-input", at: "fixture", steps: [step] });
}
const file = path.join(scratch, "chats", `${record.id}.json`);
const before = fs.readFileSync(file, "utf8");
const response = await GET(new Request(`http://localhost/api/chats/${record.id}`), { params: Promise.resolve({ id: record.id }) });
assert.equal(response.status, 200);
const body = await response.json();
assert.deepEqual(body.messages, chatStore.getChat(record.id)!.messages);
assert.deepEqual(body.pricing, sessionCost(chatStore.getChat(record.id)!));
assert.equal(body.pricing.cost, 0.037);
assert.equal(body.pricing.subscription, 1);
assert.equal(body.pricing.unpriced, 1);
assert.deepEqual(body.pricing.turnCosts, { api: 0.037, plan: null, unknown: null });
assert.equal(fs.readFileSync(file, "utf8"), before, "read-time estimates must never rewrite the usage log");
});
+7 -5
View File
@@ -3,6 +3,7 @@ import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { ActivitySection, formatElapsed, toBlobs, type ActivityFeed, type FeedExtraction } from "../components/activity-dock.tsx";
import { formatTokens, formatUsd } from "../components/usage-format.ts";
import { jobCost } from "../src/server/pricing.ts";
// The section's pure layer: folds, formatting, and blob order — then the SSR
// render (blobs, "—" doctrine, the abort affordance, the always-present
@@ -27,6 +28,7 @@ function job(over: Partial<ActivityFeed["jobs"][number]> = {}): ActivityFeed["jo
updatedAt: new Date(NOW - 1000).toISOString(),
title: "Limits lesson",
contextLimit: null,
costUsd: jobCost({ usage: over.usage }),
...over,
};
}
@@ -57,7 +59,7 @@ describe("activity dock pure layer", () => {
assert.ok(blobs.every((b) => b.chatId === "src_a"), "abort/link identity stays source-scoped");
});
it("folds cost with jobCost; absent usage is null, never zero", () => {
it("uses the server's Pi price projection; absent pricing is null, never zero", () => {
const priced = job({
usage: {
model: "openrouter:z-ai/glm-5.3-flash",
@@ -70,10 +72,10 @@ describe("activity dock pure layer", () => {
},
});
const blob = toBlobs({ jobs: [priced], extractions: [] })[0]!;
// (10000-5000)/1e6*0.15 + 5000/1e6*0.03 + 1000/1e6*0.5 = 0.0014
// (pi-ai catalog prices — glm-5.3-flash cacheRead corrected
// 0.05 → 0.03 in the pi 1.0.2 catalog refresh)
assert.ok(blob.cost !== null && blob.cost > 0.0013 && blob.cost < 0.0015);
// Pi-normalized fresh input is priced directly, never reduced by reads.
assert.equal(blob.cost, 0.00215);
const missingProjection = toBlobs({ jobs: [{ ...priced, costUsd: undefined }], extractions: [] })[0]!;
assert.equal(missingProjection.cost, null, "the browser must not invent a fallback price");
const absent = toBlobs({ jobs: [job()], extractions: [] })[0]!;
assert.equal(absent.cost, null);
});
+15 -3
View File
@@ -1,7 +1,8 @@
import { renderToStaticMarkup } from "react-dom/server";
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { StatsBar } from "../components/stats-bar.tsx";
import { StatsBar, formatContextMeasurement } from "../components/stats-bar.tsx";
import { sessionCost } from "../src/server/pricing.ts";
import { catalogLimits } from "../components/model-picker.tsx";
import type { ModelCatalogEntry, ModelCatalogResponse } from "../src/core/contracts.ts";
import { prettyModel } from "../components/usage-format.ts";
@@ -50,7 +51,7 @@ describe("stats bar", () => {
it("folds context + cost from the turns log (core/prices, no local math)", () => {
const html = renderToStaticMarkup(
<StatsBar turns={[turn(), turn()]} />,
<StatsBar turns={[turn(), turn()]} contextTokens={11_000} pricing={sessionCost({ turns: [turn(), turn()] })} />,
);
// last turn's last call: 10k in + 1k out = 11k. The LIMIT is the
// live catalog's — and the bar doesn't hold the index yet (the
@@ -68,7 +69,7 @@ describe("stats bar", () => {
it("shows — for cost when every turn is unpriceable, never a fake $0", () => {
const html = renderToStaticMarkup(
<StatsBar turns={[turn({ model: "nope/unknown" })]} />,
<StatsBar turns={[turn({ model: "nope/unknown" })]} contextTokens={11_000} pricing={sessionCost({ turns: [turn({ model: "nope/unknown" })] })} />,
);
assert.match(html, /data-testid="stats-cost">—</);
// unknown model: no catalog limit exists, but the footprint is
@@ -77,6 +78,17 @@ describe("stats bar", () => {
assert.match(html, /data-testid="stats-context">11k</);
});
it("unknown context is explicit; changing the selected limit preserves the measurement", () => {
assert.equal(formatContextMeasurement(null, 100_000), "? / 100k");
assert.equal(formatContextMeasurement(83_000, 100_000), "83k / 100k");
assert.equal(formatContextMeasurement(83_000, 200_000), "83k / 200k");
assert.equal(formatContextMeasurement(83_000, null), "83k");
const html = renderToStaticMarkup(<StatsBar chatId="fixture" turns={[turn()]} contextTokens={null} />);
assert.match(html, /data-testid="stats-context">\?</);
const empty = renderToStaticMarkup(<StatsBar chatId="fixture" turns={[]} />);
assert.match(empty, /data-testid="stats-context">\?</);
});
it("prettyModel strips vendor + date suffix, keeps dashes", () => {
assert.equal(prettyModel("z-ai/glm-5.3-flash"), "glm-5.3-flash"); // dashes kept (owner round 5)
assert.equal(prettyModel("qwen/qwen3.8-max-0902"), "qwen3.8-max");
+46
View File
@@ -0,0 +1,46 @@
import React, { act } from "react";
import assert from "node:assert/strict";
import test from "node:test";
import { createRoot } from "react-dom/client";
import { parseHTML } from "linkedom";
import { StatsBar } from "../components/stats-bar.tsx";
import { notifySettingsSaved } from "../components/settings-signal.ts";
import type { ModelCatalogResponse, TurnUsage } from "../src/core/index.ts";
(globalThis as Record<string, unknown>).React = React;
const dom = parseHTML("<html><body></body></html>");
Object.assign(globalThis, { window: dom.window, document: dom.document, Element: dom.window.Element, HTMLElement: dom.window.HTMLElement, CustomEvent: dom.window.CustomEvent, IS_REACT_ACT_ENVIRONMENT: true });
test("meter keeps its measured numerator across selected-model/catalog changes, and renders post-compaction unknown", async () => {
const originalFetch = globalThis.fetch;
let selected = "a";
const catalog: ModelCatalogResponse = {
models: ["a", "b"].map((id, i) => ({ id, provider: "fixture", name: id, input: ["text"], reasoning: false, supportedLevels: ["off"], maxTokens: 1000, contextWindow: 100_000 * (i + 1), cost: { input: 1, output: 1, cacheRead: 1, cacheWrite: 1 } })),
providers: [], defaults: null, slots: { strong: null, cheap: null, vision: null }, rolePins: {} as ModelCatalogResponse["rolePins"],
};
globalThis.fetch = async (input) => {
if (input === "/api/models") return Response.json(catalog);
if (input === "/api/settings") return Response.json({ settings: { models: {} } });
if (input === "/api/chats/fixture") return Response.json({ settings: { model: { provider: "fixture", id: selected }, thinkingLevel: "off" } });
throw new Error(`unexpected fixture request ${String(input)}`);
};
const host = document.createElement("div"); document.body.append(host);
const root = createRoot(host);
const turn: TurnUsage = { id: "summary", messageId: "marker", at: "fixture", model: "fixture:a", steps: [{ inputTokens: 4300, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0 }] };
const pricing = { cost: 0.003, unpriced: 0, turnCosts: { summary: 0.003 } };
try {
await act(async () => root.render(<StatsBar chatId="fixture" turns={[turn]} contextTokens={83_000} pricing={pricing} />));
const context = () => host.querySelector('[data-testid="stats-context"]')?.textContent;
assert.equal(context(), "83k / 100k", "summarizer's 4.3k money log is not the meter source");
selected = "b";
await act(async () => notifySettingsSaved("models"));
assert.equal(context(), "83k / 200k", "only denominator changes");
await act(async () => root.render(<StatsBar chatId="fixture" turns={[turn]} contextTokens={null} pricing={pricing} />));
assert.equal(context(), "? / 200k");
assert.equal(host.querySelector('[data-testid="stats-cost"]')?.textContent, "$0.0030", "summary spending is still visible");
await act(async () => root.render(<StatsBar chatId="fixture" turns={[turn]} contextTokens={72_000} pricing={pricing} />));
assert.equal(context(), "72k / 200k");
} finally {
await act(async () => root.unmount()); host.remove(); globalThis.fetch = originalFetch;
}
});
+69 -262
View File
@@ -1,307 +1,114 @@
/**
* Usage-accounting tests — the money channel's pure helpers (core/prices.ts)
* and the ChatStore usage-log seam (appendTurnUsage).
*
* Pure helper tests are pure. The store test uses the real fs chatStore with
* a fixture chat, cleaned up in after().
*/
import assert from "node:assert/strict";
import test from "node:test";
import * as fs from "node:fs";
import { chatStore } from "../src/store/chats.ts";
import { graphStore } from "../src/store/graph.ts";
import { PATHS, SOURCES_FILE } from "../config.ts";
import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts";
import { callCost, jobCost, sessionCost, turnCost } from "../src/server/pricing.ts";
import { sumSteps } from "../src/core/prices.ts";
import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts";
const call = (over: Partial<CallUsage> = {}): CallUsage => ({
inputTokens: 0,
outputTokens: 0,
cacheReadTokens: undefined,
cacheWriteTokens: undefined,
reasoningTokens: undefined,
...over,
inputTokens: 0, outputTokens: 0, cacheReadTokens: undefined,
cacheWriteTokens: undefined, reasoningTokens: undefined, ...over,
});
test("sumSteps folds steps; absent stays absent until a value appears", () => {
const sum = sumSteps([
call({ inputTokens: 100, outputTokens: 20 }),
call({ inputTokens: 50, outputTokens: 10, cacheReadTokens: 30 }),
]);
test("sumSteps: absent stays absent; unknown long-cache detail stays unknown", () => {
const sum = sumSteps([call({ inputTokens: 100, outputTokens: 20 }), call({ inputTokens: 50, outputTokens: 10, cacheReadTokens: 30 })]);
assert.equal(sum.inputTokens, 150);
assert.equal(sum.outputTokens, 30);
assert.equal(sum.cacheReadTokens, 30);
assert.equal(sum.cacheWriteTokens, undefined); // never reported → stays undefined
// all-absent → all-undefined (helper defaults input/output to 0, so
// explicit undefined here)
const empty = sumSteps([call({ inputTokens: undefined, outputTokens: undefined }), call({ inputTokens: undefined, outputTokens: undefined })]);
assert.equal(empty.inputTokens, undefined);
assert.equal(sum.cacheWriteTokens, undefined);
assert.equal(sumSteps([call({ inputTokens: undefined, outputTokens: undefined })]).inputTokens, undefined);
assert.equal(sumSteps([call({ cacheWriteTokens: 10 }), call({ cacheWriteTokens: 20, cacheWrite1hTokens: 5 })]).cacheWrite1hTokens, undefined);
});
test("callCost: glm-5.3-flash math — fresh input, cached read, output", () => {
test("callCost prices normalized fresh input directly, without subtracting caches again", () => {
const cost = callCost(call({ inputTokens: 100_000, outputTokens: 20_000, cacheReadTokens: 10_000 }), "openrouter:z-ai/glm-5.3-flash");
assert.ok(cost !== null);
// fresh 90k @ $0.15/M + cached 10k @ $0.03/M + output 20k @ $0.5/M
// (pi-ai catalog prices — regenerated tables, scripts/sync-pricing;
// glm-5.3-flash cacheRead corrected 0.05 → 0.03 in pi 1.0.2)
assert.ok(Math.abs(cost! - (0.0135 + 0.0003 + 0.01)) < 1e-9);
assert.ok(Math.abs(cost - (0.015 + 0.0003 + 0.01)) < 1e-9);
assert.equal(callCost(call({ inputTokens: 10_000, outputTokens: 1_000, cacheReadTokens: 70_000 }), "openrouter:openai/gpt-6.1-sol"), 0.037);
});
test("callCost: cache writes bill at the cacheWrite rate", () => {
const cost = callCost(
call({ inputTokens: 1_000_000, outputTokens: 0, cacheWriteTokens: 500_000 }),
"anthropic:claude-haiku-4-5",
);
assert.ok(cost !== null);
// Native 1.4 catalog: fresh 1M @ $1/M + cache write 500k @ $1.25/M.
assert.ok(Math.abs(cost! - (1 + 0.625)) < 1e-9);
test("cache writes: precise short/one-hour costs and honest unknown historical splits", () => {
const usage = call({ inputTokens: 1_000_000, cacheWriteTokens: 500_000, cacheWrite1hTokens: 0 });
assert.equal(callCost(usage, "anthropic:claude-haiku-4-5"), 1.625);
assert.equal(callCost({ ...usage, cacheWrite1hTokens: 200_000 }, "anthropic:claude-haiku-4-5"), 1.775);
assert.equal(callCost({ ...usage, cacheWrite1hTokens: undefined }, "anthropic:claude-haiku-4-5"), null);
});
test("callCost: unknown model and absent fields are null, never zero", () => {
assert.equal(callCost(call({ inputTokens: 100, outputTokens: 5 }), "not/a-model"), null);
assert.equal(callCost(call({ inputTokens: 100, outputTokens: undefined }), "openrouter:z-ai/glm-5.3-flash"), null);
test("unknown model, absent or invalid required usage is null, never fake zero", () => {
assert.equal(callCost(call(), "not/a-model"), null);
assert.equal(callCost(call({ outputTokens: undefined }), "openai:gpt-5"), null);
for (const value of [-1, NaN, Infinity]) assert.equal(callCost(call({ inputTokens: value }), "openai:gpt-5"), null);
});
test("turnCost sums steps; sessionCost reports unpriced turns", () => {
const turn: TurnUsage = {
id: "t1",
messageId: "m1",
at: "2026-09-09T00:00:00Z",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [call({ inputTokens: 10_000, outputTokens: 2_000 }), call({ inputTokens: 12_000, outputTokens: 1_000 })],
};
assert.ok(turnCost(turn) !== null);
const totals = sessionCost({
turns: [turn, { id: "t2", messageId: "m2", at: "x", model: "not/a-model", steps: [call({ inputTokens: 1, outputTokens: 1 })] }],
});
test("turn/session costs sum calls and count unpriced separately", () => {
const turn: TurnUsage = { id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 10_000, outputTokens: 2_000 }), call({ inputTokens: 12_000, outputTokens: 1_000 })] };
const totals = sessionCost({ turns: [turn, { ...turn, id: "t2", model: "not/a-model" }] });
assert.equal(totals.unpriced, 1);
assert.ok(totals.cost > 0);
assert.equal(totals.cost, turnCost(turn));
assert.equal(totals.turnCosts.t1, totals.cost);
assert.equal(totals.turnCosts.t2, null);
});
test("contextWindow: last call's footprint against the live index's limit", () => {
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
const cw = contextWindow(
{
turns: [
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
{ id: "t2", messageId: "m2", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
],
},
limits,
);
assert.ok(cw);
assert.equal(cw.model, "openrouter:z-ai/glm-5.3-flash");
assert.equal(cw.usedTokens, 100_000);
assert.equal(cw.limit, 1_048_576);
assert.equal(contextWindow({ turns: [] }, limits), null);
});
test("contextWindow NUMERATOR: cache tokens count (owner audit, 2026-09-23)", () => {
// The audit's live case: a cache-heavy session read 40k on the meter
// while the real footprint was ~190k — provider `input` EXCLUDES the
// cached portion, so input+output alone is wildly low. The fold is
// pi's totalTokens semantics: all four fields.
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
const cw = contextWindow(
{
turns: [
{
id: "t1",
messageId: "m1",
at: "x",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [call({ inputTokens: 40_000, outputTokens: 1_000, cacheReadTokens: 145_000, cacheWriteTokens: 4_000 })],
},
],
},
limits,
);
assert.ok(cw);
assert.equal(cw.usedTokens, 40_000 + 1_000 + 145_000 + 4_000, "input + output + cacheRead + cacheWrite");
// Cache fields absent (non-caching providers): the fold is unchanged.
const plain = contextWindow(
{ turns: [{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] }] },
limits,
);
assert.ok(plain);
assert.equal(plain.usedTokens, 10_000);
});
test("contextWindow DENOMINATOR: the live index is the ONLY source — no fallback (owner amendment, 2026-09-23)", () => {
const turns = [
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 10_000, outputTokens: 1_000 })] },
];
const live = new Map([["openrouter:z-ai/glm-5.3-flash", 65_536]]);
const cw = contextWindow({ turns }, live);
assert.ok(cw);
assert.equal(cw.limit, 65_536, "pi's catalog, via the index, is the one source");
// No index at all (a client that hasn't fetched the catalog): the
// meter renders usage-only — limit null, never a guessed window.
const noIndex = contextWindow({ turns });
assert.ok(noIndex);
assert.equal(noIndex.usedTokens, 11_000);
assert.equal(noIndex.limit, null);
const emptyIndex = contextWindow({ turns }, new Map<string, number>());
assert.ok(emptyIndex);
assert.equal(emptyIndex.limit, null, "an empty index (failed fetch) is usage-only too");
// A model missing from the live catalog (historical session, package
// update removed it): usage-only, never a fake window.
const missing = contextWindow({ turns: [{ ...turns[0]!, model: "nope/gone-model" }] }, live);
assert.ok(missing);
assert.equal(missing.limit, null);
assert.equal(missing.usedTokens, 11_000, "the footprint still reads honestly");
});
test("contextWindow ride-along: a zeroed/aborted last step falls back to the previous step", () => {
// An aborted last call reports zeroed usage — that is not the session
// footprint. The honest reading is the previous step's usage.
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
const cw = contextWindow(
{
turns: [
{
id: "t1",
messageId: "m1",
at: "x",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [
call({ inputTokens: 80_000, outputTokens: 5_000 }),
call({ inputTokens: 0, outputTokens: 0 }), // aborted — zeroed
],
},
],
},
limits,
);
assert.ok(cw);
assert.equal(cw.usedTokens, 85_000, "the previous step's footprint, not a bogus tiny meter");
// ALL steps zeroed (a first-call abort): no reading at all — null.
assert.equal(
contextWindow(
{ turns: [{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 0, outputTokens: 0 })] }] },
limits,
),
null,
);
});
test("jobCost: null without usage, computed with it", () => {
test("legacy flat-rate jobs remain priceable; missing usage stays absent", () => {
assert.equal(jobCost({}), null);
const job: Pick<JobRecord, "usage"> = {
usage: {
model: "openrouter:z-ai/glm-5.3-flash",
inputTokens: 1_000_000,
outputTokens: 100_000,
cacheReadTokens: 0,
cacheWriteTokens: 0,
reasoningTokens: 0,
steps: 4,
},
};
assert.ok(jobCost(job) !== null);
const job: Pick<JobRecord, "usage"> = { usage: { model: "openrouter:z-ai/glm-5.3-flash", inputTokens: 1_000_000, outputTokens: 100_000, cacheReadTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0, steps: 4 } };
assert.ok(jobCost(job)! > 0);
});
// ─── the store seam ──────────────────────────────────────────────────────────
test("appendTurnUsage: appends, idempotent per turn id, continuation turns (same message) all kept, survives saveMessages", () => {
const record = chatStore.createChat("usage test chat");
const entry: TurnUsage = {
id: "turn_1",
messageId: "msg_a",
at: "2026-09-09T00:00:00Z",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [call({ inputTokens: 100, outputTokens: 10 })],
};
chatStore.appendTurnUsage(record.id, entry);
chatStore.appendTurnUsage(record.id, entry); // replay (abort/retry) → no-op
// continuation turn: SAME response message (SDK coalesces), new turn id —
// usage must survive, not collapse into the first entry
chatStore.appendTurnUsage(record.id, { ...entry, id: "turn_2" });
let reread = chatStore.getChat(record.id);
assert.equal(reread?.turns?.length, 2);
// saveMessages must not clobber the log (read-mutate-write record) —
// Wave B: a NATIVE message list (the record holds agent history).
chatStore.saveMessages(record.id, [
{ id: "msg_a", role: "user", content: "native", timestamp: Date.now() } as never,
]);
reread = chatStore.getChat(record.id);
assert.equal(reread?.turns?.length, 2);
test("appendTurnUsage: idempotent per turn; continuations survive saveMessages; list omits log", () => {
const record = chatStore.createChat("usage fixture");
try {
const entry: TurnUsage = { id: "turn_1", messageId: "msg_a", at: "x", model: "openai:gpt-5", steps: [call({ inputTokens: 100, outputTokens: 10, cacheWrite1hTokens: 0 })] };
chatStore.appendTurnUsage(record.id, entry);
chatStore.appendTurnUsage(record.id, entry);
chatStore.appendTurnUsage(record.id, { ...entry, id: "turn_2" });
assert.equal(chatStore.getChat(record.id)?.turns?.length, 2);
chatStore.saveMessages(record.id, [{ id: "msg_a", role: "user", content: "fixture", timestamp: 1 }]);
assert.deepEqual(chatStore.getChat(record.id)?.turns?.[0]?.steps, JSON.parse(JSON.stringify(entry.steps)));
assert.equal(chatStore.getChat(record.id)?.turns?.length, 2);
assert.equal("turns" in chatStore.listChats().find((c) => c.id === record.id)!, false);
} finally { fs.unlinkSync(`${PATHS.chats}/${record.id}.json`); }
});
// and the list payload must NOT leak turns
const listed = chatStore.listChats().find((c) => c.id === record.id);
assert.ok(listed);
assert.equal("turns" in listed, false);
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
const runEntry = (startedAt: number): ExtractionRunUsage => ({
startedAt, at: "2026-09-09T00:00:00Z", durationMs: 42_000,
stats: { nodes: 1, chains: 2 }, stages: {
extractor: { model: "openai:gpt-5", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "openai:gpt-5", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
},
});
test("appendExtractionUsage: appends, idempotent per startedAt, not in list payload", () => {
const record = chatStore.createChat("extraction usage test chat");
const runEntry = (startedAt: number): ExtractionRunUsage => ({
startedAt,
at: "2026-09-09T00:00:00Z",
durationMs: 42_000,
stats: { nodes: 1, chains: 2 },
stages: {
extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
},
});
chatStore.appendExtractionUsage(record.id, runEntry(1));
chatStore.appendExtractionUsage(record.id, runEntry(1)); // same run appended twice → no-op
chatStore.appendExtractionUsage(record.id, runEntry(2)); // a retried run is a NEW spend
let reread = chatStore.getChat(record.id);
assert.equal(reread?.extractions?.length, 2);
// failed runs (stats absent) persist with usage — money was spent
chatStore.appendExtractionUsage(record.id, { ...runEntry(3), stats: undefined });
reread = chatStore.getChat(record.id);
assert.equal(reread?.extractions?.length, 3);
assert.equal(reread?.extractions?.at(-1)?.stats, undefined);
// the list payload must NOT leak the extractions log
const listed = chatStore.listChats().find((c) => c.id === record.id);
assert.ok(listed);
assert.equal("extractions" in listed, false);
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
test("appendExtractionUsage: idempotent; failed runs persist; list omits log", () => {
const record = chatStore.createChat("extraction usage fixture");
try {
chatStore.appendExtractionUsage(record.id, runEntry(1));
chatStore.appendExtractionUsage(record.id, runEntry(1));
chatStore.appendExtractionUsage(record.id, runEntry(2));
chatStore.appendExtractionUsage(record.id, { ...runEntry(3), stats: undefined });
assert.equal(chatStore.getChat(record.id)?.extractions?.length, 3);
assert.equal(chatStore.getChat(record.id)?.extractions?.at(-1)?.stats, undefined);
assert.equal("extractions" in chatStore.listChats().find((c) => c.id === record.id)!, false);
} finally { fs.unlinkSync(`${PATHS.chats}/${record.id}.json`); }
});
test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => {
// The real fs graphStore — sources.json backed up and restored around
// this test (same discipline as the chat fixture files).
test("appendSourceUsage: idempotent; reimport keeps history; absent source is a no-op", () => {
fs.mkdirSync(PATHS.graph, { recursive: true });
let bak: string | undefined;
if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`;
if (bak) fs.renameSync(SOURCES_FILE, bak);
const backup = fs.existsSync(SOURCES_FILE) ? fs.readFileSync(SOURCES_FILE) : undefined;
try {
const runEntry = (startedAt: number): ExtractionRunUsage => ({
startedAt,
at: "2026-09-13T00:00:00Z",
durationMs: 42_000,
stats: { nodes: 1, chains: 2 },
stages: {
extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
},
});
const id = "src_usagetest01";
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" });
graphStore.upsertSource({ id, filePath: "/fixture/notes.md", importedAt: "2026-09-13T00:00:00Z" });
graphStore.appendSourceUsage(id, runEntry(1));
graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op
graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend
graphStore.appendSourceUsage(id, runEntry(1));
graphStore.appendSourceUsage(id, runEntry(2));
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
// An upsert (re-import) KEEPS the usage history.
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" });
graphStore.upsertSource({ id, filePath: "/fixture/notes.md", importedAt: "2026-09-14T00:00:00Z" });
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
// Unknown ids are a silent no-op (nothing to bill against).
graphStore.appendSourceUsage("src_nope", runEntry(1));
graphStore.appendSourceUsage("src_missing", runEntry(1));
} finally {
fs.rmSync(SOURCES_FILE, { force: true });
if (bak) fs.renameSync(bak, SOURCES_FILE);
if (backup) fs.writeFileSync(SOURCES_FILE, backup);
else fs.rmSync(SOURCES_FILE, { force: true });
}
});