Merge compaction and native usage accounting fixes
This commit is contained in:
@@ -88,10 +88,13 @@ test("activityFeed: contextLimit is the server's live-catalog join (owner amendm
|
||||
// pi 1.0.2's catalog: glm-5.3-flash contextWindow 1_048_575 (was
|
||||
// 1_048_576 at 0.87.1 — upstream models.dev refresh).
|
||||
assert.equal(knownJob.contextLimit, 1_048_575, "pi's catalog, joined server-side onto the feed");
|
||||
assert.ok(knownJob.costUsd !== null && Math.abs(knownJob.costUsd - 0.0002) < 1e-12, "Pi's price is projected on the server");
|
||||
const unknownJob = feed.jobs.find((j) => j.id === unknown.id)!;
|
||||
assert.equal(unknownJob.contextLimit, null, "missing from the live catalog → null, never a guessed window");
|
||||
assert.equal(unknownJob.costUsd, null);
|
||||
const bareJob = feed.jobs.find((j) => j.id === bare.id)!;
|
||||
assert.equal(bareJob.contextLimit, null, "no usage yet → no context line");
|
||||
assert.equal(bareJob.costUsd, null);
|
||||
} finally {
|
||||
fs.unlinkSync(`${PATHS.chats}/${chat.id}.json`);
|
||||
const jobsDir = path.join(PATHS.data, "jobs");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { test, after } from "node:test";
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
@@ -21,7 +21,20 @@ let revision = 1;
|
||||
function connection(background: boolean) {
|
||||
ipc({ channel: "lh:chatgpt", kind: "state", stamp: { epoch: 1, authorityRevision: 1, revision: revision++ }, invalidate: background ? "none" : "background", status: { available: true, pending: false, allowBackgroundUse: background, activeId: "scratch-account", accounts: [{ id: "scratch-account", connected: true, planEnabled: true, background: !background }], error: null } });
|
||||
}
|
||||
after(() => { fs.rmSync(dir, { recursive: true, force: true }); });
|
||||
const originalSend = Object.getOwnPropertyDescriptor(process, "send"), originalConnected = Object.getOwnPropertyDescriptor(process, "connected");
|
||||
const desktopMarker = process.env.LH_DESKTOP_SERVER, desktopSecret = process.env.LH_DESKTOP_API_SECRET;
|
||||
before(() => {
|
||||
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
|
||||
Object.defineProperty(process, "connected", { configurable: true, value: true });
|
||||
Object.defineProperty(process, "send", { configurable: true, value: (_message: unknown, callback?: (error: Error) => void) => { callback?.(new Error("unexpected fixture desktop request")); return false; } });
|
||||
});
|
||||
after(() => {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
if (originalSend) Object.defineProperty(process, "send", originalSend); else delete process.send;
|
||||
if (originalConnected) Object.defineProperty(process, "connected", originalConnected); else delete (process as { connected?: boolean }).connected;
|
||||
if (desktopMarker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = desktopMarker;
|
||||
if (desktopSecret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = desktopSecret;
|
||||
});
|
||||
|
||||
for (const revokedAfterQueue of [false, true]) {
|
||||
test(`mixed learner+automatic batch: foreground runs, automatic result stays owned/deferred without consent (${revokedAfterQueue ? "revoked after handoff" : "initially off"})`, { timeout: 5000 }, async () => {
|
||||
|
||||
@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
|
||||
import { piModels } from "../src/agent/models.ts";
|
||||
import { refreshChatGPTModels, chatGPTModels, chatGPTStatus, chatGPTScope, chatGPTScopeCurrent, chatGPTLease, chatGPTAbortController } from "../src/agent/chatgpt-bridge.ts";
|
||||
const send = Object.getOwnPropertyDescriptor(process, "send"), connected = Object.getOwnPropertyDescriptor(process, "connected");
|
||||
const desktopMarker = process.env.LH_DESKTOP_SERVER, desktopSecret = process.env.LH_DESKTOP_API_SECRET;
|
||||
const pending: any[] = [];
|
||||
const model = piModels().getModels("openai").find((m) => m.api === "openai-responses")!;
|
||||
const ipc = (m: unknown) => (process.emit as (name: string, value: unknown) => boolean)("message", m);
|
||||
@@ -10,10 +11,13 @@ const status = (account: string, background = true) => ({ available: true, pendi
|
||||
const value = (name: string, revision: number, account = "A", authorityRevision = 1) => ({ epoch: account === "A" ? 1 : 2, authorityRevision, revision, status: status(account), accountId: account, models: [{ id: model.id, name }] });
|
||||
function reply(request: any, v: unknown) { ipc({ channel: "lh:chatgpt", id: request.id, op: request.op, ok: true, value: v }); }
|
||||
before(() => {
|
||||
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
|
||||
Object.defineProperty(process, "connected", { configurable: true, value: true });
|
||||
Object.defineProperty(process, "send", { configurable: true, value: (m: unknown, cb?: () => void) => { pending.push(m); cb?.(); return true; } });
|
||||
});
|
||||
after(() => {
|
||||
if (desktopMarker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = desktopMarker;
|
||||
if (desktopSecret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = desktopSecret;
|
||||
if (send) Object.defineProperty(process, "send", send); else delete process.send;
|
||||
if (connected) Object.defineProperty(process, "connected", connected); else delete (process as { connected?: boolean }).connected;
|
||||
});
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test, { after } from "node:test";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-dev-ipc-"));
|
||||
process.env.LH_DATA_DIR = scratch;
|
||||
process.env.LH_AUTH_FILE = path.join(scratch, "auth.json");
|
||||
fs.writeFileSync(process.env.LH_AUTH_FILE, "{}", { mode: 0o600 });
|
||||
process.env.OPENAI_API_KEY = "sk-fixture-not-used-for-generation";
|
||||
const { GET } = await import("../app/api/models/route.ts");
|
||||
const { chatGPTRequest, chatGPTConnected, chatGPTModels } = await import("../src/agent/chatgpt-bridge.ts");
|
||||
after(() => fs.rmSync(scratch, { recursive: true, force: true }));
|
||||
|
||||
test("Next-style connected IPC is not desktop transport and never delays keyed API catalogs", async () => {
|
||||
const send = Object.getOwnPropertyDescriptor(process, "send");
|
||||
const connected = Object.getOwnPropertyDescriptor(process, "connected");
|
||||
const marker = process.env.LH_DESKTOP_SERVER, secret = process.env.LH_DESKTOP_API_SECRET;
|
||||
const originalFetch = globalThis.fetch;
|
||||
let sends = 0;
|
||||
try {
|
||||
globalThis.fetch = async () => { throw new Error("provider/network calls forbidden in catalog fixture"); };
|
||||
Object.defineProperty(process, "send", { configurable: true, value: (_message: unknown, callback?: (error: Error) => void) => { sends++; callback?.(new Error("not a desktop parent")); return false; } });
|
||||
Object.defineProperty(process, "connected", { configurable: true, value: true });
|
||||
for (const [mode, capability] of [[undefined, undefined], ["1", undefined], [undefined, "d".repeat(64)], ["1", "short"]]) {
|
||||
if (mode === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = mode;
|
||||
if (capability === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = capability;
|
||||
// A framework parent cannot establish ChatGPT state by sharing only IPC.
|
||||
(process.emit as (name: string, value: unknown) => boolean)("message", { channel: "lh:chatgpt", kind: "state", stamp: { epoch: 1, authorityRevision: 1, revision: 1 }, status: { available: true, activeId: "fixture", accounts: [{ id: "fixture", connected: true, planEnabled: true }] } });
|
||||
const response = await GET();
|
||||
assert.equal(response.status, 200);
|
||||
const catalog = await response.json();
|
||||
assert.ok(catalog.models.some((m: { provider: string }) => m.provider === "openai"));
|
||||
assert.equal(catalog.providers.find((p: { id: string }) => p.id === "openai").keyPresent, true);
|
||||
assert.equal(catalog.providers.find((p: { id: string }) => p.id === "chatgpt").keyPresent, false);
|
||||
assert.equal(chatGPTConnected(), false);
|
||||
assert.deepEqual(chatGPTModels(), []);
|
||||
await assert.rejects(chatGPTRequest("status"), /available only in the desktop app/);
|
||||
assert.equal(sends, 0, "no desktop request/timer may be started over framework IPC");
|
||||
}
|
||||
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
|
||||
Object.defineProperty(process, "connected", { configurable: true, value: false });
|
||||
await GET();
|
||||
await assert.rejects(chatGPTRequest("status"), /available only in the desktop app/);
|
||||
assert.equal(sends, 0, "desktop markers without connected IPC do not suffice");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
if (send) Object.defineProperty(process, "send", send); else delete process.send;
|
||||
if (connected) Object.defineProperty(process, "connected", connected); else delete (process as { connected?: boolean }).connected;
|
||||
if (marker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = marker;
|
||||
if (secret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = secret;
|
||||
}
|
||||
});
|
||||
@@ -20,6 +20,7 @@ import { newUserMessage, newNextMessage, newThinkingMessage, newAnswerResult, re
|
||||
const originalSend = Object.getOwnPropertyDescriptor(process, "send");
|
||||
const originalConnected = Object.getOwnPropertyDescriptor(process, "connected");
|
||||
const realFetch = globalThis.fetch;
|
||||
const desktopMarker = process.env.LH_DESKTOP_SERVER, desktopSecret = process.env.LH_DESKTOP_API_SECRET;
|
||||
const model = piModels().getModel("openai", "gpt-5") as Model<"openai-responses">;
|
||||
let epoch = 1, revision = 1, background = true;
|
||||
let cut: "lease" | "payload" | "full" | "afterFetch" | "none" = "none";
|
||||
@@ -31,6 +32,7 @@ function revoke(full = false) {
|
||||
ipc({ channel: "lh:chatgpt", kind: "state", stamp: { epoch, authorityRevision: epoch, revision: ++revision }, status: status(), invalidate: full ? "all" : "background" });
|
||||
}
|
||||
before(() => {
|
||||
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
|
||||
Object.defineProperty(process, "connected", { configurable: true, value: true });
|
||||
Object.defineProperty(process, "send", { configurable: true, value: (m: any, callback?: () => void) => {
|
||||
if (m.op === "lease") {
|
||||
@@ -59,6 +61,8 @@ before(() => {
|
||||
};
|
||||
});
|
||||
after(() => {
|
||||
if (desktopMarker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = desktopMarker;
|
||||
if (desktopSecret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = desktopSecret;
|
||||
globalThis.fetch = realFetch;
|
||||
fs.rmSync(scratch, { recursive: true, force: true });
|
||||
if (originalSend) Object.defineProperty(process, "send", originalSend); else delete process.send;
|
||||
|
||||
@@ -9,6 +9,7 @@ import { chatGPTHistory, chatGPTPayload } from "../src/agent/chatgpt-stream.ts";
|
||||
const originalSend = Object.getOwnPropertyDescriptor(process, "send");
|
||||
const originalConnected = Object.getOwnPropertyDescriptor(process, "connected");
|
||||
const realFetch = globalThis.fetch;
|
||||
const desktopMarker = process.env.LH_DESKTOP_SERVER, desktopSecret = process.env.LH_DESKTOP_API_SECRET;
|
||||
const originalKey = process.env.OPENAI_API_KEY;
|
||||
let leaseError: string | null = null;
|
||||
let requests: { authorization: string | null; body: Record<string, any>; url: string }[] = [];
|
||||
@@ -17,6 +18,7 @@ const ipc = (value: unknown) => (process.emit as (name: string, value: unknown)
|
||||
const model = piModels().getModel("openai", "gpt-5") as Model<"openai-responses">;
|
||||
const status = { available: true, pending: false, allowBackgroundUse: false, activeId: "scratch-account", accounts: [{ id: "scratch-account", email: "dummy@example.invalid", registration: "oaiapp_scratch", connected: true, planEnabled: true, background: false }], error: null };
|
||||
before(async () => {
|
||||
process.env.LH_DESKTOP_SERVER = "1"; process.env.LH_DESKTOP_API_SECRET = "d".repeat(64);
|
||||
assert(model);
|
||||
process.env.OPENAI_API_KEY = "sk-scratch-must-never-be-used";
|
||||
Object.defineProperty(process, "connected", { configurable: true, value: true });
|
||||
@@ -36,6 +38,8 @@ before(async () => {
|
||||
};
|
||||
});
|
||||
after(() => {
|
||||
if (desktopMarker === undefined) delete process.env.LH_DESKTOP_SERVER; else process.env.LH_DESKTOP_SERVER = desktopMarker;
|
||||
if (desktopSecret === undefined) delete process.env.LH_DESKTOP_API_SECRET; else process.env.LH_DESKTOP_API_SECRET = desktopSecret;
|
||||
globalThis.fetch = realFetch;
|
||||
if (originalSend) Object.defineProperty(process, "send", originalSend); else delete process.send;
|
||||
if (originalConnected) Object.defineProperty(process, "connected", originalConnected); else delete (process as { connected?: boolean }).connected;
|
||||
|
||||
@@ -10,7 +10,8 @@ import { notifySettingsSaved } from "../components/settings-signal.ts";
|
||||
import { SidebarProvider } from "../components/sidebar.tsx";
|
||||
import { ChatGPTConnection, ChatGPTUIProvider } from "../components/settings/chatgpt-connection.tsx";
|
||||
import { GraphCostControl } from "../components/graph-view.tsx";
|
||||
import { graphCostSummary } from "../src/core/cost-summary.ts";
|
||||
import { graphCostSummary as foldGraphCostSummary, type CostSummaryInput } from "../src/core/cost-summary.ts";
|
||||
const graphCostSummary = (input: CostSummaryInput) => foldGraphCostSummary(input, () => null);
|
||||
import type { ModelCatalogResponse, ChatGPTStatus } from "../src/core/index.ts";
|
||||
const dom = parseHTML("<html><body></body></html>");
|
||||
Object.assign(globalThis, { window: dom.window, document: dom.document, Element: dom.window.Element, HTMLElement: dom.window.HTMLElement, CustomEvent: dom.window.CustomEvent, IS_REACT_ACT_ENVIRONMENT: true, localStorage: { getItem: () => null, setItem() {} } });
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { callCost, sessionCost, turnCost, jobCost } from "../src/core/prices.ts";
|
||||
import { graphCostSummary } from "../src/core/cost-summary.ts";
|
||||
import { callCost, sessionCost, turnCost, jobCost } from "../src/server/pricing.ts";
|
||||
import { graphCostSummary as foldGraphCostSummary, type CostSummaryInput } from "../src/core/cost-summary.ts";
|
||||
const graphCostSummary = (input: CostSummaryInput) => foldGraphCostSummary(input, callCost);
|
||||
import type { TurnUsage } from "../src/core/contracts.ts";
|
||||
import { planCostLabel } from "../components/usage-format.ts";
|
||||
const step = { inputTokens: 1000, outputTokens: 100, cacheReadTokens: undefined, cacheWriteTokens: undefined, reasoningTokens: undefined };
|
||||
@@ -12,7 +13,7 @@ test("subscription usage is neither API dollars nor an unpriced-model mistake, i
|
||||
assert.equal(callCost(step, plan.model), null);
|
||||
assert.equal(turnCost(turn(plan.model, [])), null);
|
||||
assert.equal(jobCost({ usage: { model: plan.model, ...step, steps: 1 } }), null);
|
||||
assert.deepEqual(sessionCost({ turns: [plan, turn(plan.model, [])] }), { cost: 0, unpriced: 0, subscription: 2 });
|
||||
assert.deepEqual(sessionCost({ turns: [plan, turn(plan.model, [])] }), { cost: 0, unpriced: 0, subscription: 2, turnCosts: { "scratch-turn": null } });
|
||||
const api = turn("openai:gpt-5");
|
||||
const mixed = sessionCost({ turns: [plan, api, turn("unknown:missing")] });
|
||||
assert.equal(mixed.cost, turnCost(api)); assert.equal(mixed.unpriced, 1); assert.equal(mixed.subscription, 1);
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/** Golden outputs from the verified published pi-coding-agent 1.4.0
|
||||
* findCutPoint + estimateTokens helpers, executed in isolation on synthetic
|
||||
* message/metadata entries. No coding-agent dependency or private data.
|
||||
* Source compaction.js SHA256:
|
||||
* f51e78ef1da66b5803fc91e185f07b968f8f989ab9326b7744958416aad3f6f6
|
||||
* Update deliberately alongside the documented local port, not automatically.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { compactionCutIndex, estimateMessageTokens, newCompactionMarker, KEEP_RECENT_TOKENS, type LhMessage } from "../src/core/index.ts";
|
||||
|
||||
let serial = 0;
|
||||
const user = (chars: number): LhMessage => ({ id: `u${serial++}`, role: "user", content: "u".repeat(chars), timestamp: 1 });
|
||||
const assistant = (chars: number): LhMessage => ({ id: `a${serial++}`, role: "assistant", content: [{ type: "text", text: "a".repeat(chars) }], timestamp: 1 }) as LhMessage;
|
||||
const call = (chars: number): LhMessage => ({ ...assistant(chars), content: [{ type: "text", text: "a".repeat(chars) }, { type: "toolCall", name: "quiz", arguments: {}, id: "c" }] }) as LhMessage;
|
||||
const result = (chars: number): LhMessage => ({ id: `r${serial++}`, role: "toolResult", content: [{ type: "text", text: "r".repeat(chars) }], toolCallId: "c", toolName: "quiz", isError: false, timestamp: 1 });
|
||||
const image = (): LhMessage => ({ id: `i${serial++}`, role: "user", content: [{ type: "image", data: "fixture", mimeType: "image/png" }], timestamp: 1 });
|
||||
const metadata = (): LhMessage => ({ id: `m${serial++}`, role: "custom", customType: "plan-block", content: "invisible".repeat(1000), display: false, details: {}, timestamp: 1 });
|
||||
|
||||
test("local cutoff matches published native golden boundaries and estimator values", () => {
|
||||
const cases: [string, LhMessage[], number, number, number[]][] = [
|
||||
["exact threshold", [user(40), user(40), user(40)], 20, 1, [10, 10, 10]],
|
||||
["under budget", [user(4), assistant(4)], 20, 0, [1, 1]],
|
||||
["split turn", [user(400), assistant(40), assistant(40)], 15, 1, [100, 10, 10]],
|
||||
["completed result boundary", [user(400), call(4), result(400), user(40)], 30, 3, [100, 3, 100, 10]],
|
||||
["oversized trailing result", [user(400), user(40), call(4), result(400)], 30, 2, [100, 10, 3, 100]],
|
||||
["multiple trailing results", [user(400), call(4), result(200), result(200)], 30, 1, [100, 3, 50, 50]],
|
||||
["image allowance", [user(400), image(), user(40)], 1000, 1, [100, 1200, 10]],
|
||||
["empty trailing text", [user(400), assistant(0), user(0)], 30, 0, [100, 0, 0]],
|
||||
["pending call", [user(400), user(400), call(4)], 10, 1, [100, 100, 3]],
|
||||
];
|
||||
for (const [name, messages, budget, expected, tokens] of cases) {
|
||||
assert.equal(compactionCutIndex(messages, budget), expected, name);
|
||||
assert.deepEqual(messages.map(estimateMessageTokens), tokens, name);
|
||||
}
|
||||
assert.equal(compactionCutIndex([user(400), metadata(), assistant(40), metadata(), user(40)], 15), 1, "adjacent invisible metadata stays with the cut");
|
||||
assert.equal(KEEP_RECENT_TOKENS, 20_000);
|
||||
});
|
||||
|
||||
test("chained cutoff operates on active retained history, never the previous summary or discarded prefix", () => {
|
||||
const old = user(400_000);
|
||||
const kept = user(40);
|
||||
const marker = newCompactionMarker("summary".repeat(100_000), { firstKeptMessageId: kept.id, tokensBefore: 100_000, at: "fixture" });
|
||||
const messages = [old, kept, assistant(40), marker, user(40)];
|
||||
assert.equal(compactionCutIndex(messages, 100), 1, "retained tail fits; no progress");
|
||||
assert.equal(compactionCutIndex(messages, 15), 2, "old summary does not consume the kept-tail budget");
|
||||
});
|
||||
|
||||
test("opaque signatures remain unmeasured and preserved; system/image allowances match native", () => {
|
||||
const signed = { ...assistant(4), content: [{ type: "thinking", thinking: "", thinkingSignature: "opaque".repeat(100_000) }] } as LhMessage;
|
||||
assert.equal(estimateMessageTokens(signed), 0);
|
||||
assert.equal((signed as { content: { thinkingSignature: string }[] }).content[0]!.thinkingSignature.length, 600_000);
|
||||
const system = { role: "system", content: [{ type: "image", data: "fixture", mimeType: "image/png" }], sections: { a: "abcd" }, toolsAdded: [{ name: "fixture" }] };
|
||||
assert.equal(estimateMessageTokens(system as never), Math.ceil((4800 + 4 + JSON.stringify(system.toolsAdded).length) / 4));
|
||||
});
|
||||
@@ -32,7 +32,7 @@ const {
|
||||
} = await import("../src/server/compaction.ts");
|
||||
import type { CompactionDeps } from "../src/server/compaction.ts";
|
||||
const { subscribe } = await import("../src/core/events.ts");
|
||||
const { activeCompactionOf, newUserMessage, COMPACTED_HISTORY_HEADER } = await import("../src/core/index.ts");
|
||||
const { activeCompactionOf, newUserMessage, measuredContextTokens, COMPACTED_HISTORY_HEADER } = await import("../src/core/index.ts");
|
||||
import type { LhMessage } from "../src/core/index.ts";
|
||||
|
||||
after(() => {
|
||||
@@ -204,6 +204,37 @@ describe("the compaction service", () => {
|
||||
kickCompaction(() => null, chatId);
|
||||
});
|
||||
|
||||
it("a usage-heavy zero cut does not spend or append a marker", async () => {
|
||||
const chat = chatStore.createChat("no-progress fixture");
|
||||
const assistant = { id: "measured", role: "assistant", content: [{ type: "text", text: "ok" }], stopReason: "stop", usage: { input: 120_000, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 120_001 }, timestamp: 1 } as LhMessage;
|
||||
chatStore.saveMessages(chat.id, [newUserMessage("tiny visible history"), assistant]);
|
||||
const before = summarizeCalls;
|
||||
await ensureCompacted(depsOf(chat.id, 40_000), chat.id);
|
||||
assert.equal(summarizeCalls, before);
|
||||
assert.equal(activeCompactionOf(chatStore.getChat(chat.id)!.messages), null);
|
||||
assert.equal(measuredContextTokens(chatStore.getChat(chat.id)!.messages), 120_001);
|
||||
});
|
||||
|
||||
it("starting/failing compaction preserves the meter; committing its marker clears it, regardless of summary usage", async () => {
|
||||
const id = overThresholdChat("meter validity fixture");
|
||||
const assistant = { id: "measured", role: "assistant", content: [{ type: "text", text: "ok" }], stopReason: "stop", usage: { input: 120_000, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 120_001 }, timestamp: 1 } as LhMessage;
|
||||
chatStore.saveMessages(id, [...chatStore.getChat(id)!.messages, assistant]);
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((done) => { release = done; });
|
||||
const deps = depsOf(id, 40_000);
|
||||
const running = ensureCompacted({ ...deps, summarize: async () => {
|
||||
await gate;
|
||||
throw new Error("fixture failure");
|
||||
} }, id);
|
||||
assert.equal(measuredContextTokens(chatStore.getChat(id)!.messages), 120_001);
|
||||
release(); await running;
|
||||
assert.equal(measuredContextTokens(chatStore.getChat(id)!.messages), 120_001);
|
||||
await ensureCompacted(deps, id);
|
||||
const record = chatStore.getChat(id)!;
|
||||
assert.ok(record.turns!.at(-1)!.steps[0]!.inputTokens! > 0);
|
||||
assert.equal(measuredContextTokens(record.messages), null);
|
||||
});
|
||||
|
||||
it("a summarizer failure leaves the record untouched and the gate re-arms", async () => {
|
||||
const chatId = overThresholdChat("compact: fail");
|
||||
const before = summarizeCalls;
|
||||
|
||||
@@ -156,7 +156,7 @@ describe("compactionCutIndex", () => {
|
||||
assert.equal((messages[cut] as { content: string }).content.startsWith("m2"), true, "the oldest kept message sits at the cut");
|
||||
});
|
||||
|
||||
it("NEVER separates a tool result from its call — the cut extends to the call's assistant message", () => {
|
||||
it("native cut after an oversized result drops both call and result, never keeps an orphan", () => {
|
||||
const messages = [
|
||||
big("m1"),
|
||||
big("m2"),
|
||||
@@ -165,13 +165,13 @@ describe("compactionCutIndex", () => {
|
||||
big("m3"),
|
||||
big("m4"),
|
||||
];
|
||||
// Budget lands such that the naive stop is ON the toolResult (m2 kept,
|
||||
// result next): the walk must extend to a1's assistant message.
|
||||
// Budget lands on the result. Pi chooses the next valid cut AT OR
|
||||
// AFTER it, dropping the complete exchange rather than extending.
|
||||
const cut = compactionCutIndex(messages, estimateTokens([big("m3"), big("m4")]) + 1);
|
||||
assert.ok(cut <= 2, `the cut must not sit between the call (idx 2) and its result (idx 3) — got ${cut}`);
|
||||
const kept = messages.slice(cut);
|
||||
const keptIds = new Set(kept.map((m) => m.id));
|
||||
assert.ok(keptIds.has("a1") && keptIds.has("r_call_1"), "call and result stay together");
|
||||
assert.equal(cut, 4);
|
||||
const keptIds = new Set(messages.slice(cut).map((m) => m.id));
|
||||
assert.equal(keptIds.has("a1"), false);
|
||||
assert.equal(keptIds.has("r_call_1"), false);
|
||||
});
|
||||
|
||||
it("a PENDING question call (no result — the pose→answer window) always survives", () => {
|
||||
@@ -181,7 +181,7 @@ describe("compactionCutIndex", () => {
|
||||
assert.ok(kept.some((m) => m.id === "a_pending"), "the pending pose is never dropped");
|
||||
});
|
||||
|
||||
it("zero budget summarizes everything; whole-history-in-budget returns 0 only when nothing precedes", () => {
|
||||
it("whole history within budget keeps everything (zero cut)", () => {
|
||||
assert.equal(compactionCutIndex([newUserMessage("hi")], 100), 0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import type { AssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { measuredContextTokens, newCompactionMarker, type LhMessage } from "../src/core/index.ts";
|
||||
import { SessionModel } from "../components/session-core.ts";
|
||||
|
||||
function assistant(id: string, stopReason: AssistantMessage["stopReason"] = "stop", total = 999_999): LhMessage {
|
||||
return { id, role: "assistant", content: [{ type: "text", text: "fixture" }], api: "openai-completions", provider: "openrouter", model: "fixture", stopReason, timestamp: 1,
|
||||
usage: { input: 10_000, output: 1_000, cacheRead: 70_000, cacheWrite: 2_000, reasoning: 500, totalTokens: total, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } } as LhMessage;
|
||||
}
|
||||
const marker = () => newCompactionMarker("educational summary", { firstKeptMessageId: "old", tokensBefore: 110_001, at: "fixture" });
|
||||
|
||||
test("meter sums four disjoint fields, ignoring reasoning subset and totalTokens overrides", () => {
|
||||
assert.equal(measuredContextTokens([assistant("old")]), 83_000);
|
||||
assert.equal(measuredContextTokens([]), null);
|
||||
for (const stop of ["stop", "length", "toolUse"] as const) assert.equal(measuredContextTokens([assistant("a", stop)]), 83_000);
|
||||
});
|
||||
|
||||
test("newest marker invalidates retained pre-cut usage; summarizer money log cannot supply a measurement", () => {
|
||||
const old = assistant("old");
|
||||
assert.equal(measuredContextTokens([old, marker()]), null);
|
||||
assert.equal(measuredContextTokens([old, marker(), assistant("fresh")]), 83_000);
|
||||
assert.equal(measuredContextTokens([old, marker(), assistant("fresh"), marker()]), null);
|
||||
// The summarizer is logged as a money-channel turn, never an assistant
|
||||
// in the native conversation list. No money-channel input exists here.
|
||||
});
|
||||
|
||||
test("errors, aborts, all-zero/invalid usage and unfinished slots preserve the previous valid reading", () => {
|
||||
const old = assistant("old");
|
||||
for (const stop of ["error", "aborted"] as const) assert.equal(measuredContextTokens([old, assistant("bad", stop)]), 83_000);
|
||||
const zero = assistant("zero") as AssistantMessage & { id: string };
|
||||
zero.usage = { ...zero.usage, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
|
||||
assert.equal(measuredContextTokens([old, zero]), 83_000);
|
||||
assert.equal(measuredContextTokens([zero]), null);
|
||||
assert.equal(measuredContextTokens([old, assistant("partial")], new Set(["partial"])), 83_000);
|
||||
assert.equal(measuredContextTokens([marker(), assistant("bad", "error")]), null);
|
||||
});
|
||||
|
||||
test("live terminal-looking message_update is not completion; each message_end updates immediately", () => {
|
||||
const model = new SessionModel({ chatId: "fixture", initialMessages: [assistant("old")] });
|
||||
const fresh = assistant("live") as AssistantMessage & { id: string };
|
||||
fresh.usage = { ...fresh.usage, input: 20_000 };
|
||||
model.applyWireEvent({ type: "message_start", message: fresh });
|
||||
model.applyWireEvent({ type: "message_update", message: fresh });
|
||||
assert.equal(model.getState().contextTokens, 83_000);
|
||||
model.applyWireEvent({ type: "message_end", message: fresh });
|
||||
assert.equal(model.getState().contextTokens, 93_000);
|
||||
model.applyWireEvent({ type: "message_start", message: assistant("next") });
|
||||
assert.equal(model.getState().contextTokens, 93_000);
|
||||
model.applyWireEvent({ type: "message_end", message: assistant("next", "toolUse") });
|
||||
assert.equal(model.getState().contextTokens, 83_000);
|
||||
model.setMessages([assistant("old"), marker()]);
|
||||
assert.equal(model.getState().contextTokens, null);
|
||||
});
|
||||
@@ -8,16 +8,17 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it as test } from "node:test";
|
||||
import {
|
||||
graphCostSummary,
|
||||
graphCostSummary as foldGraphCostSummary,
|
||||
type CostSummaryInput,
|
||||
type GraphCostSummary,
|
||||
} from "../src/core/cost-summary.ts";
|
||||
import type { CallUsage, ExtractionRunUsage } from "../src/core/contracts.ts";
|
||||
import { PRICES, callCost } from "../src/core/prices.ts";
|
||||
import { callCost } from "../src/server/pricing.ts";
|
||||
const graphCostSummary = (input: CostSummaryInput) => foldGraphCostSummary(input, callCost);
|
||||
|
||||
/** A model with full published pricing (from the generated registry). */
|
||||
const PRICED = "openrouter:z-ai/glm-5.3-flash";
|
||||
assert.ok(PRICES[PRICED], "fixture model must exist in PRICES");
|
||||
assert.notEqual(callCost({ inputTokens: 1, outputTokens: 1 }, PRICED), null);
|
||||
|
||||
const step = (input: number, output: number): CallUsage => ({
|
||||
inputTokens: input,
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import React from "react";
|
||||
(globalThis as Record<string, unknown>).React = React;
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { GraphCostControl } from "../components/graph-view.tsx";
|
||||
import { graphCostSummary } from "../src/core/cost-summary.ts";
|
||||
import { callCost } from "../src/server/pricing.ts";
|
||||
import type { CallUsage, ExtractionRunUsage } from "../src/core/contracts.ts";
|
||||
|
||||
const usage = (over: Partial<CallUsage> = {}): CallUsage => ({ inputTokens: 1000, outputTokens: 100, cacheReadTokens: 0, cacheWriteTokens: 20_000, reasoningTokens: 0, ...over });
|
||||
const run = (steps: CallUsage[], model = "anthropic:claude-haiku-4-5", at = 1): ExtractionRunUsage => ({ startedAt: at, at: "fixture", durationMs: 1, stages: { extractor: { model, steps }, auditor: { model, steps } } });
|
||||
const summary = (runs: ExtractionRunUsage[]) => graphCostSummary({ chats: [{ title: "fixture", extractions: runs }], sources: [] }, callCost);
|
||||
const headline = (runs: ExtractionRunUsage[]) => renderToStaticMarkup(<GraphCostControl costs={summary(runs)} />);
|
||||
|
||||
test("wholly unpriced historical graph costs render unknown, not known-zero dollars", () => {
|
||||
const historical = run([usage()]); // missing long-cache split
|
||||
const costs = summary([historical]);
|
||||
assert.equal(costs.totalDollars, 0);
|
||||
assert.equal(costs.unpricedRuns, 1);
|
||||
assert.equal(costs.stages.extraction.unpricedRuns, 1);
|
||||
const html = headline([historical]);
|
||||
assert.match(html, />—<\/button>/);
|
||||
assert.doesNotMatch(html, /\$0\.00/);
|
||||
});
|
||||
|
||||
test("known zero, mixed priced/unknown, and subscription headline semantics stay distinct", () => {
|
||||
const zero = run([usage({ inputTokens: 0, outputTokens: 0, cacheWriteTokens: 0 })]);
|
||||
assert.match(headline([zero]), />\$0\.00<\/button>/, "known zero is not unknown");
|
||||
const priced = run([usage({ cacheWrite1hTokens: 0 })]);
|
||||
const historical = run([usage()], undefined, 2);
|
||||
const mixed = summary([priced, historical]);
|
||||
assert.equal(mixed.unpricedRuns, 1);
|
||||
assert.ok(mixed.totalDollars > 0);
|
||||
assert.match(headline([priced, historical]), new RegExp(`>\\$${mixed.totalDollars.toFixed(2).replace(".", "\\.")}<\\/button>`));
|
||||
const plan = run([usage()], "chatgpt:fixture");
|
||||
assert.match(headline([plan]), />ChatGPT plan<\/button>/);
|
||||
assert.match(headline([plan, zero]), />\$0\.00 API \+ ChatGPT plan<\/button>/);
|
||||
});
|
||||
|
||||
test("graph stage and recent-run aggregates preserve known one-hour cache writes", () => {
|
||||
const costs = summary([run([usage({ cacheWrite1hTokens: 20_000 })]), run([usage({ cacheWrite1hTokens: 5000 })], undefined, 2)]);
|
||||
assert.equal(costs.stages.extraction.tokens.cacheWrite1hTokens, 25_000);
|
||||
assert.equal(costs.stages.audit.tokens.cacheWrite1hTokens, 25_000);
|
||||
assert.deepEqual(costs.runs.map((r) => r.tokens.cacheWrite1hTokens), [10_000, 40_000]);
|
||||
});
|
||||
|
||||
test("unknown historical long-cache splits stay unknown regardless of merge order", () => {
|
||||
const known = run([usage({ cacheWrite1hTokens: 20_000 })]);
|
||||
const historical = run([usage()], undefined, 2);
|
||||
for (const runs of [[known, historical], [historical, known]]) {
|
||||
const costs = summary(runs);
|
||||
assert.equal(costs.stages.extraction.tokens.cacheWrite1hTokens, undefined);
|
||||
assert.equal(costs.stages.audit.tokens.cacheWrite1hTokens, undefined);
|
||||
}
|
||||
const mixedWithinRun = summary([run([usage({ cacheWrite1hTokens: 20_000 }), usage()])]);
|
||||
assert.equal(mixedWithinRun.stages.extraction.tokens.cacheWrite1hTokens, undefined);
|
||||
const absentNoWrites = run([usage({ cacheWriteTokens: 0 })], undefined, 2);
|
||||
assert.equal(summary([absentNoWrites, known]).stages.extraction.tokens.cacheWrite1hTokens, 20_000, "absence without writes must not poison known write detail");
|
||||
});
|
||||
@@ -346,8 +346,11 @@ describe("steer helpers (contracts.ts)", () => {
|
||||
cacheReadTokens: 4,
|
||||
cacheWriteTokens: undefined,
|
||||
reasoningTokens: undefined,
|
||||
cacheWrite1hTokens: undefined,
|
||||
calls: [{ inputTokens: 10, outputTokens: 5, cacheReadTokens: 4, cacheWriteTokens: undefined, cacheWrite1hTokens: undefined, reasoningTokens: undefined }],
|
||||
steps: 1,
|
||||
lastStep: {
|
||||
cacheWrite1hTokens: undefined,
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
cacheReadTokens: 4,
|
||||
@@ -364,7 +367,9 @@ describe("steer helpers (contracts.ts)", () => {
|
||||
assert.equal(total.cacheWriteTokens, undefined, "never reported stays absent — never zero");
|
||||
assert.equal(total.reasoningTokens, undefined);
|
||||
assert.equal(total.steps, 2, "one step per model call, reported or not");
|
||||
assert.equal(total.calls?.length, 2);
|
||||
assert.deepEqual(total.lastStep, {
|
||||
cacheWrite1hTokens: undefined,
|
||||
inputTokens: 20,
|
||||
outputTokens: 0,
|
||||
cacheReadTokens: undefined,
|
||||
|
||||
@@ -4,7 +4,7 @@ import { normalizeContext } from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { piEngineForSpec, piModels } from "../src/agent/models.ts";
|
||||
import { modelCatalog, validateChatSettings, validateModelsSettings } from "../src/server/model-catalog.ts";
|
||||
import { callCost } from "../src/core/prices.ts";
|
||||
import { callCost } from "../src/server/pricing.ts";
|
||||
|
||||
const specimens = [
|
||||
{ id: "recursive/flint-v1.0:free", input: ["text"] },
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
/**
|
||||
* THE PRICE-TABLE DRIFT GUARD (owner, 2026-09-29).
|
||||
*
|
||||
* Failure mode this kills: bumping @earendil-works/pi-ai adds new models
|
||||
* (and price corrections) to the shipped catalogs, but nobody re-runs
|
||||
* scripts/sync-pricing.ts — src/core/prices.ts stays a stale generation
|
||||
* and every run on a new model renders cost as "—" (deepseek-v4.1-flash,
|
||||
* 2026-09-29, commit df654fb). The table must stay a generated static
|
||||
* file (core is client-safe: activity-dock and stats-bar fold costs in
|
||||
* the renderer, and reading the catalog JSON needs fs), so staleness is
|
||||
* guarded instead of eliminated.
|
||||
*
|
||||
* The test asserts PRICES equals EXACTLY what the sync script would
|
||||
* generate right now — it imports the script's own builder, so the guard
|
||||
* and the generator can never disagree:
|
||||
* - a catalog model with pricing missing from PRICES → "run the sync"
|
||||
* - a price field that drifted in the catalog → "run the sync"
|
||||
* - a PRICES key whose model left the catalog → "run the sync"
|
||||
*
|
||||
* On failure: npx tsx scripts/sync-pricing.ts, commit the regenerated
|
||||
* file, re-derive any pinned-cost tests.
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { PRICES } from "../src/core/prices.ts";
|
||||
import { buildPrices } from "../scripts/sync-pricing.ts";
|
||||
|
||||
test("PRICES matches pi-ai's shipped catalogs exactly — the sync was run after the last bump", () => {
|
||||
const catalog = buildPrices();
|
||||
|
||||
const missing = [...catalog.keys()].filter((k) => !(k in PRICES));
|
||||
const stale = Object.keys(PRICES).filter((k) => !catalog.has(k));
|
||||
const drifted: string[] = [];
|
||||
for (const [key, cost] of catalog) {
|
||||
const price = PRICES[key];
|
||||
if (!price) continue;
|
||||
if (
|
||||
price.input !== cost.input ||
|
||||
price.output !== cost.output ||
|
||||
price.cacheRead !== cost.cacheRead ||
|
||||
price.cacheWrite !== cost.cacheWrite
|
||||
) {
|
||||
drifted.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
assert.equal(
|
||||
`${missing.length} / ${stale.length} / ${drifted.length}`,
|
||||
"0 / 0 / 0",
|
||||
`price table is stale: ${missing.length} unpriced catalog models (e.g. ${missing[0] ?? "—"}), ` +
|
||||
`${stale.length} keys gone from the catalog (e.g. ${stale[0] ?? "—"}), ` +
|
||||
`${drifted.length} drifted prices (e.g. ${drifted[0] ?? "—"}). ` +
|
||||
"Run: npx tsx scripts/sync-pricing.ts, commit the regenerated file, re-derive pinned-cost tests.",
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
/** Catalog/calculator parity replaces the old generated-table drift test. */
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { calculateCost, type Usage } from "@earendil-works/pi-ai";
|
||||
import { piModels } from "../src/agent/models.ts";
|
||||
import { callCost, jobCost } from "../src/server/pricing.ts";
|
||||
import { accumulateJobUsage, piUsageToStep } from "../src/core/contracts/jobs.ts";
|
||||
|
||||
const models = piModels().getProviders().flatMap((p) => piModels().getModels(p.id));
|
||||
const nativeUsage = (input: number): Usage => ({ input, output: 1_000, cacheRead: 10_000, cacheWrite: 2_000, cacheWrite1h: 500, totalTokens: input + 13_000, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } });
|
||||
const saved = (u: Usage) => ({ inputTokens: u.input, outputTokens: u.output, cacheReadTokens: u.cacheRead, cacheWriteTokens: u.cacheWrite, cacheWrite1hTokens: u.cacheWrite1h, reasoningTokens: 500 });
|
||||
|
||||
test("all catalog models price through Pi, including every tier boundary; saved usage is never mutated", () => {
|
||||
assert.ok(models.length > 100);
|
||||
for (const model of models) {
|
||||
const sizes = [1_000, ...(model.cost.tiers ?? []).flatMap((t) => [Math.max(0, t.inputTokensAbove - 12_000), Math.max(0, t.inputTokensAbove - 12_000 + 1)])];
|
||||
for (const input of sizes) {
|
||||
const usage = nativeUsage(input);
|
||||
const call = Object.freeze(saved(usage));
|
||||
const before = structuredClone(call);
|
||||
const expected = calculateCost(model, usage).total;
|
||||
assert.equal(callCost(call, `${model.provider}:${model.id}`), expected, `${model.provider}:${model.id} input=${input}`);
|
||||
assert.deepEqual(call, before);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("job capture preserves individual tier sizes and one-hour writes; old tier aggregates are unknown", () => {
|
||||
const model = models.find((m) => m.cost.tiers?.length)!;
|
||||
assert.ok(model);
|
||||
const id = `${model.provider}:${model.id}`;
|
||||
const threshold = model.cost.tiers![0]!.inputTokensAbove;
|
||||
const first = nativeUsage(Math.ceil(threshold / 2));
|
||||
const second = nativeUsage(Math.ceil(threshold / 2));
|
||||
const total = accumulateJobUsage(accumulateJobUsage(undefined, id, piUsageToStep(first)), id, piUsageToStep(second));
|
||||
assert.equal(total.calls?.length, 2);
|
||||
assert.equal(total.cacheWrite1hTokens, 1000);
|
||||
assert.equal(total.lastStep?.cacheWrite1hTokens, 500);
|
||||
assert.equal(jobCost({ usage: total }), calculateCost(model, first).total + calculateCost(model, second).total);
|
||||
assert.equal(jobCost({ usage: { ...total, calls: undefined } }), null);
|
||||
const legacyOne = { ...total, ...saved(first), calls: undefined, steps: 1 };
|
||||
assert.equal(jobCost({ usage: legacyOne }), calculateCost(model, first).total);
|
||||
const resumedLegacy = accumulateJobUsage({ ...total, calls: undefined }, id, piUsageToStep(first));
|
||||
assert.equal(resumedLegacy.calls, undefined, "never pretend the missing old calls were reconstructed");
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test, { after } from "node:test";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-pricing-transport-"));
|
||||
process.env.LH_DATA_DIR = scratch;
|
||||
process.env.LH_AUTH_FILE = path.join(scratch, "empty-auth.json");
|
||||
fs.writeFileSync(process.env.LH_AUTH_FILE, "{}", { mode: 0o600 });
|
||||
const { chatStore } = await import("../src/store/chats.ts");
|
||||
const { GET } = await import("../app/api/chats/[id]/route.ts");
|
||||
const { sessionCost } = await import("../src/server/pricing.ts");
|
||||
after(() => fs.rmSync(scratch, { recursive: true, force: true }));
|
||||
|
||||
test("chat GET supplies derived Pi costs without persisting dollars or changing native history", async () => {
|
||||
const record = chatStore.createChat("price projection fixture");
|
||||
chatStore.saveMessages(record.id, [{ id: "fixture-input", role: "user", content: "synthetic", timestamp: 1 }]);
|
||||
const step = { inputTokens: 10_000, outputTokens: 1_000, cacheReadTokens: 70_000, cacheWriteTokens: 0, cacheWrite1hTokens: 0, reasoningTokens: 500 };
|
||||
for (const [id, model] of [["api", "openrouter:openai/gpt-6.1-sol"], ["plan", "chatgpt:fixture"], ["unknown", "missing:fixture"]]) {
|
||||
chatStore.appendTurnUsage(record.id, { id: id!, model: model!, messageId: "fixture-input", at: "fixture", steps: [step] });
|
||||
}
|
||||
const file = path.join(scratch, "chats", `${record.id}.json`);
|
||||
const before = fs.readFileSync(file, "utf8");
|
||||
const response = await GET(new Request(`http://localhost/api/chats/${record.id}`), { params: Promise.resolve({ id: record.id }) });
|
||||
assert.equal(response.status, 200);
|
||||
const body = await response.json();
|
||||
assert.deepEqual(body.messages, chatStore.getChat(record.id)!.messages);
|
||||
assert.deepEqual(body.pricing, sessionCost(chatStore.getChat(record.id)!));
|
||||
assert.equal(body.pricing.cost, 0.037);
|
||||
assert.equal(body.pricing.subscription, 1);
|
||||
assert.equal(body.pricing.unpriced, 1);
|
||||
assert.deepEqual(body.pricing.turnCosts, { api: 0.037, plan: null, unknown: null });
|
||||
assert.equal(fs.readFileSync(file, "utf8"), before, "read-time estimates must never rewrite the usage log");
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { ActivitySection, formatElapsed, toBlobs, type ActivityFeed, type FeedExtraction } from "../components/activity-dock.tsx";
|
||||
import { formatTokens, formatUsd } from "../components/usage-format.ts";
|
||||
import { jobCost } from "../src/server/pricing.ts";
|
||||
|
||||
// The section's pure layer: folds, formatting, and blob order — then the SSR
|
||||
// render (blobs, "—" doctrine, the abort affordance, the always-present
|
||||
@@ -27,6 +28,7 @@ function job(over: Partial<ActivityFeed["jobs"][number]> = {}): ActivityFeed["jo
|
||||
updatedAt: new Date(NOW - 1000).toISOString(),
|
||||
title: "Limits lesson",
|
||||
contextLimit: null,
|
||||
costUsd: jobCost({ usage: over.usage }),
|
||||
...over,
|
||||
};
|
||||
}
|
||||
@@ -57,7 +59,7 @@ describe("activity dock pure layer", () => {
|
||||
assert.ok(blobs.every((b) => b.chatId === "src_a"), "abort/link identity stays source-scoped");
|
||||
});
|
||||
|
||||
it("folds cost with jobCost; absent usage is null, never zero", () => {
|
||||
it("uses the server's Pi price projection; absent pricing is null, never zero", () => {
|
||||
const priced = job({
|
||||
usage: {
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
@@ -70,10 +72,10 @@ describe("activity dock pure layer", () => {
|
||||
},
|
||||
});
|
||||
const blob = toBlobs({ jobs: [priced], extractions: [] })[0]!;
|
||||
// (10000-5000)/1e6*0.15 + 5000/1e6*0.03 + 1000/1e6*0.5 = 0.0014
|
||||
// (pi-ai catalog prices — glm-5.3-flash cacheRead corrected
|
||||
// 0.05 → 0.03 in the pi 1.0.2 catalog refresh)
|
||||
assert.ok(blob.cost !== null && blob.cost > 0.0013 && blob.cost < 0.0015);
|
||||
// Pi-normalized fresh input is priced directly, never reduced by reads.
|
||||
assert.equal(blob.cost, 0.00215);
|
||||
const missingProjection = toBlobs({ jobs: [{ ...priced, costUsd: undefined }], extractions: [] })[0]!;
|
||||
assert.equal(missingProjection.cost, null, "the browser must not invent a fallback price");
|
||||
const absent = toBlobs({ jobs: [job()], extractions: [] })[0]!;
|
||||
assert.equal(absent.cost, null);
|
||||
});
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { StatsBar } from "../components/stats-bar.tsx";
|
||||
import { StatsBar, formatContextMeasurement } from "../components/stats-bar.tsx";
|
||||
import { sessionCost } from "../src/server/pricing.ts";
|
||||
import { catalogLimits } from "../components/model-picker.tsx";
|
||||
import type { ModelCatalogEntry, ModelCatalogResponse } from "../src/core/contracts.ts";
|
||||
import { prettyModel } from "../components/usage-format.ts";
|
||||
@@ -50,7 +51,7 @@ describe("stats bar", () => {
|
||||
|
||||
it("folds context + cost from the turns log (core/prices, no local math)", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<StatsBar turns={[turn(), turn()]} />,
|
||||
<StatsBar turns={[turn(), turn()]} contextTokens={11_000} pricing={sessionCost({ turns: [turn(), turn()] })} />,
|
||||
);
|
||||
// last turn's last call: 10k in + 1k out = 11k. The LIMIT is the
|
||||
// live catalog's — and the bar doesn't hold the index yet (the
|
||||
@@ -68,7 +69,7 @@ describe("stats bar", () => {
|
||||
|
||||
it("shows — for cost when every turn is unpriceable, never a fake $0", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<StatsBar turns={[turn({ model: "nope/unknown" })]} />,
|
||||
<StatsBar turns={[turn({ model: "nope/unknown" })]} contextTokens={11_000} pricing={sessionCost({ turns: [turn({ model: "nope/unknown" })] })} />,
|
||||
);
|
||||
assert.match(html, /data-testid="stats-cost">—</);
|
||||
// unknown model: no catalog limit exists, but the footprint is
|
||||
@@ -77,6 +78,17 @@ describe("stats bar", () => {
|
||||
assert.match(html, /data-testid="stats-context">11k</);
|
||||
});
|
||||
|
||||
it("unknown context is explicit; changing the selected limit preserves the measurement", () => {
|
||||
assert.equal(formatContextMeasurement(null, 100_000), "? / 100k");
|
||||
assert.equal(formatContextMeasurement(83_000, 100_000), "83k / 100k");
|
||||
assert.equal(formatContextMeasurement(83_000, 200_000), "83k / 200k");
|
||||
assert.equal(formatContextMeasurement(83_000, null), "83k");
|
||||
const html = renderToStaticMarkup(<StatsBar chatId="fixture" turns={[turn()]} contextTokens={null} />);
|
||||
assert.match(html, /data-testid="stats-context">\?</);
|
||||
const empty = renderToStaticMarkup(<StatsBar chatId="fixture" turns={[]} />);
|
||||
assert.match(empty, /data-testid="stats-context">\?</);
|
||||
});
|
||||
|
||||
it("prettyModel strips vendor + date suffix, keeps dashes", () => {
|
||||
assert.equal(prettyModel("z-ai/glm-5.3-flash"), "glm-5.3-flash"); // dashes kept (owner round 5)
|
||||
assert.equal(prettyModel("qwen/qwen3.8-max-0902"), "qwen3.8-max");
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import React, { act } from "react";
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { parseHTML } from "linkedom";
|
||||
import { StatsBar } from "../components/stats-bar.tsx";
|
||||
import { notifySettingsSaved } from "../components/settings-signal.ts";
|
||||
import type { ModelCatalogResponse, TurnUsage } from "../src/core/index.ts";
|
||||
|
||||
(globalThis as Record<string, unknown>).React = React;
|
||||
const dom = parseHTML("<html><body></body></html>");
|
||||
Object.assign(globalThis, { window: dom.window, document: dom.document, Element: dom.window.Element, HTMLElement: dom.window.HTMLElement, CustomEvent: dom.window.CustomEvent, IS_REACT_ACT_ENVIRONMENT: true });
|
||||
|
||||
test("meter keeps its measured numerator across selected-model/catalog changes, and renders post-compaction unknown", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let selected = "a";
|
||||
const catalog: ModelCatalogResponse = {
|
||||
models: ["a", "b"].map((id, i) => ({ id, provider: "fixture", name: id, input: ["text"], reasoning: false, supportedLevels: ["off"], maxTokens: 1000, contextWindow: 100_000 * (i + 1), cost: { input: 1, output: 1, cacheRead: 1, cacheWrite: 1 } })),
|
||||
providers: [], defaults: null, slots: { strong: null, cheap: null, vision: null }, rolePins: {} as ModelCatalogResponse["rolePins"],
|
||||
};
|
||||
globalThis.fetch = async (input) => {
|
||||
if (input === "/api/models") return Response.json(catalog);
|
||||
if (input === "/api/settings") return Response.json({ settings: { models: {} } });
|
||||
if (input === "/api/chats/fixture") return Response.json({ settings: { model: { provider: "fixture", id: selected }, thinkingLevel: "off" } });
|
||||
throw new Error(`unexpected fixture request ${String(input)}`);
|
||||
};
|
||||
const host = document.createElement("div"); document.body.append(host);
|
||||
const root = createRoot(host);
|
||||
const turn: TurnUsage = { id: "summary", messageId: "marker", at: "fixture", model: "fixture:a", steps: [{ inputTokens: 4300, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0 }] };
|
||||
const pricing = { cost: 0.003, unpriced: 0, turnCosts: { summary: 0.003 } };
|
||||
try {
|
||||
await act(async () => root.render(<StatsBar chatId="fixture" turns={[turn]} contextTokens={83_000} pricing={pricing} />));
|
||||
const context = () => host.querySelector('[data-testid="stats-context"]')?.textContent;
|
||||
assert.equal(context(), "83k / 100k", "summarizer's 4.3k money log is not the meter source");
|
||||
selected = "b";
|
||||
await act(async () => notifySettingsSaved("models"));
|
||||
assert.equal(context(), "83k / 200k", "only denominator changes");
|
||||
await act(async () => root.render(<StatsBar chatId="fixture" turns={[turn]} contextTokens={null} pricing={pricing} />));
|
||||
assert.equal(context(), "? / 200k");
|
||||
assert.equal(host.querySelector('[data-testid="stats-cost"]')?.textContent, "$0.0030", "summary spending is still visible");
|
||||
await act(async () => root.render(<StatsBar chatId="fixture" turns={[turn]} contextTokens={72_000} pricing={pricing} />));
|
||||
assert.equal(context(), "72k / 200k");
|
||||
} finally {
|
||||
await act(async () => root.unmount()); host.remove(); globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
+69
-262
@@ -1,307 +1,114 @@
|
||||
/**
|
||||
* Usage-accounting tests — the money channel's pure helpers (core/prices.ts)
|
||||
* and the ChatStore usage-log seam (appendTurnUsage).
|
||||
*
|
||||
* Pure helper tests are pure. The store test uses the real fs chatStore with
|
||||
* a fixture chat, cleaned up in after().
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import * as fs from "node:fs";
|
||||
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { graphStore } from "../src/store/graph.ts";
|
||||
import { PATHS, SOURCES_FILE } from "../config.ts";
|
||||
import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts";
|
||||
import { callCost, jobCost, sessionCost, turnCost } from "../src/server/pricing.ts";
|
||||
import { sumSteps } from "../src/core/prices.ts";
|
||||
import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts";
|
||||
|
||||
const call = (over: Partial<CallUsage> = {}): CallUsage => ({
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cacheReadTokens: undefined,
|
||||
cacheWriteTokens: undefined,
|
||||
reasoningTokens: undefined,
|
||||
...over,
|
||||
inputTokens: 0, outputTokens: 0, cacheReadTokens: undefined,
|
||||
cacheWriteTokens: undefined, reasoningTokens: undefined, ...over,
|
||||
});
|
||||
|
||||
test("sumSteps folds steps; absent stays absent until a value appears", () => {
|
||||
const sum = sumSteps([
|
||||
call({ inputTokens: 100, outputTokens: 20 }),
|
||||
call({ inputTokens: 50, outputTokens: 10, cacheReadTokens: 30 }),
|
||||
]);
|
||||
test("sumSteps: absent stays absent; unknown long-cache detail stays unknown", () => {
|
||||
const sum = sumSteps([call({ inputTokens: 100, outputTokens: 20 }), call({ inputTokens: 50, outputTokens: 10, cacheReadTokens: 30 })]);
|
||||
assert.equal(sum.inputTokens, 150);
|
||||
assert.equal(sum.outputTokens, 30);
|
||||
assert.equal(sum.cacheReadTokens, 30);
|
||||
assert.equal(sum.cacheWriteTokens, undefined); // never reported → stays undefined
|
||||
// all-absent → all-undefined (helper defaults input/output to 0, so
|
||||
// explicit undefined here)
|
||||
const empty = sumSteps([call({ inputTokens: undefined, outputTokens: undefined }), call({ inputTokens: undefined, outputTokens: undefined })]);
|
||||
assert.equal(empty.inputTokens, undefined);
|
||||
assert.equal(sum.cacheWriteTokens, undefined);
|
||||
assert.equal(sumSteps([call({ inputTokens: undefined, outputTokens: undefined })]).inputTokens, undefined);
|
||||
assert.equal(sumSteps([call({ cacheWriteTokens: 10 }), call({ cacheWriteTokens: 20, cacheWrite1hTokens: 5 })]).cacheWrite1hTokens, undefined);
|
||||
});
|
||||
|
||||
test("callCost: glm-5.3-flash math — fresh input, cached read, output", () => {
|
||||
test("callCost prices normalized fresh input directly, without subtracting caches again", () => {
|
||||
const cost = callCost(call({ inputTokens: 100_000, outputTokens: 20_000, cacheReadTokens: 10_000 }), "openrouter:z-ai/glm-5.3-flash");
|
||||
assert.ok(cost !== null);
|
||||
// fresh 90k @ $0.15/M + cached 10k @ $0.03/M + output 20k @ $0.5/M
|
||||
// (pi-ai catalog prices — regenerated tables, scripts/sync-pricing;
|
||||
// glm-5.3-flash cacheRead corrected 0.05 → 0.03 in pi 1.0.2)
|
||||
assert.ok(Math.abs(cost! - (0.0135 + 0.0003 + 0.01)) < 1e-9);
|
||||
assert.ok(Math.abs(cost - (0.015 + 0.0003 + 0.01)) < 1e-9);
|
||||
assert.equal(callCost(call({ inputTokens: 10_000, outputTokens: 1_000, cacheReadTokens: 70_000 }), "openrouter:openai/gpt-6.1-sol"), 0.037);
|
||||
});
|
||||
|
||||
test("callCost: cache writes bill at the cacheWrite rate", () => {
|
||||
const cost = callCost(
|
||||
call({ inputTokens: 1_000_000, outputTokens: 0, cacheWriteTokens: 500_000 }),
|
||||
"anthropic:claude-haiku-4-5",
|
||||
);
|
||||
assert.ok(cost !== null);
|
||||
// Native 1.4 catalog: fresh 1M @ $1/M + cache write 500k @ $1.25/M.
|
||||
assert.ok(Math.abs(cost! - (1 + 0.625)) < 1e-9);
|
||||
test("cache writes: precise short/one-hour costs and honest unknown historical splits", () => {
|
||||
const usage = call({ inputTokens: 1_000_000, cacheWriteTokens: 500_000, cacheWrite1hTokens: 0 });
|
||||
assert.equal(callCost(usage, "anthropic:claude-haiku-4-5"), 1.625);
|
||||
assert.equal(callCost({ ...usage, cacheWrite1hTokens: 200_000 }, "anthropic:claude-haiku-4-5"), 1.775);
|
||||
assert.equal(callCost({ ...usage, cacheWrite1hTokens: undefined }, "anthropic:claude-haiku-4-5"), null);
|
||||
});
|
||||
|
||||
test("callCost: unknown model and absent fields are null, never zero", () => {
|
||||
assert.equal(callCost(call({ inputTokens: 100, outputTokens: 5 }), "not/a-model"), null);
|
||||
assert.equal(callCost(call({ inputTokens: 100, outputTokens: undefined }), "openrouter:z-ai/glm-5.3-flash"), null);
|
||||
test("unknown model, absent or invalid required usage is null, never fake zero", () => {
|
||||
assert.equal(callCost(call(), "not/a-model"), null);
|
||||
assert.equal(callCost(call({ outputTokens: undefined }), "openai:gpt-5"), null);
|
||||
for (const value of [-1, NaN, Infinity]) assert.equal(callCost(call({ inputTokens: value }), "openai:gpt-5"), null);
|
||||
});
|
||||
|
||||
test("turnCost sums steps; sessionCost reports unpriced turns", () => {
|
||||
const turn: TurnUsage = {
|
||||
id: "t1",
|
||||
messageId: "m1",
|
||||
at: "2026-09-09T00:00:00Z",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
steps: [call({ inputTokens: 10_000, outputTokens: 2_000 }), call({ inputTokens: 12_000, outputTokens: 1_000 })],
|
||||
};
|
||||
assert.ok(turnCost(turn) !== null);
|
||||
const totals = sessionCost({
|
||||
turns: [turn, { id: "t2", messageId: "m2", at: "x", model: "not/a-model", steps: [call({ inputTokens: 1, outputTokens: 1 })] }],
|
||||
});
|
||||
test("turn/session costs sum calls and count unpriced separately", () => {
|
||||
const turn: TurnUsage = { id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 10_000, outputTokens: 2_000 }), call({ inputTokens: 12_000, outputTokens: 1_000 })] };
|
||||
const totals = sessionCost({ turns: [turn, { ...turn, id: "t2", model: "not/a-model" }] });
|
||||
assert.equal(totals.unpriced, 1);
|
||||
assert.ok(totals.cost > 0);
|
||||
assert.equal(totals.cost, turnCost(turn));
|
||||
assert.equal(totals.turnCosts.t1, totals.cost);
|
||||
assert.equal(totals.turnCosts.t2, null);
|
||||
});
|
||||
|
||||
test("contextWindow: last call's footprint against the live index's limit", () => {
|
||||
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
|
||||
const cw = contextWindow(
|
||||
{
|
||||
turns: [
|
||||
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
|
||||
{ id: "t2", messageId: "m2", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
|
||||
],
|
||||
},
|
||||
limits,
|
||||
);
|
||||
assert.ok(cw);
|
||||
assert.equal(cw.model, "openrouter:z-ai/glm-5.3-flash");
|
||||
assert.equal(cw.usedTokens, 100_000);
|
||||
assert.equal(cw.limit, 1_048_576);
|
||||
assert.equal(contextWindow({ turns: [] }, limits), null);
|
||||
});
|
||||
|
||||
test("contextWindow NUMERATOR: cache tokens count (owner audit, 2026-09-23)", () => {
|
||||
// The audit's live case: a cache-heavy session read 40k on the meter
|
||||
// while the real footprint was ~190k — provider `input` EXCLUDES the
|
||||
// cached portion, so input+output alone is wildly low. The fold is
|
||||
// pi's totalTokens semantics: all four fields.
|
||||
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
|
||||
const cw = contextWindow(
|
||||
{
|
||||
turns: [
|
||||
{
|
||||
id: "t1",
|
||||
messageId: "m1",
|
||||
at: "x",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
steps: [call({ inputTokens: 40_000, outputTokens: 1_000, cacheReadTokens: 145_000, cacheWriteTokens: 4_000 })],
|
||||
},
|
||||
],
|
||||
},
|
||||
limits,
|
||||
);
|
||||
assert.ok(cw);
|
||||
assert.equal(cw.usedTokens, 40_000 + 1_000 + 145_000 + 4_000, "input + output + cacheRead + cacheWrite");
|
||||
// Cache fields absent (non-caching providers): the fold is unchanged.
|
||||
const plain = contextWindow(
|
||||
{ turns: [{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] }] },
|
||||
limits,
|
||||
);
|
||||
assert.ok(plain);
|
||||
assert.equal(plain.usedTokens, 10_000);
|
||||
});
|
||||
|
||||
test("contextWindow DENOMINATOR: the live index is the ONLY source — no fallback (owner amendment, 2026-09-23)", () => {
|
||||
const turns = [
|
||||
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 10_000, outputTokens: 1_000 })] },
|
||||
];
|
||||
const live = new Map([["openrouter:z-ai/glm-5.3-flash", 65_536]]);
|
||||
const cw = contextWindow({ turns }, live);
|
||||
assert.ok(cw);
|
||||
assert.equal(cw.limit, 65_536, "pi's catalog, via the index, is the one source");
|
||||
// No index at all (a client that hasn't fetched the catalog): the
|
||||
// meter renders usage-only — limit null, never a guessed window.
|
||||
const noIndex = contextWindow({ turns });
|
||||
assert.ok(noIndex);
|
||||
assert.equal(noIndex.usedTokens, 11_000);
|
||||
assert.equal(noIndex.limit, null);
|
||||
const emptyIndex = contextWindow({ turns }, new Map<string, number>());
|
||||
assert.ok(emptyIndex);
|
||||
assert.equal(emptyIndex.limit, null, "an empty index (failed fetch) is usage-only too");
|
||||
// A model missing from the live catalog (historical session, package
|
||||
// update removed it): usage-only, never a fake window.
|
||||
const missing = contextWindow({ turns: [{ ...turns[0]!, model: "nope/gone-model" }] }, live);
|
||||
assert.ok(missing);
|
||||
assert.equal(missing.limit, null);
|
||||
assert.equal(missing.usedTokens, 11_000, "the footprint still reads honestly");
|
||||
});
|
||||
|
||||
test("contextWindow ride-along: a zeroed/aborted last step falls back to the previous step", () => {
|
||||
// An aborted last call reports zeroed usage — that is not the session
|
||||
// footprint. The honest reading is the previous step's usage.
|
||||
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
|
||||
const cw = contextWindow(
|
||||
{
|
||||
turns: [
|
||||
{
|
||||
id: "t1",
|
||||
messageId: "m1",
|
||||
at: "x",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
steps: [
|
||||
call({ inputTokens: 80_000, outputTokens: 5_000 }),
|
||||
call({ inputTokens: 0, outputTokens: 0 }), // aborted — zeroed
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
limits,
|
||||
);
|
||||
assert.ok(cw);
|
||||
assert.equal(cw.usedTokens, 85_000, "the previous step's footprint, not a bogus tiny meter");
|
||||
// ALL steps zeroed (a first-call abort): no reading at all — null.
|
||||
assert.equal(
|
||||
contextWindow(
|
||||
{ turns: [{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 0, outputTokens: 0 })] }] },
|
||||
limits,
|
||||
),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test("jobCost: null without usage, computed with it", () => {
|
||||
test("legacy flat-rate jobs remain priceable; missing usage stays absent", () => {
|
||||
assert.equal(jobCost({}), null);
|
||||
const job: Pick<JobRecord, "usage"> = {
|
||||
usage: {
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
inputTokens: 1_000_000,
|
||||
outputTokens: 100_000,
|
||||
cacheReadTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
steps: 4,
|
||||
},
|
||||
};
|
||||
assert.ok(jobCost(job) !== null);
|
||||
const job: Pick<JobRecord, "usage"> = { usage: { model: "openrouter:z-ai/glm-5.3-flash", inputTokens: 1_000_000, outputTokens: 100_000, cacheReadTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0, steps: 4 } };
|
||||
assert.ok(jobCost(job)! > 0);
|
||||
});
|
||||
|
||||
// ─── the store seam ──────────────────────────────────────────────────────────
|
||||
|
||||
test("appendTurnUsage: appends, idempotent per turn id, continuation turns (same message) all kept, survives saveMessages", () => {
|
||||
const record = chatStore.createChat("usage test chat");
|
||||
const entry: TurnUsage = {
|
||||
id: "turn_1",
|
||||
messageId: "msg_a",
|
||||
at: "2026-09-09T00:00:00Z",
|
||||
model: "openrouter:z-ai/glm-5.3-flash",
|
||||
steps: [call({ inputTokens: 100, outputTokens: 10 })],
|
||||
};
|
||||
chatStore.appendTurnUsage(record.id, entry);
|
||||
chatStore.appendTurnUsage(record.id, entry); // replay (abort/retry) → no-op
|
||||
// continuation turn: SAME response message (SDK coalesces), new turn id —
|
||||
// usage must survive, not collapse into the first entry
|
||||
chatStore.appendTurnUsage(record.id, { ...entry, id: "turn_2" });
|
||||
let reread = chatStore.getChat(record.id);
|
||||
assert.equal(reread?.turns?.length, 2);
|
||||
|
||||
// saveMessages must not clobber the log (read-mutate-write record) —
|
||||
// Wave B: a NATIVE message list (the record holds agent history).
|
||||
chatStore.saveMessages(record.id, [
|
||||
{ id: "msg_a", role: "user", content: "native", timestamp: Date.now() } as never,
|
||||
]);
|
||||
reread = chatStore.getChat(record.id);
|
||||
assert.equal(reread?.turns?.length, 2);
|
||||
test("appendTurnUsage: idempotent per turn; continuations survive saveMessages; list omits log", () => {
|
||||
const record = chatStore.createChat("usage fixture");
|
||||
try {
|
||||
const entry: TurnUsage = { id: "turn_1", messageId: "msg_a", at: "x", model: "openai:gpt-5", steps: [call({ inputTokens: 100, outputTokens: 10, cacheWrite1hTokens: 0 })] };
|
||||
chatStore.appendTurnUsage(record.id, entry);
|
||||
chatStore.appendTurnUsage(record.id, entry);
|
||||
chatStore.appendTurnUsage(record.id, { ...entry, id: "turn_2" });
|
||||
assert.equal(chatStore.getChat(record.id)?.turns?.length, 2);
|
||||
chatStore.saveMessages(record.id, [{ id: "msg_a", role: "user", content: "fixture", timestamp: 1 }]);
|
||||
assert.deepEqual(chatStore.getChat(record.id)?.turns?.[0]?.steps, JSON.parse(JSON.stringify(entry.steps)));
|
||||
assert.equal(chatStore.getChat(record.id)?.turns?.length, 2);
|
||||
assert.equal("turns" in chatStore.listChats().find((c) => c.id === record.id)!, false);
|
||||
} finally { fs.unlinkSync(`${PATHS.chats}/${record.id}.json`); }
|
||||
});
|
||||
|
||||
// and the list payload must NOT leak turns
|
||||
const listed = chatStore.listChats().find((c) => c.id === record.id);
|
||||
assert.ok(listed);
|
||||
assert.equal("turns" in listed, false);
|
||||
|
||||
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
|
||||
const runEntry = (startedAt: number): ExtractionRunUsage => ({
|
||||
startedAt, at: "2026-09-09T00:00:00Z", durationMs: 42_000,
|
||||
stats: { nodes: 1, chains: 2 }, stages: {
|
||||
extractor: { model: "openai:gpt-5", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "openai:gpt-5", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
},
|
||||
});
|
||||
|
||||
test("appendExtractionUsage: appends, idempotent per startedAt, not in list payload", () => {
|
||||
const record = chatStore.createChat("extraction usage test chat");
|
||||
const runEntry = (startedAt: number): ExtractionRunUsage => ({
|
||||
startedAt,
|
||||
at: "2026-09-09T00:00:00Z",
|
||||
durationMs: 42_000,
|
||||
stats: { nodes: 1, chains: 2 },
|
||||
stages: {
|
||||
extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
},
|
||||
});
|
||||
chatStore.appendExtractionUsage(record.id, runEntry(1));
|
||||
chatStore.appendExtractionUsage(record.id, runEntry(1)); // same run appended twice → no-op
|
||||
chatStore.appendExtractionUsage(record.id, runEntry(2)); // a retried run is a NEW spend
|
||||
let reread = chatStore.getChat(record.id);
|
||||
assert.equal(reread?.extractions?.length, 2);
|
||||
// failed runs (stats absent) persist with usage — money was spent
|
||||
chatStore.appendExtractionUsage(record.id, { ...runEntry(3), stats: undefined });
|
||||
reread = chatStore.getChat(record.id);
|
||||
assert.equal(reread?.extractions?.length, 3);
|
||||
assert.equal(reread?.extractions?.at(-1)?.stats, undefined);
|
||||
|
||||
// the list payload must NOT leak the extractions log
|
||||
const listed = chatStore.listChats().find((c) => c.id === record.id);
|
||||
assert.ok(listed);
|
||||
assert.equal("extractions" in listed, false);
|
||||
|
||||
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
|
||||
test("appendExtractionUsage: idempotent; failed runs persist; list omits log", () => {
|
||||
const record = chatStore.createChat("extraction usage fixture");
|
||||
try {
|
||||
chatStore.appendExtractionUsage(record.id, runEntry(1));
|
||||
chatStore.appendExtractionUsage(record.id, runEntry(1));
|
||||
chatStore.appendExtractionUsage(record.id, runEntry(2));
|
||||
chatStore.appendExtractionUsage(record.id, { ...runEntry(3), stats: undefined });
|
||||
assert.equal(chatStore.getChat(record.id)?.extractions?.length, 3);
|
||||
assert.equal(chatStore.getChat(record.id)?.extractions?.at(-1)?.stats, undefined);
|
||||
assert.equal("extractions" in chatStore.listChats().find((c) => c.id === record.id)!, false);
|
||||
} finally { fs.unlinkSync(`${PATHS.chats}/${record.id}.json`); }
|
||||
});
|
||||
|
||||
test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => {
|
||||
// The real fs graphStore — sources.json backed up and restored around
|
||||
// this test (same discipline as the chat fixture files).
|
||||
test("appendSourceUsage: idempotent; reimport keeps history; absent source is a no-op", () => {
|
||||
fs.mkdirSync(PATHS.graph, { recursive: true });
|
||||
let bak: string | undefined;
|
||||
if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`;
|
||||
if (bak) fs.renameSync(SOURCES_FILE, bak);
|
||||
const backup = fs.existsSync(SOURCES_FILE) ? fs.readFileSync(SOURCES_FILE) : undefined;
|
||||
try {
|
||||
const runEntry = (startedAt: number): ExtractionRunUsage => ({
|
||||
startedAt,
|
||||
at: "2026-09-13T00:00:00Z",
|
||||
durationMs: 42_000,
|
||||
stats: { nodes: 1, chains: 2 },
|
||||
stages: {
|
||||
extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
|
||||
auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
|
||||
},
|
||||
});
|
||||
const id = "src_usagetest01";
|
||||
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" });
|
||||
graphStore.upsertSource({ id, filePath: "/fixture/notes.md", importedAt: "2026-09-13T00:00:00Z" });
|
||||
graphStore.appendSourceUsage(id, runEntry(1));
|
||||
graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op
|
||||
graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend
|
||||
graphStore.appendSourceUsage(id, runEntry(1));
|
||||
graphStore.appendSourceUsage(id, runEntry(2));
|
||||
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
|
||||
|
||||
// An upsert (re-import) KEEPS the usage history.
|
||||
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" });
|
||||
graphStore.upsertSource({ id, filePath: "/fixture/notes.md", importedAt: "2026-09-14T00:00:00Z" });
|
||||
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
|
||||
|
||||
// Unknown ids are a silent no-op (nothing to bill against).
|
||||
graphStore.appendSourceUsage("src_nope", runEntry(1));
|
||||
graphStore.appendSourceUsage("src_missing", runEntry(1));
|
||||
} finally {
|
||||
fs.rmSync(SOURCES_FILE, { force: true });
|
||||
if (bak) fs.renameSync(bak, SOURCES_FILE);
|
||||
if (backup) fs.writeFileSync(SOURCES_FILE, backup);
|
||||
else fs.rmSync(SOURCES_FILE, { force: true });
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user