price drift-guard — the gates fail when a pi-ai bump isn't followed by a sync
PRICES must stay a generated static table (core is client-safe; the renderer folds costs and reading the catalog JSON needs fs), so the forgetting-the-sync failure mode is guarded instead of eliminated. - sync-pricing.ts: buildPrices() extracted (the pure catalog walk), exported for the test; main() now guarded by an entry-point check so importing the module runs nothing - tests/prices-sync.test.ts: asserts PRICES equals EXACTLY what the script would generate now — unpriced new models, drifted prices, and keys gone from the catalog all fail with 'run npx tsx scripts/sync-pricing.ts'. Verified it fires by deleting the deepseek-v4.1-flash entry (1 unpriced, actionable message) - sync script's embedded header template corrected to match prices.ts (no bare-slug legacy claim) + drift-guard note, so the next regeneration keeps the honest docs
This commit is contained in:
+25
-10
@@ -39,10 +39,12 @@ interface CatalogModel {
|
||||
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
// Every data file is {api: {modelId: entry}} — ids are per-provider
|
||||
// namespaced already (openrouter's are slugs, native ones plain).
|
||||
const priced = new Map<string, { cost: CatalogModel["cost"] }>();
|
||||
/** The pure walk: every catalog entry that carries pricing, keyed
|
||||
* "provider:id". Exported for the drift-guard test (tests/prices-sync)
|
||||
* so the guard asserts EXACTLY what this script would generate — the
|
||||
* test and the generator can never disagree (owner, 2026-09-29). */
|
||||
export function buildPrices(): Map<string, CatalogModel["cost"]> {
|
||||
const priced = new Map<string, CatalogModel["cost"]>();
|
||||
for (const file of fs.readdirSync(DATA_DIR).filter((f) => f.endsWith(".json"))) {
|
||||
const provider = file.slice(0, -".json".length);
|
||||
const catalog = JSON.parse(fs.readFileSync(path.join(DATA_DIR, file), "utf-8")) as Record<
|
||||
@@ -57,10 +59,17 @@ function main(): void {
|
||||
// First writer wins (an id can appear under two apis within one
|
||||
// file with identical pricing; cross-file collisions cannot
|
||||
// happen — keys are provider-qualified).
|
||||
priced.set(`${provider}:${id}`, { cost: m.cost });
|
||||
priced.set(`${provider}:${id}`, m.cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
return priced;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const priced = new Map<string, { cost: CatalogModel["cost"] }>(
|
||||
[...buildPrices()].map(([k, cost]) => [k, { cost }] as const),
|
||||
);
|
||||
|
||||
// Sort by key for a stable, diff-friendly generated table.
|
||||
const rows = [...priced.entries()]
|
||||
@@ -70,15 +79,18 @@ function main(): void {
|
||||
}));
|
||||
|
||||
const tables = `/** Token prices, USD per 1M tokens. A missing model makes cost render
|
||||
* as absent ("—"), never zero. Keys are provider-qualified ("provider:id")
|
||||
* since the model slots wave; openrouter ids ALSO appear under their bare
|
||||
* slug (the legacy stamp format, kept so pre-wave records keep pricing).
|
||||
* as absent ("—"), never zero. Keys are uniformly provider-qualified
|
||||
* ("provider:id") — NO legacy bare-slug compat (owner decision, sync
|
||||
* script 2026-09-20): pre-wave records stamped with bare slugs render
|
||||
* unpriced.
|
||||
* Context-window limits are NOT a table (owner amendment, 2026-09-23):
|
||||
* every meter's denominator reads pi's LIVE catalog (GET /api/models /
|
||||
* the activity feed's server-side join) — missing model, usage-only.
|
||||
* Refresh: bump @earendil-works/pi-ai, then npx tsx scripts/sync-pricing.ts
|
||||
* (reads pi-ai's shipped catalogs — the same source the engine and the
|
||||
* picker use). */
|
||||
* picker use). Forgetting the re-run is exactly how models go unpriced
|
||||
* (deepseek-v4.1-flash, 2026-09-29) — the drift-guard test
|
||||
* (tests/prices-sync.test.ts) fails the gates until you do. */
|
||||
export const PRICES: Record<string, ModelPrice> = {
|
||||
${rows.map((r) => r.price).join("\n")}
|
||||
};
|
||||
@@ -93,4 +105,7 @@ ${rows.map((r) => r.price).join("\n")}
|
||||
console.log(`prices.ts regenerated from pi-ai's catalogs: ${priced.size} keys priced.`);
|
||||
}
|
||||
|
||||
main();
|
||||
// Run only as a script, not when imported (the drift-guard test).
|
||||
if (process.argv[1] && import.meta.filename && path.resolve(process.argv[1]) === import.meta.filename) {
|
||||
main();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* THE PRICE-TABLE DRIFT GUARD (owner, 2026-09-29).
|
||||
*
|
||||
* Failure mode this kills: bumping @earendil-works/pi-ai adds new models
|
||||
* (and price corrections) to the shipped catalogs, but nobody re-runs
|
||||
* scripts/sync-pricing.ts — src/core/prices.ts stays a stale generation
|
||||
* and every run on a new model renders cost as "—" (deepseek-v4.1-flash,
|
||||
* 2026-09-29, commit df654fb). The table must stay a generated static
|
||||
* file (core is client-safe: activity-dock and stats-bar fold costs in
|
||||
* the renderer, and reading the catalog JSON needs fs), so staleness is
|
||||
* guarded instead of eliminated.
|
||||
*
|
||||
* The test asserts PRICES equals EXACTLY what the sync script would
|
||||
* generate right now — it imports the script's own builder, so the guard
|
||||
* and the generator can never disagree:
|
||||
* - a catalog model with pricing missing from PRICES → "run the sync"
|
||||
* - a price field that drifted in the catalog → "run the sync"
|
||||
* - a PRICES key whose model left the catalog → "run the sync"
|
||||
*
|
||||
* On failure: npx tsx scripts/sync-pricing.ts, commit the regenerated
|
||||
* file, re-derive any pinned-cost tests.
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { PRICES } from "../src/core/prices.ts";
|
||||
import { buildPrices } from "../scripts/sync-pricing.ts";
|
||||
|
||||
test("PRICES matches pi-ai's shipped catalogs exactly — the sync was run after the last bump", () => {
|
||||
const catalog = buildPrices();
|
||||
|
||||
const missing = [...catalog.keys()].filter((k) => !(k in PRICES));
|
||||
const stale = Object.keys(PRICES).filter((k) => !catalog.has(k));
|
||||
const drifted: string[] = [];
|
||||
for (const [key, cost] of catalog) {
|
||||
const price = PRICES[key];
|
||||
if (!price) continue;
|
||||
if (
|
||||
price.input !== cost.input ||
|
||||
price.output !== cost.output ||
|
||||
price.cacheRead !== cost.cacheRead ||
|
||||
price.cacheWrite !== cost.cacheWrite
|
||||
) {
|
||||
drifted.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
assert.equal(
|
||||
`${missing.length} / ${stale.length} / ${drifted.length}`,
|
||||
"0 / 0 / 0",
|
||||
`price table is stale: ${missing.length} unpriced catalog models (e.g. ${missing[0] ?? "—"}), ` +
|
||||
`${stale.length} keys gone from the catalog (e.g. ${stale[0] ?? "—"}), ` +
|
||||
`${drifted.length} drifted prices (e.g. ${drifted[0] ?? "—"}). ` +
|
||||
"Run: npx tsx scripts/sync-pricing.ts, commit the regenerated file, re-derive pinned-cost tests.",
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user