price drift-guard — the gates fail when a pi-ai bump isn't followed by a sync
PRICES must stay a generated static table (core is client-safe; the renderer folds costs and reading the catalog JSON needs fs), so the forgetting-the-sync failure mode is guarded instead of eliminated. - sync-pricing.ts: buildPrices() extracted (the pure catalog walk), exported for the test; main() now guarded by an entry-point check so importing the module runs nothing - tests/prices-sync.test.ts: asserts PRICES equals EXACTLY what the script would generate now — unpriced new models, drifted prices, and keys gone from the catalog all fail with 'run npx tsx scripts/sync-pricing.ts'. Verified it fires by deleting the deepseek-v4.1-flash entry (1 unpriced, actionable message) - sync script's embedded header template corrected to match prices.ts (no bare-slug legacy claim) + drift-guard note, so the next regeneration keeps the honest docs
This commit is contained in:
+25
-10
@@ -39,10 +39,12 @@ interface CatalogModel {
|
|||||||
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
||||||
}
|
}
|
||||||
|
|
||||||
function main(): void {
|
/** The pure walk: every catalog entry that carries pricing, keyed
|
||||||
// Every data file is {api: {modelId: entry}} — ids are per-provider
|
* "provider:id". Exported for the drift-guard test (tests/prices-sync)
|
||||||
// namespaced already (openrouter's are slugs, native ones plain).
|
* so the guard asserts EXACTLY what this script would generate — the
|
||||||
const priced = new Map<string, { cost: CatalogModel["cost"] }>();
|
* test and the generator can never disagree (owner, 2026-09-29). */
|
||||||
|
export function buildPrices(): Map<string, CatalogModel["cost"]> {
|
||||||
|
const priced = new Map<string, CatalogModel["cost"]>();
|
||||||
for (const file of fs.readdirSync(DATA_DIR).filter((f) => f.endsWith(".json"))) {
|
for (const file of fs.readdirSync(DATA_DIR).filter((f) => f.endsWith(".json"))) {
|
||||||
const provider = file.slice(0, -".json".length);
|
const provider = file.slice(0, -".json".length);
|
||||||
const catalog = JSON.parse(fs.readFileSync(path.join(DATA_DIR, file), "utf-8")) as Record<
|
const catalog = JSON.parse(fs.readFileSync(path.join(DATA_DIR, file), "utf-8")) as Record<
|
||||||
@@ -57,10 +59,17 @@ function main(): void {
|
|||||||
// First writer wins (an id can appear under two apis within one
|
// First writer wins (an id can appear under two apis within one
|
||||||
// file with identical pricing; cross-file collisions cannot
|
// file with identical pricing; cross-file collisions cannot
|
||||||
// happen — keys are provider-qualified).
|
// happen — keys are provider-qualified).
|
||||||
priced.set(`${provider}:${id}`, { cost: m.cost });
|
priced.set(`${provider}:${id}`, m.cost);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return priced;
|
||||||
|
}
|
||||||
|
|
||||||
|
function main(): void {
|
||||||
|
const priced = new Map<string, { cost: CatalogModel["cost"] }>(
|
||||||
|
[...buildPrices()].map(([k, cost]) => [k, { cost }] as const),
|
||||||
|
);
|
||||||
|
|
||||||
// Sort by key for a stable, diff-friendly generated table.
|
// Sort by key for a stable, diff-friendly generated table.
|
||||||
const rows = [...priced.entries()]
|
const rows = [...priced.entries()]
|
||||||
@@ -70,15 +79,18 @@ function main(): void {
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
const tables = `/** Token prices, USD per 1M tokens. A missing model makes cost render
|
const tables = `/** Token prices, USD per 1M tokens. A missing model makes cost render
|
||||||
* as absent ("—"), never zero. Keys are provider-qualified ("provider:id")
|
* as absent ("—"), never zero. Keys are uniformly provider-qualified
|
||||||
* since the model slots wave; openrouter ids ALSO appear under their bare
|
* ("provider:id") — NO legacy bare-slug compat (owner decision, sync
|
||||||
* slug (the legacy stamp format, kept so pre-wave records keep pricing).
|
* script 2026-09-20): pre-wave records stamped with bare slugs render
|
||||||
|
* unpriced.
|
||||||
* Context-window limits are NOT a table (owner amendment, 2026-09-23):
|
* Context-window limits are NOT a table (owner amendment, 2026-09-23):
|
||||||
* every meter's denominator reads pi's LIVE catalog (GET /api/models /
|
* every meter's denominator reads pi's LIVE catalog (GET /api/models /
|
||||||
* the activity feed's server-side join) — missing model, usage-only.
|
* the activity feed's server-side join) — missing model, usage-only.
|
||||||
* Refresh: bump @earendil-works/pi-ai, then npx tsx scripts/sync-pricing.ts
|
* Refresh: bump @earendil-works/pi-ai, then npx tsx scripts/sync-pricing.ts
|
||||||
* (reads pi-ai's shipped catalogs — the same source the engine and the
|
* (reads pi-ai's shipped catalogs — the same source the engine and the
|
||||||
* picker use). */
|
* picker use). Forgetting the re-run is exactly how models go unpriced
|
||||||
|
* (deepseek-v4.1-flash, 2026-09-29) — the drift-guard test
|
||||||
|
* (tests/prices-sync.test.ts) fails the gates until you do. */
|
||||||
export const PRICES: Record<string, ModelPrice> = {
|
export const PRICES: Record<string, ModelPrice> = {
|
||||||
${rows.map((r) => r.price).join("\n")}
|
${rows.map((r) => r.price).join("\n")}
|
||||||
};
|
};
|
||||||
@@ -93,4 +105,7 @@ ${rows.map((r) => r.price).join("\n")}
|
|||||||
console.log(`prices.ts regenerated from pi-ai's catalogs: ${priced.size} keys priced.`);
|
console.log(`prices.ts regenerated from pi-ai's catalogs: ${priced.size} keys priced.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
main();
|
// Run only as a script, not when imported (the drift-guard test).
|
||||||
|
if (process.argv[1] && import.meta.filename && path.resolve(process.argv[1]) === import.meta.filename) {
|
||||||
|
main();
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
/**
|
||||||
|
* THE PRICE-TABLE DRIFT GUARD (owner, 2026-09-29).
|
||||||
|
*
|
||||||
|
* Failure mode this kills: bumping @earendil-works/pi-ai adds new models
|
||||||
|
* (and price corrections) to the shipped catalogs, but nobody re-runs
|
||||||
|
* scripts/sync-pricing.ts — src/core/prices.ts stays a stale generation
|
||||||
|
* and every run on a new model renders cost as "—" (deepseek-v4.1-flash,
|
||||||
|
* 2026-09-29, commit df654fb). The table must stay a generated static
|
||||||
|
* file (core is client-safe: activity-dock and stats-bar fold costs in
|
||||||
|
* the renderer, and reading the catalog JSON needs fs), so staleness is
|
||||||
|
* guarded instead of eliminated.
|
||||||
|
*
|
||||||
|
* The test asserts PRICES equals EXACTLY what the sync script would
|
||||||
|
* generate right now — it imports the script's own builder, so the guard
|
||||||
|
* and the generator can never disagree:
|
||||||
|
* - a catalog model with pricing missing from PRICES → "run the sync"
|
||||||
|
* - a price field that drifted in the catalog → "run the sync"
|
||||||
|
* - a PRICES key whose model left the catalog → "run the sync"
|
||||||
|
*
|
||||||
|
* On failure: npx tsx scripts/sync-pricing.ts, commit the regenerated
|
||||||
|
* file, re-derive any pinned-cost tests.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import test from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { PRICES } from "../src/core/prices.ts";
|
||||||
|
import { buildPrices } from "../scripts/sync-pricing.ts";
|
||||||
|
|
||||||
|
test("PRICES matches pi-ai's shipped catalogs exactly — the sync was run after the last bump", () => {
|
||||||
|
const catalog = buildPrices();
|
||||||
|
|
||||||
|
const missing = [...catalog.keys()].filter((k) => !(k in PRICES));
|
||||||
|
const stale = Object.keys(PRICES).filter((k) => !catalog.has(k));
|
||||||
|
const drifted: string[] = [];
|
||||||
|
for (const [key, cost] of catalog) {
|
||||||
|
const price = PRICES[key];
|
||||||
|
if (!price) continue;
|
||||||
|
if (
|
||||||
|
price.input !== cost.input ||
|
||||||
|
price.output !== cost.output ||
|
||||||
|
price.cacheRead !== cost.cacheRead ||
|
||||||
|
price.cacheWrite !== cost.cacheWrite
|
||||||
|
) {
|
||||||
|
drifted.push(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
`${missing.length} / ${stale.length} / ${drifted.length}`,
|
||||||
|
"0 / 0 / 0",
|
||||||
|
`price table is stale: ${missing.length} unpriced catalog models (e.g. ${missing[0] ?? "—"}), ` +
|
||||||
|
`${stale.length} keys gone from the catalog (e.g. ${stale[0] ?? "—"}), ` +
|
||||||
|
`${drifted.length} drifted prices (e.g. ${drifted[0] ?? "—"}). ` +
|
||||||
|
"Run: npx tsx scripts/sync-pricing.ts, commit the regenerated file, re-derive pinned-cost tests.",
|
||||||
|
);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user