diff --git a/components/chat.tsx b/components/chat.tsx index 2d851d1..c1b97c8 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -37,6 +37,7 @@ import { useServerEvent, useServerResync } from "@/components/server-events"; import { isProviderKeyRefusal, firstSourceTextParts, + cliContextWindowOf, messageAnchorId, textPartMessageId, kbdEventForAction as kbdEventForChat, @@ -1690,6 +1691,7 @@ export function Chat({ phase={phases.at(-1) ?? "probe"} turns={turns} contextTokens={contextTokens} + cliContextWindow={cliContextWindowOf(nativeMessages)} pricing={pricing} onReserve={setDockReserve} dictationSlot={} @@ -1916,6 +1918,7 @@ function DockOverlay({ phase, turns, contextTokens, + cliContextWindow, pricing, composer, onSubmitAnswer, @@ -1952,6 +1955,7 @@ function DockOverlay({ /** The session's usage log — the stats bar beneath the composer folds it. */ turns: TurnUsage[]; contextTokens: number | null; + cliContextWindow: ReturnType; pricing?: SessionPricing; composer: React.ReactNode; onSubmitAnswer: (output: QuestionAnswerData) => void; @@ -2407,6 +2411,7 @@ function DockOverlay({ ): Map { - return new Map(catalog.models.map((m) => [`${m.provider}:${m.id}`, m.contextWindow])); + return new Map(catalog.models.flatMap((m) => m.contextWindow === null ? [] : [[`${m.provider}:${m.id}`, m.contextWindow] as const])); } /** diff --git a/components/settings/model-menu.tsx b/components/settings/model-menu.tsx index 4939672..678ba09 100644 --- a/components/settings/model-menu.tsx +++ b/components/settings/model-menu.tsx @@ -79,7 +79,7 @@ export function ModelMenuList({ {subscriptionLabel(`${m.provider}:${m.id}`) ? `${subscriptionLabel(`${m.provider}:${m.id}`)} (subscription)` : m.provider} · {m.id} - {formatTokens(m.contextWindow)} + {m.contextWindow === null ? "—" : formatTokens(m.contextWindow)} } /> diff --git a/components/stats-bar.tsx b/components/stats-bar.tsx index 6c1c05d..1abbc2b 100644 --- a/components/stats-bar.tsx +++ b/components/stats-bar.tsx @@ -1,7 +1,7 @@ "use client"; import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; -import { subscriptionLabels, subscriptionLabel, type SessionPricing, type TurnUsage } from "@/src/core"; +import { CLAUDE_CODE_PROVIDER, subscriptionLabels, subscriptionLabel, type cliContextWindowOf, type SessionPricing, type TurnUsage } from "@/src/core"; import { formatTokens, formatUsd, planCostLabel, prettyModel } from "@/components/usage-format"; import { ModelControl, catalogLimits, useModelCatalog } from "@/components/model-picker"; @@ -10,6 +10,15 @@ export function formatContextMeasurement(tokens: number | null, limit: number | return limit === null ? numerator : `${numerator} / ${formatTokens(limit)}`; } +/** CLI aliases have no global catalog limit: only this session's matching + * response can supply a denominator. Other providers keep their catalog join. */ +export function contextLimitForModel(model: string | undefined, limits: ReadonlyMap | undefined, + cliWindow: ReturnType = null): number | null { + if (!model) return null; + if (model.startsWith(`${CLAUDE_CODE_PROVIDER}:`)) return cliWindow?.model === model ? cliWindow.tokens : null; + return limits?.get(model) ?? null; +} + /** * The below-composer stats bar (wave 4 observability): the session's * money/context footprint as one quiet row beneath the composer — @@ -24,12 +33,14 @@ export function formatContextMeasurement(tokens: number | null, limit: number | * Spending arrives already priced by the server through Pi's catalog and * calculator. Missing cost stays "—", never fake zero. Context is a separate * completed-conversation measurement, never a money-log/summarizer fold or - * transcript estimate. Its catalog denominator follows the selected model; - * model changes preserve the numerator. Successful compaction clears it to ?. + * transcript estimate. Its denominator follows the selected model, from the + * catalog or the session's CLI report; model changes preserve the numerator. + * Successful compaction clears the numerator and any old CLI report. */ export function StatsBar({ turns, contextTokens = null, + cliContextWindow = null, pricing, dictationSlot, chatId, @@ -38,6 +49,7 @@ export function StatsBar({ }: { turns: TurnUsage[]; contextTokens?: number | null; + cliContextWindow?: ReturnType; pricing?: SessionPricing; /** The bound session (per-session models, owner 2026-09-28): present * on the in-session bar — the model control binds THE SESSION'S own @@ -86,11 +98,11 @@ export function StatsBar({ document.removeEventListener("keydown", onKeyDown); }; }, [open]); - // Catalog-only denominator; the picker reports its effective selection. - // Missing/pending catalog means usage-only, never a hardcoded maximum. + // The picker reports its effective selection. CLI windows are session- + // scoped; unknown metadata always means usage-only, never a guessed maximum. const { catalog } = useModelCatalog(); const limits = useMemo(() => (catalog ? catalogLimits(catalog) : undefined), [catalog]); - const limit = selectedModel ? limits?.get(selectedModel) ?? null : null; + const limit = contextLimitForModel(selectedModel, limits, cliContextWindow); const contextLabel = formatContextMeasurement(contextTokens, limit); const { cost = 0, unpriced = turns.length, subscription = 0 } = pricing ?? {}; // Honest cost: show dollars only when at least one turn priced; @@ -151,6 +163,9 @@ export function StatsBar({ {contextLabel} + {selectedModel?.startsWith(`${CLAUDE_CODE_PROVIDER}:`) && ( +

The window is reported by Claude Code’s local summary and may reflect its auto-compaction budget rather than the model’s full maximum. Unavailable windows are omitted.

+ )}
{subscription ? "Usage" : "Total cost"} {costLabel} diff --git a/src/agent/claude-code/context-window.ts b/src/agent/claude-code/context-window.ts new file mode 100644 index 0000000..9dd2354 --- /dev/null +++ b/src/agent/claude-code/context-window.ts @@ -0,0 +1,71 @@ +import { randomUUID } from "node:crypto"; + +const INITIALIZE_TIMEOUT_MS = 5_000; + +/** A best-effort, local-only CLI control query. Wire contract matches the + * official SDK0.3.259 source; this does not import or execute the SDK. + * Summary deliberately avoids the default full mode's token-count requests. + * CLI maxTokens is an effective/auto-compaction budget, not a nominal model max. */ +export class ContextWindowProbe { + private readonly initializeId = randomUUID(); + private summaryId: string | undefined; + private initialized = false; + private timer: NodeJS.Timeout | undefined; + private ready = false; + private closed = false; + + constructor( + private readonly write: (line: string) => void, + private readonly sendPrompt: () => void, + private readonly onWindow: (tokens: number) => void, + ) {} + + start(): void { + this.timer = setTimeout(() => this.proceed(false), INITIALIZE_TIMEOUT_MS); + this.timer.unref(); + this.request(this.initializeId, { subtype: "initialize" }); + } + + /** Consume control replies separately from model/tool output, including + * late/foreign replies while a native question tool is already waiting. */ + handle(value: unknown): boolean { + if (!value || typeof value !== "object" || !("type" in value) || value.type !== "control_response") return false; + if (this.closed || !("response" in value) || !value.response || typeof value.response !== "object") return true; + const response = value.response as Record; + if (response.request_id === this.initializeId) this.proceed(response.subtype === "success"); + else if (this.summaryId && response.request_id === this.summaryId) { + this.summaryId = undefined; + if (response.subtype !== "success") return true; + const body = response.response as Record | undefined; + const tokens = body?.maxTokens; + if (typeof tokens === "number" && Number.isSafeInteger(tokens) && tokens > 0) this.onWindow(tokens); + } + return true; + } + + /** Retry when the CLI has begun a model response: pre-first-turn metadata + * is not guaranteed by the protocol. Refresh also follows later responses + * within the same held process, without opening a second process. */ + refresh(): void { + if (!this.initialized || this.closed) return; + this.summaryId = randomUUID(); + this.request(this.summaryId, { subtype: "get_context_usage", detail: "summary" }); + } + + close(): void { this.closed = true; clearTimeout(this.timer); } + + private proceed(query: boolean): void { + if (this.ready || this.closed) return; + this.ready = true; + clearTimeout(this.timer); + this.initialized = query; + if (query) this.refresh(); + // Never wait for metadata to run the actual user request. No support, + // no response, or malformed metadata simply leaves the window unknown. + this.sendPrompt(); + } + + private request(id: string, request: Record): void { + this.write(JSON.stringify({ type: "control_request", request_id: id, request })); + } +} diff --git a/src/agent/claude-code/stream.ts b/src/agent/claude-code/stream.ts index 39a49ef..ba96098 100644 --- a/src/agent/claude-code/stream.ts +++ b/src/agent/claude-code/stream.ts @@ -24,9 +24,11 @@ import type { AnthropicEffort } from "@earendil-works/pi-ai/api/anthropic-messag import { parseStreamingJson, parseToolCallArguments } from "@earendil-works/pi-ai/utils/json-parse"; import { externalCliEnv } from "../../core/auth-env.ts"; +import type { LhMessage } from "../../core/native-messages.ts"; import { ClaudeLaunchError, createClaudeLauncher } from "./launch.ts"; import { parseCliLine, type CliBlock, type CliEvent, type CliUsage } from "./cli-events.ts"; import { ClaudeCodeError } from "./constants.ts"; +import { ContextWindowProbe } from "./context-window.ts"; import { addLiveProcess, getLiveProcess, removeLiveProcess, type LiveProcess } from "./live-processes.ts"; import { configurationForReuse, conversationForReuse } from "./reuse-context.ts"; import { BRIDGE_SERVER_NAME, createToolBridge, toolNameFromMcp, type PendingToolCall, type ToolBridge } from "./tool-bridge.ts"; @@ -137,7 +139,7 @@ function getAssistantErrorMessage(error: string, text: string): string { } class ClaudeTurn { - private readonly output: AssistantMessage; + private readonly output: AssistantMessage & Pick; private readonly contentIndexes = new Map(); private readonly partialJson = new Map(); private readonly toolBlocks = new Map(); @@ -233,6 +235,10 @@ class ClaudeTurn { getMessage(): AssistantMessage { return this.output; } + setContextWindow(tokens: number | undefined): void { + if (!this.finished && tokens !== undefined) this.output.cliContextWindow = tokens; + } + admit(calls: ToolCall[]): void { this.start(); for (const call of calls) { @@ -398,6 +404,8 @@ class ClaudeProcess implements LiveProcess { private admitted: PendingToolCall[] = []; private readonly cancelledIds = new Set(); private pendingError: string | undefined; + private contextWindow: number | undefined; + private contextProbe: ContextWindowProbe | undefined; constructor( private readonly child: ChildProcessWithoutNullStreams, @@ -420,12 +428,17 @@ class ClaudeProcess implements LiveProcess { this.child.stderr.setEncoding("utf8"); this.child.stderr.on("data", (chunk: string) => { this.stderr = (this.stderr + chunk).slice(-65_536); }); this.child.stdin.on("error", (error) => console.error("claude CLI stdin closed early:", error)); - this.child.stdin.end(`${stdinLine}\n`); + this.contextProbe = new ContextWindowProbe( + (line) => this.child.stdin.write(`${line}\n`), + () => { if (!this.stopping) this.child.stdin.write(`${stdinLine}\n`); }, + (tokens) => { this.contextWindow = tokens; this.turn.setContextWindow(tokens); }, + ); createInterface({ input: this.child.stdout }).on("line", (line) => { try { this.handleLine(line); } catch { this.fail("claude CLI emitted malformed stream output"); } }); this.setSignal(signal); + if (!this.stopping) this.contextProbe.start(); } canContinue(owner: symbol, sessionId: string | undefined, configuration: unknown, messages: Message[], results: ToolResultMessage[]): boolean { @@ -445,6 +458,7 @@ class ClaudeProcess implements LiveProcess { this.admitted = []; this.conversation = structuredClone(conversationForReuse(messages)); this.turn = turn; + this.turn.setContextWindow(this.contextWindow); this.sourceComplete = this.sourceCalls.length > 0; this.setSignal(signal); if (this.stopping) return; @@ -459,6 +473,7 @@ class ClaudeProcess implements LiveProcess { removeLiveProcess(this); if (this.stopping || this.closed) return; this.stopping = true; + this.contextProbe?.close(); this.child.kill(); this.killTimer = setTimeout(() => this.child.kill("SIGKILL"), 1000); this.killTimer.unref(); @@ -526,7 +541,8 @@ class ClaudeProcess implements LiveProcess { } private handleLine(line: string): void { - if (this.stopping) return; + if (this.stopping || !line.trim()) return; + if (this.contextProbe?.handle(JSON.parse(line))) return; const event = parseCliLine(line); if (!event) return; if (event.type === "toolResults") { @@ -545,6 +561,7 @@ class ClaudeProcess implements LiveProcess { this.fail("claude CLI moved on without admitting its harness tool calls"); return; } + if (event.type === "messageStart") this.contextProbe?.refresh(); this.turn.handle(event); if (this.turn.finished) { this.kill(); return; } if (event.type === "messageStop" && this.turn.waitingForAdmission) { @@ -562,6 +579,7 @@ class ClaudeProcess implements LiveProcess { private handleClose(message: string): void { if (this.closed) return; this.closed = true; + this.contextProbe?.close(); clearTimeout(this.killTimer); removeLiveProcess(this); if (this.pendingError) addLiveProcess(this); diff --git a/src/core/contracts/models.ts b/src/core/contracts/models.ts index cab7de7..03d3241 100644 --- a/src/core/contracts/models.ts +++ b/src/core/contracts/models.ts @@ -23,8 +23,9 @@ export interface ModelCatalogEntry { supportedLevels: ThinkingLevel[]; /** Null when LearnHarness cannot estimate prices for this connection. */ cost: { input: number; output: number; cacheRead: number; cacheWrite: number } | null; - contextWindow: number; - maxTokens: number; + /** Null for native CLI aliases whose effective limits are session-specific. */ + contextWindow: number | null; + maxTokens: number | null; } /** The conversation agents' resolved default (BYOK, owner 2026-09-22): the diff --git a/src/core/native-messages.ts b/src/core/native-messages.ts index c9d5a46..520fa86 100644 --- a/src/core/native-messages.ts +++ b/src/core/native-messages.ts @@ -36,6 +36,7 @@ import type { AgentMessage } from "@earendil-works/pi-agent-core"; import type { AssistantMessage, Message, ToolResultMessage, UserMessage } from "@earendil-works/pi-ai"; import { QUESTION_TOOLS } from "./pending.ts"; +import { CLAUDE_CODE_PROVIDER } from "./contracts/claude-code.ts"; import { CONCLUDE_ACK_TEXT, HANDOFF_TEXT, @@ -90,7 +91,26 @@ declare module "@earendil-works/pi-agent-core" { } /** A record message: a native pi message (or our marker) plus the record id. */ -export type LhMessage = (UserMessage | AssistantMessage | ToolResultMessage | LhMarkerMessage) & { id: string }; +export type LhMessage = (UserMessage | AssistantMessage | ToolResultMessage | LhMarkerMessage) & { + id: string; + /** CLI-reported effective context budget, not a catalog model maximum. + * App-only metadata: never transmitted as provider/model message content. */ + cliContextWindow?: number; +}; + +/** Latest response only: never borrow an older process/model's window when + * a fresh response has no metadata. Compaction invalidates the old report. */ +export function cliContextWindowOf(messages: readonly LhMessage[]): { model: string; tokens: number } | null { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]!; + if (message.role === "custom" && message.customType === "compaction") return null; + if (message.role !== "assistant") continue; + const tokens = message.cliContextWindow; + return message.provider === CLAUDE_CODE_PROVIDER && typeof tokens === "number" && Number.isSafeInteger(tokens) && tokens > 0 + ? { model: `${message.provider}:${message.model}`, tokens } : null; + } + return null; +} /** A record message list is assignable to the agent's context — that IS * the replay (ui-stream/chat-handler assign it to initialState.messages). */ @@ -617,8 +637,8 @@ export function toLlmMessages(messages: readonly AgentMessage[]): Message[] { }); continue; } - // Strip the record id: the wire carries the native shape only. - const { id: _id, ...native } = m as LhMessage; + // Strip app-only id/CLI context metadata: the wire carries the native shape only. + const { id: _id, cliContextWindow: _cliContextWindow, ...native } = m as LhMessage; out.push(native as Message); // Repair unanswered calls from completed generations, not calls in // failed generations. Pi filters error/aborted assistants itself; diff --git a/src/server/model-catalog.ts b/src/server/model-catalog.ts index 11063cd..6605788 100644 --- a/src/server/model-catalog.ts +++ b/src/server/model-catalog.ts @@ -170,15 +170,16 @@ export function toModelDto(model: Model): ModelDto { cacheRead: model.cost.cacheRead, cacheWrite: model.cost.cacheWrite, }, - contextWindow: model.contextWindow, - maxTokens: model.maxTokens, + contextWindow: model.provider === CLAUDE_CODE_PROVIDER ? null : model.contextWindow, + maxTokens: model.provider === CLAUDE_CODE_PROVIDER ? null : model.maxTokens, }; } /** The live-limits index (context-meter amendment, owner 2026-09-23): * provider:id → contextWindow from pi's catalog, over the SAME keyed - * providers the picker federates. The one source for every context - * meter's denominator — the stats-bar join holds /api/models' copy, + * providers the picker federates. Native CLI aliases are excluded: + * only a session-specific CLI report can provide their denominator. + * For other models the stats-bar join holds /api/models' copy, * and the activity feed (observability.ts) resolves job limits * server-side through this. No hardcoded fallback exists anywhere: * a model missing here reads limit-null (usage-only, honest render). */ @@ -186,7 +187,7 @@ export function catalogContextLimits(): Map { const keyed = keyedProviderIds(); const index = new Map(); for (const provider of piModels().getProviders()) { - if (!keyed.has(provider.id)) continue; + if (!keyed.has(provider.id) || provider.id === CLAUDE_CODE_PROVIDER) continue; for (const model of piModels().getModels(provider.id)) { index.set(`${provider.id}:${model.id}`, model.contextWindow); } diff --git a/tests/claude-code-context-ui.test.tsx b/tests/claude-code-context-ui.test.tsx new file mode 100644 index 0000000..3b026ac --- /dev/null +++ b/tests/claude-code-context-ui.test.tsx @@ -0,0 +1,43 @@ +import React, { act } from "react"; +(globalThis as Record).React = React; +import assert from "node:assert/strict"; +import { afterEach, test } from "node:test"; +import { createRoot, type Root } from "react-dom/client"; +import { parseHTML } from "linkedom"; +import { StatsBar } from "../components/stats-bar.tsx"; +import { CLAUDE_CODE_MODELS } from "../src/agent/claude-code/provider.ts"; +import { toModelDto } from "../src/server/model-catalog.ts"; +import type { ModelCatalogResponse } from "../src/core/index.ts"; + +const dom = parseHTML(""); +Object.assign(globalThis, { window: dom.window, document: dom.document, Element: dom.window.Element, HTMLElement: dom.window.HTMLElement, + CustomEvent: dom.window.CustomEvent, IS_REACT_ACT_ENVIRONMENT: true }); +const originalFetch = globalThis.fetch; +let root: Root | undefined, host: HTMLDivElement | undefined; +afterEach(async () => { await act(async () => root?.unmount()); host?.remove(); root = undefined; globalThis.fetch = originalFetch; }); + +test("compact CLI context is numeric only; effective-window explanation appears only in expanded session metadata", async () => { + const catalog: ModelCatalogResponse = { models: CLAUDE_CODE_MODELS.map(toModelDto), providers: [], + slots: { strong: null, cheap: null, vision: null }, defaults: null, + rolePins: { researcher: null, demoBuilder: null, visualMaker: null, extractor: null, auditor: null } }; + globalThis.fetch = (async (url) => { + if (url === "/api/models") return Response.json(catalog); + if (url === "/api/settings") return Response.json({ settings: { models: {} } }); + if (url === "/api/chats/cli-session") return Response.json({ settings: { model: { provider: "claude-code", id: "opus" } } }); + throw new Error(`unexpected request ${url}`); + }) as typeof fetch; + host = document.createElement("div"); document.body.append(host); root = createRoot(host); + const render = (window: { model: string; tokens: number } | null) => root!.render(); + await act(async () => render({ model: "claude-code:opus", tokens: 200000 })); + assert.equal(document.querySelector('[data-testid="stats-context"]')?.textContent, "42k / 200k"); + assert(!document.body.textContent?.includes("auto-compaction")); + const summary = document.querySelector('[data-testid="stats-summary"]')!; + assert(!summary.textContent?.includes("window")); assert(!summary.textContent?.includes("budget")); + await act(async () => summary.click()); + assert.match(document.querySelector('[data-testid="stats-detail"]')?.textContent ?? "", /auto-compaction budget rather than the model’s full maximum/); + assert.equal(document.querySelector('[data-testid="stats-context"]')?.textContent, "42k / 200k"); + await act(async () => render(null)); + assert.equal(document.querySelector('[data-testid="stats-context"]')?.textContent, "42k"); + await act(async () => render({ model: "claude-code:sonnet", tokens: 1000000 })); + assert.equal(document.querySelector('[data-testid="stats-context"]')?.textContent, "42k", "another alias cannot supply this session's denominator"); +}); diff --git a/tests/claude-code-context-window.test.ts b/tests/claude-code-context-window.test.ts new file mode 100644 index 0000000..5b131eb --- /dev/null +++ b/tests/claude-code-context-window.test.ts @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { ContextWindowProbe } from "../src/agent/claude-code/context-window.ts"; +import { CLAUDE_CODE_MODELS } from "../src/agent/claude-code/provider.ts"; +import { asAgentMessages, cliContextWindowOf, toLlmMessages, type LhMessage } from "../src/core/native-messages.ts"; +import { catalogLimits } from "../components/model-picker.tsx"; +import { contextLimitForModel, formatContextMeasurement } from "../components/stats-bar.tsx"; +import { toModelDto } from "../src/server/model-catalog.ts"; + +function setup() { + const writes: { type: string; request_id: string; request: Record }[] = []; + const windows: number[] = []; + let prompts = 0; + const probe = new ContextWindowProbe((line) => writes.push(JSON.parse(line)), () => prompts++, (tokens) => windows.push(tokens)); + probe.start(); + const reply = (index: number, body: unknown, subtype = "success") => probe.handle({ type: "control_response", response: { + request_id: writes[index]!.request_id, subtype, response: body, + } }); + return { probe, writes, windows, reply, prompts: () => prompts }; +} + +test("control initialization precedes a summary-only query; neither needs user input or a model request", () => { + const p = setup(); + assert.deepEqual(p.writes[0]!.request, { subtype: "initialize" }); assert.equal(p.prompts(), 0); + p.reply(0, {}); + assert.deepEqual(p.writes[1]!.request, { subtype: "get_context_usage", detail: "summary" }); + assert.equal(p.prompts(), 1, "conversation does not wait for metadata"); + assert.notEqual(p.writes[0]!.request_id, p.writes[1]!.request_id); + p.probe.handle({ type: "control_response", response: { request_id: "foreign", subtype: "success", response: { maxTokens: 999 } } }); + assert.deepEqual(p.windows, []); + p.reply(1, { maxTokens: 200000, rawMaxTokens: 1000000 }); + assert.deepEqual(p.windows, [200000], "effective maxTokens, not an assumed nominal window"); + p.reply(0, {}); p.reply(1, { maxTokens: 1 }); + assert.equal(p.prompts(), 1); assert.deepEqual(p.windows, [200000]); + p.probe.close(); +}); + +test("metadata unavailable before first response can be queried again locally; stale/closed replies are ignored", () => { + const p = setup(); p.reply(0, {}); p.reply(1, {}, "error"); + assert.equal(p.prompts(), 1); assert.deepEqual(p.windows, []); + p.probe.refresh(); + assert.deepEqual(p.writes[2]!.request, { subtype: "get_context_usage", detail: "summary" }); + p.reply(1, { maxTokens: 1 }); assert.deepEqual(p.windows, []); + p.reply(2, { maxTokens: 200000 }); assert.deepEqual(p.windows, [200000]); + p.probe.refresh(); p.probe.close(); p.reply(3, { maxTokens: 999999 }); + assert.deepEqual(p.windows, [200000]); assert.equal(p.prompts(), 1); +}); + +for (const tokens of [null, undefined, 0, -1, 1.5, Infinity, Number.MAX_SAFE_INTEGER + 1, "200000"]) test(`invalid summary budget ${tokens} remains unknown without failing the prompt`, () => { + const p = setup(); p.reply(0, {}); p.reply(1, { maxTokens: tokens }); + assert.deepEqual(p.windows, []); assert.equal(p.prompts(), 1); p.probe.close(); +}); + +test("initialization errors/timeouts fall back to ordinary prompt delivery; close cancels the timer", (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + const error = setup(); error.reply(0, undefined, "error"); + assert.equal(error.prompts(), 1); assert.equal(error.writes.length, 1); error.probe.close(); + const timeout = setup(); t.mock.timers.tick(5000); + assert.equal(timeout.prompts(), 1); assert.equal(timeout.writes.length, 1); + timeout.reply(0, {}); assert.equal(timeout.writes.length, 1); timeout.probe.close(); + const closed = setup(); closed.probe.close(); t.mock.timers.tick(5000); + assert.equal(closed.prompts(), 0); +}); + +function assistant(model = "opus", tokens: number | undefined = 200000): LhMessage & { role: "assistant" } { + return { id: "cli-assistant", role: "assistant", api: "claude-code-cli", provider: "claude-code", model, + content: [], timestamp: 0, stopReason: "stop", cliContextWindow: tokens, + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } }; +} + +test("per-session metadata survives JSON persistence, never borrows older reports, and cannot override other model selections", () => { + const first = assistant(); + const window = cliContextWindowOf(JSON.parse(JSON.stringify([first]))); + assert.deepEqual(window, { model: "claude-code:opus", tokens: 200000 }); + assert(!("cliContextWindow" in toLlmMessages(asAgentMessages([first]))[0]!)); + assert.equal(first.cliContextWindow, 200000, "provider projection never mutates the stored report"); + const catalog = new Map([["anthropic:fixture", 100000], ["claude-code:opus", 1000000]]); + assert.equal(contextLimitForModel("claude-code:opus", catalog, window), 200000); + assert.equal(contextLimitForModel("claude-code:sonnet", catalog, window), null); + assert.equal(contextLimitForModel("claude-code:opus", catalog, null), null, "no placeholder fallback"); + assert.equal(contextLimitForModel("anthropic:fixture", catalog, window), 100000); + assert.equal(contextLimitForModel(undefined, catalog, window), null); + assert.equal(formatContextMeasurement(42000, 200000), "42k / 200k"); + assert.equal(formatContextMeasurement(42000, null), "42k"); + const unreported = assistant("opus"); delete unreported.cliContextWindow; + assert.equal(cliContextWindowOf([first, unreported]), null); + const other = assistant("api"); other.provider = "anthropic"; + assert.equal(cliContextWindowOf([first, other]), null); + const compacted: LhMessage = { id: "compact", role: "custom", customType: "compaction", content: "summary", display: false, details: {}, timestamp: 1 }; + assert.equal(cliContextWindowOf([first, compacted]), null); + assert.equal(cliContextWindowOf([]), null); +}); + +test("CLI catalog limits remain unknown and cannot create a global denominator", () => { + const models = CLAUDE_CODE_MODELS.map(toModelDto); + assert(models.every((model) => model.contextWindow === null && model.maxTokens === null)); + assert.equal(catalogLimits({ models }).size, 0); +}); diff --git a/tests/claude-code-harness.test.ts b/tests/claude-code-harness.test.ts index 875c96c..78bedb2 100644 --- a/tests/claude-code-harness.test.ts +++ b/tests/claude-code-harness.test.ts @@ -23,7 +23,7 @@ import { CLAUDE_CODE_MODELS } from "../src/agent/claude-code/provider.ts"; import { createClaudeCodeStreamFunction, type RunClaude } from "../src/agent/claude-code/stream.ts"; import type { PiEngine } from "../src/agent/models.ts"; import { createTeacherAgent } from "../src/agent/teacher-agent.ts"; -import { danglingQuestionOf } from "../src/core/native-messages.ts"; +import { cliContextWindowOf, danglingQuestionOf } from "../src/core/native-messages.ts"; import { handleChatRequest } from "../src/server/chat-handler.ts"; import { isTurnRunning } from "../src/server/turn-liveness.ts"; import { runningTurn } from "../src/server/turn-runs.ts"; @@ -157,6 +157,8 @@ async function waitForExit(child: ChildProcessWithoutNullStreams): Promise test("a question answered before the idle timeout continues the waiting CLI process with the answer", async (t) => { const fakeRun = createFakeRun(t, [toAskScenario()]); + const run = fakeRun.runClaude; + fakeRun.runClaude = (args, options) => run(args, { ...options, env: { ...options.env, FAKE_CLAUDE_WINDOW: "200000" } }); const agentFactory = createAgentFactory(fakeRun); const chatId = createClaudeCodeChat(t); @@ -170,6 +172,8 @@ test("a question answered before the idle timeout continues the waiting CLI proc await runChatTurn(chatId, toAnswerInput(), agentFactory); assert.equal(fakeRun.children.length, 1); + assert.deepEqual(cliContextWindowOf(chatStore.getChat(chatId)!.messages), { model: "claude-code:sonnet", tokens: 200000 }); + assert.deepEqual(cliContextWindowOf(JSON.parse(readFileSync(join(PATHS.chats, `${chatId}.json`), "utf8")).messages), { model: "claude-code:sonnet", tokens: 200000 }); assert.ok(getLastAssistantText(chatId).includes(ANSWER_TEXT), getLastAssistantText(chatId)); await waitForExit(waiting); }); diff --git a/tests/claude-code-stream.test.ts b/tests/claude-code-stream.test.ts index 19e3a76..8da5596 100644 --- a/tests/claude-code-stream.test.ts +++ b/tests/claude-code-stream.test.ts @@ -620,13 +620,18 @@ test("does not inject credentials reflected only from LearnHarness's key store", assert.equal(process.env[name], "lh-only-key", "other LearnHarness providers keep their env reflection"); }); -test("writes the transcript as one stream-json user line on stdin", async (t) => { +test("writes one user prompt after local-only control initialization and summary request", async (t) => { const stdinFile = join(mkdtempSync(join(tmpdir(), "claude-code-stdin-")), "stdin.jsonl"); t.after(() => rmSync(dirname(stdinFile), { recursive: true, force: true })); await runScenario(t, readFixtureLines("text.jsonl"), undefined, { FAKE_CLAUDE_STDIN: stdinFile }); - assert.equal(readFileSync(stdinFile, "utf8"), `${toStreamJsonLine([{ type: "text", text: "hi" }])}\n`); + const lines = readFileSync(stdinFile, "utf8").trim().split("\n"); + assert(lines.length >= 3); + assert.deepEqual(JSON.parse(lines[0]!).request, { subtype: "initialize" }); + assert.deepEqual(JSON.parse(lines[1]!).request, { subtype: "get_context_usage", detail: "summary" }); + assert.equal(lines[2], toStreamJsonLine([{ type: "text", text: "hi" }])); + assert(lines.slice(3).every((line) => JSON.parse(line).request.detail === "summary")); }); test("parseCliLine ignores non-JSON, system, rate-limit and successful result lines", () => { @@ -1002,6 +1007,28 @@ test("a caller without a session ID uses fresh replay and retains no held proces await assertCleanedUp(getOnlyCall(fresh.calls)); }); +test("CLI window metadata stays on its own process and survives a held-tool continuation", async (t) => { + const read: ToolUse = { id: "toolu_window", name: "read", arguments: { path: "one" } }; + const one = createFakeRun(t, [...toToolUseMessage([read]), callTool(read), REPLY], { FAKE_CLAUDE_WINDOW: "200000" }); + const two = createFakeRun(t, [...toToolUseMessage([read]), callTool(read), REPLY], { FAKE_CLAUDE_WINDOW: "750000" }); + let spawns = 0; + const stream = createClaudeCodeStreamFunction((args, opts) => (++spawns === 1 ? one : two).runClaude(args, opts)); + const window = (m: AssistantMessage) => (m as AssistantMessage & { cliContextWindow?: number }).cliContextWindow; + const a = await runTurn(stream, [HI], { sessionId: "context-one" }); + const b = await runTurn(stream, [HI], { sessionId: "context-two" }); + assert.equal(window(a), 200000); assert.equal(window(b), 750000); + const ra = await runTurn(stream, [HI, a, toToolResult(read, "a")], { sessionId: "context-one" }); + const rb = await runTurn(stream, [HI, b, toToolResult(read, "b")], { sessionId: "context-two" }); + assert.equal(window(ra), 200000); assert.equal(window(rb), 750000); assert.equal(spawns, 2); +}); + +for (const extraEnv of [{ FAKE_CLAUDE_INIT: "error" }, { FAKE_CLAUDE_WINDOW: "-1" }, {}] as Record[]) test(`missing/unsupported CLI window never prevents a normal response: ${JSON.stringify(extraEnv)}`, async (t) => { + const { events } = await runScenario(t, readFixtureLines("text.jsonl"), undefined, extraEnv); + const result = getLastEvent(events); + assert.equal(result.type, "done"); + assert(!("cliContextWindow" in (result.type === "done" ? result.message : {}))); +}); + for (const sharedIdentity of [false, true]) test(`two concurrent stream owners keep separate processes${sharedIdentity ? " even with identical call IDs and arguments" : ""}`, async (t) => { const readOne: ToolUse = { id: "toolu_concurrent_one", name: "read", arguments: { path: "one" } }; const readTwo: ToolUse = sharedIdentity ? { ...readOne } : { id: "toolu_concurrent_two", name: "read", arguments: { path: "two" } }; diff --git a/tests/fixtures/claude-code/fake-claude.mjs b/tests/fixtures/claude-code/fake-claude.mjs index 7438444..2b0ba66 100644 --- a/tests/fixtures/claude-code/fake-claude.mjs +++ b/tests/fixtures/claude-code/fake-claude.mjs @@ -15,6 +15,7 @@ */ import { readFileSync, writeFileSync } from "node:fs"; import { on } from "node:events"; +import { createInterface } from "node:readline"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; @@ -56,8 +57,29 @@ async function writeReply(text) { const mcpConfig = readFileSync(process.argv[process.argv.indexOf("--mcp-config") + 1], "utf8"); let input = ""; -for await (const chunk of process.stdin) input += chunk; -if (process.env.FAKE_CLAUDE_STDIN) writeFileSync(process.env.FAKE_CLAUDE_STDIN, input); +let receiveUser; +const userReceived = new Promise((resolve) => { receiveUser = resolve; }); +createInterface({ input: process.stdin }).on("line", async (line) => { + input += `${line}\n`; + const message = JSON.parse(line); + if (message.type !== "control_request") { + if (process.env.FAKE_CLAUDE_STDIN) writeFileSync(process.env.FAKE_CLAUDE_STDIN, input); + receiveUser(); return; + } + if (message.request.subtype === "initialize") { + if (process.env.FAKE_CLAUDE_INIT === "ignore") return; + await writeLine(JSON.stringify({ type: "control_response", response: { + request_id: message.request_id, subtype: process.env.FAKE_CLAUDE_INIT === "error" ? "error" : "success", + response: {}, error: "unsupported initialize", + } })); + } else if (message.request.subtype === "get_context_usage") { + await writeLine(JSON.stringify({ type: "control_response", response: { + request_id: message.request_id, subtype: "success", + response: { maxTokens: process.env.FAKE_CLAUDE_WINDOW ? JSON.parse(process.env.FAKE_CLAUDE_WINDOW) : null }, + } })); + } +}); +await userReceived; let bridge = null; const results = []; diff --git a/tests/model-catalog.test.ts b/tests/model-catalog.test.ts index 3ab0ea6..9af9c7d 100644 --- a/tests/model-catalog.test.ts +++ b/tests/model-catalog.test.ts @@ -66,7 +66,8 @@ describe("the model catalog projection", () => { assert.ok(Array.isArray(m.supportedLevels)); if (m.provider === "claude-code") assert.equal(m.cost, null); else assert.ok(m.cost && typeof m.cost.input === "number" && typeof m.cost.output === "number"); - assert.ok(m.contextWindow > 0); + if (m.provider === "claude-code") assert.equal(m.contextWindow, null); + else assert.ok(m.contextWindow !== null && m.contextWindow > 0); } const glm = models.find((m) => m.id === "z-ai/glm-5.3-flash"); assert.ok(glm, "glm-5.3-flash in catalog"); @@ -384,9 +385,9 @@ describe("the claude-code provider (native CLI authentication, no LearnHarness k }); }); - it("catalogContextLimits() has claude-code:opus when the CLI is installed", () => { + it("catalogContextLimits() never advertises placeholder CLI limits, even when installed", () => { withClaudeInstalled(true, () => { - assert.ok((catalogContextLimits().get("claude-code:opus") ?? 0) > 0); + assert.equal(catalogContextLimits().has("claude-code:opus"), false); }); });