/** * Claude Code stream function (src/agent/claude-code/stream.ts) and CLI line * parser (src/agent/claude-code/cli-events.ts). * * The runClaude seam spawns tests/fixtures/claude-code/fake-claude.mjs, which * replays recorded or synthetic stream-json lines, so the real process * plumbing (argv, stdin, stdout lines, stderr, exit, kill) runs end to end. */ import assert from "node:assert/strict"; import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { once } from "node:events"; import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, sep } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import test, { type TestContext } from "node:test"; import type { AssistantMessage, AssistantMessageEvent, AssistantMessageEventStream, JsonObject, Message, SimpleStreamOptions, StreamFunction, Api, Tool, ToolResultMessage, } from "@earendil-works/pi-ai"; import { normalizeContext } from "@earendil-works/pi-ai/utils/transcript"; import { Agent } from "@earendil-works/pi-agent-core"; import { Type } from "typebox"; import { parseCliLine } from "../src/agent/claude-code/cli-events.ts"; import { ClaudeCodeError } from "../src/agent/claude-code/constants.ts"; import { LIVE_PROCESS_IDLE_TIMEOUT_MS, liveProcessIdleTimeoutMs } from "../src/agent/claude-code/live-processes.ts"; import { CLAUDE_CODE_MODELS } from "../src/agent/claude-code/provider.ts"; import { createClaudeCodeStreamFunction, toClaudeArgs, toEffort, type RunClaude } from "../src/agent/claude-code/stream.ts"; import { toStreamJsonLine } from "../src/agent/claude-code/transcript.ts"; const FAKE_CLAUDE = fileURLToPath(new URL("./fixtures/claude-code/fake-claude.mjs", import.meta.url)); const FIXTURES = fileURLToPath(new URL("./fixtures/claude-code/", import.meta.url)); const HOLD = JSON.stringify({ fake: "hold" }); const REPLY = JSON.stringify({ fake: "reply" }); const CLEANUP_TIMEOUT_MS = 2000; const SESSION_ID = "stream-test-chat"; const READ_TOOL: Tool = { name: "read", description: "Read a file", parameters: { type: "object", properties: {} } }; const HI: Message = { role: "user", content: "hi", timestamp: 0 }; type ToolUse = { id: string; name: string; arguments: JsonObject }; type RunCall = { args: string[]; cwd: string; cwdEntries: string[]; env: NodeJS.ProcessEnv; systemPrompt: string; mcpConfig: string; mcpConfigMode: number; child: ChildProcessWithoutNullStreams; }; type ScenarioRun = { events: AssistantMessageEvent[]; calls: RunCall[] }; function readFixtureLines(name: string): string[] { return readFileSync(join(FIXTURES, name), "utf8").split("\n").filter(Boolean); } function getFlagValue(args: string[], flag: string): string { const value = args[args.indexOf(flag) + 1]; assert.ok(args.includes(flag) && value !== undefined, `missing ${flag}`); return value; } function streamEvent(event: object): string { return JSON.stringify({ type: "stream_event", event }); } function exitWith(code: number, stderr: string): string { return JSON.stringify({ fake: "exit", code, stderr }); } function getToolScenario(): string[] { const lines = readFixtureLines("tool.jsonl"); const firstMessageStop = lines.findIndex((line) => line.includes('"type":"message_stop"')); return [...lines.slice(0, firstMessageStop + 1).filter((line) => !line.startsWith('{"type":"user"')), callTool({ id: "toolu_01JBB4cJsPa585FWb1XrfbNX", name: "get_weather", arguments: { city: "Paris" } }), HOLD]; } function createFakeRun( t: TestContext, scenario: string[], extraEnv: Record = {}, ): { runClaude: RunClaude; calls: RunCall[] } { const dir = mkdtempSync(join(tmpdir(), "claude-code-test-")); t.after(() => rmSync(dir, { recursive: true, force: true })); const scenarioFile = join(dir, "scenario.jsonl"); writeFileSync(scenarioFile, `${scenario.join("\n")}\n`); const calls: RunCall[] = []; const runClaude: RunClaude = (args, options) => { const mcpConfigFile = getFlagValue(args, "--mcp-config"); const child = spawn(process.execPath, [FAKE_CLAUDE, ...args], { cwd: options.cwd, env: { ...options.env, FAKE_CLAUDE_SCENARIO: scenarioFile, ...extraEnv }, stdio: ["pipe", "pipe", "pipe", "ipc"], }) as ChildProcessWithoutNullStreams; t.after(() => child.kill("SIGKILL")); calls.push({ args, cwd: options.cwd, cwdEntries: readdirSync(options.cwd), env: options.env, systemPrompt: readFileSync(getFlagValue(args, "--system-prompt-file"), "utf8"), mcpConfig: readFileSync(mcpConfigFile, "utf8"), mcpConfigMode: statSync(mcpConfigFile).mode & 0o777, child, }); return child; }; return { runClaude, calls }; } async function collectEvents( stream: AssistantMessageEventStream, onEvent: (event: AssistantMessageEvent) => void = () => {}, ): Promise { const events: AssistantMessageEvent[] = []; for await (const event of stream) { events.push(event); onEvent(event); } return events; } async function runScenario( t: TestContext, scenario: string[], options?: SimpleStreamOptions, extraEnv?: Record, ): Promise { const { runClaude, calls } = createFakeRun(t, scenario, extraEnv); const model = CLAUDE_CODE_MODELS[0]; assert.ok(model); const context = normalizeContext({ systemPrompt: "You are a tutor.", messages: [{ role: "user", content: "hi", timestamp: 0 }], tools: [READ_TOOL, { name: "get_weather", description: "Weather", parameters: { type: "object", properties: {} } }], }); const events = await collectEvents(createClaudeCodeStreamFunction(runClaude)(model, context, { sessionId: SESSION_ID, ...options })); return { events, calls }; } function toToolUseMessage(toolUses: ToolUse[]): string[] { return [ streamEvent({ type: "message_start", message: { usage: {} } }), ...toolUses.flatMap((toolUse, index) => [ streamEvent({ type: "content_block_start", index, content_block: { type: "tool_use", id: toolUse.id, name: `mcp__learnharness__${toolUse.name}`, input: {} }, }), streamEvent({ type: "content_block_delta", index, delta: { type: "input_json_delta", partial_json: JSON.stringify(toolUse.arguments) }, }), streamEvent({ type: "content_block_stop", index }), ]), streamEvent({ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: {} }), streamEvent({ type: "message_stop" }), ]; } function callTool(toolUse: ToolUse): string { return JSON.stringify({ fake: "call", id: toolUse.id, name: toolUse.name, arguments: toolUse.arguments }); } function toToolResult(toolUse: ToolUse, text: string): ToolResultMessage { return { role: "toolResult", toolCallId: toolUse.id, toolName: toolUse.name, content: [{ type: "text", text }], isError: false, timestamp: 0, }; } function getModel() { const model = CLAUDE_CODE_MODELS[0]; assert.ok(model); return model; } async function runTurn( streamFunction: StreamFunction, messages: Message[], options?: SimpleStreamOptions, ): Promise { const context = normalizeContext({ systemPrompt: "You are a tutor.", messages, tools: [READ_TOOL] }); const done = getLastEvent(await collectEvents(streamFunction(getModel(), context, { sessionId: SESSION_ID, ...options }))); assert.ok(done.type === "done", `turn ended with ${done.type}`); return done.message; } function getText(message: AssistantMessage): string { return message.content.flatMap((block) => (block.type === "text" ? [block.text] : [])).join(""); } function getMcpUrl(call: RunCall): string { const mcpConfig = JSON.parse(call.mcpConfig) as { mcpServers: Record }; const url = mcpConfig.mcpServers.learnharness?.url; assert.ok(url); return url; } function getOnlyCall(calls: RunCall[]): RunCall { assert.equal(calls.length, 1); const [call] = calls; assert.ok(call); return call; } function getLastEvent(events: AssistantMessageEvent[]): AssistantMessageEvent { const event = events.at(-1); assert.ok(event); return event; } function getErrorMessage(events: AssistantMessageEvent[]): string | undefined { const event = getLastEvent(events); assert.equal(event.type, "error"); return event.type === "error" ? event.error.errorMessage : undefined; } async function waitForExit(child: ChildProcessWithoutNullStreams): Promise { if (child.exitCode !== null || child.signalCode !== null) return; await assert.doesNotReject(once(child, "exit", { signal: AbortSignal.timeout(CLEANUP_TIMEOUT_MS) }), "claude CLI process was not killed"); } async function waitForRemoval(path: string): Promise { const deadline = Date.now() + CLEANUP_TIMEOUT_MS; while (existsSync(path)) { assert.ok(Date.now() < deadline, `${path} was not removed`); await sleep(10); } } async function assertCleanedUp(call: RunCall): Promise { await waitForExit(call.child); await waitForRemoval(call.cwd); await waitForRemoval(dirname(getFlagValue(call.args, "--system-prompt-file"))); } test("toClaudeArgs includes every isolation flag from Global Constraints", () => { const input = { model: "opus", effort: "high", systemPromptFile: "/private/system-prompt.md", mcpConfigFile: "/private/mcp.json", allowedTools: ["mcp__learnharness__read", "mcp__learnharness__write"], } as const; assert.deepEqual(toClaudeArgs({ ...input, allowedTools: [...input.allowedTools] }), [ "-p", "--input-format", "stream-json", "--output-format", "stream-json", "--verbose", "--include-partial-messages", "--model", "opus", "--effort", "high", "--system-prompt-file", "/private/system-prompt.md", "--tools", "", "--strict-mcp-config", "--mcp-config", "/private/mcp.json", "--setting-sources", "", "--disable-slash-commands", "--no-session-persistence", "--permission-prompts", "none", "--allowedTools", "mcp__learnharness__read,mcp__learnharness__write", ]); assert.equal(toClaudeArgs({ ...input, allowedTools: [] }).includes("--allowedTools"), false); }); test('toEffort maps low..max verbatim, minimal to "low", undefined to "medium"', () => { assert.deepEqual( (["low", "medium", "high", "xhigh", "max"] as const).map((level) => toEffort(level)), ["low", "medium", "high", "xhigh", "max"], ); assert.equal(toEffort("minimal"), "low"); assert.equal(toEffort(undefined), "medium"); }); test("text scenario streams start, text_start, text_delta…, text_end, done(stop)", async (t) => { const { events, calls } = await runScenario(t, readFixtureLines("text.jsonl")); assert.deepEqual( events.map((event) => event.type), ["start", "text_start", "text_delta", "text_end", "done"], ); const done = getLastEvent(events); assert.ok(done.type === "done"); assert.equal(done.reason, "stop"); assert.equal(done.message.stopReason, "stop"); assert.deepEqual(done.message.content, [{ type: "text", text: "hello there" }]); assert.equal(done.message.provider, "claude-code"); assert.equal(done.message.api, "claude-code-cli"); assert.equal(done.message.model, "opus"); await assertCleanedUp(getOnlyCall(calls)); }); test("thinking deltas map to thinking_*", async (t) => { const { events } = await runScenario(t, [ streamEvent({ type: "message_start", message: { usage: { input_tokens: 3, output_tokens: 1 } } }), streamEvent({ type: "content_block_start", index: 0, content_block: { type: "thinking", thinking: "" } }), streamEvent({ type: "content_block_delta", index: 0, delta: { type: "thinking_delta", thinking: "Sockets are " } }), streamEvent({ type: "content_block_delta", index: 0, delta: { type: "thinking_delta", thinking: "endpoints." } }), streamEvent({ type: "content_block_stop", index: 0 }), streamEvent({ type: "content_block_start", index: 1, content_block: { type: "text", text: "" } }), streamEvent({ type: "content_block_delta", index: 1, delta: { type: "text_delta", text: "An endpoint." } }), streamEvent({ type: "content_block_stop", index: 1 }), streamEvent({ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 9 } }), streamEvent({ type: "message_stop" }), ]); assert.deepEqual( events.map((event) => event.type), ["start", "thinking_start", "thinking_delta", "thinking_delta", "thinking_end", "text_start", "text_delta", "text_end", "done"], ); const done = getLastEvent(events); assert.ok(done.type === "done"); assert.deepEqual(done.message.content, [ { type: "thinking", thinking: "Sockets are endpoints." }, { type: "text", text: "An endpoint." }, ]); }); test("tool scenario ends with toolcall_end (Claude's tool_use id, unprefixed name) and done(toolUse)", async (t) => { const { events } = await runScenario(t, getToolScenario()); assert.deepEqual( events.map((event) => event.type), ["start", "toolcall_start", "toolcall_end", "done"], ); const toolCallEnd = events.find((event) => event.type === "toolcall_end"); assert.ok(toolCallEnd?.type === "toolcall_end"); assert.deepEqual(toolCallEnd.toolCall, { type: "toolCall", id: "toolu_01JBB4cJsPa585FWb1XrfbNX", name: "get_weather", arguments: { city: "Paris" }, }); const done = getLastEvent(events); assert.ok(done.type === "done"); assert.equal(done.reason, "toolUse"); assert.deepEqual(done.message.content, [toolCallEnd.toolCall]); }); test("unsupported native tool intent fails without exposing executable calls", async (t) => { const { events } = await runScenario(t, [ streamEvent({ type: "message_start", message: { usage: {} } }), streamEvent({ type: "content_block_start", index: 0, content_block: { type: "tool_use", id: "toolu_a", name: "mcp__learnharness__nope", input: {} }, }), streamEvent({ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: '{"path": ' } }), streamEvent({ type: "content_block_stop", index: 0 }), streamEvent({ type: "content_block_start", index: 1, content_block: { type: "tool_use", id: "toolu_b", name: "Bash", input: {} } }), streamEvent({ type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: '{"command":"ls"}' } }), streamEvent({ type: "content_block_stop", index: 1 }), streamEvent({ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: {} }), streamEvent({ type: "message_stop" }), HOLD, ]); assert.equal(events.some((event) => event.type === "toolcall_end"), false); assert.match(getErrorMessage(events) ?? "", /unsupported native tool/); }); test("usage maps input/output/cacheRead/cacheWrite with zero cost, message_delta overriding message_start", async (t) => { const { events } = await runScenario(t, getToolScenario()); const done = getLastEvent(events); assert.ok(done.type === "done"); assert.deepEqual(done.message.usage, { input: 2, output: 59, cacheRead: 0, cacheWrite: 1102, totalTokens: 1163, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }); }); test('missing binary → error "claude CLI not found"', async () => { const model = CLAUDE_CODE_MODELS[0]; assert.ok(model); let cwd = ""; const runClaude: RunClaude = (_args, options) => { cwd = options.cwd; return null; }; const context = normalizeContext({ systemPrompt: "p", messages: [{ role: "user", content: "hi", timestamp: 0 }] }); const events = await collectEvents(createClaudeCodeStreamFunction(runClaude)(model, context)); assert.deepEqual( events.map((event) => event.type), ["error"], ); assert.equal(getErrorMessage(events), ClaudeCodeError.CliNotFound); await waitForRemoval(cwd); }); test('signed-out fixture → error "not signed in"', async (t) => { const { events, calls } = await runScenario(t, [...readFixtureLines("signed-out.jsonl"), exitWith(1, "")]); assert.equal(getErrorMessage(events), ClaudeCodeError.NotSignedIn); await assertCleanedUp(getOnlyCall(calls)); }); test("other assistant errors surface their text", async (t) => { const assistantError = JSON.stringify({ type: "assistant", error: "rate_limit", message: { content: [{ type: "text", text: "API Error: Rate limit reached" }] }, }); const { events } = await runScenario(t, [assistantError, exitWith(1, "")]); assert.equal(getErrorMessage(events), "API Error: Rate limit reached"); }); test("result error before message_stop surfaces the result text", async (t) => { const resultError = JSON.stringify({ type: "result", subtype: "error_during_execution", is_error: true, result: "Credit balance is too low", }); const { events } = await runScenario(t, [resultError, HOLD]); assert.equal(getErrorMessage(events), "Credit balance is too low"); }); test("result error without result text surfaces its errors", async (t) => { const resultError = JSON.stringify({ type: "result", subtype: "error_max_turns", is_error: true, errors: ["Reached maximum number of turns"], }); const { events } = await runScenario(t, [resultError, HOLD]); assert.equal(getErrorMessage(events), "Reached maximum number of turns"); }); test("result error without result text or errors surfaces its subtype", async (t) => { const resultError = JSON.stringify({ type: "result", subtype: "error_during_execution", is_error: true }); const { events } = await runScenario(t, [resultError, HOLD]); assert.equal(getErrorMessage(events), "claude CLI failed: error_during_execution"); }); test("refusal stop reason → error", async (t) => { const { events } = await runScenario(t, [ streamEvent({ type: "message_start", message: { usage: {} } }), streamEvent({ type: "message_delta", delta: { stop_reason: "refusal" }, usage: {} }), streamEvent({ type: "message_stop" }), ]); assert.equal(getErrorMessage(events), "The model refused to complete the request"); }); test("non-zero exit mid-stream → error naming the exit code, stderr logged not shown, process and temp dirs gone", async (t) => { const consoleError = t.mock.method(console, "error", () => {}); const midStream = readFixtureLines("text.jsonl").slice(0, 5); const { events, calls } = await runScenario(t, [...midStream, exitWith(1, "Error: ENOENT /home/someone/.claude/x\n at main\n")]); assert.deepEqual( events.map((event) => event.type), ["start", "text_start", "text_delta", "error"], ); assert.equal(getErrorMessage(events), "claude CLI exited with code 1"); assert.ok( consoleError.mock.calls.some((call) => call.arguments.some((argument) => String(argument).includes("ENOENT /home/someone/.claude/x"))), ); const error = getLastEvent(events); assert.ok(error.type === "error"); assert.equal(error.reason, "error"); assert.deepEqual(error.error.content, [{ type: "text", text: "hello there" }]); await assertCleanedUp(getOnlyCall(calls)); }); test("exit without stderr names the exit code", async (t) => { const { events } = await runScenario(t, [exitWith(3, "")]); assert.equal(getErrorMessage(events), "claude CLI exited with code 3"); }); test("abort kills the process and ends with error(aborted)", async (t) => { const { runClaude, calls } = createFakeRun(t, [streamEvent({ type: "message_start", message: { usage: {} } }), HOLD]); const model = CLAUDE_CODE_MODELS[0]; assert.ok(model); const controller = new AbortController(); const context = normalizeContext({ systemPrompt: "p", messages: [{ role: "user", content: "hi", timestamp: 0 }] }); const stream = createClaudeCodeStreamFunction(runClaude)(model, context, { signal: controller.signal }); const events = await collectEvents(stream, (event) => { if (event.type === "start") controller.abort(); }); assert.deepEqual( events.map((event) => event.type), ["start", "error"], ); const error = getLastEvent(events); assert.ok(error.type === "error"); assert.equal(error.reason, "aborted"); await assertCleanedUp(getOnlyCall(calls)); }); test("cwd is an empty temp dir; prompt and mcp config live in a separate private dir", async (t) => { const { calls } = await runScenario(t, readFixtureLines("text.jsonl")); const call = getOnlyCall(calls); assert.deepEqual(call.cwdEntries, []); assert.ok(call.cwd.startsWith(tmpdir())); const privateDir = dirname(getFlagValue(call.args, "--system-prompt-file")); assert.notEqual(privateDir, call.cwd); assert.ok(!privateDir.startsWith(call.cwd + sep)); assert.equal(dirname(getFlagValue(call.args, "--mcp-config")), privateDir); assert.equal(call.systemPrompt, "You are a tutor."); assert.equal(call.mcpConfigMode, 0o600); const mcpConfig = JSON.parse(call.mcpConfig) as { mcpServers: Record }; assert.match(mcpConfig.mcpServers.learnharness?.url ?? "", /^http:\/\/127\.0\.0\.1:\d+\/mcp$/); }); test("spawns with the model alias, effort, bridge tools, MCP_TOOL_TIMEOUT and the caller's HOME", async (t) => { const { calls } = await runScenario(t, readFixtureLines("text.jsonl"), { reasoning: "xhigh" }); const call = getOnlyCall(calls); assert.equal(getFlagValue(call.args, "--model"), "opus"); assert.equal(getFlagValue(call.args, "--effort"), "xhigh"); assert.equal(getFlagValue(call.args, "--allowedTools"), "mcp__learnharness__read,mcp__learnharness__get_weather"); assert.equal(call.env.MCP_TOOL_TIMEOUT, "1800000"); assert.equal(call.env.HOME, process.env.HOME); assert.equal(call.env.CLAUDE_CONFIG_DIR, process.env.CLAUDE_CONFIG_DIR); }); test("spawns without Anthropic API credentials so turns stay on the subscription", async (t) => { const saved = { ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN: process.env.ANTHROPIC_AUTH_TOKEN }; t.after(() => { for (const [name, value] of Object.entries(saved)) { if (value === undefined) delete process.env[name]; else process.env[name] = value; } }); process.env.ANTHROPIC_API_KEY = "sk-ant-test"; process.env.ANTHROPIC_AUTH_TOKEN = "token-test"; const { calls } = await runScenario(t, readFixtureLines("text.jsonl")); const call = getOnlyCall(calls); assert.equal(call.env.ANTHROPIC_API_KEY, undefined); assert.equal(call.env.ANTHROPIC_AUTH_TOKEN, undefined); }); test("writes the transcript as one stream-json user line on stdin", async (t) => { const stdinFile = join(mkdtempSync(join(tmpdir(), "claude-code-stdin-")), "stdin.jsonl"); t.after(() => rmSync(dirname(stdinFile), { recursive: true, force: true })); await runScenario(t, readFixtureLines("text.jsonl"), undefined, { FAKE_CLAUDE_STDIN: stdinFile }); assert.equal(readFileSync(stdinFile, "utf8"), `${toStreamJsonLine([{ type: "text", text: "hi" }])}\n`); }); test("parseCliLine ignores non-JSON, system, rate-limit and successful result lines", () => { assert.equal(parseCliLine("not json"), null); assert.equal(parseCliLine("null"), null); assert.equal(parseCliLine('{"type":"system","subtype":"init"}'), null); assert.equal(parseCliLine('{"type":"rate_limit_event"}'), null); assert.equal(parseCliLine('{"type":"result","is_error":false,"result":"hello there"}'), null); assert.equal(parseCliLine('{"type":"assistant","error":null,"message":{"content":[{"type":"text","text":"hello there"}]}}'), null); }); test("trailing toolResults matching a live process continue it: held calls answered, new assistant message streamed, no new spawn", async (t) => { const readA: ToolUse = { id: "toolu_continue_a", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([readA]), callTool(readA), REPLY]); const streamFunction = createClaudeCodeStreamFunction(runClaude); const toolMessage = await runTurn(streamFunction, [HI]); const reply = await runTurn(streamFunction, [HI, toolMessage, toToolResult(readA, "contents of a")]); assert.equal(toolMessage.stopReason, "toolUse"); assert.equal(getText(reply), "contents of a"); await assertCleanedUp(getOnlyCall(calls)); }); test("sequential identical calls split one response and correlate results by id, not arguments", async (t) => { const readA: ToolUse = { id: "toolu_many_a", name: "read", arguments: { path: "a" } }; const readB: ToolUse = { id: "toolu_many_b", name: "read", arguments: { path: "a" } }; const source = toToolUseMessage([readA, readB]); source[0] = streamEvent({ type: "message_start", message: { usage: { input_tokens: 7, output_tokens: 2 } } }); const { runClaude, calls } = createFakeRun(t, [...source, callTool(readB), callTool(readA), REPLY]); const streamFunction = createClaudeCodeStreamFunction(runClaude); const toolMessage = await runTurn(streamFunction, [HI]); assert.deepEqual(toolMessage.content.filter((block) => block.type === "toolCall").map((call) => call.id), [readB.id]); const history = [HI, toolMessage, toToolResult(readB, "B")]; const second = await runTurn(streamFunction, history); assert.deepEqual(second.content.filter((block) => block.type === "toolCall").map((call) => call.id), [readA.id]); const reply = await runTurn(streamFunction, [...history, second, toToolResult(readA, "A")]); assert.equal(getText(reply), "B | A"); assert.equal(toolMessage.usage.totalTokens, 9); assert.equal(second.usage.totalTokens, 0, "splitting one response must not charge its tokens twice"); assert.equal(calls.length, 1); }); test("mismatched result id starts fresh and kills the old process", async (t) => { const readA: ToolUse = { id: "toolu_partial_a", name: "read", arguments: { path: "a" } }; const readB: ToolUse = { id: "toolu_partial_b", name: "read", arguments: { path: "b" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([readA, readB]), callTool(readA), callTool(readB), REPLY]); const streamFunction = createClaudeCodeStreamFunction(runClaude); const toolMessage = await runTurn(streamFunction, [HI]); await runTurn(streamFunction, [HI, toolMessage, toToolResult(readB, "B")]); assert.equal(calls.length, 2); const [oldCall] = calls; assert.ok(oldCall); await assertCleanedUp(oldCall); }); test("user message after tool results starts fresh with unfinished calls in the replay", async (t) => { const stdinFile = join(mkdtempSync(join(tmpdir(), "claude-code-stdin-")), "stdin.jsonl"); t.after(() => rmSync(dirname(stdinFile), { recursive: true, force: true })); const readA: ToolUse = { id: "toolu_steer_a", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([readA]), callTool(readA), REPLY], { FAKE_CLAUDE_STDIN: stdinFile }); const streamFunction = createClaudeCodeStreamFunction(runClaude); const toolMessage = await runTurn(streamFunction, [HI]); const steer: Message = { role: "user", content: "wait, explain sockets first", timestamp: 0 }; await runTurn(streamFunction, [HI, toolMessage, toToolResult(readA, "A"), steer]); assert.equal(calls.length, 2); const [oldCall] = calls; assert.ok(oldCall); await assertCleanedUp(oldCall); const stdin = readFileSync(stdinFile, "utf8"); assert.ok(stdin.includes('[tool call read] {\\"path\\":\\"a\\"}'), stdin); assert.ok(stdin.includes("wait, explain sockets first"), stdin); }); test("output while waiting for tool results kills the process", async (t) => { const readA: ToolUse = { id: "toolu_moved_on_a", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([readA]), REPLY, HOLD]); const events = await collectEvents(createClaudeCodeStreamFunction(runClaude)(getModel(), normalizeContext({ messages: [HI], tools: [READ_TOOL] }))); assert.match(getErrorMessage(events) ?? "", /moved on without admitting/); assert.equal(events.some((event) => event.type === "toolcall_end"), false); await assertCleanedUp(getOnlyCall(calls)); }); test("unadmitted tool intent never becomes an executable call", async (t) => { const readA: ToolUse = { id: "toolu_undeclared_a", name: "read", arguments: { path: "a" } }; const nope: ToolUse = { id: "toolu_undeclared_nope", name: "nope", arguments: {} }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([readA, nope]), exitWith(1, "tool denied")]); const events = await collectEvents(createClaudeCodeStreamFunction(runClaude)(getModel(), normalizeContext({ messages: [HI], tools: [READ_TOOL] }))); assert.equal(events.some((event) => event.type === "toolcall_end"), false); assert.match(getErrorMessage(events) ?? "", /exited with code 1/); await assertCleanedUp(getOnlyCall(calls)); }); test("a CLI-rejected malformed call is not exposed for execution", async (t) => { const { runClaude, calls } = createFakeRun(t, [ streamEvent({ type: "message_start", message: { usage: {} } }), streamEvent({ type: "content_block_start", index: 0, content_block: { type: "tool_use", id: "toolu_bad_json", name: "mcp__learnharness__read", input: {} }, }), streamEvent({ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: '{"path": ' } }), streamEvent({ type: "content_block_stop", index: 0 }), streamEvent({ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: {} }), streamEvent({ type: "message_stop" }), JSON.stringify({ type: "user", message: { content: [{ type: "tool_result", tool_use_id: "toolu_bad_json", is_error: true, content: "invalid arguments" }] } }), HOLD, ]); const events = await collectEvents(createClaudeCodeStreamFunction(runClaude)(getModel(), normalizeContext({ messages: [HI], tools: [READ_TOOL] }))); assert.equal(events.some((event) => event.type === "toolcall_end"), false); assert.match(getErrorMessage(events) ?? "", /refused.*before MCP admission/); await assertCleanedUp(getOnlyCall(calls)); }); test("abort while waiting kills the process and closes the bridge", async (t) => { const readA: ToolUse = { id: "toolu_abort_a", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([readA]), callTool(readA), REPLY]); const controller = new AbortController(); await runTurn(createClaudeCodeStreamFunction(runClaude), [HI], { signal: controller.signal }); controller.abort(); const call = getOnlyCall(calls); await assertCleanedUp(call); await assert.rejects(fetch(getMcpUrl(call))); }); test("idle timeout kills an orphaned waiting process", async (t) => { liveProcessIdleTimeoutMs.current = 50; t.after(() => (liveProcessIdleTimeoutMs.current = LIVE_PROCESS_IDLE_TIMEOUT_MS)); const readA: ToolUse = { id: "toolu_idle_a", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([readA]), callTool(readA), REPLY]); await runTurn(createClaudeCodeStreamFunction(runClaude), [HI]); await assertCleanedUp(getOnlyCall(calls)); }); test("a tool result arriving after the idle timeout starts fresh with the call and its result in the replay", async (t) => { liveProcessIdleTimeoutMs.current = 300; t.after(() => (liveProcessIdleTimeoutMs.current = LIVE_PROCESS_IDLE_TIMEOUT_MS)); const stdinFile = join(mkdtempSync(join(tmpdir(), "claude-code-stdin-")), "stdin.jsonl"); t.after(() => rmSync(dirname(stdinFile), { recursive: true, force: true })); const readA: ToolUse = { id: "toolu_late_a", name: "read", arguments: { path: "a" } }; const waiting = createFakeRun(t, [...toToolUseMessage([readA]), callTool(readA), REPLY]); const fresh = createFakeRun(t, readFixtureLines("text.jsonl"), { FAKE_CLAUDE_STDIN: stdinFile }); const runClaude: RunClaude = (args, options) => (waiting.calls.length === 0 ? waiting : fresh).runClaude(args, options); const streamFunction = createClaudeCodeStreamFunction(runClaude); const toolMessage = await runTurn(streamFunction, [HI]); const waitingCall = getOnlyCall(waiting.calls); assert.equal(waitingCall.child.exitCode, null, "the process should wait for the tool result until the idle timeout"); await assertCleanedUp(waitingCall); const reply = await runTurn(streamFunction, [HI, toolMessage, toToolResult(readA, "contents of a")]); assert.equal(getText(reply), "hello there"); await assertCleanedUp(getOnlyCall(fresh.calls)); const stdin = readFileSync(stdinFile, "utf8"); assert.ok(stdin.includes('[tool call read] {\\"path\\":\\"a\\"}'), stdin); assert.ok(stdin.includes("[tool result read]"), stdin); assert.ok(stdin.includes("contents of a"), stdin); }); test("a tool result arriving late but before the idle timeout still continues the waiting process", async (t) => { liveProcessIdleTimeoutMs.current = 1000; t.after(() => (liveProcessIdleTimeoutMs.current = LIVE_PROCESS_IDLE_TIMEOUT_MS)); const readA: ToolUse = { id: "toolu_slow_a", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([readA]), callTool(readA), REPLY]); const streamFunction = createClaudeCodeStreamFunction(runClaude); const toolMessage = await runTurn(streamFunction, [HI]); await sleep(500); const reply = await runTurn(streamFunction, [HI, toolMessage, toToolResult(readA, "contents of a")]); assert.equal(getText(reply), "contents of a"); await assertCleanedUp(getOnlyCall(calls)); }); test("CLI denial cannot execute a tool in the native Pi loop", async (t) => { const read: ToolUse = { id: "toolu_denied", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([read]), JSON.stringify({ type: "user", message: { content: [ { type: "tool_result", tool_use_id: read.id, is_error: true, content: "Denied by managed policy" }, ] } }), HOLD]); let executed = 0; const agent = new Agent({ initialState: { model: getModel(), tools: [{ name: "read", label: "Read", description: "Read", parameters: Type.Object({ path: Type.String() }), execute: async () => { executed++; return { content: [{ type: "text", text: "must not run" }], details: null }; }, }] }, streamFn: createClaudeCodeStreamFunction(runClaude) }); await agent.prompt("hi"); assert.equal(executed, 0); const last = agent.state.messages.at(-1); assert.ok(last?.role === "assistant"); assert.equal(last.stopReason, "error"); assert.match(last.errorMessage ?? "", /refused.*before MCP admission/); await assertCleanedUp(getOnlyCall(calls)); }); test("pre-aborted requests do not spawn a process", async () => { const controller = new AbortController(); controller.abort(); let spawned = false; const stream = createClaudeCodeStreamFunction(() => { spawned = true; return null; }); const events = await collectEvents(stream(getModel(), normalizeContext({ messages: [HI] }), { signal: controller.signal })); assert.equal(spawned, false); assert.match(getErrorMessage(events) ?? "", /aborted/); }); test("missing MCP identity fails closed without emitting executable calls", async (t) => { const read: ToolUse = { id: "toolu_no_meta", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([read]), JSON.stringify({ fake: "call", name: read.name, arguments: read.arguments })]); const events = await collectEvents(createClaudeCodeStreamFunction(runClaude)(getModel(), normalizeContext({ messages: [HI], tools: [READ_TOOL] }))); assert.equal(events.some((event) => event.type === "toolcall_end"), false); assert.match(getErrorMessage(events) ?? "", /compatibility error.*claudecode\/toolUseId/); await assertCleanedUp(getOnlyCall(calls)); }); test("a parallel identity failure after an admitted slice is reported on continuation, not hidden by replay", async (t) => { const first: ToolUse = { id: "toolu_good_parallel", name: "read", arguments: { path: "a" } }; const second: ToolUse = { id: "toolu_bad_parallel", name: "read", arguments: { path: "b" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([first, second]), JSON.stringify({ fake: "start-call", id: first.id, name: first.name, arguments: first.arguments }), JSON.stringify({ fake: "wait-for-message" }), JSON.stringify({ fake: "call", name: second.name, arguments: second.arguments }), HOLD]); const stream = createClaudeCodeStreamFunction(runClaude); const message = await runTurn(stream, [HI]); getOnlyCall(calls).child.send("continue"); await waitForExit(getOnlyCall(calls).child); const events = await collectEvents(stream(getModel(), normalizeContext({ systemPrompt: "You are a tutor.", messages: [HI, message, toToolResult(first, "A")], tools: [READ_TOOL] }), { sessionId: SESSION_ID })); assert.match(getErrorMessage(events) ?? "", /compatibility error.*claudecode\/toolUseId/); assert.equal(calls.length, 1, "must not hide the compatibility failure by spawning again"); await assertCleanedUp(getOnlyCall(calls)); }); test("output after MCP admission fails continuation instead of silently replaying", async (t) => { const call: ToolUse = { id: "toolu_moved_on", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([call]), JSON.stringify({ fake: "start-call", id: call.id, name: call.name, arguments: call.arguments }), JSON.stringify({ fake: "wait-for-message" }), ...toToolUseMessage([call]), HOLD]); const stream = createClaudeCodeStreamFunction(runClaude); const message = await runTurn(stream, [HI]); getOnlyCall(calls).child.send("continue"); await waitForExit(getOnlyCall(calls).child); const events = await collectEvents(stream(getModel(), normalizeContext({ systemPrompt: "You are a tutor.", messages: [HI, message, toToolResult(call, "A")], tools: [READ_TOOL] }), { sessionId: SESSION_ID })); assert.match(getErrorMessage(events) ?? "", /moved on while.*waiting for results/); assert.equal(calls.length, 1); await assertCleanedUp(getOnlyCall(calls)); }); test("normal completion escalates termination when the child ignores SIGTERM", async (t) => { const { events, calls } = await runScenario(t, [JSON.stringify({ fake: "ignore-term" }), ...readFixtureLines("text.jsonl"), HOLD]); assert.equal(getLastEvent(events).type, "done"); await assertCleanedUp(getOnlyCall(calls)); assert.equal(getOnlyCall(calls).child.signalCode, "SIGKILL"); }); test("a synchronous spawn failure removes both temp directories and the bridge", async () => { let resources: { cwd: string; privateDir: string; url: string } | undefined; const stream = createClaudeCodeStreamFunction((args, options) => { const config = JSON.parse(readFileSync(getFlagValue(args, "--mcp-config"), "utf8")); resources = { cwd: options.cwd, privateDir: dirname(getFlagValue(args, "--mcp-config")), url: config.mcpServers.learnharness.url }; throw new Error("synthetic spawn failure"); }); const events = await collectEvents(stream(getModel(), normalizeContext({ messages: [HI] }))); assert.match(getErrorMessage(events) ?? "", /failed to start/); assert.ok(resources); assert.equal(existsSync(resources.cwd), false); assert.equal(existsSync(resources.privateDir), false); await assert.rejects(fetch(resources.url)); }); test("a malformed JSON event shape fails the turn rather than escaping the readline handler", async (t) => { const { events, calls } = await runScenario(t, [JSON.stringify({ type: "assistant", error: "bad", message: { content: [null] } }), HOLD]); assert.match(getErrorMessage(events) ?? "", /malformed stream output/); await assertCleanedUp(getOnlyCall(calls)); }); test("a mismatched MCP identity cannot admit a streamed call", async (t) => { const read: ToolUse = { id: "toolu_intent", name: "read", arguments: { path: "a" } }; const { runClaude, calls } = createFakeRun(t, [...toToolUseMessage([read]), callTool({ ...read, id: "toolu_someone_else" })]); const events = await collectEvents(createClaudeCodeStreamFunction(runClaude)(getModel(), normalizeContext({ messages: [HI], tools: [READ_TOOL] }))); assert.equal(events.some((event) => event.type === "toolcall_end"), false); assert.match(getErrorMessage(events) ?? "", /does not match its streamed tool intent/); await assertCleanedUp(getOnlyCall(calls)); }); for (const change of ["model", "effort", "system prompt", "tool schema", "tool description", "history"] as const) { test(`a changed ${change} prevents reuse of a held process`, async (t) => { const read: ToolUse = { id: `toolu_reuse_${change}`, name: "read", arguments: { path: "a" } }; const waiting = createFakeRun(t, [...toToolUseMessage([read]), callTool(read), REPLY]); const fresh = createFakeRun(t, readFixtureLines("text.jsonl")); const stream = createClaudeCodeStreamFunction((args, options) => (waiting.calls.length === 0 ? waiting : fresh).runClaude(args, options)); const message = await runTurn(stream, [HI]); const messages = [change === "history" ? { ...HI, content: "replacement history after compaction" } : HI, message, toToolResult(read, "A")]; const context = normalizeContext({ systemPrompt: change === "system prompt" ? "Changed tutor instructions" : "You are a tutor.", messages, tools: [{ ...READ_TOOL, ...(change === "tool schema" ? { parameters: { type: "object", properties: { path: { type: "string" } } } } : {}), ...(change === "tool description" ? { description: "Changed description" } : {}), }], }); const model = change === "model" ? CLAUDE_CODE_MODELS[1]! : getModel(); const events = await collectEvents(stream(model, context, { sessionId: SESSION_ID, ...(change === "effort" ? { reasoning: "high" as const } : {}) })); assert.equal(getLastEvent(events).type, "done"); assert.equal(fresh.calls.length, 1, "must start a fresh CLI rather than silently use stale configuration"); await assertCleanedUp(getOnlyCall(waiting.calls)); }); } for (const identicalHistory of [false, true]) test(`one registered stream isolates two sessions with colliding tool IDs and ${identicalHistory ? "identical" : "different"} histories`, async (t) => { const call: ToolUse = { id: "toolu_shared_singleton", name: "read", arguments: { path: "same" } }; const one = createFakeRun(t, [...toToolUseMessage([call]), callTool(call), REPLY]); const two = createFakeRun(t, [...toToolUseMessage([call]), callTool(call), REPLY]); let spawns = 0; const stream = createClaudeCodeStreamFunction((args, options) => { const run = [one, two][spawns++]; assert.ok(run, "a valid continuation must not spawn another CLI"); return run.runClaude(args, options); }); const historyOne = [HI]; const historyTwo: Message[] = identicalHistory ? [HI] : [{ role: "user", content: "another conversation", timestamp: 0 }]; const messageOne = await runTurn(stream, historyOne, { sessionId: "chat-one" }); const messageTwo = await runTurn(stream, historyTwo, { sessionId: "chat-two" }); // B returns first while A is earlier in the registry: IDs and even full // history must not let B resolve A's authenticated MCP request. const replyTwo = await runTurn(stream, [...historyTwo, messageTwo, toToolResult(call, "from two")], { sessionId: "chat-two" }); const replyOne = await runTurn(stream, [...historyOne, messageOne, toToolResult(call, "from one")], { sessionId: "chat-one" }); assert.equal(getText(replyTwo), "from two"); assert.equal(getText(replyOne), "from one"); assert.equal(spawns, 2); await assertCleanedUp(getOnlyCall(one.calls)); await assertCleanedUp(getOnlyCall(two.calls)); }); test("a caller without a session ID uses fresh replay and retains no held process", async (t) => { const call: ToolUse = { id: "toolu_no_scope", name: "read", arguments: { path: "a" } }; const waiting = createFakeRun(t, [...toToolUseMessage([call]), callTool(call), REPLY]); const stdinDir = mkdtempSync(join(tmpdir(), "claude-unscoped-replay-")); t.after(() => rmSync(stdinDir, { recursive: true, force: true })); const stdinFile = join(stdinDir, "stdin.jsonl"); const fresh = createFakeRun(t, readFixtureLines("text.jsonl"), { FAKE_CLAUDE_STDIN: stdinFile }); const stream = createClaudeCodeStreamFunction((args, options) => (waiting.calls.length === 0 ? waiting : fresh).runClaude(args, options)); const message = await runTurn(stream, [HI], { sessionId: undefined }); await assertCleanedUp(getOnlyCall(waiting.calls)); const reply = await runTurn(stream, [HI, message, toToolResult(call, "the preserved answer")], { sessionId: undefined }); assert.equal(reply.stopReason, "stop"); assert.equal(fresh.calls.length, 1); assert.match(readFileSync(stdinFile, "utf8"), /the preserved answer/); await assertCleanedUp(getOnlyCall(fresh.calls)); }); for (const sharedIdentity of [false, true]) test(`two concurrent stream owners keep separate processes${sharedIdentity ? " even with identical call IDs and arguments" : ""}`, async (t) => { const readOne: ToolUse = { id: "toolu_concurrent_one", name: "read", arguments: { path: "one" } }; const readTwo: ToolUse = sharedIdentity ? { ...readOne } : { id: "toolu_concurrent_two", name: "read", arguments: { path: "two" } }; const one = createFakeRun(t, [...toToolUseMessage([readOne]), callTool(readOne), REPLY]); const two = createFakeRun(t, [...toToolUseMessage([readTwo]), callTool(readTwo), REPLY]); const streamOne = createClaudeCodeStreamFunction(one.runClaude); const streamTwo = createClaudeCodeStreamFunction(two.runClaude); const [toolMessageOne, toolMessageTwo] = await Promise.all([runTurn(streamOne, [HI]), runTurn(streamTwo, [HI])]); const [replyOne, replyTwo] = await Promise.all([ runTurn(streamOne, [HI, toolMessageOne, toToolResult(readOne, "from one")]), runTurn(streamTwo, [HI, toolMessageTwo, toToolResult(readTwo, "from two")]), ]); assert.equal(getText(replyOne), "from one"); assert.equal(getText(replyTwo), "from two"); assert.equal(one.calls.length, 1); assert.equal(two.calls.length, 1); });