Merge native Claude Code integration with explicit opt-in
# Conflicts: # src/agent/teacher-agent.ts # src/server/keys.ts
This commit is contained in:
amosblomqvist
@@ -77,14 +77,16 @@ test("Providers is the first/default page; General is last and other pages keep
|
||||
assert.doesNotMatch(html, />Model providers<\/button>/);
|
||||
});
|
||||
|
||||
test("Providers has exactly four ordered sections; unknown variables stay masked/read-only at the end of Model API keys", async () => {
|
||||
test("Providers has five ordered sections including External CLI; unknown variables stay masked/read-only at the end of Model API keys", async () => {
|
||||
globalThis.fetch = async (url) => {
|
||||
if (url === "/api/claude-code") return Response.json({ state: "not-found", version: null, minimumVersion: "2.1.259" });
|
||||
assert.equal(url, "/api/keys");
|
||||
return Response.json({ providers: [{ id: "openai", envVar: "OPENAI_API_KEY", label: "OpenAI" }], keys: [{ id: "FUTURE_VAR", info: null, present: true, masked: "…abcd", source: "file", shadowedByEnv: false }] });
|
||||
};
|
||||
fixture(); await mount(<KeysSection />);
|
||||
const headings = [...document.querySelectorAll('[data-testid^="group-"] > p')].map((el) => el.textContent);
|
||||
assert.deepEqual(headings, ["Model API keys", "Subscriptions", "Dictation", "Search"]);
|
||||
assert.deepEqual(headings, ["Model API keys", "Subscriptions", "External CLI", "Dictation", "Search"]);
|
||||
assert(byTest("claude-code-installation").closest('[data-testid="group-external-cli"]'));
|
||||
assert(byTest("key-deepgram_api_key").closest('[data-testid="group-dictation"]'));
|
||||
assert(byTest("key-tavily_api_key").closest('[data-testid="group-search"]'));
|
||||
assert(byTest("key-brave_api_key").closest('[data-testid="group-search"]'));
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Claude Code binary lookup (src/agent/claude-code/binary.ts).
|
||||
*
|
||||
* The packaged desktop app often starts with a minimal PATH, so lookup falls
|
||||
* back to the standard install locations. A fake isExecutable set stands in
|
||||
* for the filesystem.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { resolveClaudeBinary } from "../src/agent/claude-code/binary.ts";
|
||||
|
||||
const HOME = "/home/u";
|
||||
|
||||
function resolveWith(executables: string[], env: Record<string, string> = {}, platform: NodeJS.Platform = "linux", homeDir = HOME) {
|
||||
const found = new Set(executables);
|
||||
return resolveClaudeBinary({ env, platform, homeDir, isExecutable: (p) => found.has(p) });
|
||||
}
|
||||
|
||||
test("prefers PATH hit", () => {
|
||||
const binary = resolveWith(["/usr/bin/claude", `${HOME}/.local/bin/claude`], { PATH: "/opt/x:/usr/bin" });
|
||||
assert.equal(binary, "/usr/bin/claude");
|
||||
});
|
||||
|
||||
test("falls back to ~/.local/bin when PATH misses", () => {
|
||||
const binary = resolveWith([`${HOME}/.local/bin/claude`, "/usr/local/bin/claude"], { PATH: "/usr/bin" });
|
||||
assert.equal(binary, `${HOME}/.local/bin/claude`);
|
||||
});
|
||||
|
||||
test("checks ~/.claude/local, npm global bin, /opt/homebrew/bin, /usr/local/bin in that order", () => {
|
||||
const all = [`${HOME}/.claude/local/claude`, `${HOME}/.npm-global/bin/claude`, "/opt/homebrew/bin/claude", "/usr/local/bin/claude"];
|
||||
for (let i = 0; i < all.length; i++) {
|
||||
assert.equal(resolveWith(all.slice(i), { PATH: "" }), all[i]);
|
||||
}
|
||||
});
|
||||
|
||||
test("npm global bin follows npm_config_prefix when set", () => {
|
||||
const binary = resolveWith(["/opt/npm/bin/claude", `${HOME}/.npm-global/bin/claude`], { npm_config_prefix: "/opt/npm" });
|
||||
assert.equal(binary, "/opt/npm/bin/claude");
|
||||
});
|
||||
|
||||
test("windows finds native claude.exe and does not advertise unsupported .cmd-only installs", () => {
|
||||
const winHome = "C:\\Users\\u";
|
||||
const env = { PATH: "C:\\tools;C:\\bin", APPDATA: `${winHome}\\AppData\\Roaming` };
|
||||
assert.equal(resolveWith(["C:\\bin\\claude.cmd"], env, "win32", winHome), null);
|
||||
assert.equal(resolveWith(["C:\\bin\\claude.exe"], env, "win32", winHome), "C:\\bin\\claude.exe");
|
||||
assert.equal(resolveWith([`${winHome}\\.local\\bin\\claude.exe`], env, "win32", winHome), `${winHome}\\.local\\bin\\claude.exe`);
|
||||
assert.equal(
|
||||
resolveWith([`${winHome}\\AppData\\Roaming\\npm\\claude.cmd`], env, "win32", winHome),
|
||||
null,
|
||||
);
|
||||
assert.equal(resolveWith(["C:\\bin\\claude"], env, "win32", winHome), null);
|
||||
});
|
||||
|
||||
test("windows prefers claude.exe anywhere over claude.cmd, which Node cannot spawn without a shell", () => {
|
||||
const winHome = "C:\\Users\\u";
|
||||
const env = { PATH: "C:\\bin", APPDATA: `${winHome}\\AppData\\Roaming` };
|
||||
const executables = ["C:\\bin\\claude.cmd", `${winHome}\\.local\\bin\\claude.exe`];
|
||||
assert.equal(resolveWith(executables, env, "win32", winHome), `${winHome}\\.local\\bin\\claude.exe`);
|
||||
});
|
||||
|
||||
test("relative Windows PATH entries are fixed to an absolute discovery identity", () => {
|
||||
assert.equal(resolveClaudeBinary({ env: { PATH: ".\\bin" }, platform: "win32", homeDir: "C:\\Users\\u", cwd: "C:\\server",
|
||||
isExecutable: (candidate) => candidate === "C:\\server\\bin\\claude.exe",
|
||||
}), "C:\\server\\bin\\claude.exe");
|
||||
});
|
||||
|
||||
test("returns null when every location misses", () => {
|
||||
assert.equal(resolveWith([], { PATH: "/usr/bin:/bin" }), null);
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
import React, { act } from "react";
|
||||
(globalThis as Record<string, unknown>).React = React;
|
||||
import assert from "node:assert/strict";
|
||||
import { afterEach, test } from "node:test";
|
||||
import { createRoot, type Root } from "react-dom/client";
|
||||
import { parseHTML } from "linkedom";
|
||||
import { StatsBar } from "../components/stats-bar.tsx";
|
||||
import { CLAUDE_CODE_MODELS } from "../src/agent/claude-code/provider.ts";
|
||||
import { toModelDto } from "../src/server/model-catalog.ts";
|
||||
import type { ModelCatalogResponse, TurnUsage } from "../src/core/index.ts";
|
||||
|
||||
const dom = parseHTML("<html><body></body></html>");
|
||||
Object.assign(globalThis, { window: dom.window, document: dom.document, Element: dom.window.Element, HTMLElement: dom.window.HTMLElement,
|
||||
CustomEvent: dom.window.CustomEvent, IS_REACT_ACT_ENVIRONMENT: true });
|
||||
const originalFetch = globalThis.fetch;
|
||||
let root: Root | undefined, host: HTMLDivElement | undefined;
|
||||
afterEach(async () => { await act(async () => root?.unmount()); host?.remove(); root = undefined; globalThis.fetch = originalFetch; });
|
||||
|
||||
test("compact CLI context is numeric only; effective-window explanation appears only in expanded session metadata", async () => {
|
||||
const catalog: ModelCatalogResponse = { models: CLAUDE_CODE_MODELS.map(toModelDto), providers: [],
|
||||
slots: { strong: null, cheap: null, vision: null }, defaults: null,
|
||||
rolePins: { researcher: null, demoBuilder: null, visualMaker: null, extractor: null, auditor: null } };
|
||||
globalThis.fetch = (async (url) => {
|
||||
if (url === "/api/models") return Response.json(catalog);
|
||||
if (url === "/api/settings") return Response.json({ settings: { models: {} } });
|
||||
if (url === "/api/chats/cli-session") return Response.json({ settings: { model: { provider: "claude-code", id: "opus" } } });
|
||||
throw new Error(`unexpected request ${url}`);
|
||||
}) as typeof fetch;
|
||||
host = document.createElement("div"); document.body.append(host); root = createRoot(host);
|
||||
const turn: TurnUsage = { id: "cli-turn", messageId: "cli-message", model: "claude-code:opus", at: "2026-10-28T00:00:00Z", steps: [{ inputTokens: 42000, outputTokens: 1, cacheReadTokens: undefined, cacheWriteTokens: undefined, reasoningTokens: undefined }] };
|
||||
const render = (window: { model: string; tokens: number } | null) => root!.render(<StatsBar chatId="cli-session" turns={[turn]} contextTokens={42000} cliContextWindow={window} />);
|
||||
await act(async () => render({ model: "claude-code:opus", tokens: 200000 }));
|
||||
assert.equal(document.querySelector('[data-testid="stats-context"]')?.textContent, "42k / 200k");
|
||||
assert(!document.body.textContent?.includes("auto-compaction"));
|
||||
assert.equal(document.querySelector('[data-testid="stats-cost"]')?.textContent, "CLI");
|
||||
const summary = document.querySelector<HTMLButtonElement>('[data-testid="stats-summary"]')!;
|
||||
assert(!summary.textContent?.includes("window")); assert(!summary.textContent?.includes("budget"));
|
||||
await act(async () => summary.click());
|
||||
assert.match(document.querySelector('[data-testid="stats-detail"]')?.textContent ?? "", /Window reported by Claude Code; may be lower for auto-compaction\./);
|
||||
assert.equal(document.querySelector('[data-testid="stats-detail"] li > span:last-child > span:last-child')?.textContent, "CLI");
|
||||
assert.equal(document.querySelector('[data-testid="stats-context"]')?.textContent, "42k / 200k");
|
||||
await act(async () => render(null));
|
||||
assert.equal(document.querySelector('[data-testid="stats-context"]')?.textContent, "42k");
|
||||
await act(async () => render({ model: "claude-code:sonnet", tokens: 1000000 }));
|
||||
assert.equal(document.querySelector('[data-testid="stats-context"]')?.textContent, "42k", "another alias cannot supply this session's denominator");
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { ContextWindowProbe } from "../src/agent/claude-code/context-window.ts";
|
||||
import { CLAUDE_CODE_MODELS } from "../src/agent/claude-code/provider.ts";
|
||||
import { asAgentMessages, cliContextWindowOf, toLlmMessages, type LhMessage } from "../src/core/native-messages.ts";
|
||||
import { catalogLimits } from "../components/model-picker.tsx";
|
||||
import { contextLimitForModel, formatContextMeasurement } from "../components/stats-bar.tsx";
|
||||
import { toModelDto } from "../src/server/model-catalog.ts";
|
||||
|
||||
function setup() {
|
||||
const writes: { type: string; request_id: string; request: Record<string, string> }[] = [];
|
||||
const windows: number[] = [];
|
||||
let prompts = 0;
|
||||
const probe = new ContextWindowProbe((line) => writes.push(JSON.parse(line)), () => prompts++, (tokens) => windows.push(tokens));
|
||||
probe.start();
|
||||
const reply = (index: number, body: unknown, subtype = "success") => probe.handle({ type: "control_response", response: {
|
||||
request_id: writes[index]!.request_id, subtype, response: body,
|
||||
} });
|
||||
return { probe, writes, windows, reply, prompts: () => prompts };
|
||||
}
|
||||
|
||||
test("control initialization precedes a summary-only query; neither needs user input or a model request", () => {
|
||||
const p = setup();
|
||||
assert.deepEqual(p.writes[0]!.request, { subtype: "initialize" }); assert.equal(p.prompts(), 0);
|
||||
p.reply(0, {});
|
||||
assert.deepEqual(p.writes[1]!.request, { subtype: "get_context_usage", detail: "summary" });
|
||||
assert.equal(p.prompts(), 1, "conversation does not wait for metadata");
|
||||
assert.notEqual(p.writes[0]!.request_id, p.writes[1]!.request_id);
|
||||
p.probe.handle({ type: "control_response", response: { request_id: "foreign", subtype: "success", response: { maxTokens: 999 } } });
|
||||
assert.deepEqual(p.windows, []);
|
||||
p.reply(1, { maxTokens: 200000, rawMaxTokens: 1000000 });
|
||||
assert.deepEqual(p.windows, [200000], "effective maxTokens, not an assumed nominal window");
|
||||
p.reply(0, {}); p.reply(1, { maxTokens: 1 });
|
||||
assert.equal(p.prompts(), 1); assert.deepEqual(p.windows, [200000]);
|
||||
p.probe.close();
|
||||
});
|
||||
|
||||
test("metadata unavailable before first response can be queried again locally; stale/closed replies are ignored", () => {
|
||||
const p = setup(); p.reply(0, {}); p.reply(1, {}, "error");
|
||||
assert.equal(p.prompts(), 1); assert.deepEqual(p.windows, []);
|
||||
p.probe.refresh();
|
||||
assert.deepEqual(p.writes[2]!.request, { subtype: "get_context_usage", detail: "summary" });
|
||||
p.reply(1, { maxTokens: 1 }); assert.deepEqual(p.windows, []);
|
||||
p.reply(2, { maxTokens: 200000 }); assert.deepEqual(p.windows, [200000]);
|
||||
p.probe.refresh(); p.probe.close(); p.reply(3, { maxTokens: 999999 });
|
||||
assert.deepEqual(p.windows, [200000]); assert.equal(p.prompts(), 1);
|
||||
});
|
||||
|
||||
for (const tokens of [null, undefined, 0, -1, 1.5, Infinity, Number.MAX_SAFE_INTEGER + 1, "200000"]) test(`invalid summary budget ${tokens} remains unknown without failing the prompt`, () => {
|
||||
const p = setup(); p.reply(0, {}); p.reply(1, { maxTokens: tokens });
|
||||
assert.deepEqual(p.windows, []); assert.equal(p.prompts(), 1); p.probe.close();
|
||||
});
|
||||
|
||||
test("initialization errors/timeouts fall back to ordinary prompt delivery; close cancels the timer", (t) => {
|
||||
t.mock.timers.enable({ apis: ["setTimeout"] });
|
||||
const error = setup(); error.reply(0, undefined, "error");
|
||||
assert.equal(error.prompts(), 1); assert.equal(error.writes.length, 1); error.probe.close();
|
||||
const timeout = setup(); t.mock.timers.tick(5000);
|
||||
assert.equal(timeout.prompts(), 1); assert.equal(timeout.writes.length, 1);
|
||||
timeout.reply(0, {}); assert.equal(timeout.writes.length, 1); timeout.probe.close();
|
||||
const closed = setup(); closed.probe.close(); t.mock.timers.tick(5000);
|
||||
assert.equal(closed.prompts(), 0);
|
||||
});
|
||||
|
||||
function assistant(model = "opus", tokens: number | undefined = 200000): LhMessage & { role: "assistant" } {
|
||||
return { id: "cli-assistant", role: "assistant", api: "claude-code-cli", provider: "claude-code", model,
|
||||
content: [], timestamp: 0, stopReason: "stop", cliContextWindow: tokens,
|
||||
usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } };
|
||||
}
|
||||
|
||||
test("per-session metadata survives JSON persistence, never borrows older reports, and cannot override other model selections", () => {
|
||||
const first = assistant();
|
||||
const window = cliContextWindowOf(JSON.parse(JSON.stringify([first])));
|
||||
assert.deepEqual(window, { model: "claude-code:opus", tokens: 200000 });
|
||||
assert(!("cliContextWindow" in toLlmMessages(asAgentMessages([first]))[0]!));
|
||||
assert.equal(first.cliContextWindow, 200000, "provider projection never mutates the stored report");
|
||||
const catalog = new Map([["anthropic:fixture", 100000], ["claude-code:opus", 1000000]]);
|
||||
assert.equal(contextLimitForModel("claude-code:opus", catalog, window), 200000);
|
||||
assert.equal(contextLimitForModel("claude-code:sonnet", catalog, window), null);
|
||||
assert.equal(contextLimitForModel("claude-code:opus", catalog, null), null, "no placeholder fallback");
|
||||
assert.equal(contextLimitForModel("anthropic:fixture", catalog, window), 100000);
|
||||
assert.equal(contextLimitForModel(undefined, catalog, window), null);
|
||||
assert.equal(formatContextMeasurement(42000, 200000), "42k / 200k");
|
||||
assert.equal(formatContextMeasurement(42000, null), "42k");
|
||||
const unreported = assistant("opus"); delete unreported.cliContextWindow;
|
||||
assert.equal(cliContextWindowOf([first, unreported]), null);
|
||||
const other = assistant("api"); other.provider = "anthropic";
|
||||
assert.equal(cliContextWindowOf([first, other]), null);
|
||||
const compacted: LhMessage = { id: "compact", role: "custom", customType: "compaction", content: "summary", display: false, details: {}, timestamp: 1 };
|
||||
assert.equal(cliContextWindowOf([first, compacted]), null);
|
||||
assert.equal(cliContextWindowOf([]), null);
|
||||
});
|
||||
|
||||
test("CLI catalog limits remain unknown and cannot create a global denominator", () => {
|
||||
const models = CLAUDE_CODE_MODELS.map(toModelDto);
|
||||
assert(models.every((model) => model.contextWindow === null && model.maxTokens === null));
|
||||
assert.equal(catalogLimits({ models }).size, 0);
|
||||
});
|
||||
@@ -0,0 +1,319 @@
|
||||
/**
|
||||
* Claude Code provider driven through the real chat handler: a learner-facing
|
||||
* question ends the turn while the CLI process waits on the held MCP call, and
|
||||
* the learner's answer arrives later as that call's tool result.
|
||||
*
|
||||
* The runClaude seam spawns tests/fixtures/claude-code/fake-claude.mjs, one
|
||||
* scenario per spawn, so the spawn count shows whether the answer continued
|
||||
* the waiting process or started a fresh one with a replay.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import { mkdtempSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test, { after, type TestContext } from "node:test";
|
||||
|
||||
import type { RunClaude } from "../src/agent/claude-code/stream.ts";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
|
||||
// Opt-in writes must not race another test process's settings record.
|
||||
const testData = mkdtempSync(join(tmpdir(), "lh-cli-harness-data-"));
|
||||
process.env.LH_DATA_DIR = testData;
|
||||
after(() => rmSync(testData, { recursive: true, force: true }));
|
||||
const { PATHS, SETTINGS_FILE } = await import("../config.ts");
|
||||
const { readSettings, writeSettings } = await import("../src/store/settings.ts");
|
||||
const { LIVE_PROCESS_IDLE_TIMEOUT_MS, liveProcessIdleTimeoutMs } = await import("../src/agent/claude-code/live-processes.ts");
|
||||
const { CLAUDE_CODE_MODELS } = await import("../src/agent/claude-code/provider.ts");
|
||||
const { createClaudeCodeStreamFunction } = await import("../src/agent/claude-code/stream.ts");
|
||||
const { createTeacherAgent } = await import("../src/agent/teacher-agent.ts");
|
||||
const { cliContextWindowOf, danglingQuestionOf } = await import("../src/core/native-messages.ts");
|
||||
const { handleChatRequest } = await import("../src/server/chat-handler.ts");
|
||||
const { isTurnRunning } = await import("../src/server/turn-liveness.ts");
|
||||
const { runningTurn } = await import("../src/server/turn-runs.ts");
|
||||
const { chatStore } = await import("../src/store/chats.ts");
|
||||
|
||||
const FAKE_CLAUDE = fileURLToPath(new URL("./fixtures/claude-code/fake-claude.mjs", import.meta.url));
|
||||
const TEXT_FIXTURE = fileURLToPath(new URL("./fixtures/claude-code/text.jsonl", import.meta.url));
|
||||
const TURN_TIMEOUT_MS = 8000;
|
||||
|
||||
const ASK_ID = "toolu_harness_ask";
|
||||
const ASK_INPUT = { question: "What do you already know about sockets?" };
|
||||
const ANSWER_TEXT = "Only that they carry bytes between programs";
|
||||
|
||||
type FakeRun = { runClaude: RunClaude; children: ChildProcessWithoutNullStreams[]; stdinFiles: string[] };
|
||||
|
||||
function streamEvent(event: object): string {
|
||||
return JSON.stringify({ type: "stream_event", event });
|
||||
}
|
||||
|
||||
function toAskScenario(): string[] {
|
||||
return [
|
||||
streamEvent({ type: "message_start", message: { usage: {} } }),
|
||||
streamEvent({
|
||||
type: "content_block_start",
|
||||
index: 0,
|
||||
content_block: { type: "tool_use", id: ASK_ID, name: "mcp__learnharness__ask_user_question", input: {} },
|
||||
}),
|
||||
streamEvent({ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: JSON.stringify(ASK_INPUT) } }),
|
||||
streamEvent({ type: "content_block_stop", index: 0 }),
|
||||
streamEvent({ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: {} }),
|
||||
streamEvent({ type: "message_stop" }),
|
||||
JSON.stringify({ fake: "call", id: ASK_ID, name: "ask_user_question", arguments: ASK_INPUT }),
|
||||
JSON.stringify({ fake: "reply" }),
|
||||
];
|
||||
}
|
||||
|
||||
function createFakeRun(t: TestContext, scenarios: string[][]): FakeRun {
|
||||
const settings = readSettings();
|
||||
writeSettings({ ...(settings ?? { version: 1 }), claudeCode: { enabled: true } });
|
||||
t.after(() => { if (settings) writeSettings(settings); else rmSync(SETTINGS_FILE, { force: true }); });
|
||||
const dir = mkdtempSync(join(tmpdir(), "claude-code-harness-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
// Satisfy the real presence gate; this marker is NEVER executed. All work
|
||||
// below goes through the injected Node fake-CLI seam, not an installed CLI.
|
||||
writeFileSync(join(dir, process.platform === "win32" ? "claude.exe" : "claude"), "", { mode: 0o755 });
|
||||
const savedPath = process.env.PATH;
|
||||
process.env.PATH = `${dir}${delimiter}${savedPath ?? ""}`;
|
||||
t.after(() => { if (savedPath === undefined) delete process.env.PATH; else process.env.PATH = savedPath; });
|
||||
|
||||
const children: ChildProcessWithoutNullStreams[] = [];
|
||||
const stdinFiles: string[] = [];
|
||||
|
||||
const runClaude: RunClaude = (args, options) => {
|
||||
const scenario = scenarios[children.length];
|
||||
assert.ok(scenario, `unexpected spawn #${children.length + 1}`);
|
||||
|
||||
const scenarioFile = join(dir, `scenario-${children.length}.jsonl`);
|
||||
const stdinFile = join(dir, `stdin-${children.length}.jsonl`);
|
||||
writeFileSync(scenarioFile, `${scenario.join("\n")}\n`);
|
||||
|
||||
const child = spawn(process.execPath, [FAKE_CLAUDE, ...args], {
|
||||
cwd: options.cwd,
|
||||
env: { ...options.env, FAKE_CLAUDE_SCENARIO: scenarioFile, FAKE_CLAUDE_STDIN: stdinFile },
|
||||
stdio: ["pipe", "pipe", "pipe", "ipc"],
|
||||
}) as ChildProcessWithoutNullStreams;
|
||||
t.after(() => child.kill("SIGKILL"));
|
||||
|
||||
children.push(child);
|
||||
stdinFiles.push(stdinFile);
|
||||
return child;
|
||||
};
|
||||
|
||||
return { runClaude, children, stdinFiles };
|
||||
}
|
||||
|
||||
function createClaudeCodeChat(t: TestContext): string {
|
||||
const chat = chatStore.createChat("Claude Code: question answered later");
|
||||
chatStore.saveSettings(chat.id, { model: { provider: "claude-code", id: "sonnet" } });
|
||||
t.after(() => unlinkSync(join(PATHS.chats, `${chat.id}.json`)));
|
||||
|
||||
return chat.id;
|
||||
}
|
||||
|
||||
function createAgentFactory(fakeRun: FakeRun): Parameters<typeof handleChatRequest>[1] {
|
||||
const model = CLAUDE_CODE_MODELS.find((candidate) => candidate.id === "sonnet");
|
||||
assert.ok(model);
|
||||
|
||||
const engine: PiEngine = { model, streamFn: createClaudeCodeStreamFunction(fakeRun.runClaude) };
|
||||
|
||||
return (options) => createTeacherAgent(undefined, { engine, chatId: options.chatId, messages: options.messages });
|
||||
}
|
||||
|
||||
async function runChatTurn(chatId: string, input: object, agentFactory: Parameters<typeof handleChatRequest>[1]): Promise<void> {
|
||||
const request = new Request("http://test/api/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ chatId, input }),
|
||||
});
|
||||
|
||||
const ack = await handleChatRequest(request, agentFactory);
|
||||
assert.equal(ack.status, 200, await ack.clone().text());
|
||||
|
||||
const handle = runningTurn(chatId);
|
||||
if (handle) await new Response(handle.subscribe()).text();
|
||||
|
||||
const deadline = Date.now() + TURN_TIMEOUT_MS;
|
||||
while (isTurnRunning(chatId)) {
|
||||
assert.ok(Date.now() < deadline, "turn did not settle");
|
||||
await sleep(10);
|
||||
}
|
||||
}
|
||||
|
||||
function toAnswerInput(): object {
|
||||
return {
|
||||
type: "answer",
|
||||
toolCallId: ASK_ID,
|
||||
answer: { kind: "ask", questionText: ASK_INPUT.question, dontKnow: false, answers: [], text: ANSWER_TEXT, answeredAt: new Date().toISOString() },
|
||||
};
|
||||
}
|
||||
|
||||
function getLastAssistantText(chatId: string): string {
|
||||
const messages = chatStore.getChat(chatId)?.messages ?? [];
|
||||
const assistant = messages.findLast((message) => message.role === "assistant");
|
||||
assert.ok(assistant?.role === "assistant");
|
||||
|
||||
return assistant.content.flatMap((block) => (block.type === "text" ? [block.text] : [])).join("");
|
||||
}
|
||||
|
||||
async function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
|
||||
await once(child, "exit", { signal: AbortSignal.timeout(TURN_TIMEOUT_MS) });
|
||||
}
|
||||
|
||||
test("disabled opt-in refuses a saved CLI chat before write-ahead, liveness, or agent creation", async (t) => {
|
||||
const fakeRun = createFakeRun(t, [toAskScenario()]);
|
||||
const chatId = createClaudeCodeChat(t);
|
||||
writeSettings({ ...(readSettings() ?? { version: 1 }), claudeCode: { enabled: false } });
|
||||
const before = chatStore.getChat(chatId)!.messages;
|
||||
const response = await handleChatRequest(new Request("http://test/api/chat", { method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ chatId, input: { type: "text", text: "must not start" } }) }), () => assert.fail("disabled work must not build an agent"));
|
||||
assert.equal(response.status, 503);
|
||||
assert.match((await response.json()).error, /Enable and set it up in Settings/);
|
||||
assert.equal(isTurnRunning(chatId), false); assert.equal(fakeRun.children.length, 0);
|
||||
assert.deepEqual(chatStore.getChat(chatId)!.messages, before);
|
||||
assert.deepEqual(chatStore.getChat(chatId)!.settings?.model, { provider: "claude-code", id: "sonnet" });
|
||||
});
|
||||
|
||||
test("switching off blocks future requests but does not interrupt an already admitted CLI turn", async (t) => {
|
||||
const scenario = [JSON.stringify({ fake: "checkpoint" }), JSON.stringify({ fake: "wait-for-message" }), ...readFileSync(TEXT_FIXTURE, "utf8").trim().split("\n")];
|
||||
const fakeRun = createFakeRun(t, [scenario]);
|
||||
const run = fakeRun.runClaude;
|
||||
let ready!: (child: ChildProcessWithoutNullStreams) => void;
|
||||
let failed!: (error: unknown) => void;
|
||||
const checkpoint = new Promise<ChildProcessWithoutNullStreams>((resolve, reject) => { ready = resolve; failed = reject; });
|
||||
fakeRun.runClaude = async (args, options) => {
|
||||
const child = await run(args, options); assert(child);
|
||||
void once(child, "message", { signal: AbortSignal.timeout(TURN_TIMEOUT_MS) }).then(() => ready(child), failed);
|
||||
return child;
|
||||
};
|
||||
const chatId = createClaudeCodeChat(t);
|
||||
const response = await handleChatRequest(new Request("http://test/api/chat", { method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ chatId, input: { type: "text", text: "complete this admitted fixture" } }) }), createAgentFactory(fakeRun));
|
||||
assert.equal(response.status, 200);
|
||||
const child = await checkpoint;
|
||||
const handle = runningTurn(chatId); assert(handle);
|
||||
const finished = new Response(handle.subscribe()).text();
|
||||
writeSettings({ ...(readSettings() ?? { version: 1 }), claudeCode: { enabled: false } });
|
||||
assert.equal(child.signalCode, null, "opt-out must not kill an already running process");
|
||||
child.send("continue"); await finished;
|
||||
assert(getLastAssistantText(chatId).length > 0, "the admitted turn completes normally");
|
||||
assert.equal(readSettings()?.claudeCode?.enabled, false);
|
||||
await waitForExit(child);
|
||||
});
|
||||
|
||||
test("a question answered before the idle timeout continues the waiting CLI process with the answer", async (t) => {
|
||||
const fakeRun = createFakeRun(t, [toAskScenario()]);
|
||||
const run = fakeRun.runClaude;
|
||||
fakeRun.runClaude = (args, options) => run(args, { ...options, env: { ...options.env, FAKE_CLAUDE_WINDOW: "200000" } });
|
||||
const agentFactory = createAgentFactory(fakeRun);
|
||||
const chatId = createClaudeCodeChat(t);
|
||||
|
||||
await runChatTurn(chatId, { type: "text", text: "teach me sockets" }, agentFactory);
|
||||
|
||||
assert.equal(danglingQuestionOf(chatStore.getChat(chatId)?.messages ?? [])?.toolCallId, ASK_ID);
|
||||
const [waiting] = fakeRun.children;
|
||||
assert.ok(waiting);
|
||||
assert.equal(waiting.exitCode, null, "the CLI process should wait on the held question call");
|
||||
|
||||
await runChatTurn(chatId, toAnswerInput(), agentFactory);
|
||||
|
||||
assert.equal(fakeRun.children.length, 1);
|
||||
assert.deepEqual(cliContextWindowOf(chatStore.getChat(chatId)!.messages), { model: "claude-code:sonnet", tokens: 200000 });
|
||||
assert.deepEqual(cliContextWindowOf(JSON.parse(readFileSync(join(PATHS.chats, `${chatId}.json`), "utf8")).messages), { model: "claude-code:sonnet", tokens: 200000 });
|
||||
assert.ok(getLastAssistantText(chatId).includes(ANSWER_TEXT), getLastAssistantText(chatId));
|
||||
await waitForExit(waiting);
|
||||
});
|
||||
|
||||
test("a question pauses later calls from the same CLI response until the learner answers", async (t) => {
|
||||
const secondId = "toolu_second_question";
|
||||
const secondQuestion = "What do you want to learn next?";
|
||||
const scenario = toAskScenario();
|
||||
scenario.splice(4, 0,
|
||||
streamEvent({ type: "content_block_start", index: 1, content_block: { type: "tool_use", id: secondId, name: "mcp__learnharness__ask_user_question", input: {} } }),
|
||||
streamEvent({ type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: JSON.stringify({ question: secondQuestion }) } }),
|
||||
streamEvent({ type: "content_block_stop", index: 1 }),
|
||||
);
|
||||
scenario.splice(scenario.length - 1, 0, JSON.stringify({ fake: "call", id: secondId, name: "ask_user_question", arguments: { question: secondQuestion } }));
|
||||
const fakeRun = createFakeRun(t, [scenario]);
|
||||
const factory = createAgentFactory(fakeRun);
|
||||
const chatId = createClaudeCodeChat(t);
|
||||
await runChatTurn(chatId, { type: "text", text: "teach me sockets" }, factory);
|
||||
assert.equal(danglingQuestionOf(chatStore.getChat(chatId)?.messages ?? [])?.toolCallId, ASK_ID);
|
||||
assert.equal(JSON.stringify(chatStore.getChat(chatId)?.messages).includes(secondId), false, "unadmitted second question must not be posed or persisted");
|
||||
await runChatTurn(chatId, toAnswerInput(), factory);
|
||||
assert.equal(danglingQuestionOf(chatStore.getChat(chatId)?.messages ?? [])?.toolCallId, secondId);
|
||||
await runChatTurn(chatId, {
|
||||
type: "answer", toolCallId: secondId,
|
||||
answer: { kind: "ask", questionText: secondQuestion, dontKnow: false, answers: [], text: "HTTP next", answeredAt: new Date().toISOString() },
|
||||
}, factory);
|
||||
assert.equal(fakeRun.children.length, 1);
|
||||
assert.ok(getLastAssistantText(chatId).includes(ANSWER_TEXT));
|
||||
assert.ok(getLastAssistantText(chatId).includes("HTTP next"));
|
||||
await waitForExit(fakeRun.children[0]!);
|
||||
});
|
||||
|
||||
test("an individually cancelled MCP question queued behind a learner wait never executes", async (t) => {
|
||||
const cancelledId = "toolu_cancelled_question";
|
||||
const question = "This cancelled question must never be posed";
|
||||
const scenario = toAskScenario();
|
||||
scenario.splice(4, 0,
|
||||
streamEvent({ type: "content_block_start", index: 1, content_block: { type: "tool_use", id: cancelledId, name: "mcp__learnharness__ask_user_question", input: {} } }),
|
||||
streamEvent({ type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: JSON.stringify({ question }) } }),
|
||||
streamEvent({ type: "content_block_stop", index: 1 }),
|
||||
);
|
||||
scenario.splice(scenario.length - 2, 2,
|
||||
JSON.stringify({ fake: "start-call", id: ASK_ID, name: "ask_user_question", arguments: ASK_INPUT }),
|
||||
JSON.stringify({ fake: "wait-for-message" }),
|
||||
JSON.stringify({ fake: "start-call", id: cancelledId, name: "ask_user_question", arguments: { question } }),
|
||||
JSON.stringify({ fake: "cancel-call", id: cancelledId }),
|
||||
JSON.stringify({ fake: "checkpoint" }),
|
||||
JSON.stringify({ fake: "reply" }),
|
||||
);
|
||||
const fakeRun = createFakeRun(t, [scenario]);
|
||||
const factory = createAgentFactory(fakeRun);
|
||||
const chatId = createClaudeCodeChat(t);
|
||||
await runChatTurn(chatId, { type: "text", text: "teach me sockets" }, factory);
|
||||
assert.equal(danglingQuestionOf(chatStore.getChat(chatId)?.messages ?? [])?.toolCallId, ASK_ID);
|
||||
const child = fakeRun.children[0]!;
|
||||
const cancelledOnServer = once(child, "message", { signal: AbortSignal.timeout(TURN_TIMEOUT_MS) });
|
||||
child.send("queue and cancel B");
|
||||
assert.deepEqual((await cancelledOnServer)[0], { fake: "checkpoint" });
|
||||
await runChatTurn(chatId, toAnswerInput(), factory);
|
||||
assert.equal(fakeRun.children.length, 1);
|
||||
assert.equal(danglingQuestionOf(chatStore.getChat(chatId)?.messages ?? []), null);
|
||||
assert.equal(JSON.stringify(chatStore.getChat(chatId)?.messages).includes(cancelledId), false, "cancelled B must never reach native tool execution or persistence");
|
||||
assert.ok(getLastAssistantText(chatId).includes(ANSWER_TEXT));
|
||||
await waitForExit(child);
|
||||
});
|
||||
|
||||
test("a question answered after the idle timeout starts a fresh CLI process that replays the question and answer", async (t) => {
|
||||
liveProcessIdleTimeoutMs.current = 300;
|
||||
t.after(() => (liveProcessIdleTimeoutMs.current = LIVE_PROCESS_IDLE_TIMEOUT_MS));
|
||||
|
||||
const fakeRun = createFakeRun(t, [toAskScenario(), readFileSync(TEXT_FIXTURE, "utf8").split("\n").filter(Boolean)]);
|
||||
const agentFactory = createAgentFactory(fakeRun);
|
||||
const chatId = createClaudeCodeChat(t);
|
||||
|
||||
await runChatTurn(chatId, { type: "text", text: "teach me sockets" }, agentFactory);
|
||||
|
||||
const [waiting] = fakeRun.children;
|
||||
assert.ok(waiting);
|
||||
assert.equal(waiting.exitCode, null, "the CLI process should wait on the held question call");
|
||||
await waitForExit(waiting);
|
||||
|
||||
await runChatTurn(chatId, toAnswerInput(), agentFactory);
|
||||
|
||||
assert.equal(fakeRun.children.length, 2);
|
||||
assert.equal(getLastAssistantText(chatId), "hello there");
|
||||
|
||||
const replay = readFileSync(fakeRun.stdinFiles[1] ?? "", "utf8");
|
||||
assert.ok(replay.includes("[tool call ask_user_question]"), replay);
|
||||
assert.ok(replay.includes(ASK_INPUT.question), replay);
|
||||
assert.ok(replay.includes("[tool result ask_user_question]"), replay);
|
||||
assert.ok(replay.includes(ANSWER_TEXT), replay);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { type ChildProcessWithoutNullStreams, type spawn } from "node:child_process";
|
||||
import { test } from "node:test";
|
||||
import { inspectClaudeInstallation } from "../src/agent/claude-code/installation.ts";
|
||||
import { ClaudeLaunchError, createClaudeLauncher } from "../src/agent/claude-code/launch.ts";
|
||||
|
||||
const binary = "/fake/claude";
|
||||
const options = { cwd: "/fake/work", env: { NODE_ENV: "test" as const, ANTHROPIC_API_KEY: "native-test-key" } };
|
||||
|
||||
test("missing installation does not execute a command", async () => {
|
||||
assert.deepEqual(await inspectClaudeInstallation(() => null, async () => { throw new Error("must not run"); }), {
|
||||
state: "not-found", version: null, minimumVersion: "2.1.259",
|
||||
});
|
||||
});
|
||||
|
||||
for (const [output, state, version] of [
|
||||
["2.1.258 (Claude Code)\n", "update-required", "2.1.258"],
|
||||
["2.1.259 (Claude Code)", "meets-minimum", "2.1.259"],
|
||||
["2.1.310 (Claude Code)", "meets-minimum", "2.1.310"],
|
||||
["2.2.0", "meets-minimum", "2.2.0"],
|
||||
["2.0.999", "update-required", "2.0.999"],
|
||||
["3.0.0", "meets-minimum", "3.0.0"],
|
||||
["unexpected output containing private details", "version-unchecked", null],
|
||||
["2.1.310-preview", "version-unchecked", null],
|
||||
["99999999999999999999999999.1.1", "version-unchecked", null],
|
||||
] as const) test(`installation version output ${JSON.stringify(output)} gives ${state}`, async () => {
|
||||
const status = await inspectClaudeInstallation(() => binary, async (path) => { assert.equal(path, binary); return output; });
|
||||
assert.deepEqual(status, { state, version, minimumVersion: "2.1.259" });
|
||||
});
|
||||
|
||||
test("probe failures expose neither raw errors nor binary paths", async () => {
|
||||
const status = await inspectClaudeInstallation(() => binary, async () => { throw new Error("timeout /private/path secret-token"); });
|
||||
assert.deepEqual(status, { state: "version-unchecked", version: null, minimumVersion: "2.1.259" });
|
||||
});
|
||||
|
||||
for (const output of ["2.1.258", "garbage", null]) test(`launcher refuses ${output ?? "failed"} version check before model work`, async () => {
|
||||
const launch = createClaudeLauncher({ findBinary: () => binary,
|
||||
readVersion: async () => { if (output === null) throw new Error("failed probe"); return output; },
|
||||
spawnProcess: (() => { assert.fail("must not start model work"); }) as typeof spawn,
|
||||
});
|
||||
await assert.rejects(launch(["-p"], options) as Promise<unknown>, ClaudeLaunchError);
|
||||
});
|
||||
|
||||
test("launcher uses the verified executable directly and preserves model args and native env", async () => {
|
||||
const child = {} as ChildProcessWithoutNullStreams;
|
||||
let calls = 0;
|
||||
const launch = createClaudeLauncher({ findBinary: () => binary, readVersion: async () => "2.1.310 (Claude Code)",
|
||||
spawnProcess: ((path: string, args: string[], opts: unknown) => {
|
||||
calls++;
|
||||
assert.equal(path, binary);
|
||||
assert.deepEqual(args, ["-p", "--tools", ""]);
|
||||
assert.deepEqual(opts, { ...options, stdio: "pipe" });
|
||||
return child;
|
||||
}) as typeof spawn,
|
||||
});
|
||||
assert.equal(await launch(["-p", "--tools", ""], options), child);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test("aborting during a version check prevents model process creation", async () => {
|
||||
const controller = new AbortController();
|
||||
const launch = createClaudeLauncher({ findBinary: () => binary,
|
||||
readVersion: async () => { controller.abort(); return "2.1.310"; },
|
||||
spawnProcess: (() => { assert.fail("must not start after abort"); }) as typeof spawn,
|
||||
});
|
||||
await assert.rejects(launch([], { ...options, signal: controller.signal }) as Promise<unknown>, /aborted/);
|
||||
});
|
||||
|
||||
test("missing executable is not mistaken for an unchecked version", async () => {
|
||||
const launch = createClaudeLauncher({ findBinary: () => null, readVersion: async () => assert.fail("must not probe") });
|
||||
assert.equal(await launch([], options), null);
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
import test, { after, type TestContext } from "node:test";
|
||||
|
||||
const testData = mkdtempSync(join(tmpdir(), "lh-cli-opt-in-data-"));
|
||||
process.env.LH_DATA_DIR = testData;
|
||||
after(() => rmSync(testData, { recursive: true, force: true }));
|
||||
const { SETTINGS_FILE } = await import("../config.ts");
|
||||
const { PUT } = await import("../app/api/settings/[section]/route.ts");
|
||||
const { sanitizeUserSettings, providerKeyMissing } = await import("../src/core/settings.ts");
|
||||
const { parseClaudeCodeSettingsBody } = await import("../src/server/settings-body.ts");
|
||||
const { claudeCodeAvailable, claudeCodeEnabled } = await import("../src/server/claude-code-config.ts");
|
||||
const { claudeCodeInstallationResponse } = await import("../src/server/claude-code.ts");
|
||||
const { modelCatalog, validateModelsSettings } = await import("../src/server/model-catalog.ts");
|
||||
const { providerKeyPresent } = await import("../src/server/keys.ts");
|
||||
const { readSettings, writeSettings } = await import("../src/store/settings.ts");
|
||||
const { chatStore } = await import("../src/store/chats.ts");
|
||||
const { jobStore } = await import("../src/store/jobs.ts");
|
||||
const { createJobRunner } = await import("../src/agent/subagents/runner.ts");
|
||||
const { CLAUDE_CODE_MODELS } = await import("../src/agent/claude-code/provider.ts");
|
||||
|
||||
function setup(t: TestContext) {
|
||||
const previous = readSettings();
|
||||
const path = process.env.PATH;
|
||||
const dir = mkdtempSync(join(tmpdir(), "claude-opt-in-"));
|
||||
// Never executed: only installation discovery observes this marker.
|
||||
writeFileSync(join(dir, process.platform === "win32" ? "claude.exe" : "claude"), "", { mode: 0o755 });
|
||||
process.env.PATH = `${dir}${delimiter}${path ?? ""}`;
|
||||
t.after(() => {
|
||||
if (previous) writeSettings(previous); else rmSync(SETTINGS_FILE, { force: true });
|
||||
if (path === undefined) delete process.env.PATH; else process.env.PATH = path;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
writeSettings({ version: 1 });
|
||||
}
|
||||
async function save(body: unknown) {
|
||||
return PUT(new Request("http://test/api/settings/claudeCode", { method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }),
|
||||
{ params: Promise.resolve({ section: "claudeCode" }) });
|
||||
}
|
||||
|
||||
test("opt-in parses strictly and malformed stored values never enable the CLI", () => {
|
||||
assert.deepEqual(parseClaudeCodeSettingsBody({}), { ok: true, settings: {} });
|
||||
assert.deepEqual(parseClaudeCodeSettingsBody({ enabled: true, ignored: true }), { ok: true, settings: { enabled: true } });
|
||||
assert.deepEqual(parseClaudeCodeSettingsBody({ enabled: false }), { ok: true, settings: { enabled: false } });
|
||||
for (const raw of [true, null, [], "on", { enabled: "true" }, { enabled: 1 }]) assert.equal(parseClaudeCodeSettingsBody(raw).ok, false);
|
||||
for (const raw of [null, [], true, "on", { enabled: "true" }, { enabled: 1 }]) {
|
||||
assert.notEqual(sanitizeUserSettings({ version: 1, claudeCode: raw })?.claudeCode?.enabled, true);
|
||||
}
|
||||
assert.equal(sanitizeUserSettings({ version: 1 })?.claudeCode, undefined);
|
||||
});
|
||||
|
||||
test("default off even with an installed binary; section writes toggle availability but preserve saved model pins and unrelated settings", async (t) => {
|
||||
setup(t);
|
||||
const models = { strong: { model: { provider: "claude-code", id: "sonnet" } }, cheap: { model: { provider: "claude-code", id: "haiku" } } };
|
||||
writeSettings({ version: 1, models, graph: { enabled: false }, futureSection: { keep: 1 } });
|
||||
assert.equal(claudeCodeEnabled(), false); assert.equal(claudeCodeAvailable(), false);
|
||||
assert.equal(providerKeyPresent("claude-code"), false);
|
||||
assert(!modelCatalog().models.some((m) => m.provider === "claude-code"));
|
||||
assert.equal((await save({ enabled: true })).status, 200);
|
||||
assert.equal(claudeCodeEnabled(), true); assert.equal(claudeCodeAvailable(), true);
|
||||
assert.equal(modelCatalog().models.filter((m) => m.provider === "claude-code").length, 3);
|
||||
assert.equal((await save({ enabled: false })).status, 200);
|
||||
assert.equal(claudeCodeAvailable(), false);
|
||||
assert(!modelCatalog().models.some((m) => m.provider === "claude-code"));
|
||||
assert.deepEqual(readSettings()?.models, models);
|
||||
assert.deepEqual(readSettings()?.graph, { enabled: false });
|
||||
assert.deepEqual(readSettings()?.futureSection, { keep: 1 });
|
||||
assert.equal(validateModelsSettings(models, models).ok, true, "saved unavailable pins remain repairable, not silently removed");
|
||||
assert.equal((await save({ enabled: "true" })).status, 400); assert.equal(claudeCodeEnabled(), false);
|
||||
assert.equal((await save({})).status, 200); assert.equal(claudeCodeEnabled(), false);
|
||||
});
|
||||
|
||||
test("off cannot launch a version probe even through an otherwise authorized HTTP request", async (t) => {
|
||||
setup(t);
|
||||
const request = new Request("http://test/api/claude-code", { headers: { "x-learnharness-claude-code": "1" } });
|
||||
const response = await claudeCodeInstallationResponse(request, async () => assert.fail("disabled CLI must not execute"));
|
||||
assert.equal(response.status, 403);
|
||||
});
|
||||
|
||||
test("a saved CLI background job is refused before model execution while opt-in is off", async (t) => {
|
||||
setup(t);
|
||||
const chat = chatStore.createChat("CLI opt-in job fixture");
|
||||
t.after(() => { jobStore.deleteJobsForChat(chat.id); chatStore.deleteChat(chat.id); });
|
||||
const job = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "cli-off-job" });
|
||||
let calls = 0;
|
||||
const runner = createJobRunner({ chatStore, jobStore, deliver: async () => {},
|
||||
engineFor: () => ({ model: CLAUDE_CODE_MODELS[0]!, streamFn: () => { calls++; assert.fail("must not run a disabled CLI model"); } }),
|
||||
providerKeyMissing: (provider, tier) => providerKeyPresent(provider) ? null : providerKeyMissing(provider, [tier]),
|
||||
});
|
||||
await runner.runJob(job.id, { question: "Synthetic fixture only" });
|
||||
assert.equal(calls, 0);
|
||||
const refused = jobStore.getJob(job.id)!;
|
||||
assert.equal(refused.status, "error");
|
||||
assert.match(JSON.stringify(refused.result), /Enable and set it up in Settings/);
|
||||
assert(!JSON.stringify(refused.result).includes("add one in Settings → Keys"));
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Claude Code provider catalog (src/agent/claude-code/provider.ts).
|
||||
*
|
||||
* Registered on a fresh pi model collection with a stub stream function, so
|
||||
* only the catalog and auth surface are under test here.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { clampThinkingLevel, createAssistantMessageEventStream, createModels, getSupportedThinkingLevels } from "@earendil-works/pi-ai";
|
||||
|
||||
import { CLAUDE_CODE_PROVIDER } from "../src/agent/claude-code/constants.ts";
|
||||
import { createClaudeCodeProvider } from "../src/agent/claude-code/provider.ts";
|
||||
|
||||
function createCollection() {
|
||||
const collection = createModels();
|
||||
collection.setProvider(createClaudeCodeProvider(() => createAssistantMessageEventStream()));
|
||||
return collection;
|
||||
}
|
||||
|
||||
test("lists exactly the opus, sonnet and haiku aliases", () => {
|
||||
const models = createCollection().getModels(CLAUDE_CODE_PROVIDER);
|
||||
assert.deepEqual(
|
||||
models.map((m) => m.id),
|
||||
["opus", "sonnet", "haiku"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
models.map((m) => m.name),
|
||||
["Claude Opus", "Claude Sonnet", "Claude Haiku"],
|
||||
);
|
||||
});
|
||||
|
||||
test("supports low through max thinking and clamps off to low", () => {
|
||||
for (const model of createCollection().getModels(CLAUDE_CODE_PROVIDER)) {
|
||||
assert.deepEqual(getSupportedThinkingLevels(model), ["low", "medium", "high", "xhigh", "max"]);
|
||||
assert.equal(clampThinkingLevel(model, "off"), "low");
|
||||
assert.equal(clampThinkingLevel(model, "minimal"), "low");
|
||||
}
|
||||
});
|
||||
|
||||
test("accepts text and image input", () => {
|
||||
for (const model of createCollection().getModels(CLAUDE_CODE_PROVIDER)) {
|
||||
assert.deepEqual(model.input, ["text", "image"]);
|
||||
}
|
||||
});
|
||||
|
||||
test("every cost field is zero", () => {
|
||||
for (const model of createCollection().getModels(CLAUDE_CODE_PROVIDER)) {
|
||||
assert.deepEqual(model.cost, { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 });
|
||||
}
|
||||
});
|
||||
|
||||
test("auth resolves as configured without any key", async () => {
|
||||
const collection = createCollection();
|
||||
assert.ok(await collection.checkAuth(CLAUDE_CODE_PROVIDER));
|
||||
assert.equal((await collection.getAvailable(CLAUDE_CODE_PROVIDER)).length, 3);
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { claudeCodeInstallationResponse as respond } from "../src/server/claude-code.ts";
|
||||
const claudeCodeInstallationResponse: typeof respond = (request, inspect) => respond(request, inspect, () => true);
|
||||
|
||||
function request(headers: Record<string, string> = {}) {
|
||||
return new Request("http://localhost:3000/api/claude-code", { headers: { "x-learnharness-claude-code": "1", ...headers } });
|
||||
}
|
||||
const status = { state: "not-found" as const, version: null, minimumVersion: "2.1.259" };
|
||||
|
||||
for (const headers of [
|
||||
{ origin: "https://attacker.invalid" },
|
||||
{ "sec-fetch-site": "cross-site" },
|
||||
{ origin: "http://localhost:3000", "sec-fetch-site": "cross-site" },
|
||||
{ origin: "null" },
|
||||
{ "x-learnharness-claude-code": "" },
|
||||
] as Record<string, string>[]) test(`unsafe status request ${JSON.stringify(headers)} never invokes the probe`, async () => {
|
||||
let probes = 0;
|
||||
const response = await claudeCodeInstallationResponse(request(headers), async () => { probes++; return status; });
|
||||
assert.equal(response.status, 403); assert.equal(probes, 0);
|
||||
assert.equal(response.headers.get("access-control-allow-origin"), null);
|
||||
});
|
||||
|
||||
test("disabled opt-in never starts even a version probe", async () => {
|
||||
let probes = 0;
|
||||
const response = await respond(request(), async () => { probes++; return status; }, () => false);
|
||||
assert.equal(response.status, 403); assert.equal(probes, 0);
|
||||
});
|
||||
|
||||
test("ordinary image/subresource GET without intent header cannot start a probe", async () => {
|
||||
const response = await claudeCodeInstallationResponse(new Request("http://localhost:3000/api/claude-code"), async () => assert.fail("must not probe"));
|
||||
assert.equal(response.status, 403);
|
||||
});
|
||||
|
||||
for (const headers of [{}, { origin: "http://localhost:3000", "sec-fetch-site": "same-origin" }] as Record<string, string>[]) test(`legitimate Providers GET ${JSON.stringify(headers)} is supported`, async () => {
|
||||
let probes = 0;
|
||||
const response = await claudeCodeInstallationResponse(request(headers), async () => { probes++; return status; });
|
||||
assert.equal(response.status, 200); assert.equal(probes, 1);
|
||||
assert.equal(response.headers.get("cache-control"), "no-store");
|
||||
assert.deepEqual(await response.json(), status);
|
||||
});
|
||||
|
||||
test("a forged forwarded origin cannot authorize a foreign Origin header", async () => {
|
||||
const response = await claudeCodeInstallationResponse(request({ origin: "https://attacker.invalid", "x-forwarded-host": "attacker.invalid", "x-forwarded-proto": "https" }), async () => assert.fail("must not probe"));
|
||||
assert.equal(response.status, 403);
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import React, { act } from "react";
|
||||
(globalThis as Record<string, unknown>).React = React;
|
||||
import assert from "node:assert/strict";
|
||||
import { afterEach, test } from "node:test";
|
||||
import { createRoot, type Root } from "react-dom/client";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { parseHTML } from "linkedom";
|
||||
import { ClaudeCodeInstallationStatus, claudeCodeStatusLabel } from "../components/settings/claude-code-status.tsx";
|
||||
import { ConnectionRow } from "../components/settings/connection-controls.tsx";
|
||||
import { useModelCatalog } from "../components/model-catalog.ts";
|
||||
import type { ClaudeCodeStatus } from "../src/core/contracts/claude-code.ts";
|
||||
|
||||
const dom = parseHTML("<html><body></body></html>");
|
||||
Object.assign(globalThis, { window: dom.window, document: dom.document, Element: dom.window.Element, HTMLElement: dom.window.HTMLElement,
|
||||
CustomEvent: dom.window.CustomEvent, IS_REACT_ACT_ENVIRONMENT: true });
|
||||
const originalFetch = globalThis.fetch;
|
||||
let root: Root | undefined, host: HTMLDivElement | undefined;
|
||||
const status = (state: ClaudeCodeStatus["state"], version: string | null = null): ClaudeCodeStatus => ({ state, version, minimumVersion: "2.1.259" });
|
||||
const button = () => document.querySelector<HTMLButtonElement>('[data-testid="claude-code-check-again"]')!;
|
||||
const toggle = () => document.querySelector<HTMLButtonElement>('[role="switch"]')!;
|
||||
const label = () => document.querySelector('[role="status"]')?.textContent;
|
||||
function fixture(probe: (init?: RequestInit) => Promise<Response>, enabled = true, failSave = false) {
|
||||
const calls: string[] = [];
|
||||
let on = enabled;
|
||||
globalThis.fetch = (async (url, init) => {
|
||||
calls.push(`${init?.method ?? "GET"} ${url}`);
|
||||
if (url === "/api/settings") return Response.json({ settings: { version: 1, claudeCode: { enabled: on } } });
|
||||
if (url === "/api/settings/claudeCode") {
|
||||
assert.equal(init?.method, "PUT"); assert.equal(new Headers(init.headers).get("Content-Type"), "application/json");
|
||||
if (failSave) return new Response(null, { status: 500 });
|
||||
on = JSON.parse(String(init.body)).enabled;
|
||||
return Response.json({ settings: { version: 1, claudeCode: { enabled: on } } });
|
||||
}
|
||||
if (url === "/api/models") return Response.json({ models: on ? [{ provider: "claude-code", id: "sonnet" }] : [] });
|
||||
assert.equal(url, "/api/claude-code");
|
||||
assert.equal(init?.cache, "no-store");
|
||||
assert.equal(new Headers(init?.headers).get("x-learnharness-claude-code"), "1");
|
||||
return probe(init);
|
||||
}) as typeof fetch;
|
||||
return calls;
|
||||
}
|
||||
async function mount(extra?: React.ReactNode) {
|
||||
host = document.createElement("div"); document.body.append(host); root = createRoot(host);
|
||||
await act(async () => root!.render(<><ClaudeCodeInstallationStatus />{extra}</>));
|
||||
}
|
||||
afterEach(async () => {
|
||||
await act(async () => root?.unmount()); host?.remove(); root = undefined; host = undefined;
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
test("initial render is collapsed and off, with no setup prose or version probe action", () => {
|
||||
const html = renderToStaticMarkup(<ClaudeCodeInstallationStatus />);
|
||||
assert.match(html, /Claude Code/); assert.match(html, /aria-checked="false"/);
|
||||
assert.equal((html.match(/<button/g) ?? []).length, 1);
|
||||
assert(!html.includes("Check again")); assert(!html.includes("Authenticate")); assert(!html.includes("Setup ↗"));
|
||||
});
|
||||
|
||||
test("off by default; enabling persists before probing and reveals the version/action row below the toggle", async () => {
|
||||
const calls = fixture(async () => Response.json(status("not-found")), false);
|
||||
await mount();
|
||||
assert.equal(toggle().getAttribute("aria-checked"), "false"); assert(!button());
|
||||
assert.deepEqual(calls, ["GET /api/settings"]);
|
||||
await act(async () => toggle().click());
|
||||
assert.equal(toggle().getAttribute("aria-checked"), "true"); assert.equal(label(), "Not found");
|
||||
assert(calls.indexOf("PUT /api/settings/claudeCode") < calls.indexOf("GET /api/claude-code"));
|
||||
assert.notEqual(toggle().parentElement, button().parentElement);
|
||||
assert.equal(document.querySelector('[role="status"]')?.parentElement, button().parentElement);
|
||||
assert.match(document.body.textContent ?? "", /no background process required/);
|
||||
assert.match(document.body.textContent ?? "", /not authentication, model access, or quota/);
|
||||
assert.match(document.body.textContent ?? "", /Claude Code controls billing/);
|
||||
assert.match(document.body.textContent ?? "", /Verify AI claims independently—they may be inaccurate, incomplete, misleading, or outdated\./);
|
||||
assert(document.querySelector("strong")?.textContent?.includes("Install and authenticate"));
|
||||
assert.equal(document.querySelector("a")?.getAttribute("href"), "https://code.claude.com/docs/en/setup");
|
||||
await act(async () => toggle().click());
|
||||
assert.equal(toggle().getAttribute("aria-checked"), "false"); assert(!button()); assert.equal(label(), undefined);
|
||||
assert.equal(calls.filter((call) => call === "GET /api/claude-code").length, 1);
|
||||
});
|
||||
|
||||
test("recheck matches Connect styling; Manage remains ghost", async () => {
|
||||
fixture(async () => Response.json(status("not-found"))); await mount();
|
||||
const row = (action: string) => parseHTML(renderToStaticMarkup(<ConnectionRow label="Fixture" action={action} disabled={false} onAction={() => {}} />)).document.querySelector("button")!;
|
||||
assert.equal(button().getAttribute("class"), row("Connect").getAttribute("class"));
|
||||
assert.equal(button().getAttribute("data-variant"), "outline");
|
||||
assert.equal(row("Manage").getAttribute("data-variant"), "ghost");
|
||||
});
|
||||
|
||||
for (const [value, text] of [
|
||||
[status("not-found"), "Not found"],
|
||||
[status("meets-minimum", "2.1.310"), "Version 2.1.310 · meets minimum"],
|
||||
[status("update-required", "2.1.258"), "Version 2.1.258 · update required"],
|
||||
[status("version-unchecked"), "Version could not be checked"],
|
||||
] as const) test(`installation display: ${text}`, () => assert.equal(claudeCodeStatusLabel(value), text));
|
||||
|
||||
test("enabled mount checks once; outlined Check again refreshes without probing authentication", async () => {
|
||||
let probes = 0;
|
||||
const calls = fixture(async () => Response.json(++probes === 1 ? status("not-found") : status("meets-minimum", "2.1.310")));
|
||||
await mount(); assert.equal(label(), "Not found");
|
||||
await act(async () => button().click()); assert.equal(label(), "Version 2.1.310 · meets minimum");
|
||||
assert.equal(calls.filter((call) => call === "GET /api/claude-code").length, 2);
|
||||
});
|
||||
|
||||
test("failed opt-in save stays collapsed/off and never probes", async () => {
|
||||
const calls = fixture(async () => assert.fail("must not probe"), false, true);
|
||||
await mount(); await act(async () => toggle().click());
|
||||
assert.equal(toggle().getAttribute("aria-checked"), "false"); assert(!button());
|
||||
assert.match(document.querySelector('[role="alert"]')?.textContent ?? "", /Could not save/);
|
||||
assert(!calls.includes("GET /api/claude-code"));
|
||||
});
|
||||
|
||||
test("slow checks disable recheck; switching off aborts pending version work without starting another probe", async () => {
|
||||
let signal: AbortSignal | null | undefined;
|
||||
let resolve!: (response: Response) => void;
|
||||
fixture((init) => { signal = init?.signal; return new Promise<Response>((done) => { resolve = done; }); });
|
||||
await mount(); assert.equal(label(), "Checking…"); assert(button().disabled); assert(!toggle().disabled);
|
||||
await act(async () => toggle().click()); assert(signal?.aborted); assert(!button());
|
||||
await act(async () => resolve(Response.json(status("meets-minimum", "2.1.310")))); assert.equal(label(), undefined);
|
||||
});
|
||||
|
||||
for (const failure of ["network", "http", "payload"] as const) test(`${failure} failure never retains a stale meets-minimum status`, async () => {
|
||||
let calls = 0;
|
||||
fixture(async () => {
|
||||
if (++calls === 1) return Response.json(status("meets-minimum", "2.1.310"));
|
||||
if (failure === "network") throw new Error("offline");
|
||||
return failure === "http" ? new Response(null, { status: 503 }) : Response.json({ state: "authenticated" });
|
||||
});
|
||||
await mount(); assert.equal(label(), "Version 2.1.310 · meets minimum");
|
||||
await act(async () => button().click()); assert.equal(label(), "Version could not be checked"); assert(!button().disabled);
|
||||
});
|
||||
|
||||
function CatalogReader() {
|
||||
const { catalog } = useModelCatalog();
|
||||
return <span data-testid="cli-catalog-models">{catalog?.models.map((m) => `${m.provider}:${m.id}`).join(",") ?? "loading"}</span>;
|
||||
}
|
||||
test("persisted toggle updates invalidate shared model availability in both directions", async () => {
|
||||
fixture(async () => Response.json(status("not-found")), false);
|
||||
await mount(<CatalogReader />);
|
||||
const models = () => document.querySelector('[data-testid="cli-catalog-models"]')?.textContent;
|
||||
assert.equal(models(), "");
|
||||
await act(async () => toggle().click()); assert.equal(models(), "claude-code:sonnet");
|
||||
await act(async () => toggle().click()); assert.equal(models(), "");
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* Claude Code MCP tool bridge (src/agent/claude-code/tool-bridge.ts).
|
||||
*
|
||||
* A real MCP SDK client talks to the bridge over loopback HTTP, the same way
|
||||
* the claude CLI does, so listing, auth, pending calls and their results
|
||||
* are all exercised end to end.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import * as net from "node:net";
|
||||
import * as os from "node:os";
|
||||
import test from "node:test";
|
||||
|
||||
import type { Tool, ToolResultMessage } from "@earendil-works/pi-ai";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
||||
import { Type } from "typebox";
|
||||
|
||||
import { BRIDGE_SERVER_NAME, createToolBridge, toolNameFromMcp, type ToolBridge } from "../src/agent/claude-code/tool-bridge.ts";
|
||||
|
||||
const READ_TOOL: Tool = { name: "read", description: "Read a file", parameters: Type.Object({ path: Type.String() }) };
|
||||
const SCREENSHOT_TOOL: Tool = { name: "screenshot", description: "Capture the screen", parameters: Type.Object({}) };
|
||||
|
||||
type BridgeServerConfig = { type: string; url: string; headers: { Authorization: string } };
|
||||
|
||||
function getServerConfig(bridge: ToolBridge): BridgeServerConfig {
|
||||
const config = JSON.parse(bridge.mcpConfig) as { mcpServers: Record<string, BridgeServerConfig> };
|
||||
const server = config.mcpServers[BRIDGE_SERVER_NAME];
|
||||
assert.ok(server);
|
||||
return server;
|
||||
}
|
||||
|
||||
async function connectClient(bridge: ToolBridge, authorization = getServerConfig(bridge).headers.Authorization): Promise<Client> {
|
||||
const transport = new StreamableHTTPClientTransport(new URL(getServerConfig(bridge).url), {
|
||||
requestInit: { headers: { Authorization: authorization } },
|
||||
});
|
||||
const client = new Client({ name: "test", version: "0" });
|
||||
await client.connect(transport);
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
function toolResult(content: ToolResultMessage["content"], isError = false): ToolResultMessage {
|
||||
return { role: "toolResult", toolCallId: "call-1", toolName: "read", content, isError, timestamp: 0 };
|
||||
}
|
||||
|
||||
test("lists exactly the given tools with their JSON schemas", async (t) => {
|
||||
const bridge = await createToolBridge([READ_TOOL, SCREENSHOT_TOOL]);
|
||||
t.after(() => bridge.close());
|
||||
const client = await connectClient(bridge);
|
||||
t.after(() => client.close());
|
||||
|
||||
const { tools } = await client.listTools();
|
||||
|
||||
assert.deepEqual(
|
||||
tools.map((tool) => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema })),
|
||||
[
|
||||
{ name: "read", description: "Read a file", inputSchema: JSON.parse(JSON.stringify(READ_TOOL.parameters)) },
|
||||
{ name: "screenshot", description: "Capture the screen", inputSchema: JSON.parse(JSON.stringify(SCREENSHOT_TOOL.parameters)) },
|
||||
],
|
||||
);
|
||||
|
||||
assert.deepEqual(bridge.allowedTools, ["mcp__learnharness__read", "mcp__learnharness__screenshot"]);
|
||||
});
|
||||
|
||||
test("rejects requests without the bearer token", async (t) => {
|
||||
const bridge = await createToolBridge([READ_TOOL]);
|
||||
t.after(() => bridge.close());
|
||||
|
||||
await assert.rejects(connectClient(bridge, "Bearer wrong"), (error: { code?: number }) => error.code === 401);
|
||||
const response = await fetch(getServerConfig(bridge).url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: "{}",
|
||||
});
|
||||
assert.equal(response.status, 401);
|
||||
});
|
||||
|
||||
test("binds 127.0.0.1", async (t) => {
|
||||
const bridge = await createToolBridge([READ_TOOL]);
|
||||
t.after(() => bridge.close());
|
||||
|
||||
const url = new URL(getServerConfig(bridge).url);
|
||||
assert.equal(url.hostname, "127.0.0.1");
|
||||
|
||||
const externalAddress = Object.values(os.networkInterfaces())
|
||||
.flat()
|
||||
.find((address) => address && !address.internal && address.family === "IPv4");
|
||||
|
||||
if (!externalAddress) {
|
||||
t.diagnostic("no non-loopback IPv4 interface; skipped external connect check");
|
||||
return;
|
||||
}
|
||||
|
||||
await assert.rejects(
|
||||
new Promise((resolve, reject) => {
|
||||
const socket = net.connect(Number(url.port), externalAddress.address, () => resolve(socket.end()));
|
||||
socket.once("error", reject);
|
||||
}),
|
||||
{ code: "ECONNREFUSED" },
|
||||
);
|
||||
});
|
||||
|
||||
test("tools/call stays pending until resolved and returns its text and image content", async (t) => {
|
||||
const bridge = await createToolBridge([READ_TOOL]);
|
||||
t.after(() => bridge.close());
|
||||
const client = await connectClient(bridge);
|
||||
t.after(() => client.close());
|
||||
|
||||
let settled = false;
|
||||
const pending = client.callTool({ name: "read", arguments: { path: "notes.md" }, _meta: { "claudecode/toolUseId": "call-1" } }).finally(() => {
|
||||
settled = true;
|
||||
});
|
||||
|
||||
const [call] = await bridge.waitForToolCalls(1);
|
||||
assert.ok(call);
|
||||
assert.deepEqual(call, { id: "call-1", name: "read", arguments: { path: "notes.md" } });
|
||||
assert.deepEqual(bridge.getPendingToolCalls(), [call]);
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.equal(settled, false);
|
||||
|
||||
bridge.resolveToolCall(
|
||||
call,
|
||||
toolResult([
|
||||
{ type: "text", text: "# Notes" },
|
||||
{ type: "image", data: "AAAA", mimeType: "image/png" },
|
||||
]),
|
||||
);
|
||||
|
||||
const result = await pending;
|
||||
assert.deepEqual(result.content, [
|
||||
{ type: "text", text: "# Notes" },
|
||||
{ type: "image", data: "AAAA", mimeType: "image/png" },
|
||||
]);
|
||||
assert.notEqual(result.isError, true);
|
||||
assert.deepEqual(bridge.getPendingToolCalls(), []);
|
||||
});
|
||||
|
||||
test("individual MCP cancellation removes only that request and leaves the other request resolvable", async (t) => {
|
||||
let acknowledgeCancel!: () => void;
|
||||
const cancelled = new Promise<void>((resolve) => { acknowledgeCancel = resolve; });
|
||||
const cancellations: string[] = [];
|
||||
const bridge = await createToolBridge([READ_TOOL], {
|
||||
onCall() {}, onError(error) { throw error; },
|
||||
onCancel(call) { cancellations.push(call.id); acknowledgeCancel(); },
|
||||
});
|
||||
t.after(() => bridge.close());
|
||||
const client = await connectClient(bridge);
|
||||
t.after(() => client.close());
|
||||
const first = client.callTool({ name: "read", arguments: { path: "a" }, _meta: { "claudecode/toolUseId": "call-a" } });
|
||||
const controller = new AbortController();
|
||||
const second = client.callTool({ name: "read", arguments: { path: "b" }, _meta: { "claudecode/toolUseId": "call-b" } }, undefined, { signal: controller.signal });
|
||||
const rejected = assert.rejects(second, /cancelled B/);
|
||||
await bridge.waitForToolCalls(2);
|
||||
controller.abort("cancelled B");
|
||||
await cancelled;
|
||||
await rejected;
|
||||
assert.deepEqual(cancellations, ["call-b"]);
|
||||
const pending = bridge.getPendingToolCalls();
|
||||
assert.deepEqual(pending.map((call) => call.id), ["call-a"]);
|
||||
bridge.resolveToolCall(pending[0]!, toolResult([{ type: "text", text: "A" }]));
|
||||
assert.deepEqual((await first).content, [{ type: "text", text: "A" }]);
|
||||
assert.deepEqual(bridge.getPendingToolCalls(), []);
|
||||
});
|
||||
|
||||
test("waitForToolCalls resolves once that many calls are pending, in arrival order", async (t) => {
|
||||
const bridge = await createToolBridge([READ_TOOL, SCREENSHOT_TOOL]);
|
||||
t.after(() => bridge.close());
|
||||
const client = await connectClient(bridge);
|
||||
t.after(() => client.close());
|
||||
|
||||
let waited = false;
|
||||
const waiting = bridge.waitForToolCalls(2).finally(() => {
|
||||
waited = true;
|
||||
});
|
||||
|
||||
void client.callTool({ name: "read", arguments: { path: "a.md" }, _meta: { "claudecode/toolUseId": "call-1" } }).catch(() => {});
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.equal(waited, false);
|
||||
|
||||
void client.callTool({ name: "screenshot", arguments: {}, _meta: { "claudecode/toolUseId": "call-2" } }).catch(() => {});
|
||||
|
||||
assert.deepEqual(await waiting, [
|
||||
{ id: "call-1", name: "read", arguments: { path: "a.md" } },
|
||||
{ id: "call-2", name: "screenshot", arguments: {} },
|
||||
]);
|
||||
});
|
||||
|
||||
test("isError result becomes MCP isError true", async (t) => {
|
||||
const bridge = await createToolBridge([READ_TOOL]);
|
||||
t.after(() => bridge.close());
|
||||
const client = await connectClient(bridge);
|
||||
t.after(() => client.close());
|
||||
|
||||
const pending = client.callTool({ name: "read", arguments: { path: "missing.md" }, _meta: { "claudecode/toolUseId": "call-1" } });
|
||||
const [call] = await bridge.waitForToolCalls(1);
|
||||
assert.ok(call);
|
||||
|
||||
bridge.resolveToolCall(call, toolResult([{ type: "text", text: "ENOENT" }], true));
|
||||
|
||||
const result = await pending;
|
||||
assert.equal(result.isError, true);
|
||||
assert.deepEqual(result.content, [{ type: "text", text: "ENOENT" }]);
|
||||
});
|
||||
|
||||
test("close fails pending calls and frees the port", async () => {
|
||||
const bridge = await createToolBridge([READ_TOOL]);
|
||||
const port = Number(new URL(getServerConfig(bridge).url).port);
|
||||
const client = await connectClient(bridge);
|
||||
const pending = client.callTool({ name: "read", arguments: { path: "notes.md" }, _meta: { "claudecode/toolUseId": "call-1" } });
|
||||
await bridge.waitForToolCalls(1);
|
||||
|
||||
await bridge.close();
|
||||
|
||||
await assert.rejects(pending);
|
||||
assert.deepEqual(bridge.getPendingToolCalls(), []);
|
||||
await client.close();
|
||||
|
||||
const probe = net.createServer();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
probe.once("error", reject);
|
||||
probe.listen(port, "127.0.0.1", resolve);
|
||||
});
|
||||
await new Promise((resolve) => probe.close(resolve));
|
||||
});
|
||||
|
||||
test("close rejects consumers still waiting for tool calls", { timeout: 1000 }, async () => {
|
||||
const bridge = await createToolBridge([READ_TOOL]);
|
||||
const waiting = bridge.waitForToolCalls(1);
|
||||
|
||||
await bridge.close();
|
||||
await assert.rejects(waiting, /tool bridge closed/);
|
||||
});
|
||||
|
||||
test('mcpConfig is {"mcpServers":{"learnharness":{"type":"http","url":...,"headers":{"Authorization":"Bearer ..."}}}}', async (t) => {
|
||||
const bridge = await createToolBridge([READ_TOOL]);
|
||||
t.after(() => bridge.close());
|
||||
|
||||
const config = JSON.parse(bridge.mcpConfig) as { mcpServers: Record<string, unknown> };
|
||||
assert.deepEqual(Object.keys(config.mcpServers), ["learnharness"]);
|
||||
|
||||
const server = getServerConfig(bridge);
|
||||
assert.deepEqual(Object.keys(server).sort(), ["headers", "type", "url"]);
|
||||
assert.equal(server.type, "http");
|
||||
assert.match(server.url, /^http:\/\/127\.0\.0\.1:\d+\/mcp$/);
|
||||
assert.match(server.headers.Authorization, /^Bearer [0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
test("missing and reused Claude tool IDs are rejected rather than matched by arguments", async (t) => {
|
||||
const failures: string[] = [];
|
||||
const bridge = await createToolBridge([READ_TOOL], { onCall: () => {}, onError: (error) => failures.push(error.message) });
|
||||
t.after(() => bridge.close());
|
||||
const client = await connectClient(bridge);
|
||||
t.after(() => client.close());
|
||||
await assert.rejects(client.callTool({ name: "read", arguments: { path: "a" } }), /claudecode\/toolUseId/);
|
||||
assert.deepEqual(bridge.getPendingToolCalls(), []);
|
||||
const params = { name: "read", arguments: { path: "a" }, _meta: { "claudecode/toolUseId": "same-id" } };
|
||||
const first = client.callTool(params);
|
||||
const [call] = await bridge.waitForToolCalls(1);
|
||||
assert.ok(call);
|
||||
bridge.resolveToolCall(call, toolResult([{ type: "text", text: "first" }]));
|
||||
await first;
|
||||
await assert.rejects(client.callTool(params), /claudecode\/toolUseId/);
|
||||
assert.deepEqual(bridge.getPendingToolCalls(), []);
|
||||
assert.equal(failures.length, 2);
|
||||
});
|
||||
|
||||
test("toolNameFromMcp strips the bridge prefix", () => {
|
||||
assert.equal(toolNameFromMcp("mcp__learnharness__read"), "read");
|
||||
assert.equal(toolNameFromMcp("read"), "read");
|
||||
});
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* Claude Code transcript replay (src/agent/claude-code/transcript.ts).
|
||||
*
|
||||
* The CLI makes one model call per input user message, so earlier turns are
|
||||
* rendered as labelled text inside the single user message it receives.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import type { AssistantMessage, Message, SystemMessage, Tool, ToolResultMessage, UserMessage } from "@earendil-works/pi-ai";
|
||||
|
||||
import { toClaudeInput, toStreamJsonLine } from "../src/agent/claude-code/transcript.ts";
|
||||
|
||||
const READ_TOOL: Tool = { name: "read", description: "Read a file", parameters: { type: "object", properties: {} } };
|
||||
const WRITE_TOOL: Tool = { name: "write", description: "Write a file", parameters: { type: "object", properties: {} } };
|
||||
|
||||
function system(fields: Partial<SystemMessage>): SystemMessage {
|
||||
return { role: "system", content: "", timestamp: 0, ...fields };
|
||||
}
|
||||
|
||||
function user(content: UserMessage["content"]): UserMessage {
|
||||
return { role: "user", content, timestamp: 0 };
|
||||
}
|
||||
|
||||
function assistant(content: AssistantMessage["content"]): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content,
|
||||
api: "claude-code-cli",
|
||||
provider: "claude-code",
|
||||
model: "opus",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function toolResult(toolName: string, content: ToolResultMessage["content"], isError = false): ToolResultMessage {
|
||||
return { role: "toolResult", toolCallId: `${toolName}-1`, toolName, content, isError, timestamp: 0 };
|
||||
}
|
||||
|
||||
test("system prompt joins all system content, then the live sections", () => {
|
||||
const messages: Message[] = [
|
||||
system({ content: "You are a tutor.", sections: { goals: "Teach TCP.", style: "Be brief." } }),
|
||||
user("hi"),
|
||||
system({ content: "The learner switched topics.", sections: { goals: "Teach TLS.", style: null } }),
|
||||
user("next"),
|
||||
];
|
||||
assert.equal(toClaudeInput(messages).systemPrompt, "You are a tutor.\n\nThe learner switched topics.\n\nTeach TLS.");
|
||||
});
|
||||
|
||||
test("tools reflect toolsAdded minus toolsRemoved", () => {
|
||||
const messages: Message[] = [
|
||||
system({ content: "p", toolsAdded: [READ_TOOL, WRITE_TOOL] }),
|
||||
user("hi"),
|
||||
system({ toolsRemoved: [{ name: "read" }] }),
|
||||
user("next"),
|
||||
];
|
||||
assert.deepEqual(
|
||||
toClaudeInput(messages).tools.map((t) => t.name),
|
||||
["write"],
|
||||
);
|
||||
});
|
||||
|
||||
test("single user message passes through as text", () => {
|
||||
const input = toClaudeInput([system({ content: "p" }), user("what is a socket?")]);
|
||||
assert.deepEqual(input.userContent, [{ type: "text", text: "what is a socket?" }]);
|
||||
});
|
||||
|
||||
test("history renders user, assistant text, tool call name+JSON args and tool result text in order before the latest user message", () => {
|
||||
const messages: Message[] = [
|
||||
user([{ type: "text", text: "open notes.md" }]),
|
||||
assistant([
|
||||
{ type: "text", text: "Reading it." },
|
||||
{ type: "toolCall", id: "read-1", name: "read", arguments: { path: "notes.md" } },
|
||||
]),
|
||||
toolResult("read", [{ type: "text", text: "# Notes" }]),
|
||||
assistant([{ type: "text", text: "It has a heading." }]),
|
||||
user("summarise it"),
|
||||
];
|
||||
const expected = [
|
||||
"[user]\nopen notes.md",
|
||||
"[assistant]\nReading it.",
|
||||
'[tool call read] {"path":"notes.md"}',
|
||||
"[tool result read]\n# Notes",
|
||||
"[assistant]\nIt has a heading.",
|
||||
"[user]\nsummarise it",
|
||||
].join("\n\n");
|
||||
assert.deepEqual(toClaudeInput(messages).userContent, [{ type: "text", text: expected }]);
|
||||
});
|
||||
|
||||
test("error tool results are marked as errors in the replay", () => {
|
||||
const messages: Message[] = [
|
||||
user("open missing.md"),
|
||||
assistant([{ type: "toolCall", id: "read-1", name: "read", arguments: { path: "missing.md" } }]),
|
||||
toolResult("read", [{ type: "text", text: "ENOENT" }], true),
|
||||
user("why?"),
|
||||
];
|
||||
const expected = [
|
||||
"[user]\nopen missing.md",
|
||||
'[tool call read] {"path":"missing.md"}',
|
||||
"[tool result read error]\nENOENT",
|
||||
"[user]\nwhy?",
|
||||
].join("\n\n");
|
||||
assert.deepEqual(toClaudeInput(messages).userContent, [{ type: "text", text: expected }]);
|
||||
});
|
||||
|
||||
test("images in user and tool-result content become base64 image blocks", () => {
|
||||
const messages: Message[] = [
|
||||
user([
|
||||
{ type: "text", text: "look" },
|
||||
{ type: "image", data: "AAAA", mimeType: "image/png" },
|
||||
]),
|
||||
assistant([{ type: "toolCall", id: "shot-1", name: "screenshot", arguments: {} }]),
|
||||
toolResult("screenshot", [{ type: "image", data: "BBBB", mimeType: "image/jpeg" }]),
|
||||
user("compare"),
|
||||
];
|
||||
assert.deepEqual(toClaudeInput(messages).userContent, [
|
||||
{ type: "text", text: "[user]\nlook" },
|
||||
{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAAA" } },
|
||||
{ type: "text", text: "[tool call screenshot] {}\n\n[tool result screenshot]" },
|
||||
{ type: "image", source: { type: "base64", media_type: "image/jpeg", data: "BBBB" } },
|
||||
{ type: "text", text: "[user]\ncompare" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("transcript ending on a tool result renders as history with no trailing user turn", () => {
|
||||
const messages: Message[] = [
|
||||
user("open notes.md"),
|
||||
assistant([{ type: "toolCall", id: "read-1", name: "read", arguments: { path: "notes.md" } }]),
|
||||
toolResult("read", [{ type: "text", text: "# Notes" }]),
|
||||
];
|
||||
const expected = ["[user]\nopen notes.md", '[tool call read] {"path":"notes.md"}', "[tool result read]\n# Notes"].join("\n\n");
|
||||
assert.deepEqual(toClaudeInput(messages).userContent, [{ type: "text", text: expected }]);
|
||||
});
|
||||
|
||||
test("thinking blocks are omitted from replay", () => {
|
||||
const messages: Message[] = [
|
||||
user("hi"),
|
||||
assistant([
|
||||
{ type: "thinking", thinking: "secret plan" },
|
||||
{ type: "text", text: "Hello." },
|
||||
]),
|
||||
user("bye"),
|
||||
];
|
||||
assert.deepEqual(toClaudeInput(messages).userContent, [{ type: "text", text: "[user]\nhi\n\n[assistant]\nHello.\n\n[user]\nbye" }]);
|
||||
});
|
||||
|
||||
test('toStreamJsonLine emits {"type":"user","message":{"role":"user","content":[...]}}', () => {
|
||||
assert.deepEqual(JSON.parse(toStreamJsonLine([{ type: "text", text: "hi" }])), {
|
||||
type: "user",
|
||||
message: { role: "user", content: [{ type: "text", text: "hi" }] },
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import React from "react";
|
||||
(globalThis as Record<string, unknown>).React = React;
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { CLAUDE_CODE_MODELS } from "../src/agent/claude-code/provider.ts";
|
||||
import { isSubscriptionModel, sumSteps, type TurnUsage } from "../src/core/index.ts";
|
||||
import { graphCostSummary } from "../src/core/cost-summary.ts";
|
||||
import { accumulateJobUsage } from "../src/core/contracts/jobs.ts";
|
||||
import { callCost, jobCost, sessionCost } from "../src/server/pricing.ts";
|
||||
import { toModelDto } from "../src/server/model-catalog.ts";
|
||||
import { ModelMenuList } from "../components/settings/model-menu.tsx";
|
||||
import { StatsBar } from "../components/stats-bar.tsx";
|
||||
|
||||
const step = { inputTokens: 1000, outputTokens: 100, cacheReadTokens: 50, cacheWriteTokens: 0, reasoningTokens: undefined };
|
||||
for (const model of CLAUDE_CODE_MODELS) test(`${model.id}: token usage is retained but dollars are unpriced, not free or subscription use`, () => {
|
||||
const id = `${model.provider}:${model.id}`;
|
||||
const turn: TurnUsage = { id: "cli-turn", messageId: "cli-message", at: "2026-10-28T00:00:00Z", model: id, steps: [step] };
|
||||
assert(!isSubscriptionModel(id));
|
||||
assert.equal(callCost(step, id), null);
|
||||
const pricing = sessionCost({ turns: [turn] });
|
||||
assert.equal(pricing.unpriced, 1); assert.equal(pricing.subscription, undefined);
|
||||
assert.equal(pricing.turnCosts[turn.id], null);
|
||||
assert.deepEqual(sumSteps(turn.steps), { ...step, cacheWrite1hTokens: undefined, reasoningTokens: undefined });
|
||||
assert.equal(jobCost({ usage: accumulateJobUsage(undefined, id, step) }), null);
|
||||
const html = renderToStaticMarkup(<StatsBar turns={[turn]} pricing={pricing} />);
|
||||
assert(!html.includes("$0"));
|
||||
assert.match(html, /data-testid="stats-cost">CLI<\/span>/);
|
||||
const summary = graphCostSummary({ chats: [{ title: "CLI fixture", extractions: [{ startedAt: 1, at: turn.at, durationMs: 1,
|
||||
stages: { extractor: { model: id, steps: [step] }, auditor: { model: id, steps: [step] } },
|
||||
}] }], sources: [] }, callCost);
|
||||
assert.equal(summary.unpricedRuns, 1); assert.equal(summary.subscriptionRuns, undefined);
|
||||
assert.equal(summary.runs[0]!.dollars, null);
|
||||
});
|
||||
|
||||
test("mixed session totals retain known API dollars and subscription labels alongside CLI billing", () => {
|
||||
const cli: TurnUsage = { id: "cli", messageId: "cli", at: "2026-10-28T00:00:00Z", model: "claude-code:sonnet", steps: [step] };
|
||||
const api = { ...cli, id: "api", messageId: "api", model: "openai:fixture" };
|
||||
const pricing = { cost: 0.12, unpriced: 1, turnCosts: { cli: null, api: 0.12 } };
|
||||
const html = renderToStaticMarkup(<StatsBar turns={[api, cli]} pricing={pricing} />);
|
||||
assert.match(html, /data-testid="stats-cost">\$0\.1200 \+ CLI<\/span>/);
|
||||
const subscription = { ...cli, id: "plan", messageId: "plan", model: "chatgpt:fixture" };
|
||||
const mixed = renderToStaticMarkup(<StatsBar turns={[api, cli, subscription]} pricing={{ ...pricing, subscription: 1, subscriptionProviders: ["ChatGPT"] }} />);
|
||||
assert.match(mixed, /data-testid="stats-cost">\$0\.1200 \+ ChatGPT \+ CLI<\/span>/);
|
||||
assert.equal(callCost(step, cli.model), null, "CLI stays unpriced in accounting");
|
||||
});
|
||||
|
||||
test("model-catalog API and menus never advertise the CLI adapter's placeholder zero rates", () => {
|
||||
const models = CLAUDE_CODE_MODELS.map(toModelDto);
|
||||
assert(models.every((model) => model.cost === null));
|
||||
const html = renderToStaticMarkup(<ModelMenuList models={models} effectiveId={undefined} onPick={() => {}} />);
|
||||
assert.equal((html.match(/>CLI billing<\/span>/g) ?? []).length, 3);
|
||||
assert.match(html, /does not estimate it/);
|
||||
assert(!html.includes("$0")); assert(!html.includes("USD per 1M")); assert(!html.includes("(subscription)"));
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
/** Real execFile boundary, authored fixture executables only. Never the installed CLI. */
|
||||
import assert from "node:assert/strict";
|
||||
import { type spawn } from "node:child_process";
|
||||
import { accessSync, constants, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { resolveClaudeBinary } from "../src/agent/claude-code/binary.ts";
|
||||
import { readClaudeVersion } from "../src/agent/claude-code/installation.ts";
|
||||
import { createClaudeLauncher } from "../src/agent/claude-code/launch.ts";
|
||||
|
||||
const unsupportedShebang = process.platform === "win32" || /\s/.test(process.execPath);
|
||||
function fixture(t: TestContext, mode: "normal" | "oversize" | "hang", name = "claude") {
|
||||
const dir = mkdtempSync(join(tmpdir(), "lh-version-runner-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
const bin = join(dir, "bin"); mkdirSync(bin);
|
||||
const binary = join(bin, name);
|
||||
const body = mode === "hang" ? `
|
||||
import { writeFileSync } from "node:fs";
|
||||
writeFileSync(process.argv[1] + ".pid", String(process.pid));
|
||||
process.on("SIGTERM", () => {});
|
||||
setInterval(() => {}, 1000);
|
||||
` : mode === "oversize" ? 'process.stdout.write("x".repeat(100000));' : `
|
||||
if (JSON.stringify(process.argv.slice(2)) !== '["--version"]') process.exit(2);
|
||||
process.stdout.write("2.1.310 (Claude Code)\\n");
|
||||
`;
|
||||
writeFileSync(binary, `#!${process.execPath}\n${body}`, { mode: 0o755 });
|
||||
return { dir, binary };
|
||||
}
|
||||
|
||||
test("production version runner directly executes a spaced/metacharacter filename with only --version", { skip: unsupportedShebang }, async (t) => {
|
||||
const { binary } = fixture(t, "normal", "claude ; fixture");
|
||||
assert.equal(await readClaudeVersion(binary), "2.1.310 (Claude Code)\n");
|
||||
});
|
||||
|
||||
test("production version runner bounds stdout", { skip: unsupportedShebang }, async (t) => {
|
||||
const { binary } = fixture(t, "oversize");
|
||||
await assert.rejects(readClaudeVersion(binary), (error: NodeJS.ErrnoException) => error.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER");
|
||||
});
|
||||
|
||||
test("production version runner times out and SIGKILLs an uncooperative fixture", { skip: unsupportedShebang, timeout: 10000 }, async (t) => {
|
||||
const { binary } = fixture(t, "hang");
|
||||
const start = Date.now();
|
||||
await assert.rejects(readClaudeVersion(binary), (error: Error & { signal?: string }) => error.signal === "SIGKILL");
|
||||
assert(Date.now() - start < 8500, "bounded five-second timeout");
|
||||
const pid = Number(readFileSync(`${binary}.pid`, "utf8"));
|
||||
assert.throws(() => process.kill(pid, 0), /ESRCH/, "fixture no longer exists");
|
||||
});
|
||||
|
||||
test("relative PATH is resolved at server cwd and stays the same for probe and model launch", { skip: unsupportedShebang }, async (t) => {
|
||||
const { dir, binary } = fixture(t, "normal");
|
||||
const found = resolveClaudeBinary({ env: { PATH: "./bin" }, homeDir: dir, platform: process.platform, cwd: dir,
|
||||
isExecutable: (path) => { try { accessSync(path, constants.X_OK); return statSync(path).isFile(); } catch { return false; } },
|
||||
});
|
||||
assert.equal(found, binary);
|
||||
assert.equal(await readClaudeVersion(found!), "2.1.310 (Claude Code)\n", "probe changes cwd to tmpdir but executes the discovered file");
|
||||
let spawned = false;
|
||||
const launch = createClaudeLauncher({ findBinary: () => found,
|
||||
spawnProcess: ((path: string) => { assert.equal(path, binary); spawned = true; return null; }) as unknown as typeof spawn,
|
||||
});
|
||||
await launch(["-p"], { cwd: tmpdir(), env: { NODE_ENV: "test" } });
|
||||
assert(spawned, "same verified absolute identity passed to the model launcher");
|
||||
});
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Stand-in for the claude CLI in tests. Reads the stream-json input from
|
||||
* stdin, then replays the scenario file named by FAKE_CLAUDE_SCENARIO line by
|
||||
* line. Lines with a "fake" key are directives instead of output:
|
||||
* {"fake":"hold"} stay alive until killed, like the CLI waiting on a held MCP call
|
||||
* {"fake":"exit","code":1,"stderr":"..."} write stderr and exit with that code
|
||||
* {"fake":"call","id":"toolu_...","name":"read","arguments":{...}} call the MCP bridge and wait for its result
|
||||
* {"fake":"start-call",...} start the same request without waiting for its result
|
||||
* {"fake":"cancel-call","id":"toolu_..."} cancel an individual pending MCP request
|
||||
* {"fake":"checkpoint"} notify the parent after previous sends finish
|
||||
* {"fake":"wait-for-message"} pause until the test releases the child over IPC
|
||||
* {"fake":"ignore-term"} ignore SIGTERM to exercise kill escalation
|
||||
* {"fake":"reply"} stream one assistant message whose text is every result so far, joined by " | "
|
||||
* FAKE_CLAUDE_STDIN, when set, receives the stdin it read.
|
||||
*/
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { on } from "node:events";
|
||||
import { createInterface } from "node:readline";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
||||
|
||||
const parentMessages = on(process, "message");
|
||||
const HOLD_INTERVAL_MS = 60_000;
|
||||
const BRIDGE_SERVER_NAME = "learnharness";
|
||||
|
||||
function writeLine(line) {
|
||||
return new Promise((resolve) => process.stdout.write(`${line}\n`, resolve));
|
||||
}
|
||||
|
||||
function streamEvent(event) {
|
||||
return JSON.stringify({ type: "stream_event", event });
|
||||
}
|
||||
|
||||
let lastSend = Promise.resolve();
|
||||
|
||||
async function connectBridge(mcpConfig) {
|
||||
const { url, headers } = JSON.parse(mcpConfig).mcpServers[BRIDGE_SERVER_NAME];
|
||||
const client = new Client({ name: "fake-claude", version: "1.0.0" });
|
||||
|
||||
const transport = new StreamableHTTPClientTransport(new URL(url), { requestInit: { headers } });
|
||||
const send = transport.send.bind(transport);
|
||||
transport.send = (...args) => (lastSend = send(...args));
|
||||
await client.connect(transport);
|
||||
return client;
|
||||
}
|
||||
|
||||
async function writeReply(text) {
|
||||
await writeLine(streamEvent({ type: "message_start", message: { usage: {} } }));
|
||||
await writeLine(streamEvent({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }));
|
||||
await writeLine(streamEvent({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text } }));
|
||||
await writeLine(streamEvent({ type: "content_block_stop", index: 0 }));
|
||||
await writeLine(streamEvent({ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: {} }));
|
||||
await writeLine(streamEvent({ type: "message_stop" }));
|
||||
}
|
||||
|
||||
const mcpConfig = readFileSync(process.argv[process.argv.indexOf("--mcp-config") + 1], "utf8");
|
||||
|
||||
let input = "";
|
||||
let receiveUser;
|
||||
const userReceived = new Promise((resolve) => { receiveUser = resolve; });
|
||||
createInterface({ input: process.stdin }).on("line", async (line) => {
|
||||
input += `${line}\n`;
|
||||
const message = JSON.parse(line);
|
||||
if (message.type !== "control_request") {
|
||||
if (process.env.FAKE_CLAUDE_STDIN) writeFileSync(process.env.FAKE_CLAUDE_STDIN, input);
|
||||
receiveUser(); return;
|
||||
}
|
||||
if (message.request.subtype === "initialize") {
|
||||
if (process.env.FAKE_CLAUDE_INIT === "ignore") return;
|
||||
await writeLine(JSON.stringify({ type: "control_response", response: {
|
||||
request_id: message.request_id, subtype: process.env.FAKE_CLAUDE_INIT === "error" ? "error" : "success",
|
||||
response: {}, error: "unsupported initialize",
|
||||
} }));
|
||||
} else if (message.request.subtype === "get_context_usage") {
|
||||
await writeLine(JSON.stringify({ type: "control_response", response: {
|
||||
request_id: message.request_id, subtype: "success",
|
||||
response: { maxTokens: process.env.FAKE_CLAUDE_WINDOW ? JSON.parse(process.env.FAKE_CLAUDE_WINDOW) : null },
|
||||
} }));
|
||||
}
|
||||
});
|
||||
await userReceived;
|
||||
|
||||
let bridge = null;
|
||||
const results = [];
|
||||
const pendingResults = [];
|
||||
const controllers = new Map();
|
||||
const cancelled = new Set();
|
||||
const lines = readFileSync(process.env.FAKE_CLAUDE_SCENARIO, "utf8").split("\n").filter(Boolean);
|
||||
for (const line of lines) {
|
||||
const directive = JSON.parse(line);
|
||||
|
||||
if (directive.fake === "cancel-call") {
|
||||
cancelled.add(directive.id);
|
||||
controllers.get(directive.id)?.abort("cancelled by fake CLI");
|
||||
await lastSend;
|
||||
continue;
|
||||
}
|
||||
if (directive.fake === "checkpoint") {
|
||||
await lastSend;
|
||||
process.send?.({ fake: "checkpoint" });
|
||||
continue;
|
||||
}
|
||||
if (directive.fake === "wait-for-message") {
|
||||
await parentMessages.next();
|
||||
continue;
|
||||
}
|
||||
if (directive.fake === "hold") {
|
||||
setInterval(() => {}, HOLD_INTERVAL_MS);
|
||||
break;
|
||||
}
|
||||
|
||||
if (directive.fake === "exit") {
|
||||
process.stderr.write(directive.stderr ?? "");
|
||||
process.exit(directive.code);
|
||||
}
|
||||
|
||||
if (directive.fake === "ignore-term") {
|
||||
process.on("SIGTERM", () => {});
|
||||
continue;
|
||||
}
|
||||
if (directive.fake === "call" || directive.fake === "start-call") {
|
||||
bridge ??= await connectBridge(mcpConfig);
|
||||
const controller = new AbortController();
|
||||
controllers.set(directive.id, controller);
|
||||
const request = bridge.callTool({
|
||||
name: directive.name,
|
||||
arguments: directive.arguments,
|
||||
...(directive.id !== undefined ? { _meta: { "claudecode/toolUseId": directive.id } } : {}),
|
||||
}, undefined, { signal: controller.signal });
|
||||
if (directive.fake === "start-call") {
|
||||
pendingResults.push({ id: directive.id, request });
|
||||
void request.catch(() => {});
|
||||
await lastSend;
|
||||
} else {
|
||||
const result = await request;
|
||||
results.push(result.content.map((block) => block.text).join(""));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (directive.fake === "reply") {
|
||||
for (const result of await Promise.all(pendingResults.splice(0).filter(({ id }) => !cancelled.has(id)).map(({ request }) => request))) results.push(result.content.map((block) => block.text).join(""));
|
||||
await writeReply(results.join(" | "));
|
||||
continue;
|
||||
}
|
||||
|
||||
await writeLine(line);
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{"type":"system","subtype":"init","cwd":"/tmp/claude-code-empty","session_id":"433f00b9-adec-48b0-8eb7-c5499f88cc6b","model":"claude-haiku-5-5","tools":[],"mcp_servers":[],"apiKeySource":"none","permissionMode":"default","claude_code_version":"2.1.308","uuid":"941c8a75-e5cc-4014-93ec-4b065e619d15"}
|
||||
{"type":"system","subtype":"status","status":"requesting","session_id":"433f00b9-adec-48b0-8eb7-c5499f88cc6b","uuid":"1cbc259e-ad27-49ae-8a03-1dd0bc7d04ae"}
|
||||
{"type":"assistant","message":{"diagnostics":null,"id":"a4eeead1-04bb-47fd-9567-bebc78052a3e","container":null,"model":"<synthetic>","role":"assistant","stop_details":null,"stop_reason":"stop_sequence","stop_sequence":"","type":"message","usage":{"output_tokens_details":null,"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":null,"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":null,"iterations":null,"speed":null,"fallback_credit":null},"content":[{"type":"text","text":"Not logged in · Please run /login"}],"context_management":null},"parent_tool_use_id":null,"session_id":"433f00b9-adec-48b0-8eb7-c5499f88cc6b","uuid":"58c2fe59-d155-4d2d-a0e1-f64e6aa09199","timestamp":"2026-10-24T14:03:24.319Z","error":"authentication_failed","is_api_error_message":true}
|
||||
{"duration_api_ms":0,"stop_reason":"stop_sequence","session_id":"433f00b9-adec-48b0-8eb7-c5499f88cc6b","total_cost_usd":0,"usage":{"output_tokens_details":{"thinking_tokens":0},"input_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{},"permission_denials":[],"terminal_reason":"api_error","fast_mode_state":"off","fast_mode_disabled_reason":"sdk_opt_in_required","subagent_stats":{"spawned":0,"requested":{"background":0,"foreground":0,"unset":0},"started_in_background":0,"max_depth":0,"spawned_by_subagents":0,"completed":0,"failed":0,"killed":{"parent":0,"user":0,"system":0},"refused":{"depth_limit":0,"concurrency_limit":0,"budget":0},"by_type":{}},"safety_stops":0,"is_error":true,"num_turns":1,"subtype":"success","api_error_status":null,"result":"Not logged in · Please run /login","type":"result","duration_ms":21,"uuid":"7e41c29d-1912-4c14-8955-9f176fef725f","queued_turn_count":0,"result_index":0}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{"type":"system","subtype":"init","cwd":"/tmp/claude-code-empty","session_id":"a489e311-9d94-4990-b539-63a765dd0edd","model":"claude-haiku-5-5","tools":[],"mcp_servers":[],"apiKeySource":"none","permissionMode":"default","claude_code_version":"2.1.308","uuid":"c7e564b7-af08-4024-a89e-66736710e2c1"}
|
||||
{"type":"system","subtype":"status","status":"requesting","session_id":"a489e311-9d94-4990-b539-63a765dd0edd","uuid":"0738b666-b249-4cca-8c36-f92df8326546"}
|
||||
{"type":"stream_event","event":{"type":"message_start","message":{"model":"claude-haiku-5-5","id":"msg_011CgMBka5NLhDVNVihRY4Lm","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":607,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":607},"output_tokens":5,"service_tier":"standard","inference_geo":"global","speed":"standard"},"input_transformations":[],"diagnostics":null,"context_management":null}},"session_id":"a489e311-9d94-4990-b539-63a765dd0edd","parent_tool_use_id":null,"uuid":"512e3a21-2563-457b-addb-adade96b3d2b","ttft_ms":1224,"thinking_display":"updates","api_message_id":"msg_011CgMBka5NLhDVNVihRY4Lm"}
|
||||
{"type":"stream_event","event":{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}},"session_id":"a489e311-9d94-4990-b539-63a765dd0edd","parent_tool_use_id":null,"uuid":"0b2a75fb-819f-43ef-9fc7-839f1a5c5a01","api_message_id":"msg_011CgMBka5NLhDVNVihRY4Lm"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello there"}},"session_id":"a489e311-9d94-4990-b539-63a765dd0edd","parent_tool_use_id":null,"uuid":"85823b9b-d5e0-4973-93f9-8525a8894762","api_message_id":"msg_011CgMBka5NLhDVNVihRY4Lm"}
|
||||
{"type":"assistant","message":{"model":"claude-haiku-5-5","id":"msg_011CgMBka5NLhDVNVihRY4Lm","type":"message","role":"assistant","content":[{"type":"text","text":"hello there"}],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":607,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":607},"output_tokens":5,"service_tier":"standard","inference_geo":"global","speed":"standard"},"input_transformations":[],"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"a489e311-9d94-4990-b539-63a765dd0edd","uuid":"92acba82-60ca-4502-a5a9-bafd75bd7277","timestamp":"2026-10-24T14:02:46.324Z","request_id":"req_011CgMBkZrxiAPxuioCwrRRu"}
|
||||
{"type":"stream_event","event":{"type":"content_block_stop","index":0},"session_id":"a489e311-9d94-4990-b539-63a765dd0edd","parent_tool_use_id":null,"uuid":"1d19a483-51e6-45b6-96a2-bf445a9ab04b","api_message_id":"msg_011CgMBka5NLhDVNVihRY4Lm"}
|
||||
{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":2,"cache_creation_input_tokens":607,"cache_read_input_tokens":0,"output_tokens":5,"output_tokens_details":{"thinking_tokens":0},"iterations":[{"input_tokens":2,"output_tokens":5,"cache_read_input_tokens":0,"cache_creation_input_tokens":607,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":607},"type":"message"}]},"context_management":{"applied_edits":[]}},"session_id":"a489e311-9d94-4990-b539-63a765dd0edd","parent_tool_use_id":null,"uuid":"85ded13f-99fc-4ac2-b211-2c71a40bb817","api_message_id":"msg_011CgMBka5NLhDVNVihRY4Lm"}
|
||||
{"type":"stream_event","event":{"type":"message_stop"},"session_id":"a489e311-9d94-4990-b539-63a765dd0edd","parent_tool_use_id":null,"uuid":"d9085ac3-b00c-4e22-a17e-0fafcf7782a2","api_message_id":"msg_011CgMBka5NLhDVNVihRY4Lm"}
|
||||
{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"679f75df-0571-4b65-bc3e-c4cee96c7061","session_id":"a489e311-9d94-4990-b539-63a765dd0edd"}
|
||||
{"duration_api_ms":1236,"stop_reason":"end_turn","session_id":"a489e311-9d94-4990-b539-63a765dd0edd","total_cost_usd":0.0001241,"usage":{"input_tokens":2,"cache_creation_input_tokens":607,"cache_read_input_tokens":0,"output_tokens":5,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":607,"ephemeral_5m_input_tokens":0},"inference_geo":"global","iterations":[{"input_tokens":2,"output_tokens":5,"cache_read_input_tokens":0,"cache_creation_input_tokens":607,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":607},"type":"message"}],"speed":"standard","fallback_credit":null},"modelUsage":{"claude-haiku-5-5":{"inputTokens":2,"outputTokens":5,"cacheReadInputTokens":0,"cacheCreationInputTokens":607,"webSearchRequests":0,"costUSD":0.0001241,"contextWindow":1000000,"maxOutputTokens":128000,"thinkingTokens":0,"canonicalModel":"claude-haiku-5-5","provider":"firstParty","costBasis":"list"}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","fast_mode_disabled_reason":"sdk_opt_in_required","subagent_stats":{"spawned":0,"requested":{"background":0,"foreground":0,"unset":0},"started_in_background":0,"max_depth":0,"spawned_by_subagents":0,"completed":0,"failed":0,"killed":{"parent":0,"user":0,"system":0},"refused":{"depth_limit":0,"concurrency_limit":0,"budget":0},"by_type":{}},"safety_stops":0,"is_error":false,"num_turns":1,"subtype":"success","api_error_status":null,"result":"hello there","ttft_ms":1245,"type":"result","duration_ms":1258,"uuid":"faf44ee1-84e1-40eb-914e-654abb6c2286","ttft_stream_ms":1244,"time_to_request_ms":21,"first_content_frame_ms":1245,"queued_turn_count":0,"result_index":0}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{"type":"system","subtype":"init","cwd":"/tmp/claude-code-empty","session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","model":"claude-haiku-5-5","tools":["mcp__learnharness__get_weather"],"mcp_servers":[{"name":"learnharness","status":"connected","source":"dynamic"}],"apiKeySource":"none","permissionMode":"default","claude_code_version":"2.1.308","uuid":"9524e7cd-f580-4019-8e63-861e36aa190d"}
|
||||
{"type":"system","subtype":"status","status":"requesting","session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","uuid":"6a55f520-5449-4845-bfe7-d3d74e4f5dda"}
|
||||
{"type":"stream_event","event":{"type":"message_start","message":{"model":"claude-haiku-5-5","id":"msg_011CgMBnEgMnaPp7zJQPnYig","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":1102,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":1102},"output_tokens":32,"service_tier":"standard","inference_geo":"global","speed":"standard"},"input_transformations":[],"diagnostics":null,"context_management":null}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"16a84391-efcc-4221-9029-1fbea99edbab","ttft_ms":1170,"thinking_display":"updates","api_message_id":"msg_011CgMBnEgMnaPp7zJQPnYig"}
|
||||
{"type":"stream_event","event":{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01JBB4cJsPa585FWb1XrfbNX","name":"mcp__learnharness__get_weather","input":{},"caller":{"type":"direct"}}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"460ff54c-2ab3-46a0-b2bf-c0d780d04e58","api_message_id":"msg_011CgMBnEgMnaPp7zJQPnYig"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":""}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"e73df731-5ecc-439c-811a-7b6cc5219a7f","api_message_id":"msg_011CgMBnEgMnaPp7zJQPnYig"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\": \"Paris"}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"1ae8e01b-8cfe-4282-a008-a9ef3f2e05fa","api_message_id":"msg_011CgMBnEgMnaPp7zJQPnYig"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"}"}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"b7cf01be-dc67-463e-ad06-19267ea81cc3","api_message_id":"msg_011CgMBnEgMnaPp7zJQPnYig"}
|
||||
{"type":"assistant","message":{"model":"claude-haiku-5-5","id":"msg_011CgMBnEgMnaPp7zJQPnYig","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01JBB4cJsPa585FWb1XrfbNX","name":"mcp__learnharness__get_weather","input":{"city":"Paris"},"caller":{"type":"direct"}}],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":1102,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":1102},"output_tokens":32,"service_tier":"standard","inference_geo":"global","speed":"standard"},"input_transformations":[],"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","uuid":"da987ff6-9b30-4c1e-8f5b-20bda7386663","timestamp":"2026-10-24T14:03:09.115Z","request_id":"req_011CgMBnETxUPY4jwbRZ3rFy","wire_tool_inputs":{"toolu_01JBB4cJsPa585FWb1XrfbNX":{"city":"Paris"}},"tool_use_meta":[{"id":"toolu_01JBB4cJsPa585FWb1XrfbNX","display_name":"Get Weather","server_display_name":"learnharness","tool_name":"get_weather"}]}
|
||||
{"type":"stream_event","event":{"type":"content_block_stop","index":0},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"f438dbab-5bac-4b68-a237-ef9564abd25c","api_message_id":"msg_011CgMBnEgMnaPp7zJQPnYig"}
|
||||
{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01JBB4cJsPa585FWb1XrfbNX","type":"tool_result","content":[{"type":"text","text":"Sunny, 21C in Paris"}]}]},"parent_tool_use_id":null,"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","uuid":"f44480b5-8e02-4958-888e-fbbe3b54dc80","timestamp":"2026-10-24T14:03:09.124Z","tool_use_result":[{"type":"text","text":"Sunny, 21C in Paris"}],"tool_result_meta":[{"id":"toolu_01JBB4cJsPa585FWb1XrfbNX","permission_decision":{"decision":"accept","source":"config","reason_type":"rule"}}]}
|
||||
{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":2,"cache_creation_input_tokens":1102,"cache_read_input_tokens":0,"output_tokens":59,"output_tokens_details":{"thinking_tokens":0},"iterations":[{"input_tokens":2,"output_tokens":59,"cache_read_input_tokens":0,"cache_creation_input_tokens":1102,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":1102},"type":"message"}]},"context_management":{"applied_edits":[]}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"7fec4e77-f338-4f24-810c-d6fa959d154f","api_message_id":"msg_011CgMBnEgMnaPp7zJQPnYig"}
|
||||
{"type":"stream_event","event":{"type":"message_stop"},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"5d3b3311-08d0-4af6-be82-b0d3f472e99d","api_message_id":"msg_011CgMBnEgMnaPp7zJQPnYig"}
|
||||
{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"f21e5357-4ffc-4346-b9fc-6543e910eab3","session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038"}
|
||||
{"type":"system","subtype":"status","status":"requesting","session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","uuid":"22297606-28e0-4b2f-9208-bc4be5487a80"}
|
||||
{"type":"stream_event","event":{"type":"message_start","message":{"model":"claude-haiku-5-5","id":"msg_011CgMBnLsfzVJKssny6FQZm","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":100,"cache_read_input_tokens":1102,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":100},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"input_transformations":[],"diagnostics":null,"context_management":null}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"90d16779-8e65-40d2-a55e-a68a9786a927","ttft_ms":1274,"thinking_display":"updates","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"stream_event","event":{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"1f8d1832-7837-444e-a07e-2598927888bb","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"It"}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"29baece7-f120-4088-8901-9ee4c16f9ffa","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s sunny in Paris"}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"7189b574-320a-4730-9cf7-c92c089bdf95","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" right"}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"1db451a5-7027-4b25-bb69-2b8b6b94d600","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" now, with a temperature of"}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"7ca48472-b631-4117-a204-9f6fe7854f4a","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" 21°C."}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"86930cda-4e2b-405c-a45f-29ee42d2efca","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"assistant","message":{"model":"claude-haiku-5-5","id":"msg_011CgMBnLsfzVJKssny6FQZm","type":"message","role":"assistant","content":[{"type":"text","text":"It's sunny in Paris right now, with a temperature of 21°C."}],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":100,"cache_read_input_tokens":1102,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":100},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"input_transformations":[],"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","uuid":"69cad926-aa35-4a82-a24c-599cc5422d99","timestamp":"2026-10-24T14:03:10.497Z","request_id":"req_011CgMBnLc4EeExeJ32dkAoH"}
|
||||
{"type":"stream_event","event":{"type":"content_block_stop","index":0},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"c05c7c31-f90a-4f4b-8a5b-31f2dae60294","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":2,"cache_creation_input_tokens":100,"cache_read_input_tokens":1102,"output_tokens":25,"output_tokens_details":{"thinking_tokens":0},"iterations":[{"input_tokens":2,"output_tokens":25,"cache_read_input_tokens":1102,"cache_creation_input_tokens":100,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":100},"type":"message"}]},"context_management":{"applied_edits":[]}},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"6193aeb9-61b1-41e8-959a-9ae646429d11","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"type":"stream_event","event":{"type":"message_stop"},"session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","parent_tool_use_id":null,"uuid":"22fbda30-76c1-4527-9819-cc3d00fb4e68","api_message_id":"msg_011CgMBnLsfzVJKssny6FQZm"}
|
||||
{"duration_api_ms":2827,"stop_reason":"end_turn","session_id":"0385b444-f8b1-47bd-b1f9-addd4b300038","total_cost_usd":0.00029382000000000003,"usage":{"input_tokens":4,"cache_creation_input_tokens":1202,"cache_read_input_tokens":1102,"output_tokens":84,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":1202,"ephemeral_5m_input_tokens":0},"inference_geo":"global","iterations":[{"input_tokens":2,"output_tokens":25,"cache_read_input_tokens":1102,"cache_creation_input_tokens":100,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":100},"type":"message"}],"speed":"standard","fallback_credit":null},"modelUsage":{"claude-haiku-5-5":{"inputTokens":4,"outputTokens":84,"cacheReadInputTokens":1102,"cacheCreationInputTokens":1202,"webSearchRequests":0,"costUSD":0.00029382000000000003,"contextWindow":1000000,"maxOutputTokens":128000,"thinkingTokens":0,"canonicalModel":"claude-haiku-5-5","provider":"firstParty","costBasis":"list"}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","fast_mode_disabled_reason":"sdk_opt_in_required","subagent_stats":{"spawned":0,"requested":{"background":0,"foreground":0,"unset":0},"started_in_background":0,"max_depth":0,"spawned_by_subagents":0,"completed":0,"failed":0,"killed":{"parent":0,"user":0,"system":0},"refused":{"depth_limit":0,"concurrency_limit":0,"budget":0},"by_type":{}},"safety_stops":0,"is_error":false,"num_turns":2,"subtype":"success","api_error_status":null,"result":"It's sunny in Paris right now, with a temperature of 21°C.","ttft_ms":1457,"type":"result","duration_ms":2854,"uuid":"0be743a1-83e1-45ab-96f5-926ffee7a85b","ttft_stream_ms":1191,"time_to_request_ms":21,"first_content_frame_ms":1191,"queued_turn_count":0,"result_index":0}
|
||||
@@ -1,6 +1,13 @@
|
||||
import { describe, it } from "node:test";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
const testData = fs.mkdtempSync(path.join(os.tmpdir(), "lh-cli-catalog-data-"));
|
||||
process.env.LH_DATA_DIR = testData;
|
||||
after(() => fs.rmSync(testData, { recursive: true, force: true }));
|
||||
const {
|
||||
catalogContextLimits,
|
||||
catalogDefaultsFor,
|
||||
catalogProviders,
|
||||
catalogRolePins,
|
||||
@@ -12,16 +19,18 @@ import {
|
||||
toModelDto,
|
||||
validateChatSettings,
|
||||
validateModelsSettings,
|
||||
} from "../src/server/model-catalog.ts";
|
||||
} = await import("../src/server/model-catalog.ts");
|
||||
import type { EffectiveSlot } from "../src/core/settings-resolution.ts";
|
||||
import type { SettingsValidation } from "../src/server/model-catalog.ts";
|
||||
import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai";
|
||||
import { DEFAULT_THINKING_ORDER } from "../src/core/settings.ts";
|
||||
import { piEngineForSpec, piModels } from "../src/agent/models.ts";
|
||||
const { piEngineForSpec, piModels } = await import("../src/agent/models.ts");
|
||||
import { BACKGROUND_MODEL_ROLES } from "../src/core/settings.ts";
|
||||
import { resolveSettings } from "../src/core/settings-resolution.ts";
|
||||
import { readSettings } from "../src/store/settings.ts";
|
||||
import { effectiveSettings } from "../src/server/stores.ts";
|
||||
const { readSettings, writeSettings } = await import("../src/store/settings.ts");
|
||||
const { SETTINGS_FILE } = await import("../config.ts");
|
||||
const { effectiveSettings } = await import("../src/server/stores.ts");
|
||||
const { providerKeyPresent } = await import("../src/server/keys.ts");
|
||||
|
||||
// The model-control wave (2026-09-12): pi-ai owns the registry — these pins
|
||||
// hold our projection + validation honest against the shipped catalog, and
|
||||
@@ -59,8 +68,10 @@ describe("the model catalog projection", () => {
|
||||
assert.equal(typeof m.id, "string");
|
||||
assert.equal(typeof m.name, "string");
|
||||
assert.ok(Array.isArray(m.supportedLevels));
|
||||
assert.ok(typeof m.cost.input === "number" && typeof m.cost.output === "number");
|
||||
assert.ok(m.contextWindow > 0);
|
||||
if (m.provider === "claude-code") assert.equal(m.cost, null);
|
||||
else assert.ok(m.cost && typeof m.cost.input === "number" && typeof m.cost.output === "number");
|
||||
if (m.provider === "claude-code") assert.equal(m.contextWindow, null);
|
||||
else assert.ok(m.contextWindow !== null && m.contextWindow > 0);
|
||||
}
|
||||
const glm = models.find((m) => m.id === "z-ai/glm-5.3-flash");
|
||||
assert.ok(glm, "glm-5.3-flash in catalog");
|
||||
@@ -121,6 +132,7 @@ describe("the model catalog projection", () => {
|
||||
assert.equal(dto.id, raw.id);
|
||||
assert.equal(dto.name, raw.name);
|
||||
assert.deepEqual(dto.supportedLevels, getSupportedThinkingLevels(raw));
|
||||
assert.ok(dto.cost);
|
||||
assert.equal(dto.cost.input, raw.cost.input);
|
||||
assert.equal(dto.contextWindow, raw.contextWindow);
|
||||
});
|
||||
@@ -326,4 +338,81 @@ describe("pi thinking truth (pinned against pi-ai upgrades)", () => {
|
||||
const { piEngineFor } = await import("../src/agent/models.ts");
|
||||
assert.throws(() => piEngineFor("prober"), /no configured model for role "prober"/);
|
||||
});
|
||||
});
|
||||
});
|
||||
/** Runs fn with PATH and HOME pointed at a temp dir, so the CLI lookup only
|
||||
* finds a claude binary when installed is true. */
|
||||
function withClaudeInstalled(installed: boolean, fn: () => void, enabled = true): void {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "claude-catalog-"));
|
||||
const saved = { PATH: process.env.PATH, HOME: process.env.HOME, npm_config_prefix: process.env.npm_config_prefix };
|
||||
const settings = readSettings();
|
||||
writeSettings({ ...(settings ?? { version: 1 }), claudeCode: { enabled } });
|
||||
|
||||
if (installed) fs.writeFileSync(path.join(dir, "claude"), "#!/bin/sh\n", { mode: 0o755 });
|
||||
process.env.PATH = dir;
|
||||
process.env.HOME = dir;
|
||||
delete process.env.npm_config_prefix;
|
||||
|
||||
try {
|
||||
fn();
|
||||
} finally {
|
||||
for (const [name, value] of Object.entries(saved)) {
|
||||
if (value === undefined) delete process.env[name];
|
||||
else process.env[name] = value;
|
||||
}
|
||||
if (settings) writeSettings(settings);
|
||||
else fs.rmSync(SETTINGS_FILE, { force: true });
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe("the claude-code provider (native CLI authentication, no LearnHarness key)", () => {
|
||||
it("modelCatalog() lists claude-code opus/sonnet/haiku when the CLI is installed", () => {
|
||||
withClaudeInstalled(true, () => {
|
||||
const ids = modelCatalog()
|
||||
.models.filter((m) => m.provider === "claude-code")
|
||||
.map((m) => m.id);
|
||||
assert.deepEqual(ids, ["haiku", "opus", "sonnet"]);
|
||||
});
|
||||
});
|
||||
|
||||
it("catalogProviders() marks claude-code present when the CLI is installed", () => {
|
||||
withClaudeInstalled(true, () => {
|
||||
const row = catalogProviders().find((p) => p.id === "claude-code");
|
||||
assert.deepEqual(row, { id: "claude-code", envVar: "", keyPresent: true });
|
||||
});
|
||||
});
|
||||
|
||||
it("catalogProviders() marks claude-code absent and modelCatalog() hides it when the CLI is missing", () => {
|
||||
withClaudeInstalled(false, () => {
|
||||
const row = catalogProviders().find((p) => p.id === "claude-code");
|
||||
assert.equal(row?.keyPresent, false);
|
||||
assert.equal(
|
||||
modelCatalog().models.some((m) => m.provider === "claude-code"),
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("catalogContextLimits() never advertises placeholder CLI limits, even when installed", () => {
|
||||
withClaudeInstalled(true, () => {
|
||||
assert.equal(catalogContextLimits().has("claude-code:opus"), false);
|
||||
});
|
||||
});
|
||||
|
||||
it('piEngineForSpec({ provider: "claude-code", id: "sonnet" }) resolves', () => {
|
||||
const engine = piEngineForSpec({ provider: "claude-code", id: "sonnet" });
|
||||
assert.equal(engine.model.provider, "claude-code");
|
||||
assert.equal(engine.model.id, "sonnet");
|
||||
});
|
||||
|
||||
it('providerKeyPresent("claude-code") requires both explicit opt-in and executable presence', () => {
|
||||
withClaudeInstalled(true, () => assert.equal(providerKeyPresent("claude-code"), true));
|
||||
withClaudeInstalled(false, () => assert.equal(providerKeyPresent("claude-code"), false));
|
||||
withClaudeInstalled(true, () => {
|
||||
assert.equal(providerKeyPresent("claude-code"), false);
|
||||
assert.equal(catalogProviders().find((p) => p.id === "claude-code")?.keyPresent, false);
|
||||
assert(!modelCatalog().models.some((m) => m.provider === "claude-code"));
|
||||
assert.equal(validateChatSettings({ model: { provider: "claude-code", id: "sonnet" } }).ok, false);
|
||||
}, false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,7 +10,7 @@ const models = piModels().getProviders().flatMap((p) => piModels().getModels(p.i
|
||||
const nativeUsage = (input: number): Usage => ({ input, output: 1_000, cacheRead: 10_000, cacheWrite: 2_000, cacheWrite1h: 500, totalTokens: input + 13_000, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } });
|
||||
const saved = (u: Usage) => ({ inputTokens: u.input, outputTokens: u.output, cacheReadTokens: u.cacheRead, cacheWriteTokens: u.cacheWrite, cacheWrite1hTokens: u.cacheWrite1h, reasoningTokens: 500 });
|
||||
|
||||
test("all catalog models price through Pi, including every tier boundary; saved usage is never mutated", () => {
|
||||
test("API catalog models price through Pi at every tier boundary; native CLI stays unpriced and saved usage is never mutated", () => {
|
||||
assert.ok(models.length > 100);
|
||||
for (const model of models) {
|
||||
const sizes = [1_000, ...(model.cost.tiers ?? []).flatMap((t) => [Math.max(0, t.inputTokensAbove - 12_000), Math.max(0, t.inputTokensAbove - 12_000 + 1)])];
|
||||
@@ -18,7 +18,7 @@ test("all catalog models price through Pi, including every tier boundary; saved
|
||||
const usage = nativeUsage(input);
|
||||
const call = Object.freeze(saved(usage));
|
||||
const before = structuredClone(call);
|
||||
const expected = calculateCost(model, usage).total;
|
||||
const expected = model.provider === "claude-code" ? null : calculateCost(model, usage).total;
|
||||
assert.equal(callCost(call, `${model.provider}:${model.id}`), expected, `${model.provider}:${model.id} input=${input}`);
|
||||
assert.deepEqual(call, before);
|
||||
}
|
||||
|
||||
@@ -88,6 +88,9 @@ describe("send_to_user model gate", () => {
|
||||
["openrouter", "anthropic/claude-sonnet-5.5", true],
|
||||
["amazon-bedrock", "us.anthropic.claude-opus-5-5-v1:0", true],
|
||||
["fixture", "CLAUDE-fixture", true],
|
||||
["claude-code", "opus", true],
|
||||
["claude-code", "sonnet", true],
|
||||
["claude-code", "haiku", true],
|
||||
["openrouter", "z-ai/glm-5.3-flash", false],
|
||||
["openai", "gpt-4o-mini", false],
|
||||
["anthropic", "opaque-alias", false],
|
||||
|
||||
@@ -17,6 +17,7 @@ import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { externalCliEnv } from "../src/core/auth-env.ts";
|
||||
|
||||
// THIS PROCESS'S ENV IS THE TEST SUBJECT — strip any inherited catalog vars
|
||||
// (a developer shell may export REAL keys; tests use throwaway values only,
|
||||
@@ -72,10 +73,12 @@ describe("boot fill-gap + precedence", () => {
|
||||
assert.ok(row.shadowedByEnv, "file value differs from the env value");
|
||||
assert.equal(row.masked, `…${ENV_OWNED.slice(-4)} (${ENV_OWNED.length} chars)`);
|
||||
assert.equal(authStore.authVars().OPENROUTER_API_KEY, "sk-or-v1-filefilefile", "file copy untouched");
|
||||
assert.equal(externalCliEnv().OPENROUTER_API_KEY, ENV_OWNED, "external CLIs retain launch-env ownership");
|
||||
});
|
||||
|
||||
it("unknown file var: fill-gapped into env, surfaced as a generic row", () => {
|
||||
assert.equal(process.env.FUTURE_PLANS_TOKEN, "opaque-plans-token", "reflected at boot");
|
||||
assert.equal(externalCliEnv().FUTURE_PLANS_TOKEN, undefined, "external CLIs exclude unknown file-only injections too");
|
||||
const row = keyStatuses().find((s) => s.id === "FUTURE_PLANS_TOKEN")!;
|
||||
assert.equal(row.info, null, "no curated form");
|
||||
assert.equal(row.source, "file");
|
||||
@@ -103,6 +106,7 @@ describe("saveKey / deleteKey (immediate effect, no restart)", () => {
|
||||
const r = saveKey("DEEPGRAM_API_KEY", value);
|
||||
assert.ok(r.ok);
|
||||
assert.equal(process.env.DEEPGRAM_API_KEY, value, "env updated immediately");
|
||||
assert.equal(externalCliEnv().DEEPGRAM_API_KEY, undefined);
|
||||
assert.equal(authStore.authVars().DEEPGRAM_API_KEY, value, "file updated");
|
||||
assert.equal(r.status.source, "file", "the reflected copy is the store's — the file is its origin of truth");
|
||||
assert.ok(!r.status.shadowedByEnv);
|
||||
@@ -112,6 +116,7 @@ describe("saveKey / deleteKey (immediate effect, no restart)", () => {
|
||||
const r = saveKey("OPENROUTER_API_KEY", "sk-or-v1-newvalue12345");
|
||||
assert.ok(r.ok);
|
||||
assert.equal(process.env.OPENROUTER_API_KEY, ENV_OWNED, "real env untouched");
|
||||
assert.equal(externalCliEnv().OPENROUTER_API_KEY, ENV_OWNED, "saving a shadowed key cannot remove real env ownership");
|
||||
assert.equal(authStore.authVars().OPENROUTER_API_KEY, "sk-or-v1-newvalue12345", "file updated for later");
|
||||
assert.ok(r.status.shadowedByEnv);
|
||||
});
|
||||
|
||||
@@ -239,7 +239,7 @@ describe("SETTINGS_SECTIONS (the route's section map)", () => {
|
||||
// threshold is engine behavior, not an agent-section key.
|
||||
assert.deepEqual(
|
||||
Object.keys(SETTINGS_SECTIONS),
|
||||
["general", "chat", "agent", "appearance", "graph", "compaction", "models", "dictation"],
|
||||
["general", "chat", "agent", "appearance", "graph", "compaction", "models", "dictation", "claudeCode"],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user