Wave A: pi-native tool layer — delete LhTool/toPiTool, TypeBox everywhere, terminate on questions
- src/agent/lh-tool.ts deleted: the zod-wrapped LhTool + toPiTool bridge (z.toJSONSchema per call, clientQuestion flag, hand-rolled result mapping) is gone. Tools are pi-agent-core native AgentTools now. - New src/agent/tool-execute.ts owns what the wrapper added: the abort guard (runWithAbortGuard: run signal + 30s per-call timeout + 250ms grace, 'Operation aborted' / timeout wording preserved), ToolInputError (the retryable '- problem at path' format), result mapping (string verbatim / object JSON / image parts, details rides the record's tool part output), and questionResult() — placeholder + terminate:true (owner Option A: belt of terminate + slim makeQuestionStopper with the clean-question rule; maxTurns deleted from the teaching turn entirely per owner). - All 11 tool files + tools.ts registries converted to TypeBox parameters (typebox@1.3.7, pinned to pi-agent-core's copy). zod refinements (quizInputProblems, askOptionProblems, live-nodeId) moved from schemas into execute — the model sees the same retryable error shape. - question-core + contracts/probe TypeBox-authored (single source shared with the UI); tool-parts parse seam runs the boundary's full chain (Convert → Check → Clean → refinement), fixing the live-smoke lockout: pi's boundary Value.Converts correctAnswer: 17 → '17' but the record keeps raw args, so a parse without Convert rejected a posed quiz (invisible pending question, turn ended). Pinned with the real smoke emission shape. - Subagents converted: demo-builder/researcher AgentTool-native, visual-maker/pdf-reader declaration-only pi-ai Tools with local execute fns, extraction submit schemas re-authored in TypeBox (owner option 2: zod defaults → Type.Optional + fill at the submit seam). - Accepted delta, flagged: TypeBox-native schemas skip pi's JSON-Schema primitive coercion fast path... (boundary still Value.Converts), and prober-quiz unknown keys ride through (Value.Clean strips at display). - Tests: 988 passing (was 967 at the recon baseline); tool-abort and read-tool rewritten against the native seams; question-core/question- tools/tool-parts/demos/web-search/graph-tools/plan-tree/probe-done/ tool-toggles converted to Value.Check/execute pins. Live smoke (PORT=3004, real model): native ls turn streams end-to-end; a real quiz pose (first attempt missing correctAnswer → tool-input-error → model retry → posed) terminates the turn cleanly and persists pending.
This commit is contained in:
@@ -28,6 +28,7 @@ import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { Value } from "typebox/value";
|
||||
import { createFauxCore, fauxAssistantMessage, fauxToolCall, type FauxResponseStep } from "@earendil-works/pi-ai";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { PATHS } from "../config.ts";
|
||||
@@ -845,13 +846,13 @@ describe("tool + renderer registry wiring", () => {
|
||||
for (const name of ["demo", "visual", "research", "pdf"] as const) {
|
||||
assert.equal(typeof (tools[name] as { execute?: unknown }).execute, "function", `${name} executes server-side — fire-and-forget ack per THE ASYNC PROTOCOL`);
|
||||
}
|
||||
assert.equal(demoInputSchema.safeParse({ title: "T", brief: "b" }).success, true);
|
||||
assert.equal(demoInputSchema.safeParse({ brief: "b" }).success, false, "demo requires a title");
|
||||
assert.equal(visualInputSchema.safeParse({ brief: "b" }).success, true);
|
||||
assert.equal(visualInputSchema.safeParse({}).success, false, "visual requires a brief");
|
||||
assert.equal(pdfInputSchema.safeParse({ path: "/tmp/doc.pdf", question: "q" }).success, true);
|
||||
assert.equal(pdfInputSchema.safeParse({ question: "q" }).success, false, "pdf requires a path");
|
||||
assert.equal(pdfInputSchema.safeParse({ path: "/tmp/doc.pdf" }).success, false, "pdf requires a question");
|
||||
assert.equal(Value.Check(demoInputSchema, { title: "T", brief: "b" }), true);
|
||||
assert.equal(Value.Check(demoInputSchema, { brief: "b" }), false, "demo requires a title");
|
||||
assert.equal(Value.Check(visualInputSchema, { brief: "b" }), true);
|
||||
assert.equal(Value.Check(visualInputSchema, {}), false, "visual requires a brief");
|
||||
assert.equal(Value.Check(pdfInputSchema, { path: "/tmp/doc.pdf", question: "q" }), true);
|
||||
assert.equal(Value.Check(pdfInputSchema, { question: "q" }), false, "pdf requires a path");
|
||||
assert.equal(Value.Check(pdfInputSchema, { path: "/tmp/doc.pdf" }), false, "pdf requires a question");
|
||||
});
|
||||
|
||||
it("the artifact renderer registry maps demo/visual to the W6 cards", () => {
|
||||
|
||||
+31
-24
@@ -4,6 +4,7 @@ import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, it } from "node:test";
|
||||
|
||||
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import { expandBraces, globToRegex, matchGlob, walkFiles } from "../src/agent/tools/fs-shared.ts";
|
||||
import { findTool } from "../src/agent/tools/find.ts";
|
||||
import { grepTool } from "../src/agent/tools/grep.ts";
|
||||
@@ -28,6 +29,15 @@ import { readTool } from "../src/agent/tools/read.ts";
|
||||
|
||||
let FIXTURE: string;
|
||||
|
||||
/** Drive one tool's native execute and return the model-facing text
|
||||
* (pi signature: execute(toolCallId, params, signal) → AgentToolResult). */
|
||||
async function runText(tool: AgentTool<any>, params: unknown, signal?: AbortSignal): Promise<string> {
|
||||
const result = await tool.execute!("c", params, signal);
|
||||
const first = result.content[0];
|
||||
assert.ok(first !== undefined && first.type === "text", "first content part is text");
|
||||
return (first as { type: "text"; text: string }).text;
|
||||
}
|
||||
|
||||
async function setup(): Promise<string> {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "lh-fs-"));
|
||||
await fs.mkdir(path.join(root, "notes", "sub"), { recursive: true });
|
||||
@@ -79,7 +89,7 @@ describe("glob helpers (fs-shared)", () => {
|
||||
describe("ls (pi port)", () => {
|
||||
it("lists sorted, dirs suffixed, dotfiles included", async () => {
|
||||
FIXTURE = await setup();
|
||||
const out = (await lsTool.execute!({ path: FIXTURE }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(lsTool, { path: FIXTURE });
|
||||
assert.ok(out.includes("code/"));
|
||||
assert.ok(out.includes("notes/"));
|
||||
assert.ok(out.includes("binary.bin"));
|
||||
@@ -90,14 +100,14 @@ describe("ls (pi port)", () => {
|
||||
});
|
||||
|
||||
it("honors the entry limit with pi's truncation notice", async () => {
|
||||
const out = (await lsTool.execute!({ path: FIXTURE, limit: 2 }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(lsTool, { path: FIXTURE, limit: 2 });
|
||||
assert.match(out, /\[Truncated: 2 entries limit\]/);
|
||||
});
|
||||
|
||||
it("missing path and non-directory throw (the model sees the error)", async () => {
|
||||
await assert.rejects(async () => lsTool.execute!({ path: "/nonexistent/xyz" }, { toolCallId: "c" }), /Path not found/);
|
||||
await assert.rejects(async () => lsTool.execute!("c", { path: "/nonexistent/xyz" }), /Path not found/);
|
||||
await assert.rejects(
|
||||
async () => lsTool.execute!({ path: path.join(FIXTURE, "binary.bin") }, { toolCallId: "c" }),
|
||||
async () => lsTool.execute!("c", { path: path.join(FIXTURE, "binary.bin") }),
|
||||
/Not a directory/,
|
||||
);
|
||||
});
|
||||
@@ -105,31 +115,31 @@ describe("ls (pi port)", () => {
|
||||
|
||||
describe("find (pi port)", () => {
|
||||
it("finds by glob, relativized to the search root", async () => {
|
||||
const out = (await findTool.execute!({ pattern: "*.md", path: FIXTURE }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(findTool, { pattern: "*.md", path: FIXTURE });
|
||||
assert.ok(out.includes("notes/calculus.md"));
|
||||
assert.ok(out.includes("notes/sub/hidden.md"));
|
||||
assert.ok(!out.includes(FIXTURE), "paths are relative, not absolute");
|
||||
});
|
||||
|
||||
it("never enters node_modules or Library", async () => {
|
||||
const out = (await findTool.execute!({ pattern: "*.txt", path: FIXTURE }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(findTool, { pattern: "*.txt", path: FIXTURE });
|
||||
assert.ok(!out.includes("node_modules"));
|
||||
assert.ok(!out.includes("Library"));
|
||||
assert.ok(out.includes("notes/linear.txt"));
|
||||
});
|
||||
|
||||
it("result limit carries pi's notice", async () => {
|
||||
const out = (await findTool.execute!({ pattern: "*", path: FIXTURE, limit: 2 }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(findTool, { pattern: "*", path: FIXTURE, limit: 2 });
|
||||
assert.match(out, /\[2 results limit reached\]/);
|
||||
});
|
||||
|
||||
it("no-match message matches pi", async () => {
|
||||
const out = (await findTool.execute!({ pattern: "*.nomatch", path: FIXTURE }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(findTool, { pattern: "*.nomatch", path: FIXTURE });
|
||||
assert.equal(out, "No files found matching pattern");
|
||||
});
|
||||
|
||||
it("slash patterns match paths, not basenames", async () => {
|
||||
const out = (await findTool.execute!({ pattern: "notes/**/*.md", path: FIXTURE }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(findTool, { pattern: "notes/**/*.md", path: FIXTURE });
|
||||
assert.ok(out.includes("notes/sub/hidden.md"));
|
||||
assert.ok(!out.includes("code/"));
|
||||
});
|
||||
@@ -139,7 +149,7 @@ describe("grep (pi port)", () => {
|
||||
it("formats matches pi-style: rel:line: text", async () => {
|
||||
// case-sensitive: the lowercase "eigenvector" hits main.ts but NOT
|
||||
// linear.txt's capital-E "Eigenvectors" (that case lives below).
|
||||
const out = (await grepTool.execute!({ pattern: "eigenvector", path: FIXTURE }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(grepTool, { pattern: "eigenvector", path: FIXTURE });
|
||||
assert.equal(out, "code/main.ts:2: const eigenvector = 'here';");
|
||||
assert.ok(!out.includes("node_modules"), "excluded dirs are not searched");
|
||||
assert.ok(!out.includes("Library"));
|
||||
@@ -147,42 +157,42 @@ describe("grep (pi port)", () => {
|
||||
});
|
||||
|
||||
it("ignoreCase and literal flags", async () => {
|
||||
const icase = (await grepTool.execute!({ pattern: "EIGENVECTOR", path: FIXTURE, ignoreCase: true }, { toolCallId: "c" })) as string;
|
||||
const icase = await runText(grepTool, { pattern: "EIGENVECTOR", path: FIXTURE, ignoreCase: true });
|
||||
assert.match(icase, /linear\.txt:1:/);
|
||||
const literal = (await grepTool.execute!({ pattern: "e.gen", literal: true, path: path.join(FIXTURE, "code") }, { toolCallId: "c" })) as string;
|
||||
const literal = await runText(grepTool, { pattern: "e.gen", literal: true, path: path.join(FIXTURE, "code") });
|
||||
assert.equal(literal, "No matches found", "dots are literal when literal=true");
|
||||
});
|
||||
|
||||
it("context lines use the dash format", async () => {
|
||||
const out = (await grepTool.execute!({ pattern: "scaling", path: FIXTURE, context: 1 }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(grepTool, { pattern: "scaling", path: FIXTURE, context: 1 });
|
||||
assert.match(out, /notes\/linear\.txt:2: only scaling remains/, "the match line uses colons");
|
||||
assert.match(out, /notes\/linear\.txt-1- Eigenvectors keep their direction/, "context before uses dashes");
|
||||
assert.match(out, /notes\/linear\.txt-3- $/, "context after (empty line) uses dashes");
|
||||
});
|
||||
|
||||
it("glob filter restricts files", async () => {
|
||||
const out = (await grepTool.execute!({ pattern: "eigenvector", path: FIXTURE, glob: "*.ts" }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(grepTool, { pattern: "eigenvector", path: FIXTURE, glob: "*.ts" });
|
||||
assert.match(out, /main\.ts:2:/);
|
||||
assert.ok(!out.includes("linear.txt"));
|
||||
});
|
||||
|
||||
it("single-file search prints basename; match limit notices pi-style", async () => {
|
||||
const file = path.join(FIXTURE, "notes", "calculus.md");
|
||||
const out = (await grepTool.execute!({ pattern: "e", path: file }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(grepTool, { pattern: "e", path: file });
|
||||
assert.match(out, /^calculus\.md:1: /, "basename for a single-file search");
|
||||
const limited = (await grepTool.execute!({ pattern: "e", path: file, limit: 1 }, { toolCallId: "c" })) as string;
|
||||
const limited = await runText(grepTool, { pattern: "e", path: file, limit: 1 });
|
||||
assert.match(limited, /\[1 matches limit reached\]/);
|
||||
});
|
||||
|
||||
it("no-match message matches pi", async () => {
|
||||
const out = (await grepTool.execute!({ pattern: "zzzz", path: FIXTURE }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(grepTool, { pattern: "zzzz", path: FIXTURE });
|
||||
assert.equal(out, "No matches found");
|
||||
});
|
||||
});
|
||||
|
||||
describe("read (home-anchored re-port)", () => {
|
||||
it("reads a fixture file verbatim through the shared slice machinery", async () => {
|
||||
const out = (await readTool.execute!({ path: path.join(FIXTURE, "notes", "linear.txt") }, { toolCallId: "c" })) as string;
|
||||
const out = await runText(readTool, { path: path.join(FIXTURE, "notes", "linear.txt") });
|
||||
assert.match(out, /Eigenvectors keep their direction/);
|
||||
});
|
||||
});
|
||||
@@ -211,14 +221,14 @@ describe("abort plumbing (the 2026-09-19 cascade fix)", () => {
|
||||
FIXTURE = await setup();
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const out = (await findTool.execute!({ pattern: "*.md" }, { toolCallId: "c", signal: ac.signal })) as string;
|
||||
const out = await runText(findTool, { pattern: "*.md" }, ac.signal);
|
||||
assert.equal(out, "Operation aborted");
|
||||
});
|
||||
|
||||
it("grep honors a pre-aborted signal without scanning", async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const out = (await grepTool.execute!({ pattern: "eigenvector" }, { toolCallId: "c", signal: ac.signal })) as string;
|
||||
const out = await runText(grepTool, { pattern: "eigenvector" }, ac.signal);
|
||||
assert.equal(out, "Operation aborted");
|
||||
});
|
||||
|
||||
@@ -235,10 +245,7 @@ describe("abort plumbing (the 2026-09-19 cascade fix)", () => {
|
||||
const ac = new AbortController();
|
||||
setTimeout(() => ac.abort(), 1);
|
||||
const started = Date.now();
|
||||
const out = (await grepTool.execute!(
|
||||
{ pattern: "needle", path: root },
|
||||
{ toolCallId: "c", signal: ac.signal },
|
||||
)) as string;
|
||||
const out = await runText(grepTool, { pattern: "needle", path: root }, ac.signal);
|
||||
assert.match(out, /aborted/);
|
||||
assert.ok(Date.now() - started < 5_000, "returned promptly");
|
||||
});
|
||||
|
||||
@@ -138,7 +138,7 @@ describe("list_sessions", () => {
|
||||
},
|
||||
);
|
||||
const tools = createProberTools({ ...baseCtx, chatStore, graphStore });
|
||||
const out = (await tools.list_sessions!.execute!({}, { toolCallId: "tc" })) as string;
|
||||
const out = (await tools.list_sessions!.execute!("tc", {})).details as string;
|
||||
assert.match(out, /2 session\(s\) in the knowledge graph, most recent first:/);
|
||||
// The chat: name, kind, born count, dates, id — the full resume.
|
||||
assert.match(out, /First lesson \(chat, 2 propositions, born 2026-09-20T10:00:00.000Z, last active 2026-09-21T10:00:00.000Z\) — id: c1/);
|
||||
@@ -151,7 +151,7 @@ describe("list_sessions", () => {
|
||||
it("an empty graph gets an honest answer, not an empty string or a throw", async () => {
|
||||
const { chatStore, graphStore } = fakeStores([]);
|
||||
const tools = createProberTools({ ...baseCtx, chatStore, graphStore });
|
||||
const out = (await tools.list_sessions!.execute!({}, { toolCallId: "tc" })) as string;
|
||||
const out = (await tools.list_sessions!.execute!("tc", {})).details as string;
|
||||
assert.equal(out, "The knowledge graph is empty — no session has contributed propositions yet.");
|
||||
});
|
||||
});
|
||||
@@ -172,7 +172,7 @@ describe("list_propositions", () => {
|
||||
},
|
||||
);
|
||||
const tools = createProberTools({ ...baseCtx, chatStore, graphStore });
|
||||
const out = (await tools.list_propositions!.execute!({ session: "c1" }, { toolCallId: "tc" })) as string;
|
||||
const out = (await tools.list_propositions!.execute!("tc", { session: "c1" })).details as string;
|
||||
assert.match(out, /2 proposition\(s\) born in this session:/);
|
||||
assert.match(out, /- Water cycle — solid \(as of 2026-09-22T12:00:00.000Z\)/);
|
||||
assert.match(out, /- Evaporation — not yet engaged \(as of 2026-09-21T12:00:00.000Z\)/);
|
||||
@@ -186,7 +186,7 @@ describe("list_propositions", () => {
|
||||
events: [],
|
||||
});
|
||||
const tools = createProberTools({ ...baseCtx, chatStore, graphStore });
|
||||
const out = (await tools.list_propositions!.execute!({ session: "c1" }, { toolCallId: "tc" })) as string;
|
||||
const out = (await tools.list_propositions!.execute!("tc", { session: "c1" })).details as string;
|
||||
assert.match(out, /- Water cycle — never evaluated/);
|
||||
assert.ok(!out.includes("(as of"));
|
||||
});
|
||||
@@ -194,7 +194,7 @@ describe("list_propositions", () => {
|
||||
it("an unknown session id — honest redirect, not a throw", async () => {
|
||||
const { chatStore, graphStore } = fakeStores([node("n1", "Water cycle", [pt("c1")])]);
|
||||
const tools = createProberTools({ ...baseCtx, chatStore, graphStore });
|
||||
const out = (await tools.list_propositions!.execute!({ session: "no-such" }, { toolCallId: "tc" })) as string;
|
||||
const out = (await tools.list_propositions!.execute!("tc", { session: "no-such" })).details as string;
|
||||
assert.equal(out, 'No propositions born in session "no-such". Call list_sessions for the current ids.');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -101,7 +101,7 @@ after(() => {
|
||||
async function exec(tools: ReturnType<typeof planTools>, name: string, input: unknown): Promise<PlanToolResult> {
|
||||
const t = tools[name]!;
|
||||
assert.ok(typeof t.execute === "function", `${name} must be a server-side tool with execute`);
|
||||
return t.execute(input as never, { toolCallId: "test", messages: [] } as never) as Promise<PlanToolResult>;
|
||||
return t.execute("test", input).then((r) => r.details as PlanToolResult);
|
||||
}
|
||||
|
||||
function newChat(title: string): string {
|
||||
|
||||
@@ -43,10 +43,7 @@ const ASSESSMENT = {
|
||||
async function execute(tools: ReturnType<typeof probeDoneTool>, input: unknown) {
|
||||
const t = tools.probe_done!;
|
||||
assert.ok(typeof t.execute === "function");
|
||||
return t.execute(input as never, { toolCallId: "test", messages: [] } as never) as Promise<{
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
}>;
|
||||
return t.execute("test", input).then((r) => r.details as { ok: boolean; error?: string });
|
||||
}
|
||||
|
||||
describe("probe_done: the probe→planning seam", () => {
|
||||
|
||||
+27
-30
@@ -4,7 +4,8 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { z } from "zod";
|
||||
import { Value } from "typebox/value";
|
||||
import type { TSchema } from "typebox";
|
||||
import {
|
||||
askInputSchema,
|
||||
askOptionProblems,
|
||||
@@ -29,24 +30,25 @@ import {
|
||||
toClientQuestion,
|
||||
} from "../src/core/question-core.ts";
|
||||
|
||||
// ─── the authoring contract (tool-schema refinements) ───────────────────
|
||||
// ─── the authoring contract (TypeBox schemas + execute-time refinements) ──
|
||||
// Schema-valid ⇔ gradeable: a quiz whose correctAnswer can't be resolved
|
||||
// dies at the tool-call boundary, and the SDK returns the error to the
|
||||
// teacher to retry — it must never reach the learner's dock.
|
||||
// dies at the tool-call boundary (structural TypeBox check in pi's loop +
|
||||
// the refinement problems in the tool's execute — a ToolInputError the
|
||||
// model retries on), and never reaches the learner's dock. The boundary
|
||||
// predicate is BOTH: structural check AND zero authoring problems.
|
||||
const boundaryOk = (schema: TSchema, input: unknown, problems: string[]): boolean =>
|
||||
Value.Check(schema, input) && problems.length === 0;
|
||||
|
||||
describe("the schema's argument order (reasoning before the pick)", () => {
|
||||
// Argument-order ruling (owner, 2026-09-25): the explanation is
|
||||
// reasoning, and reasoning happens BEFORE the answer is chosen — models
|
||||
// emit tool args in schema order, so explanation sits immediately before
|
||||
// correctAnswer. z.toJSONSchema preserves zod's declaration order into
|
||||
// the model-facing properties, so this pin guards the emission end to
|
||||
// end. All consumers read by key — the order is authoring guidance.
|
||||
// correctAnswer. The TypeBox authoring order IS the model-facing JSON
|
||||
// Schema property order, so this pin guards the emission end to end.
|
||||
// All consumers read by key — the order is authoring guidance.
|
||||
it("emits explanation immediately before correctAnswer (both agents' schemas)", () => {
|
||||
for (const schema of [quizInputSchema, probeQuizInputSchema]) {
|
||||
const json = z.toJSONSchema(schema as z.ZodType, { io: "input" }) as {
|
||||
properties: Record<string, unknown>;
|
||||
};
|
||||
const keys = Object.keys(json.properties);
|
||||
const keys = Object.keys((schema as { properties: Record<string, unknown> }).properties);
|
||||
assert.ok(
|
||||
keys.indexOf("explanation") < keys.indexOf("correctAnswer"),
|
||||
`explanation must precede correctAnswer in the emitted schema (got: ${keys.join(", ")})`,
|
||||
@@ -73,8 +75,7 @@ describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
|
||||
};
|
||||
|
||||
it("accepts a valid single-select quiz (label as correctAnswer)", () => {
|
||||
const parsed = quizInputSchema.safeParse(valid);
|
||||
assert.equal(parsed.success, true);
|
||||
assert.equal(boundaryOk(quizInputSchema, valid, quizInputProblems(valid)), true);
|
||||
assert.deepEqual(quizInputProblems(valid), []);
|
||||
});
|
||||
|
||||
@@ -89,10 +90,10 @@ describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
|
||||
correctAnswer: ["a", "b"],
|
||||
multiSelect: true,
|
||||
};
|
||||
assert.equal(quizInputSchema.safeParse(multi).success, true);
|
||||
assert.equal(boundaryOk(quizInputSchema, multi, quizInputProblems(multi)), true);
|
||||
// The union lists String first, so providers sometimes stringify arrays.
|
||||
const stringified = { ...multi, correctAnswer: JSON.stringify(multi.correctAnswer) };
|
||||
assert.equal(quizInputSchema.safeParse(stringified).success, true);
|
||||
assert.equal(boundaryOk(quizInputSchema, stringified, quizInputProblems(stringified)), true);
|
||||
assert.deepEqual(quizInputProblems(stringified), []);
|
||||
});
|
||||
|
||||
@@ -108,20 +109,16 @@ describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
|
||||
correctAnswer: "cultural-diagnosis",
|
||||
explanation: "...",
|
||||
};
|
||||
const parsed = quizInputSchema.safeParse(bad);
|
||||
assert.equal(parsed.success, false);
|
||||
assert.equal(boundaryOk(quizInputSchema, bad, quizInputProblems(bad)), false);
|
||||
const problems = quizInputProblems(bad);
|
||||
assert.equal(problems.length, 1);
|
||||
assert.match(problems[0]!, /does not match any option value/);
|
||||
assert.match(problems[0]!, /"A cultural diagnosis"/); // the valid values are listed for the retry
|
||||
// The schema's error message carries the problems (the SDK returns them to the teacher).
|
||||
const msg = JSON.stringify(parsed.error?.issues);
|
||||
assert.match(msg, /does not match any option value/);
|
||||
});
|
||||
|
||||
it("rejects multiple correct answers without multiSelect (ungradeable single-select)", () => {
|
||||
const bad = { ...valid, correctAnswer: ["Venus", "Mercury"] };
|
||||
assert.equal(quizInputSchema.safeParse(bad).success, false);
|
||||
assert.equal(boundaryOk(quizInputSchema, bad, quizInputProblems(bad)), false);
|
||||
assert.match(quizInputProblems(bad).join(" "), /multiSelect/);
|
||||
});
|
||||
|
||||
@@ -133,38 +130,38 @@ describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
|
||||
{ label: "Venus", value: "same" },
|
||||
],
|
||||
};
|
||||
assert.equal(quizInputSchema.safeParse(bad).success, false);
|
||||
assert.equal(boundaryOk(quizInputSchema, bad, quizInputProblems(bad)), false);
|
||||
assert.match(quizInputProblems(bad).join(" "), /same value/);
|
||||
});
|
||||
|
||||
it("rejects an empty option label", () => {
|
||||
const bad = { ...valid, options: [{ label: " " }, { label: "Mercury" }, { label: "Mars" }] };
|
||||
assert.equal(quizInputSchema.safeParse(bad).success, false);
|
||||
assert.equal(boundaryOk(quizInputSchema, bad, quizInputProblems(bad)), false);
|
||||
assert.match(quizInputProblems(bad).join(" "), /label is empty/);
|
||||
});
|
||||
|
||||
it("requires a resolvable correctAnswer", () => {
|
||||
assert.equal(quizInputSchema.safeParse({ ...valid, correctAnswer: "" }).success, false);
|
||||
assert.equal(quizInputSchema.safeParse({ ...valid, correctAnswer: "[]" }).success, false);
|
||||
assert.equal(boundaryOk(quizInputSchema, { ...valid, correctAnswer: "" }, quizInputProblems({ ...valid, correctAnswer: "" })), false);
|
||||
assert.equal(boundaryOk(quizInputSchema, { ...valid, correctAnswer: "[]" }, quizInputProblems({ ...valid, correctAnswer: "[]" })), false);
|
||||
assert.match(quizInputProblems({ ...valid, correctAnswer: "" }).join(" "), /required/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("askInputSchema / askOptionProblems", () => {
|
||||
it("accepts a valid ask", () => {
|
||||
const parsed = askInputSchema.safeParse({
|
||||
const ask = {
|
||||
question: "Which topic first?",
|
||||
options: [{ label: "Limits" }, { label: "Chain rule" }],
|
||||
});
|
||||
assert.equal(parsed.success, true);
|
||||
};
|
||||
assert.equal(boundaryOk(askInputSchema, ask, askOptionProblems(ask.options ?? [])), true);
|
||||
});
|
||||
|
||||
it("rejects duplicate option values and empty labels (they break the dock's keying)", () => {
|
||||
const bad = { question: "?", options: [{ label: "A", value: "x" }, { label: "B", value: "x" }] };
|
||||
assert.equal(askInputSchema.safeParse(bad).success, false);
|
||||
assert.equal(boundaryOk(askInputSchema, bad, askOptionProblems(bad.options ?? [])), false);
|
||||
assert.match(askOptionProblems(bad.options).join(" "), /same value/);
|
||||
const emptyLabel = { question: "?", options: [{ label: "" }, { label: "B" }] };
|
||||
assert.equal(askInputSchema.safeParse(emptyLabel).success, false);
|
||||
assert.equal(boundaryOk(askInputSchema, emptyLabel, askOptionProblems(emptyLabel.options ?? [])), false);
|
||||
assert.match(askOptionProblems(emptyLabel.options).join(" "), /label is empty/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@ import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { Value } from "typebox/value";
|
||||
import { createProberTools, createTeacherTools, teachQuizToolFor, type TeacherToolContext } from "../src/agent/tools.ts";
|
||||
import { TOGGLEABLE_TOOLS } from "../src/core/pending.ts";
|
||||
import { jobStore } from "../src/store/jobs.ts";
|
||||
@@ -51,15 +52,33 @@ const validQuiz = {
|
||||
explanation: "Arithmetic.",
|
||||
};
|
||||
|
||||
interface QuizToolSpec {
|
||||
schema: { safeParse: (v: unknown) => { success: boolean; error?: { issues: Array<{ message: string }> } } };
|
||||
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import { ToolInputError } from "../src/agent/tool-execute.ts";
|
||||
|
||||
/** The Wave A boundary predicate (2026-09-26): pi's loop structurally
|
||||
* validates against the TypeBox parameters, then execute re-runs the
|
||||
* authoring refinements (the old superRefine) and throws ToolInputError
|
||||
* on problems — the model sees both as retryable input errors. This
|
||||
* helper replicates the old safeParse seam end to end. */
|
||||
async function boundaryParse(
|
||||
// `input: never` param: contravariance lets any AgentTool's execute slot in.
|
||||
tool: { parameters: any; execute: (id: string, input: never) => Promise<unknown> },
|
||||
input: unknown,
|
||||
): Promise<{ success: boolean; error?: ToolInputError }> {
|
||||
if (!Value.Check(tool.parameters, input)) return { success: false };
|
||||
try {
|
||||
await tool.execute("tc", input as never);
|
||||
return { success: true };
|
||||
} catch (e) {
|
||||
assert.ok(e instanceof ToolInputError, `expected ToolInputError, got: ${e}`);
|
||||
return { success: false, error: e };
|
||||
}
|
||||
}
|
||||
/** teachQuizToolFor returns the single quiz tool; createProberTools a registry. */
|
||||
const schemaOf = (toolLike: unknown) => (toolLike as QuizToolSpec).schema;
|
||||
const quizSchemaOf = (tools: Record<string, unknown>) => schemaOf(tools.quiz);
|
||||
const problemLines = (e?: ToolInputError): string[] =>
|
||||
(e?.message.split("\n") ?? []).filter((l) => l.startsWith("- "));
|
||||
|
||||
describe("teachQuizToolFor: the dynamic plan-nodeId validation (position signal)", () => {
|
||||
it("accepts a nodeId that matches a node of the session's tree", () => {
|
||||
it("accepts a nodeId that matches a node of the session's tree", async () => {
|
||||
const chatId = newChat("NodeId: valid");
|
||||
chatStore.saveTree(chatId, {
|
||||
nodes: [
|
||||
@@ -69,46 +88,46 @@ describe("teachQuizToolFor: the dynamic plan-nodeId validation (position signal)
|
||||
edges: [{ from: "arith", to: "expressions" }],
|
||||
version: 1,
|
||||
});
|
||||
const res = schemaOf(teachQuizToolFor({ chatId, chatStore })).safeParse({
|
||||
const res = await boundaryParse(teachQuizToolFor({ chatId, chatStore }), {
|
||||
...validQuiz,
|
||||
nodeId: "expressions",
|
||||
});
|
||||
assert.equal(res.success, true);
|
||||
});
|
||||
|
||||
it("rejects a nodeId that matches no node — with a message listing the slugs", () => {
|
||||
it("rejects a nodeId that matches no node — with a message listing the slugs", async () => {
|
||||
const chatId = newChat("NodeId: typo");
|
||||
chatStore.saveTree(chatId, {
|
||||
nodes: [{ id: "arith", title: "Arithmetic" }],
|
||||
edges: [],
|
||||
version: 1,
|
||||
});
|
||||
const res = schemaOf(teachQuizToolFor({ chatId, chatStore })).safeParse({
|
||||
const res = await boundaryParse(teachQuizToolFor({ chatId, chatStore }), {
|
||||
...validQuiz,
|
||||
nodeId: "arithmetic", // the typo — slug is 'arith'
|
||||
});
|
||||
assert.equal(res.success, false);
|
||||
const msg = res.error!.issues.map((i) => i.message).join("\n");
|
||||
const msg = problemLines(res.error).join("\n");
|
||||
assert.match(msg, /matches no node/);
|
||||
assert.match(msg, /arith/);
|
||||
});
|
||||
|
||||
it("no tree yet: ANY nodeId is rejected — omit it for ad-hoc checks", () => {
|
||||
it("no tree yet: ANY nodeId is rejected — omit it for ad-hoc checks", async () => {
|
||||
const chatId = newChat("NodeId: no tree");
|
||||
const schema = schemaOf(teachQuizToolFor({ chatId, chatStore }));
|
||||
const res = schema.safeParse({ ...validQuiz, nodeId: "anything" });
|
||||
const tool = teachQuizToolFor({ chatId, chatStore });
|
||||
const res = await boundaryParse(tool, { ...validQuiz, nodeId: "anything" });
|
||||
assert.equal(res.success, false);
|
||||
assert.match(res.error!.issues.at(-1)!.message, /No plan map exists yet/);
|
||||
assert.match(problemLines(res.error).at(-1)!, /No plan map exists yet/);
|
||||
// …while an untagged ad-hoc check is fine
|
||||
const plain = schema.safeParse(validQuiz);
|
||||
const plain = await boundaryParse(tool, validQuiz);
|
||||
assert.equal(plain.success, true);
|
||||
});
|
||||
|
||||
it("reads the LIVE record — a plan_init mutation mid-turn is honored immediately", () => {
|
||||
it("reads the LIVE record — a plan_init mutation mid-turn is honored immediately", async () => {
|
||||
const chatId = newChat("NodeId: live record");
|
||||
const schema = schemaOf(teachQuizToolFor({ chatId, chatStore }));
|
||||
const tool = teachQuizToolFor({ chatId, chatStore });
|
||||
|
||||
const before = schema.safeParse({ ...validQuiz, nodeId: "fresh-node" });
|
||||
const before = await boundaryParse(tool, { ...validQuiz, nodeId: "fresh-node" });
|
||||
assert.equal(before.success, false, "no tree yet → rejected");
|
||||
|
||||
chatStore.saveTree(chatId, {
|
||||
@@ -116,31 +135,36 @@ describe("teachQuizToolFor: the dynamic plan-nodeId validation (position signal)
|
||||
edges: [],
|
||||
version: 1,
|
||||
});
|
||||
const after = schema.safeParse({ ...validQuiz, nodeId: "fresh-node" });
|
||||
const after = await boundaryParse(tool, { ...validQuiz, nodeId: "fresh-node" });
|
||||
assert.equal(after.success, true, "the same id validates once the tree exists — same turn");
|
||||
});
|
||||
|
||||
it("the shared grading rules still chain FIRST (schema-valid ⇔ gradeable)", () => {
|
||||
it("the shared grading rules still chain FIRST (schema-valid ⇔ gradeable)", async () => {
|
||||
const chatId = newChat("NodeId: grading rules intact");
|
||||
const res = schemaOf(teachQuizToolFor({ chatId, chatStore })).safeParse({
|
||||
const res = await boundaryParse(teachQuizToolFor({ chatId, chatStore }), {
|
||||
...validQuiz,
|
||||
correctAnswer: "42", // matches no option — question-core's rule
|
||||
nodeId: "ghost", // also bad — BOTH problems surface
|
||||
});
|
||||
assert.equal(res.success, false);
|
||||
const msgs = res.error!.issues.map((i) => i.message).join("\n");
|
||||
const msgs = problemLines(res.error).join("\n");
|
||||
assert.match(msgs, /correctAnswer/);
|
||||
assert.match(msgs, /matches no node/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the prober's quiz: no nodeId field at all", () => {
|
||||
it("the probe schema accepts a plain quiz and STRIPS a smuggled nodeId", () => {
|
||||
const plain = probeQuizInputSchema.safeParse(validQuiz);
|
||||
assert.equal(plain.success, true);
|
||||
const smuggled = probeQuizInputSchema.safeParse({ ...validQuiz, nodeId: "anything" });
|
||||
assert.equal(smuggled.success, true, "the field doesn't exist — zod strips it");
|
||||
assert.equal("nodeId" in smuggled.data, false, "stripped, not tolerated");
|
||||
it("the probe schema accepts a plain quiz; a smuggled nodeId is NOT in the wire schema", () => {
|
||||
assert.ok(Value.Check(probeQuizInputSchema, validQuiz));
|
||||
// Wave A delta (2026-09-26): pi's TypeBox validation tolerates unknown
|
||||
// keys rather than stripping them — but the probe schema's wire
|
||||
// properties carry no nodeId at all, and the display parse seam
|
||||
// (tool-parts parseQuizInput, Value.Clean) strips it before the dock.
|
||||
const smuggled = { ...validQuiz, nodeId: "anything" };
|
||||
assert.ok(Value.Check(probeQuizInputSchema, smuggled));
|
||||
const wireKeys = Object.keys((probeQuizInputSchema as { properties: Record<string, unknown> }).properties);
|
||||
assert.ok(!wireKeys.includes("nodeId"), "no session check to fail — the field isn't the prober's to set");
|
||||
assert.equal("nodeId" in (Value.Clean(probeQuizInputSchema, smuggled) as object), false, "the dock parse strips it");
|
||||
});
|
||||
|
||||
it("the prober's tool registry carries the nodeId-less quiz, probe_done, and the learner-filesystem set", () => {
|
||||
@@ -151,8 +175,8 @@ describe("the prober's quiz: no nodeId field at all", () => {
|
||||
assert.deepEqual(Object.keys(tools).sort(), [
|
||||
"ask_user_question", "find", "grep", "ls", "pdf", "probe_done", "quiz", "read", "research",
|
||||
]);
|
||||
const schema = quizSchemaOf(tools);
|
||||
assert.equal(schema.safeParse({ ...validQuiz, nodeId: "anything" }).success, true, "no session check to fail");
|
||||
const quiz = tools.quiz as AgentTool<any>;
|
||||
assert.equal(Value.Check(quiz.parameters, { ...validQuiz, nodeId: "anything" }), true, "no session check to fail");
|
||||
});
|
||||
});
|
||||
// ─── tool availability (2026-09 settings seam) ───────────────────────────────
|
||||
|
||||
+37
-34
@@ -1,7 +1,10 @@
|
||||
/**
|
||||
* The read tool (sources v0) — a Node port of pi's read semantics:
|
||||
* truncation with continuation offsets, image mime detection, error paths.
|
||||
* Plus the lh-tool.ts image-result escape hatch it exercises.
|
||||
* Driven through the NATIVE AgentTool execute (Wave A, 2026-09-26): text
|
||||
* reads return the raw string as details (the record's tool part output
|
||||
* reads it); image reads return native pi content parts (text + image)
|
||||
* with null details (the output-less part until the record persists).
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
@@ -9,7 +12,6 @@ import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, it } from "node:test";
|
||||
import { readTool } from "../src/agent/tools/read.ts";
|
||||
import { toPiTool } from "../src/agent/lh-tool.ts";
|
||||
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "lh-read-test-"));
|
||||
|
||||
@@ -19,16 +21,21 @@ function write(name: string, content: string): string {
|
||||
return file;
|
||||
}
|
||||
|
||||
/** The 2-line/8-byte local config, applied the same way as the real caps. */
|
||||
/** Native execute → the raw string body (text reads ride details verbatim). */
|
||||
const run = (args: { path: string; offset?: number; limit?: number }) =>
|
||||
(readTool.execute as (i: unknown, c: { toolCallId: string }) => Promise<unknown>)!(args, {
|
||||
toolCallId: "test",
|
||||
readTool.execute!("test", args).then((r) => {
|
||||
assert.ok(r.content.length >= 1);
|
||||
const first = r.content[0]!;
|
||||
assert.equal(first.type, "text");
|
||||
return { text: (first as { type: "text"; text: string }).text, details: r.details };
|
||||
});
|
||||
|
||||
describe("read tool — text semantics", () => {
|
||||
it("reads a small file verbatim", async () => {
|
||||
const file = write("small.txt", "hello\nworld");
|
||||
assert.equal(await run({ path: file }), "hello\nworld");
|
||||
const { text, details } = await run({ path: file });
|
||||
assert.equal(text, "hello\nworld");
|
||||
assert.equal(details, "hello\nworld", "the record part output carries the string verbatim");
|
||||
});
|
||||
|
||||
it("resolves relative paths against the learner's home (sources v1 anchoring)", async () => {
|
||||
@@ -39,7 +46,7 @@ describe("read tool — text semantics", () => {
|
||||
const homeFile = path.join(os.homedir(), rel);
|
||||
fs.writeFileSync(homeFile, "rel", "utf-8");
|
||||
try {
|
||||
assert.equal(await run({ path: rel }), "rel");
|
||||
assert.equal((await run({ path: rel })).text, "rel");
|
||||
} finally {
|
||||
fs.unlinkSync(homeFile);
|
||||
}
|
||||
@@ -48,7 +55,7 @@ describe("read tool — text semantics", () => {
|
||||
it("honors offset (1-indexed) and limit", async () => {
|
||||
const body = Array.from({ length: 30 }, (_, i) => `line ${i + 1}`).join("\n");
|
||||
const file = write("thirty.txt", body);
|
||||
const out = (await run({ path: file, offset: 10, limit: 3 })) as string;
|
||||
const out = (await run({ path: file, offset: 10, limit: 3 })).text;
|
||||
// pi parity: the explicit limit stopping short gets a remaining notice
|
||||
assert.match(out, /^line 10\nline 11\nline 12\n\n\[18 more lines in file\. Use offset=13 to continue\.\]$/);
|
||||
});
|
||||
@@ -65,7 +72,7 @@ describe("read tool — text semantics", () => {
|
||||
it("truncates at 2000 lines with a use-offset continuation notice", async () => {
|
||||
const body = Array.from({ length: 3000 }, (_, i) => `l${i + 1}`).join("\n");
|
||||
const file = write("big.txt", body);
|
||||
const out = (await run({ path: file })) as string;
|
||||
const out = (await run({ path: file })).text;
|
||||
assert.match(out, /Showing lines 1-2000 of 3000 \(2000 line limit\)/);
|
||||
assert.match(out, /Use offset=2001 to continue/);
|
||||
assert.ok(!out.includes("l2001\n"), "line 2001 must not ride along");
|
||||
@@ -76,7 +83,7 @@ describe("read tool — text semantics", () => {
|
||||
const line = "x".repeat(400);
|
||||
const body = Array.from({ length: 200 }, () => line).join("\n");
|
||||
const file = write("wide.txt", body);
|
||||
const out = (await run({ path: file })) as string;
|
||||
const out = (await run({ path: file })).text;
|
||||
assert.match(out, /50\.0KB limit\)\. Use offset=\d+ to continue\./);
|
||||
// and the truncation actually cut content
|
||||
assert.ok(out.length < 52_000, "the 50KB ceiling must cut the 80KB body");
|
||||
@@ -85,38 +92,34 @@ describe("read tool — text semantics", () => {
|
||||
it("an explicit limit that stops short gets a remaining-lines notice", async () => {
|
||||
const body = Array.from({ length: 10 }, (_, i) => `l${i + 1}`).join("\n");
|
||||
const file = write("ten.txt", body);
|
||||
const out = (await run({ path: file, limit: 4 })) as string;
|
||||
const out = (await run({ path: file, limit: 4 })).text;
|
||||
assert.match(out, /\[6 more lines in file\. Use offset=5 to continue\.\]/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("read tool — image semantics", () => {
|
||||
it("returns the LhToolImageResult escape-hatch shape for image files", async () => {
|
||||
describe("read tool — image semantics (native content parts)", () => {
|
||||
it("returns native text + image parts for image files, details null (output-less part)", async () => {
|
||||
const file = path.join(tmp, "dot.png");
|
||||
// minimal valid PNG header bytes are enough — detection is by extension
|
||||
fs.writeFileSync(file, Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]));
|
||||
const out = (await run({ path: file })) as { text: string; images?: { data: string; mimeType: string }[] };
|
||||
assert.match(out.text, /Read image file \[image\/png\]/);
|
||||
assert.equal(out.images?.length, 1);
|
||||
assert.equal(out.images![0]!.mimeType, "image/png");
|
||||
assert.match(out.images![0]!.data, /^[A-Za-z0-9+/]+={0,2}$/);
|
||||
const result = await readTool.execute!("test", { path: file });
|
||||
assert.equal(result.content.length, 2);
|
||||
const parts = result.content as Array<{ type: string; text?: string; data?: string; mimeType?: string }>;
|
||||
const textPart = parts[0]!;
|
||||
const imagePart = parts[1]!;
|
||||
assert.equal(textPart.type, "text");
|
||||
assert.match(textPart.text!, /Read image file \[image\/png\]/);
|
||||
assert.equal(imagePart.type, "image");
|
||||
assert.equal(imagePart.mimeType, "image/png");
|
||||
assert.match(imagePart.data!, /^[A-Za-z0-9+/]+={0,2}$/);
|
||||
assert.equal(result.details, null, "image reads keep the record part output-less");
|
||||
});
|
||||
|
||||
it("toPiTool maps the image result to native pi content parts", async () => {
|
||||
const pi = toPiTool(readTool);
|
||||
const result = await pi.execute("call-1", { path: path.join(tmp, "dot.png") }, undefined as never);
|
||||
assert.ok(!("isError" in result) || !(result as { isError?: boolean }).isError);
|
||||
const content = (result as { content: Array<{ type: string }> }).content;
|
||||
assert.equal(content[0]!.type, "text");
|
||||
assert.equal(content[1]!.type, "image");
|
||||
});
|
||||
|
||||
it("toPiTool maps plain string results to a single text part (parity)", async () => {
|
||||
it("plain string results map to a single text part (parity)", async () => {
|
||||
const file = write("plain.txt", "body");
|
||||
const pi = toPiTool(readTool);
|
||||
const result = await pi.execute("call-2", { path: file }, undefined as never);
|
||||
const content = (result as { content: Array<{ type: string }> }).content;
|
||||
assert.equal(content.length, 1);
|
||||
assert.equal(content[0]!.type, "text");
|
||||
const result = await readTool.execute!("test", { path: file });
|
||||
assert.equal(result.content.length, 1);
|
||||
assert.equal(result.content[0]!.type, "text");
|
||||
assert.equal((result.content[0] as { type: "text"; text: string }).text, "body");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+91
-61
@@ -1,17 +1,25 @@
|
||||
/**
|
||||
* The abort-cascade fix (2026-09-19): pi passes every run's AbortSignal to
|
||||
* tool execute — the bridge used to drop it, so agent.abort() only landed
|
||||
* at the NEXT step boundary and a long-running local tool (the grep that
|
||||
* outlived its turn by minutes) held the turn hostage until the dev
|
||||
* server's memory restart killed the run mid-tool, freezing the record at
|
||||
* its write-ahead state. toPiTool now threads the signal and races every
|
||||
* execute against the system-level timeout; the fs tools honor it with
|
||||
* partial results (fs-tools.test.ts covers their side).
|
||||
* The abort-cascade fix (2026-09-19), pinned against the pi-native seams
|
||||
* (Wave A, 2026-09-26): serverExecute (tool-execute.ts) races every tool
|
||||
* body against the run's AbortSignal and the system-level per-call timeout
|
||||
* — aborting the run kills a running tool, the timeout is the net so no
|
||||
* tool can hold a turn hostage (the 2.7-minute grep lesson). Question
|
||||
* tools bypass the guard entirely (questionResult is instant and carries
|
||||
* terminate). The fs tools honor the signal with partial results
|
||||
* (fs-tools.test.ts covers their side).
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { afterEach, describe, it } from "node:test";
|
||||
import { z } from "zod";
|
||||
import { lhTool, toPiTool, toolTimeoutMs, ToolInputError } from "../src/agent/lh-tool.ts";
|
||||
import { Type } from "typebox";
|
||||
import {
|
||||
questionResult,
|
||||
serverExecute,
|
||||
toolTimeoutMs,
|
||||
ToolInputError,
|
||||
toToolResult,
|
||||
type ToolExecuteContext,
|
||||
} from "../src/agent/tool-execute.ts";
|
||||
import { askUserQuestionTool } from "../src/agent/tools.ts";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
|
||||
@@ -19,34 +27,37 @@ afterEach(() => {
|
||||
toolTimeoutMs.current = DEFAULT_TIMEOUT_MS;
|
||||
});
|
||||
|
||||
/** A tool that captures ctx.signal, then either hangs or resolves `value`. */
|
||||
const probeSchema = Type.Object({});
|
||||
|
||||
/** A native tool whose execute captures ctx.signal, then hangs or resolves. */
|
||||
function probeTool(captured: { signal?: AbortSignal }, opts: { hang?: boolean; value?: string } = {}) {
|
||||
return lhTool({
|
||||
return {
|
||||
name: "probe",
|
||||
label: "probe",
|
||||
description: "test probe",
|
||||
schema: z.object({}),
|
||||
execute: (_input, ctx) => {
|
||||
captured.signal = ctx?.signal;
|
||||
parameters: probeSchema,
|
||||
execute: serverExecute("probe", probeSchema, (_input: unknown, ctx: ToolExecuteContext) => {
|
||||
captured.signal = ctx.signal;
|
||||
if (opts.hang) return new Promise<string>(() => {}); // never settles, no timer
|
||||
return opts.value ?? "ok";
|
||||
},
|
||||
});
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
describe("toPiTool abort + timeout guard", () => {
|
||||
describe("serverExecute abort + timeout guard", () => {
|
||||
it("forwards a live abort signal to the tool's ctx", async () => {
|
||||
// Even with no parent signal, the tool still gets the guard's combined
|
||||
// signal (parent+timeout) — a stable reference tools can poll.
|
||||
const captured: { signal?: AbortSignal } = {};
|
||||
const pi = toPiTool(probeTool(captured));
|
||||
await pi.execute("c1", {}, undefined as never);
|
||||
const pi = probeTool(captured);
|
||||
await pi.execute("c1", {});
|
||||
assert.ok(captured.signal, "the tool received a signal");
|
||||
assert.equal(captured.signal!.aborted, false);
|
||||
// The forwarded signal is the guard's COMBINED one — aborting the
|
||||
// parent flips it, which is what long-running tools poll.
|
||||
const ac = new AbortController();
|
||||
const captured2: { signal?: AbortSignal } = {};
|
||||
const pi2 = toPiTool(probeTool(captured2, { value: "done" }));
|
||||
const pi2 = probeTool(captured2, { value: "done" });
|
||||
const pending = pi2.execute("c1b", {}, ac.signal);
|
||||
ac.abort();
|
||||
await pending;
|
||||
@@ -56,7 +67,7 @@ describe("toPiTool abort + timeout guard", () => {
|
||||
it("a signal-wired tool observes the run's abort", async () => {
|
||||
const ac = new AbortController();
|
||||
const captured: { signal?: AbortSignal } = {};
|
||||
const pi = toPiTool(probeTool(captured, { hang: true }));
|
||||
const pi = probeTool(captured, { hang: true });
|
||||
const pending = pi.execute("c2", {}, ac.signal).catch((e: Error) => e.message);
|
||||
ac.abort();
|
||||
assert.match((await pending) as string, /Operation aborted/);
|
||||
@@ -65,68 +76,87 @@ describe("toPiTool abort + timeout guard", () => {
|
||||
|
||||
it("rejects a hanging tool at the system-level timeout", async () => {
|
||||
toolTimeoutMs.current = 50;
|
||||
const pi = toPiTool(probeTool({}, { hang: true }));
|
||||
await assert.rejects(pi.execute("c3", {}, undefined as never), /timed out/);
|
||||
const pi = probeTool({}, { hang: true });
|
||||
await assert.rejects(pi.execute("c3", {}), /timed out/);
|
||||
});
|
||||
|
||||
it("a pre-aborted run rejects immediately with pi's wording", async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const pi = toPiTool(probeTool({}, { hang: true }));
|
||||
const pi = probeTool({}, { hang: true });
|
||||
await assert.rejects(pi.execute("c4", {}, ac.signal), /Operation aborted/);
|
||||
});
|
||||
|
||||
it("a tool that honors the signal settles before the timeout wins", async () => {
|
||||
// The race resolves with the tool's own (partial) result — the fs
|
||||
// tools' contract: abort yields partial results, not an error throw.
|
||||
});
|
||||
describe("serverExecute: cooperative settle, mapping parity, question bypass", () => {
|
||||
it("a cooperative tool settles partial results before the grace", async () => {
|
||||
toolTimeoutMs.current = 2_000;
|
||||
const tool = lhTool({
|
||||
const schema = Type.Object({});
|
||||
const pi = {
|
||||
name: "partial",
|
||||
label: "partial",
|
||||
description: "test",
|
||||
schema: z.object({}),
|
||||
execute: (_input, ctx) =>
|
||||
parameters: schema,
|
||||
execute: serverExecute("partial", schema, (_input: unknown, ctx: ToolExecuteContext) =>
|
||||
new Promise((resolve) => {
|
||||
// The fs-tools contract: poll upfront (a listener on an
|
||||
// already-aborted signal never fires), else listen.
|
||||
const signal = ctx!.signal!;
|
||||
const signal = ctx.signal!;
|
||||
if (signal.aborted) return resolve("partial results");
|
||||
signal.addEventListener("abort", () => resolve("partial results"), { once: true });
|
||||
}),
|
||||
});
|
||||
})),
|
||||
};
|
||||
const ac = new AbortController();
|
||||
const pi = toPiTool(tool);
|
||||
const pending = pi.execute("c5", {}, ac.signal);
|
||||
ac.abort();
|
||||
const result = (await pending) as { content: Array<{ text?: string }> };
|
||||
assert.equal(result.content[0]!.text, "partial results");
|
||||
const result = await pending;
|
||||
assert.equal((result.content[0] as { text?: string }).text, "partial results");
|
||||
});
|
||||
|
||||
it("question tools bypass the guard — the placeholder is instant", async () => {
|
||||
it("questionResult is instant, terminates, and carries no details", async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const tool = lhTool({
|
||||
name: "quiz",
|
||||
description: "test",
|
||||
schema: z.object({}),
|
||||
clientQuestion: true,
|
||||
});
|
||||
const pi = toPiTool(tool);
|
||||
const result = (await pi.execute("c6", {}, ac.signal)) as { content: Array<{ text?: string }> };
|
||||
assert.match(result.content[0]!.text!, /Question posed/);
|
||||
ac.abort(); // a pre-aborted run signal — the placeholder must not care
|
||||
const result = await askUserQuestionTool.execute("c6", { question: "q" }, ac.signal);
|
||||
assert.match((result.content[0] as { text?: string }).text!, /Question posed/);
|
||||
assert.equal(result.details, null);
|
||||
assert.equal(result.terminate, true, "owner Option A: question results end the batch natively");
|
||||
});
|
||||
|
||||
it("validation errors pass through the guard unchanged", async () => {
|
||||
const tool = lhTool({
|
||||
name: "strict",
|
||||
description: "test",
|
||||
schema: z.object({ n: z.number() }),
|
||||
execute: () => "never",
|
||||
});
|
||||
const pi = toPiTool(tool);
|
||||
await assert.rejects(pi.execute("c7", { n: "not a number" }, undefined as never), (e: unknown) => {
|
||||
assert.ok(e instanceof ToolInputError);
|
||||
assert.equal(e.toolName, "strict");
|
||||
return true;
|
||||
it("authoring failures throw ToolInputError with the retryable problem list", async () => {
|
||||
await assert.rejects(
|
||||
askUserQuestionTool.execute("c7", {
|
||||
question: "q",
|
||||
options: [
|
||||
{ label: "A", value: "same" },
|
||||
{ label: "B", value: "same" },
|
||||
],
|
||||
}),
|
||||
(e: unknown) => {
|
||||
assert.ok(e instanceof ToolInputError);
|
||||
assert.equal(e.toolName, "ask_user_question");
|
||||
assert.match(e.message, /Invalid tool input for tool "ask_user_question":/);
|
||||
assert.match(e.message, /- .*same value/s);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("toToolResult parity: string verbatim, object as JSON, details rides the raw value", () => {
|
||||
const str = toToolResult("plain");
|
||||
assert.equal((str.content[0] as { text: string }).text, "plain");
|
||||
assert.equal(str.details, "plain");
|
||||
const obj = toToolResult({ a: 1 });
|
||||
assert.equal((obj.content[0] as { text: string }).text, '{"a":1}');
|
||||
assert.deepEqual(obj.details, { a: 1 });
|
||||
const img = toToolResult({
|
||||
text: "the picture",
|
||||
images: [{ data: "AAAA", mimeType: "image/png" }],
|
||||
details: { page: 2 },
|
||||
});
|
||||
assert.equal(img.content.length, 2);
|
||||
assert.equal(img.content[0]!.type, "text");
|
||||
assert.equal(img.content[1]!.type, "image");
|
||||
assert.deepEqual(img.details, { page: 2 });
|
||||
const imgNoDetails = toToolResult({ text: "t", images: [{ data: "AAAA", mimeType: "image/png" }] });
|
||||
assert.equal(imgNoDetails.details, null, "image reads keep the record part output-less (null details)");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { quizInputProblems } from "../src/core/question-core.ts";
|
||||
import { Value } from "typebox/value";
|
||||
import {
|
||||
parseAskInput,
|
||||
parseProbeDoneInput,
|
||||
@@ -59,15 +61,13 @@ describe("parseQuizInput", () => {
|
||||
});
|
||||
|
||||
it("boundary parity: anything the TEACH schema accepts, the display parse accepts", () => {
|
||||
const boundary = quizInputSchema.safeParse(TEACHER_QUIZ);
|
||||
assert.ok(boundary.success, "fixture must be boundary-valid for this pin to mean anything");
|
||||
assert.ok(parseQuizInput(boundary.data));
|
||||
assert.ok(Value.Check(quizInputSchema, TEACHER_QUIZ), "fixture must be boundary-valid for this pin to mean anything");
|
||||
assert.ok(parseQuizInput(TEACHER_QUIZ));
|
||||
});
|
||||
|
||||
it("boundary parity: anything the PROBE schema accepts, the display parse accepts", () => {
|
||||
const boundary = probeQuizInputSchema.safeParse(PROBER_QUIZ);
|
||||
assert.ok(boundary.success);
|
||||
assert.ok(parseQuizInput(boundary.data));
|
||||
assert.ok(Value.Check(probeQuizInputSchema, PROBER_QUIZ));
|
||||
assert.ok(parseQuizInput(PROBER_QUIZ));
|
||||
});
|
||||
|
||||
it("the live crash shape — options as a JSON string — does not parse", () => {
|
||||
@@ -75,6 +75,40 @@ describe("parseQuizInput", () => {
|
||||
assert.equal(parseQuizInput(stringified), undefined);
|
||||
});
|
||||
|
||||
it("boundary parity under Value.Convert: a NUMERIC correctAnswer parses (live-smoke lockout fix, 2026-09-26)", () => {
|
||||
// The live smoke's model emitted correctAnswer: 17 (number). pi's
|
||||
// boundary runs Value.Convert before Value.Check, so the quiz posed
|
||||
// — but the record keeps the RAW args. A parse seam without Convert
|
||||
// would reject exactly what the boundary accepted: the pending quiz
|
||||
// renders nowhere while the turn has ended = learner lockout.
|
||||
// Dock-visible parity case: the smoke's real emission — options whose
|
||||
// values ARE the numbers, correctAnswer numeric. Convert → "17",
|
||||
// refinement passes (it matches an option), the quiz parses.
|
||||
const numeric = {
|
||||
question: "What is 5 + 3 × 4?",
|
||||
options: [
|
||||
{ label: "17", value: "17" },
|
||||
{ label: "27", value: "27" },
|
||||
{ label: "9", value: "9" },
|
||||
],
|
||||
explanation: "Multiplication first.",
|
||||
correctAnswer: 17,
|
||||
};
|
||||
const parsed = parseQuizInput(numeric);
|
||||
assert.ok(parsed);
|
||||
assert.equal(parsed.correctAnswer, "17", "converted to the string the boundary saw");
|
||||
const numericArray = { ...numeric, multiSelect: true, correctAnswer: [17, "27"] };
|
||||
const parsedArray = parseQuizInput(numericArray);
|
||||
assert.ok(parsedArray);
|
||||
assert.deepEqual(parsedArray.correctAnswer, ["17", "27"]);
|
||||
// Non-convertible garbage still never parses (structure owns it)…
|
||||
assert.equal(parseQuizInput({ ...PROBER_QUIZ, correctAnswer: { bad: true } }), undefined);
|
||||
// …and Convert's widening (undefined → "") is caught by the refinement
|
||||
// chain the boundary itself uses — garbage renders as a quiet fallback.
|
||||
const widened = { ...PROBER_QUIZ, correctAnswer: undefined };
|
||||
assert.equal(parseQuizInput(widened), undefined, "the refinement owns the reject");
|
||||
});
|
||||
|
||||
it("garbage never parses: non-object, missing question, missing correctAnswer", () => {
|
||||
assert.equal(parseQuizInput("a string"), undefined);
|
||||
assert.equal(parseQuizInput({ options: [] }), undefined);
|
||||
@@ -102,9 +136,8 @@ describe("parseProbeDoneInput", () => {
|
||||
});
|
||||
|
||||
it("boundary parity: anything the tool schema accepts, the display parse accepts", () => {
|
||||
const boundary = probeDoneInputSchema.safeParse(VALID_ASSESSMENT);
|
||||
assert.ok(boundary.success);
|
||||
assert.ok(parseProbeDoneInput(boundary.data));
|
||||
assert.ok(Value.Check(probeDoneInputSchema, VALID_ASSESSMENT));
|
||||
assert.ok(parseProbeDoneInput(VALID_ASSESSMENT));
|
||||
});
|
||||
|
||||
it("the live crash shape — assessment STRINGIFIED — does not parse", () => {
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { z } from "zod";
|
||||
import { describe, it } from "node:test";
|
||||
import { Type } from "typebox";
|
||||
import { questionResult } from "../src/agent/tool-execute.ts";
|
||||
import { createProberAgent, createTeacherAgent } from "../src/agent/teacher-agent.ts";
|
||||
import type { ProberToolContext } from "../src/agent/tools.ts";
|
||||
import { jobStore } from "../src/store/jobs.ts";
|
||||
@@ -53,7 +55,13 @@ describe("AgentBuildOptions.toolToggles (the P2 bridge)", () => {
|
||||
const teacher = createTeacherAgent(undefined, {
|
||||
engine: fauxEngine(),
|
||||
tools: {
|
||||
quiz: { name: "quiz", description: "test", clientQuestion: true, schema: z.object({}) },
|
||||
quiz: {
|
||||
name: "quiz",
|
||||
label: "quiz",
|
||||
description: "test",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => questionResult(),
|
||||
},
|
||||
},
|
||||
toolToggles: { quiz: false },
|
||||
});
|
||||
|
||||
+13
-13
@@ -31,9 +31,9 @@ function stubProvider(hits: SearchHit[] = [{ title: "t", url: "https://e.com/",
|
||||
describe("web_search on the injected port (web.ts)", () => {
|
||||
it("null provider → the honest unavailability string, never a crash", async () => {
|
||||
const { web_search } = createWebTools(() => null);
|
||||
const out = await web_search.execute!({ query: "anything" }, { toolCallId: "c1" });
|
||||
assert.match(String(out), /not available: no search engine is configured/);
|
||||
assert.match(String(out), /Tavily or Brave/);
|
||||
const out = await web_search!.execute!("c1", { query: "anything" });
|
||||
assert.match(String(out.details), /not available: no search engine is configured/);
|
||||
assert.match(String(out.details), /Tavily or Brave/);
|
||||
});
|
||||
|
||||
it("a live provider returns its hits (count defaults to 5)", async () => {
|
||||
@@ -46,9 +46,9 @@ describe("web_search on the injected port (web.ts)", () => {
|
||||
},
|
||||
};
|
||||
const { web_search } = createWebTools(() => provider);
|
||||
const out = await web_search.execute!({ query: "sigma.js latest version" }, { toolCallId: "c1" });
|
||||
const out = await web_search!.execute!("c1", { query: "sigma.js latest version" });
|
||||
assert.deepEqual(seen, [{ query: "sigma.js latest version", count: 5 }]);
|
||||
assert.deepEqual(out, [{ title: "sigma.js", url: "https://www.sigmajs.org/", snippet: "v3 released" }]);
|
||||
assert.deepEqual(out.details, [{ title: "sigma.js", url: "https://www.sigmajs.org/", snippet: "v3 released" }]);
|
||||
});
|
||||
|
||||
it("provider errors surface as error strings, never thrown", async () => {
|
||||
@@ -59,26 +59,26 @@ describe("web_search on the injected port (web.ts)", () => {
|
||||
},
|
||||
};
|
||||
const { web_search } = createWebTools(() => provider);
|
||||
const out = await web_search.execute!({ query: "q" }, { toolCallId: "c1" });
|
||||
assert.match(String(out), /web_search failed: Brave search API 429: rate limited/);
|
||||
const out = await web_search!.execute!("c1", { query: "q" });
|
||||
assert.match(String(out.details), /web_search failed: Brave search API 429: rate limited/);
|
||||
});
|
||||
|
||||
it("the resolution is FRESH per call — a key appearing mid-session takes effect", async () => {
|
||||
let provider: SearchProvider | null = null;
|
||||
const { web_search } = createWebTools(() => provider);
|
||||
const first = await web_search.execute!({ query: "q" }, { toolCallId: "c1" });
|
||||
assert.match(String(first), /not available/);
|
||||
const first = await web_search!.execute!("c1", { query: "q" });
|
||||
assert.match(String(first.details), /not available/);
|
||||
provider = stubProvider();
|
||||
const second = await web_search.execute!({ query: "q" }, { toolCallId: "c2" });
|
||||
assert.ok(Array.isArray(second), "the live provider answered on the very next call");
|
||||
const second = await web_search!.execute!("c2", { query: "q" });
|
||||
assert.ok(Array.isArray(second.details), "the live provider answered on the very next call");
|
||||
});
|
||||
|
||||
it("web_fetch needs no search engine — it stays independent", async () => {
|
||||
const { web_fetch } = createWebTools(() => null);
|
||||
assert.ok(web_fetch, "web_fetch present with no provider");
|
||||
// (Full fetch behavior is out of scope here — no network in tests.)
|
||||
const out = await web_fetch.execute!({ url: "not a url" }, { toolCallId: "c1" });
|
||||
assert.match(String(out), /web_fetch failed: invalid URL/);
|
||||
const out = await web_fetch!.execute!("c1", { url: "not a url" });
|
||||
assert.match(String(out.details), /web_fetch failed: invalid URL/);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user