researcher: PDFs from the web — the pdf-reader's text tools in the loop (design B)

web_fetch_pdf downloads (25MB cap, probe-validated) into a per-job temp
dir and returns the triage summary — page count, size class (the pdf
doctrine's <=10/<=40 thresholds), flagged math/image-heavy pages. The
researcher's toolset then gains pdf_info / pdf_extract / pdf_search over
that dir, scoped by a resolve-and-startswith wall: the researcher's
filesystem ends at the temp dir. Render is deliberately absent — the
researcher slot has no vision; visual reading stays the pdf job's job.

web_fetch: a PDF URL now returns a POINTER to web_fetch_pdf instead of
dead-ending, and every output carries 'Source: <final url after
redirects> (HTTP <status>)' — the 'ill-formed request' lesson: a
200-served error body must be distinguishable from a real page.

The runner owns the temp lifecycle (mkdtemp before, recursive cleanup in
the job's finally — never enters the record) and web_fetch_pdf urls ride
the sources ledger. Doctrine step 5 added: fetch, triage, extract only
the pages that matter, cite as (p. N).

Tests build a REAL minimal PDF programmatically (correct xref offsets —
pdfjs parses it like any download): zero network, real parse.
This commit is contained in:
amosblomqvist
2026-10-07 22:36:11 +03:00
parent d450689a8b
commit 8dfe621748
5 changed files with 463 additions and 7 deletions
+14 -2
View File
@@ -1,6 +1,7 @@
import type { AgentTool } from "@earendil-works/pi-agent-core"; import type { AgentTool } from "@earendil-works/pi-agent-core";
import type { Usage } from "@earendil-works/pi-ai"; import type { Usage } from "@earendil-works/pi-ai";
import { createWebTools } from "../tools/web.ts"; import { createWebTools } from "../tools/web.ts";
import { createPdfReadTools } from "../tools/pdf-read.ts";
import type { SearchProvider } from "../../core/search.ts"; import type { SearchProvider } from "../../core/search.ts";
import type { PiEngine } from "../models.ts"; import type { PiEngine } from "../models.ts";
import { runSubagent } from "./pi-run.ts"; import { runSubagent } from "./pi-run.ts";
@@ -32,7 +33,11 @@ Process:
3. Read the results. Identify what's well-covered and what has gaps. If the first round 3. Read the results. Identify what's well-covered and what has gaps. If the first round
doesn't fully answer the question, search again with refined queries targeting the gaps. doesn't fully answer the question, search again with refined queries targeting the gaps.
4. For the 2-3 most promising source URLs, use web_fetch to get the full page content. 4. For the 2-3 most promising source URLs, use web_fetch to get the full page content.
5. Synthesize everything into a brief that directly answers the question. 5. PDF sources: when web_fetch says a URL is a PDF, call web_fetch_pdf on it — it downloads
and triages the document (page count, size class, flagged pages). Then read it with
pdf_extract — for long ones, pdf_search first and extract only the pages that hit.
Cite claims from a PDF as (p. N).
6. Synthesize everything into a brief that directly answers the question.
Evaluation — what to keep vs drop: Evaluation — what to keep vs drop:
- Official docs and primary sources outweigh blog posts and forum threads. - Official docs and primary sources outweigh blog posts and forum threads.
@@ -68,6 +73,11 @@ export interface ResearcherRunOptions {
* seam reads as unavailable: the server wiring always injects one; the * seam reads as unavailable: the server wiring always injects one; the
* dispatch-level drop (jobTools) keeps doomed jobs from even starting. */ * dispatch-level drop (jobTools) keeps doomed jobs from even starting. */
search?: () => SearchProvider | null; search?: () => SearchProvider | null;
/** The researcher's PDF dir (PDF wave, 2026-10-07): when present, the
* toolset gains web_fetch_pdf + pdf_info/pdf_extract/pdf_search scoped
* to this per-job temp dir — PDFs from the web, read page-level. The
* runner owns its lifecycle (mkdtemp before, rm after). */
pdfDir?: string;
abortSignal?: AbortSignal; abortSignal?: AbortSignal;
onUsage?: (usage: Usage) => void; onUsage?: (usage: Usage) => void;
onToolStart?: (toolName: string, args: unknown) => void; onToolStart?: (toolName: string, args: unknown) => void;
@@ -81,7 +91,9 @@ export interface ResearcherRunOptions {
* the loop scripted. * the loop scripted.
*/ */
export async function runResearcherAgent(prompt: string, opts: ResearcherRunOptions = {}): Promise<string> { export async function runResearcherAgent(prompt: string, opts: ResearcherRunOptions = {}): Promise<string> {
const { text } = await runSubagent(Object.values(createWebTools(opts.search ?? (() => null))), { const tools = Object.values(createWebTools(opts.search ?? (() => null)));
if (opts.pdfDir) tools.push(...Object.values(createPdfReadTools(opts.pdfDir)));
const { text } = await runSubagent(tools, {
role: "researcher", role: "researcher",
prompt, prompt,
engine: opts.engine, engine: opts.engine,
+12 -2
View File
@@ -19,6 +19,9 @@ import {
type VisualToolInput, type VisualToolInput,
} from "../../core/contracts.ts"; } from "../../core/contracts.ts";
import { registerJobAbort, unregisterJobAbort } from "./aborts.ts"; import { registerJobAbort, unregisterJobAbort } from "./aborts.ts";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import type { ModelRole } from "../../../config.ts"; import type { ModelRole } from "../../../config.ts";
import { ROLE_SLOT, modelsNotConfigured, type ModelSlotName, type ProviderKeyMissing } from "../../core/settings.ts"; import { ROLE_SLOT, modelsNotConfigured, type ModelSlotName, type ProviderKeyMissing } from "../../core/settings.ts";
import type { ChatStore, JobStore } from "../../core/ports.ts"; import type { ChatStore, JobStore } from "../../core/ports.ts";
@@ -353,20 +356,25 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
.join("\n\n"); .join("\n\n");
const fetchedUrls: string[] = []; const fetchedUrls: string[] = [];
// Per-job PDF dir (researcher PDF wave, 2026-10-07): web_fetch_pdf
// downloads land here; every pdf tool is scoped to it; it dies with
// the job (never enters the record).
const pdfDir = await fs.mkdtemp(path.join(os.tmpdir(), "lh-research-"));
try { try {
jobStore.updateJob(jobId, { activity: "researching" }); jobStore.updateJob(jobId, { activity: "researching" });
const brief = await runResearcherAgent(prompt, { const brief = await runResearcherAgent(prompt, {
engine: model, engine: model,
search: deps.searchProvider, search: deps.searchProvider,
pdfDir,
abortSignal: loop.abortSignal, abortSignal: loop.abortSignal,
chatId: loop.chatId, chatId: loop.chatId,
// Money channel: each agent step is one model call — fold its // Money channel: each agent step is one model call — fold its
// usage into the record as it completes (progress write). // usage into the record as it completes (progress write).
onUsage: (usage) => loop.onUsage(piUsageToStep(usage)), onUsage: (usage) => loop.onUsage(piUsageToStep(usage)),
onToolStart: (toolName, args) => { onToolStart: (toolName, args) => {
const input = args as { query?: string; url?: string } | undefined; const input = args as { query?: string; url?: string } | undefined;
if (toolName === "web_fetch" && typeof input?.url === "string") { if (typeof input?.url === "string" && (toolName === "web_fetch" || toolName === "web_fetch_pdf")) {
fetchedUrls.push(input.url); fetchedUrls.push(input.url);
} }
jobStore.updateJob(jobId, { activity: activityLine(toolName, input) }); jobStore.updateJob(jobId, { activity: activityLine(toolName, input) });
@@ -383,6 +391,8 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
const reason = err instanceof Error ? err.message : String(err); const reason = err instanceof Error ? err.message : String(err);
if (stillRunning(jobId)) jobStore.updateJob(jobId, { error: reason }); if (stillRunning(jobId)) jobStore.updateJob(jobId, { error: reason });
await finishJob(jobId, { brief: `Research failed: ${reason}`, sources: [] }); await finishJob(jobId, { brief: `Research failed: ${reason}`, sources: [] });
} finally {
await fs.rm(pdfDir, { recursive: true, force: true });
} }
} }
+221
View File
@@ -0,0 +1,221 @@
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { Type } from "typebox";
import type { AgentTool } from "@earendil-works/pi-agent-core";
import { serverExecute } from "../tool-execute.ts";
import { pdfExtract, pdfInfo, pdfProbe, pdfSearch } from "../subagents/pdf-ops.ts";
/**
* PDF reading tools for the RESEARCHER sub-agent (owner design B,
* 2026-10-07): PDFs from the web, with the pdf-reader agent's TEXT
* capabilities — triage (pdf_info), page-level extraction (pdf_extract),
* targeted lookup (pdf_search) — woven into the researcher's own loop so
* page-cited content lands in the brief directly.
*
* web_fetch_pdf acquires the file (download, size-capped, probe-validated)
* into a per-job temp dir; every path-taking tool is SCOPED to that dir
* (the researcher can never read outside it). pdf_render is deliberately
* absent: the researcher's model slot has no vision — visual reading of
* math-heavy pages stays the pdf job's job.
*
* Tool contract (the web.ts law): NEVER throw into the agent loop — every
* failure path returns a clear error STRING the researcher can adapt to.
*/
const USER_AGENT =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
const FETCH_TIMEOUT_MS = 60_000;
/** PDFs run bigger than pages — arxiv papers routinely 5-20 MB. */
const MAX_PDF_BYTES = 25 * 1024 * 1024;
/** The pdf-reader doctrine's size classes, by page count. */
function sizeClass(pageCount: number): "short" | "medium" | "long" {
if (pageCount <= 10) return "short";
if (pageCount <= 40) return "medium";
return "long";
}
/** Pages worth a look before any extraction — math- or image-heavy. */
function flaggedPages(info: Awaited<ReturnType<typeof pdfInfo>>): number[] {
return info.pages
.filter((p) => p.math_density >= 0.03 || p.image_count > 0)
.slice(0, 30)
.map((p) => p.page);
}
/** Resolve a tool-supplied path and require it to live INSIDE the job's
* pdf dir — the researcher's filesystem ends at the temp dir's wall. */
function scopedPath(dir: string, filePath: string): string | null {
const root = path.resolve(dir) + path.sep;
const resolved = path.resolve(dir, filePath);
if (!resolved.startsWith(root)) return null;
return resolved;
}
/** Filename for a downloaded PDF — slug from the URL's last segment plus a
* unique suffix (two PDFs from different URLs never collide). */
function pdfFileName(url: string): string {
try {
const last = new URL(url).pathname.split("/").filter(Boolean).pop() ?? "document";
const slug = last.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(-80) || "document";
const stem = slug.replace(/\.pdf$/i, "");
return `${stem}-${Math.random().toString(36).slice(2, 8)}.pdf`;
} catch {
return `document-${Math.random().toString(36).slice(2, 8)}.pdf`;
}
}
/** The download + triage summary the researcher sees from web_fetch_pdf. */
export interface PdfFetchSummary {
path: string;
url: string;
page_count: number;
size_class: "short" | "medium" | "long";
mean_text_length: number;
flagged_pages: number[];
}
export function createPdfReadTools(dir: string): Record<string, AgentTool<any>> {
const pdfFetchInput = Type.Object({
url: Type.String({ description: "The PDF URL to download" }),
});
const pdfPathInput = Type.Object({
path: Type.String({ description: "The local PDF path returned by web_fetch_pdf" }),
});
const pdfExtractInput = Type.Object({
path: Type.String({ description: "The local PDF path returned by web_fetch_pdf" }),
pages: Type.String({
description: "Pages to extract: 'all', '1-5', '1,3,7', or '3'. Use pdf_info first; narrow the spec for big documents.",
}),
});
const pdfSearchInput = Type.Object({
path: Type.String({ description: "The local PDF path returned by web_fetch_pdf" }),
query: Type.String({ description: "Text to find (case-insensitive substring)" }),
});
return {
web_fetch_pdf: {
name: "web_fetch_pdf",
label: "web_fetch_pdf",
description:
"Download a PDF from the web and triage it: returns its local path, page count, size class " +
"(short ≤10 pages, medium ≤40, long), and flagged (math/image-heavy) pages. " +
"Then read it with pdf_extract / pdf_search. Use when web_fetch reports a URL is a PDF.",
parameters: pdfFetchInput,
execute: serverExecute("web_fetch_pdf", pdfFetchInput, async ({ url }): Promise<PdfFetchSummary | string> => {
try {
const parsed = new URL(url);
if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
throw new Error("Only http(s) URLs are supported");
} catch {
return `web_fetch_pdf failed: invalid URL "${url}"`;
}
try {
const resp = await fetch(url, {
headers: {
"User-Agent": USER_AGENT,
Accept: "application/pdf,application/octet-stream,*/*",
},
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (!resp.ok) throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
const contentType = resp.headers.get("content-type") ?? "";
if (!/application\/pdf/i.test(contentType) && !/\.pdf(\?|$)/i.test(parsedPathname(url))) {
throw new Error(`Not a PDF (content-type ${contentType.split(";")[0] || "unknown"})`);
}
const declared = Number(resp.headers.get("content-length") ?? 0);
if (declared > MAX_PDF_BYTES) throw new Error("PDF too large (over 25 MB)");
const bytes = new Uint8Array(await resp.arrayBuffer());
if (bytes.length === 0) throw new Error("Empty response");
if (bytes.length > MAX_PDF_BYTES) throw new Error("PDF too large (over 25 MB)");
const filePath = path.join(dir, pdfFileName(url));
await fs.writeFile(filePath, bytes);
try {
await pdfProbe(filePath);
} catch {
await fs.rm(filePath, { force: true });
throw new Error("The download is not a readable PDF");
}
const info = await pdfInfo(filePath);
const summary: PdfFetchSummary = {
path: filePath,
url,
page_count: info.page_count,
size_class: sizeClass(info.page_count),
mean_text_length: info.mean_text_length,
flagged_pages: flaggedPages(info),
};
return summary;
} catch (err) {
return `web_fetch_pdf failed for ${url}: ${err instanceof Error ? err.message : String(err)}`;
}
}),
},
pdf_info: {
name: "pdf_info",
label: "pdf_info",
description:
"Triage a downloaded PDF: page count plus per-page signals (text_length, image_count, math_density).",
parameters: pdfPathInput,
execute: serverExecute("pdf_info", pdfPathInput, async ({ path: filePath }) => {
const scoped = scopedPath(dir, filePath);
if (!scoped) return `pdf_info failed: path must stay inside this job's download dir`;
try {
const info = await pdfInfo(scoped);
return {
page_count: info.page_count,
mean_text_length: info.mean_text_length,
size_class: sizeClass(info.page_count),
pages: info.pages,
};
} catch (err) {
return `pdf_info failed: ${err instanceof Error ? err.message : String(err)}`;
}
}),
},
pdf_extract: {
name: "pdf_extract",
label: "pdf_extract",
description:
"Extract text from pages of a downloaded PDF (spec: 'all', '1-5', '1,3,7', '3'). " +
"Use pdf_info first; narrow the spec for big documents.",
parameters: pdfExtractInput,
execute: serverExecute("pdf_extract", pdfExtractInput, async ({ path: filePath, pages }) => {
const scoped = scopedPath(dir, filePath);
if (!scoped) return `pdf_extract failed: path must stay inside this job's download dir`;
try {
return await pdfExtract(scoped, pages);
} catch (err) {
return `pdf_extract failed: ${err instanceof Error ? err.message : String(err)}`;
}
}),
},
pdf_search: {
name: "pdf_search",
label: "pdf_search",
description: "Find text in a downloaded PDF — returns the pages and contexts that hit.",
parameters: pdfSearchInput,
execute: serverExecute("pdf_search", pdfSearchInput, async ({ path: filePath, query }) => {
const scoped = scopedPath(dir, filePath);
if (!scoped) return `pdf_search failed: path must stay inside this job's download dir`;
try {
const { hits } = await pdfSearch(scoped, query, { literal: true });
return { hits };
} catch (err) {
return `pdf_search failed: ${err instanceof Error ? err.message : String(err)}`;
}
}),
},
};
}
function parsedPathname(url: string): string {
try {
return new URL(url).pathname;
} catch {
return "";
}
}
+12 -3
View File
@@ -91,8 +91,17 @@ async function fetchAndExtract(url: string): Promise<string> {
}); });
if (!resp.ok) throw new Error(`HTTP ${resp.status} ${resp.statusText}`); if (!resp.ok) throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
// Final URL after redirects + status — the researcher's only way to
// tell a real page from a 200-served error body (owner-reported
// "ill-formed request" case: the SITE answered, we just relayed it).
const sourceLine = `Source: ${resp.url || url} (HTTP ${resp.status})`;
const contentType = resp.headers.get("content-type") ?? ""; const contentType = resp.headers.get("content-type") ?? "";
if (contentType.includes("application/pdf")) throw new Error("PDF documents are not supported"); if (contentType.includes("application/pdf")) {
// Pointer, not dead-end (researcher PDF wave, 2026-10-07): the
// researcher's web_fetch_pdf tool downloads + triages PDFs.
return `web_fetch: this URL is a PDF document — call web_fetch_pdf on it to download and read it.`;
}
if (/image\/|audio\/|video\/|zip|octet-stream/.test(contentType)) { if (/image\/|audio\/|video\/|zip|octet-stream/.test(contentType)) {
throw new Error(`Unsupported content type: ${contentType.split(";")[0]}`); throw new Error(`Unsupported content type: ${contentType.split(";")[0]}`);
} }
@@ -100,11 +109,11 @@ async function fetchAndExtract(url: string): Promise<string> {
if (length > MAX_RESPONSE_BYTES) throw new Error("Response too large"); if (length > MAX_RESPONSE_BYTES) throw new Error("Response too large");
const text = await resp.text(); const text = await resp.text();
if (!contentType.includes("html")) return truncate(text); // plain text / markdown / code if (!contentType.includes("html")) return `${sourceLine}\n\n${truncate(text)}`; // plain text / markdown / code
const extracted = htmlToMarkdown(text); const extracted = htmlToMarkdown(text);
if (extracted) { if (extracted) {
const header = extracted.title ? `# ${extracted.title}\n\nSource: ${url}\n\n---\n\n` : ""; const header = extracted.title ? `# ${extracted.title}\n\n${sourceLine}\n\n---\n\n` : `${sourceLine}\n\n---\n\n`;
return header + truncate(extracted.content); return header + truncate(extracted.content);
} }
// Readability failed — likely JS-rendered. Try Jina Reader. // Readability failed — likely JS-rendered. Try Jina Reader.
+204
View File
@@ -0,0 +1,204 @@
/**
* Researcher PDF capability (PDF wave, 2026-10-07, design B): web_fetch_pdf
* downloads a PDF into a per-job temp dir and returns the triage summary;
* pdf_info/pdf_extract/pdf_search read it page-level, ALL scoped to the dir
* (the researcher's filesystem ends at the temp wall). web_fetch gained the
* PDF POINTER (no dead-end) and the Source line with final URL + HTTP
* status (the "ill-formed request" lesson: 200-served error bodies must be
* distinguishable from real pages).
*
* Zero network: globalThis.fetch is stubbed with a REAL minimal PDF built
* programmatically (correct xref offsets — pdfjs parses it like any
* download). Rendering is out of scope here (no vision in the researcher).
*/
import assert from "node:assert/strict";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { after, describe, it } from "node:test";
import { createPdfReadTools } from "../src/agent/tools/pdf-read.ts";
import { createWebTools } from "../src/agent/tools/web.ts";
/** Build a valid N-page PDF in memory — ASCII-only so string offsets ARE
* byte offsets; xref entries computed as we assemble. */
function buildPdf(pageTexts: string[]): Uint8Array {
const objects: string[] = []; // 1-indexed
const pageIds = pageTexts.map((_, i) => 3 + i * 2);
const fontId = 3 + pageTexts.length * 2;
objects[1] = "<</Type/Catalog/Pages 2 0 R>>";
objects[2] = `<</Type/Pages/Kids[${pageIds.map((id) => `${id} 0 R`).join(" ")}]/Count ${pageTexts.length}>>`;
pageTexts.forEach((text, i) => {
const pageId = pageIds[i]!;
const contentId = pageId + 1;
const escaped = text.replace(/([()\\])/g, "\\$1");
const stream = `BT /F1 12 Tf 72 720 Td (${escaped}) Tj ET`;
objects[pageId] =
`<</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]/Contents ${contentId} 0 R` +
`/Resources<</Font<</F1 ${fontId} 0 R>>>>>>`;
objects[contentId] = `<</Length ${stream.length}>>\nstream\n${stream}\nendstream`;
});
objects[fontId] = "<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
let pdf = "%PDF-1.4\n";
const offsets: number[] = [];
for (let id = 1; id < objects.length; id++) {
offsets[id] = pdf.length;
pdf += `${id} 0 obj\n${objects[id]}\nendobj\n`;
}
const xrefStart = pdf.length;
pdf += `xref\n0 ${objects.length}\n0000000000 65535 f \n`;
for (let id = 1; id < objects.length; id++) {
pdf += `${String(offsets[id]!).padStart(10, "0")} 00000 n \n`;
}
pdf += `trailer\n<</Size ${objects.length}/Root 1 0 R>>\nstartxref\n${xrefStart}\n%%EOF\n`;
return new TextEncoder().encode(pdf);
}
/** A fresh Response per call (bodies are single-use). */
const pdfResponse = () =>
new Response(PDF_BYTES.slice().buffer as ArrayBuffer, { headers: { "content-type": "application/pdf" } });
const PDF_BYTES = buildPdf([
"Hello PDF page one — the triage summary should see this.",
"Page two holds the orthogonal complement theorem statement.",
]);
const tempDirs: string[] = [];
async function newDir(): Promise<string> {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "lh-test-pdf-"));
tempDirs.push(dir);
return dir;
}
after(async () => {
await Promise.all(tempDirs.map((d) => fs.rm(d, { recursive: true, force: true })));
});
/** Tool-result .details → displayable string (the agent sees this shape). */
function asText(result: unknown): string {
const r = result as { details?: unknown };
if (typeof r?.details === "string") return r.details;
return JSON.stringify(r?.details ?? r);
}
describe("researcher pdf tools — download + triage", () => {
it("web_fetch_pdf downloads, triages, and returns a usable summary", async () => {
const dir = await newDir();
const tools = createPdfReadTools(dir);
const realFetch = globalThis.fetch;
globalThis.fetch = (async () =>
pdfResponse()) as typeof fetch;
try {
const out = await tools.web_fetch_pdf!.execute!("t1", { url: "https://arxiv.org/pdf/2406.14058" });
const text = asText(out);
assert.match(text, /"page_count":2/);
assert.match(text, /"size_class":"short"/);
assert.match(text, /"path":"[^"]+\.pdf"/);
const summary = (out as { details: { path: string } }).details;
const written = await fs.readFile(summary.path);
assert.equal(written.byteLength, PDF_BYTES.byteLength, "the PDF bytes land on disk");
} finally {
globalThis.fetch = realFetch;
}
});
it("the follow-up tools read the download page-level (extract + search)", async () => {
const dir = await newDir();
const tools = createPdfReadTools(dir);
const realFetch = globalThis.fetch;
globalThis.fetch = (async () =>
pdfResponse()) as typeof fetch;
try {
const out = await tools.web_fetch_pdf!.execute!("t2", { url: "https://example.com/paper.pdf" });
const { path: pdfPath } = (out as { details: { path: string } }).details;
const extract = asText(await tools.pdf_extract!.execute!("t3", { path: pdfPath, pages: "2" }));
assert.match(extract, /--- Page 2 ---/);
assert.match(extract, /orthogonal complement/);
assert.ok(!extract.includes("page one"), "narrow specs extract only the asked pages");
const search = asText(await tools.pdf_search!.execute!("t4", { path: pdfPath, query: "theorem" }));
assert.match(search, /"page":2/);
} finally {
globalThis.fetch = realFetch;
}
});
it("rejects non-PDF responses and non-http(s) schemes with error strings", async () => {
const dir = await newDir();
const tools = createPdfReadTools(dir);
const realFetch = globalThis.fetch;
globalThis.fetch = (async () =>
new Response("<html>just a page</html>", { headers: { "content-type": "text/html" } })) as typeof fetch;
try {
const notPdf = asText(await tools.web_fetch_pdf!.execute!("t5", { url: "https://example.com/doc" }));
assert.match(notPdf, /web_fetch_pdf failed/);
assert.match(notPdf, /Not a PDF/);
} finally {
globalThis.fetch = realFetch;
}
const ftp = asText(await tools.web_fetch_pdf!.execute!("t6", { url: "ftp://example.com/x.pdf" }));
assert.match(ftp, /invalid URL/);
});
});
describe("researcher pdf tools — the temp-dir wall", () => {
it("path-taking tools refuse anything outside the job dir", async () => {
const dir = await newDir();
const tools = createPdfReadTools(dir);
const outside = path.join(os.tmpdir(), "escape-attempt.pdf");
const rel = asText(await tools.pdf_info!.execute!("t7", { path: outside }));
assert.match(rel, /pdf_info failed: path must stay inside/);
const traversal = asText(await tools.pdf_extract!.execute!("t8", { path: "../../etc/passwd", pages: "1" }));
assert.match(traversal, /pdf_extract failed: path must stay inside/);
});
it("pdf_info reports per-page triage signals for a real download", async () => {
const dir = await newDir();
const tools = createPdfReadTools(dir);
const realFetch = globalThis.fetch;
globalThis.fetch = (async () =>
pdfResponse()) as typeof fetch;
try {
const out = await tools.web_fetch_pdf!.execute!("t9", { url: "https://example.com/x.pdf" });
const { path: pdfPath } = (out as { details: { path: string } }).details;
const info = asText(await tools.pdf_info!.execute!("t10", { path: pdfPath }));
assert.match(info, /"page_count":2/);
assert.match(info, /"pages":\[/);
assert.match(info, /"text_length":/);
} finally {
globalThis.fetch = realFetch;
}
});
});
describe("web_fetch — PDF pointer + source header", () => {
it("a PDF URL points at web_fetch_pdf instead of dead-ending", async () => {
const web = createWebTools(() => null);
const realFetch = globalThis.fetch;
globalThis.fetch = (async () =>
pdfResponse()) as typeof fetch;
try {
const out = asText(await web.web_fetch!.execute!("t11", { url: "https://example.com/paper.pdf" }));
assert.match(out, /PDF document/);
assert.match(out, /web_fetch_pdf/);
} finally {
globalThis.fetch = realFetch;
}
});
it("output carries the source line with HTTP status", async () => {
const web = createWebTools(() => null);
const realFetch = globalThis.fetch;
globalThis.fetch = (async () =>
new Response("A plain text page with enough content to matter.", {
headers: { "content-type": "text/plain" },
})) as typeof fetch;
try {
const out = asText(await web.web_fetch!.execute!("t12", { url: "https://example.com/page" }));
assert.match(out, /^Source: https:\/\/example\.com\/page \(HTTP 200\)/);
} finally {
globalThis.fetch = realFetch;
}
});
});