researcher: PDFs from the web — the pdf-reader's text tools in the loop (design B)
web_fetch_pdf downloads (25MB cap, probe-validated) into a per-job temp dir and returns the triage summary — page count, size class (the pdf doctrine's <=10/<=40 thresholds), flagged math/image-heavy pages. The researcher's toolset then gains pdf_info / pdf_extract / pdf_search over that dir, scoped by a resolve-and-startswith wall: the researcher's filesystem ends at the temp dir. Render is deliberately absent — the researcher slot has no vision; visual reading stays the pdf job's job. web_fetch: a PDF URL now returns a POINTER to web_fetch_pdf instead of dead-ending, and every output carries 'Source: <final url after redirects> (HTTP <status>)' — the 'ill-formed request' lesson: a 200-served error body must be distinguishable from a real page. The runner owns the temp lifecycle (mkdtemp before, recursive cleanup in the job's finally — never enters the record) and web_fetch_pdf urls ride the sources ledger. Doctrine step 5 added: fetch, triage, extract only the pages that matter, cite as (p. N). Tests build a REAL minimal PDF programmatically (correct xref offsets — pdfjs parses it like any download): zero network, real parse.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import type { Usage } from "@earendil-works/pi-ai";
|
||||
import { createWebTools } from "../tools/web.ts";
|
||||
import { createPdfReadTools } from "../tools/pdf-read.ts";
|
||||
import type { SearchProvider } from "../../core/search.ts";
|
||||
import type { PiEngine } from "../models.ts";
|
||||
import { runSubagent } from "./pi-run.ts";
|
||||
@@ -32,7 +33,11 @@ Process:
|
||||
3. Read the results. Identify what's well-covered and what has gaps. If the first round
|
||||
doesn't fully answer the question, search again with refined queries targeting the gaps.
|
||||
4. For the 2-3 most promising source URLs, use web_fetch to get the full page content.
|
||||
5. Synthesize everything into a brief that directly answers the question.
|
||||
5. PDF sources: when web_fetch says a URL is a PDF, call web_fetch_pdf on it — it downloads
|
||||
and triages the document (page count, size class, flagged pages). Then read it with
|
||||
pdf_extract — for long ones, pdf_search first and extract only the pages that hit.
|
||||
Cite claims from a PDF as (p. N).
|
||||
6. Synthesize everything into a brief that directly answers the question.
|
||||
|
||||
Evaluation — what to keep vs drop:
|
||||
- Official docs and primary sources outweigh blog posts and forum threads.
|
||||
@@ -68,6 +73,11 @@ export interface ResearcherRunOptions {
|
||||
* seam reads as unavailable: the server wiring always injects one; the
|
||||
* dispatch-level drop (jobTools) keeps doomed jobs from even starting. */
|
||||
search?: () => SearchProvider | null;
|
||||
/** The researcher's PDF dir (PDF wave, 2026-10-07): when present, the
|
||||
* toolset gains web_fetch_pdf + pdf_info/pdf_extract/pdf_search scoped
|
||||
* to this per-job temp dir — PDFs from the web, read page-level. The
|
||||
* runner owns its lifecycle (mkdtemp before, rm after). */
|
||||
pdfDir?: string;
|
||||
abortSignal?: AbortSignal;
|
||||
onUsage?: (usage: Usage) => void;
|
||||
onToolStart?: (toolName: string, args: unknown) => void;
|
||||
@@ -81,7 +91,9 @@ export interface ResearcherRunOptions {
|
||||
* the loop scripted.
|
||||
*/
|
||||
export async function runResearcherAgent(prompt: string, opts: ResearcherRunOptions = {}): Promise<string> {
|
||||
const { text } = await runSubagent(Object.values(createWebTools(opts.search ?? (() => null))), {
|
||||
const tools = Object.values(createWebTools(opts.search ?? (() => null)));
|
||||
if (opts.pdfDir) tools.push(...Object.values(createPdfReadTools(opts.pdfDir)));
|
||||
const { text } = await runSubagent(tools, {
|
||||
role: "researcher",
|
||||
prompt,
|
||||
engine: opts.engine,
|
||||
|
||||
@@ -19,6 +19,9 @@ import {
|
||||
type VisualToolInput,
|
||||
} from "../../core/contracts.ts";
|
||||
import { registerJobAbort, unregisterJobAbort } from "./aborts.ts";
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import type { ModelRole } from "../../../config.ts";
|
||||
import { ROLE_SLOT, modelsNotConfigured, type ModelSlotName, type ProviderKeyMissing } from "../../core/settings.ts";
|
||||
import type { ChatStore, JobStore } from "../../core/ports.ts";
|
||||
@@ -353,20 +356,25 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
|
||||
.join("\n\n");
|
||||
|
||||
const fetchedUrls: string[] = [];
|
||||
// Per-job PDF dir (researcher PDF wave, 2026-10-07): web_fetch_pdf
|
||||
// downloads land here; every pdf tool is scoped to it; it dies with
|
||||
// the job (never enters the record).
|
||||
const pdfDir = await fs.mkdtemp(path.join(os.tmpdir(), "lh-research-"));
|
||||
|
||||
try {
|
||||
jobStore.updateJob(jobId, { activity: "researching" });
|
||||
const brief = await runResearcherAgent(prompt, {
|
||||
engine: model,
|
||||
search: deps.searchProvider,
|
||||
pdfDir,
|
||||
abortSignal: loop.abortSignal,
|
||||
chatId: loop.chatId,
|
||||
// Money channel: each agent step is one model call — fold its
|
||||
// usage into the record as it completes (progress write).
|
||||
onUsage: (usage) => loop.onUsage(piUsageToStep(usage)),
|
||||
onToolStart: (toolName, args) => {
|
||||
onToolStart: (toolName, args) => {
|
||||
const input = args as { query?: string; url?: string } | undefined;
|
||||
if (toolName === "web_fetch" && typeof input?.url === "string") {
|
||||
if (typeof input?.url === "string" && (toolName === "web_fetch" || toolName === "web_fetch_pdf")) {
|
||||
fetchedUrls.push(input.url);
|
||||
}
|
||||
jobStore.updateJob(jobId, { activity: activityLine(toolName, input) });
|
||||
@@ -383,6 +391,8 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
if (stillRunning(jobId)) jobStore.updateJob(jobId, { error: reason });
|
||||
await finishJob(jobId, { brief: `Research failed: ${reason}`, sources: [] });
|
||||
} finally {
|
||||
await fs.rm(pdfDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user