feat(T2b): sources v0 — read tool + pdf-reader sub-agent

Learner source files, on the teacher's own terms (nothing auto-injects):

- read tool (src/agent/tools/read.ts): pi-read port — path/offset/limit,
  2000-line/50KB truncation with use-offset continuation notices, image
  files (jpg/png/gif/webp/bmp) returned as native image content parts
  (new LhToolImageResult escape hatch in lh-tool.ts). Always-on in both
  the prober and teacher registries (user-decided 2026-09).
- pdf-reader sub-agent (JobKind 'pdf', toggleable, ack-and-steer): triage
  via pdf_info (pdfjs text layer: per-page text/image/math-density),
  targeted pdf_extract, regex pdf_search (hits count as consultation),
  and pdf_render — pages painted by pdfjs onto a canvas shim in headless
  Chrome (the pdfium viewer doesn't paint headless), screenshot fed back
  image-in-the-loop (visual-maker idiom, providerSafePng applied). The
  unread-finish gate bounces a zero-consultation brief once. Teacher gets
  a text brief with page citations only; renders never leave the loop.
- MODELS.pdfReader = qwen/qwen3.8-flash (user override 2026-09-19: the
  look-at-renders workload wants the vision-proven model; catalog listed
  glm-5.3-flash as image-capable but qwen is the known-good daily driver).
- UI: PdfCompleteBadge + PdfBriefCard (steer-card, job-card), dispatch
  card + activity-dock kind entries (type-forced, flagged for UI review).
- Prompts: 'Learner source files' section in teacher/30-jobs.md; read
  line in prober/10-probe-doctrine.md. Doctrine rides tool descriptions
  per the self-contained-tools principle.
This commit is contained in:
amosblomqvist
2026-09-19 15:24:26 +03:00
parent bc2c02b865
commit 9a90dd51d9
27 changed files with 1970 additions and 55 deletions
+107 -3
View File
@@ -1,7 +1,7 @@
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { pathToFileURL } from "node:url";
import { fileURLToPath, pathToFileURL } from "node:url";
/**
* Shared headless-browser helper for the verifying sub-agents (W6):
@@ -46,6 +46,16 @@ export interface RenderSvgOptions {
height?: number;
}
export interface RenderPdfOptions {
/** Render DPI — 72 is 100% zoom; the skill's guidance is 100/150/200. */
dpi?: number;
}
export interface RenderedPdfPage {
page: number;
png: Buffer;
}
export interface HeadlessApi {
/** True when a Chrome executable is resolvable (cheap, no launch). */
available(): boolean;
@@ -55,6 +65,9 @@ export interface HeadlessApi {
runDemo(html: string, opts?: RunDemoOptions): Promise<DemoRunResult>;
/** Rasterize an SVG to a PNG buffer (for verify-by-looking). */
renderSvg(svg: string, opts?: RenderSvgOptions): Promise<Buffer>;
/** Render PDF pages to PNG buffers via Chrome's pdfium viewer (for the
* pdf-reader sub-agent's look-at-the-page loop). */
renderPdfPages(pdfPath: string, pages: number[], opts?: RenderPdfOptions): Promise<RenderedPdfPage[]>;
}
/** Test seam — undefined in production. */
@@ -94,7 +107,7 @@ const NO_CHROME_MESSAGE =
"No Chrome/Chromium found for headless testing. Install Google Chrome or set CHROME_PATH to a Chrome executable. " +
"(Verification is mandatory — demos and visuals are never delivered untested.)";
async function launchBrowser(): Promise<import("puppeteer-core").Browser> {
async function launchBrowser(extraArgs: string[] = []): Promise<import("puppeteer-core").Browser> {
const puppeteer = (await import("puppeteer-core")).default;
const executablePath = resolveChromeExecutablePath();
const browser = await puppeteer.launch({
@@ -102,7 +115,7 @@ async function launchBrowser(): Promise<import("puppeteer-core").Browser> {
// Explicit path when we found one; otherwise let puppeteer resolve the
// system Chrome install itself via the "chrome" channel.
...(executablePath ? { executablePath } : { channel: "chrome" as const }),
args: ["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"],
args: ["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage", ...extraArgs],
});
return browser;
}
@@ -275,6 +288,96 @@ async function renderSvg(svg: string, opts: RenderSvgOptions = {}): Promise<Buff
}
}
// ─── PDF page rasterization (pdfjs paints, Chrome hosts) ───────────────
/** Render PDF pages to PNGs — the sources-v0 pdf-reader's eyes.
*
* The engine (user-chosen hybrid, live-verified 2026-09-19): pdfjs paints
* each page into a <canvas> (scale = dpi/72, the skill's zoom arithmetic)
* inside a headless Chrome tab, and the canvas pixels come back via
* toDataURL. Chrome's OWN PDF viewer plugin does not paint in headless
* mode (verified: identical blank screenshots regardless of page), so the
* canvas shim is the pixel path — Chrome still hosts and encodes, no
* native node-canvas anywhere. PDF bytes pass in as base64 (fetch() is
* blocked on file://); pdfjs runs via its real worker module under
* --allow-file-access-from-files. */
async function renderPdfPages(
pdfPath: string,
pages: number[],
opts: RenderPdfOptions = {},
): Promise<RenderedPdfPage[]> {
const dpi = opts.dpi ?? 150;
const pdfAbs = path.resolve(pdfPath);
const pdfjsUrl = pathToFileURL(
path.join(PATHS_nodeModules(), "pdfjs-dist/legacy/build/pdf.mjs"),
).href;
const b64 = fs.readFileSync(pdfAbs).toString("base64");
const dir = tempDir("lh-pdf-");
const file = path.join(dir, "shim.html");
fs.writeFileSync(
file,
"<!doctype html><html><head><meta charset='utf-8'><style>html,body{margin:0;padding:0;background:#fff}canvas{display:block}</style></head><body></body></html>",
"utf-8",
);
const browser = await launchBrowser(["--allow-file-access-from-files"]);
try {
const page = await browser.newPage();
await page.setViewport({ width: 1200, height: 1600, deviceScaleFactor: 1 });
await page.goto(pathToFileURL(file).href, { waitUntil: "load", timeout: 30_000 });
const results = await page.evaluate(
async (b64pdf, workerUrl, pageNos, dpi) => {
// window-scope dynamic import (Chrome evaluates this string in the
// shim page, not under Node)
const pdfjs = (await import(workerUrl)) as typeof import("pdfjs-dist");
pdfjs.GlobalWorkerOptions.workerSrc = workerUrl.replace("pdf.mjs", "pdf.worker.mjs");
const bin = atob(b64pdf);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
const doc = await pdfjs.getDocument({ data: bytes }).promise;
const out: Array<{ page: number; dataUrl: string; w: number; h: number }> = [];
for (const pageNo of pageNos) {
if (pageNo < 1 || pageNo > doc.numPages) continue;
const pg = await doc.getPage(pageNo);
const vp = pg.getViewport({ scale: dpi / 72 });
const canvas = document.createElement("canvas");
canvas.width = Math.ceil(vp.width);
canvas.height = Math.ceil(vp.height);
const ctx = canvas.getContext("2d");
if (!ctx) throw new Error("canvas 2d context unavailable");
await pg.render({ canvasContext: ctx, canvas, viewport: vp }).promise;
out.push({ page: pageNo, dataUrl: canvas.toDataURL("image/png"), w: canvas.width, h: canvas.height });
}
return out;
},
b64,
pdfjsUrl,
pages,
dpi,
);
return results.map((r) => ({
page: r.page,
// canvas-encoded PNG → providerSafePng for the same single-IDAT
// canonical form every other vision-model image gets.
png: providerSafePng(Buffer.from(r.dataUrl.slice(r.dataUrl.indexOf(",") + 1), "base64")),
}));
} finally {
await browser.close().catch(() => {});
try {
fs.rmSync(dir, { recursive: true, force: true });
} catch {
/* best-effort cleanup */
}
}
}
/** node_modules — resolved from this module's location. */
function PATHS_nodeModules(): string {
// import.meta.url = .../src/agent/subagents/headless.ts → three up + node_modules
return path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "node_modules");
}
// ─── the real API ────────────────────────────────────────────────────────────
export const realHeadless: HeadlessApi = {
@@ -286,4 +389,5 @@ export const realHeadless: HeadlessApi = {
},
runDemo,
renderSvg,
renderPdfPages,
};
+227
View File
@@ -0,0 +1,227 @@
import * as fs from "node:fs/promises";
import * as path from "node:path";
/**
* PDF plumbing for the pdf-reader sub-agent — sources v0 (2026-09): Node
* ports of the four pdf-reader skill scripts (pdf_info / pdf_extract /
* pdf_search / pdf_render), on the user-chosen hybrid stack:
*
* - TEXT (info / extract / search): pdfjs-dist — pure JS, no native deps.
* - RENDER: the shared headless-Chrome infra (subagents/headless.ts,
* puppeteer + system Chrome) screenshots pdfium-rendered pages.
*
* Doctrine (the skill's reading strategy, mirrored in the sub-agent's system
* prompt): triage with pdf_info FIRST, then size-based strategy — short PDFs
* render all pages, medium render flagged (math/image-heavy) pages, long
* ones search for targeted lookups before reading anything at length.
*
* These are pure library functions — no model in sight. The pdf-reader
* agent (pdf-reader.ts) drives them through its own tool loop.
*/
export interface PdfPageInfo {
page: number;
text_length: number;
image_count: number;
/** Fraction of math-flavored chars — the render-this-page signal (port
* of the skill's unicode-range density heuristic). */
math_density: number;
}
export interface PdfInfo {
path: string;
page_count: number;
/** Mean chars/page — the size class the strategy branches on. */
mean_text_length: number;
pages: PdfPageInfo[];
}
/** Load the legacy pdfjs build (runs headless in plain node, no worker). */
async function loadPdfjs() {
const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
return pdfjs;
}
async function openDocument(filePath: string) {
const pdfjs = await loadPdfjs();
const data = new Uint8Array(await fs.readFile(filePath));
// The legacy build in plain node runs on a main-thread fake worker (no
// workerSrc needed); teardown is the loading task's destroy().
const task = pdfjs.getDocument({ data });
return { doc: await task.promise, task };
}
/** Concatenate a page's text items into a body string. */
function itemsToText(items: unknown[]): string {
let out = "";
for (const it of items) {
if (it && typeof it === "object" && "str" in (it as Record<string, unknown>)) {
const str = (it as { str?: unknown }).str;
if (typeof str === "string") out += str;
}
}
return out;
}
/** Math-flavored char fraction — the skill's heuristic, verbatim ranges. */
function mathDensity(text: string): number {
let mathChars = 0;
for (const c of text) {
const code = c.codePointAt(0)!;
if (
(code >= 0x2200 && code <= 0x22ff) || // Mathematical Operators
(code >= 0x2100 && code <= 0x214f) || // Letterlike Symbols
(code >= 0x2190 && code <= 0x21ff) || // Arrows
(code >= 0x27c0 && code <= 0x27ef) || // Misc Mathematical Symbols-A
(code >= 0x2980 && code <= 0x29ff) || // Misc Mathematical Symbols-B
(code >= 0x2a00 && code <= 0x2aff) || // Supplemental Mathematical Operators
(code >= 0x0370 && code <= 0x03ff) || // Greek
(code >= 0x1d400 && code <= 0x1d7ff) || // Mathematical Alphanumeric Symbols
"∫∑∏√∂∇∞≈≠≤≥±×÷∈∉⊂⊃∪∩∧∨¬∀∃∅".includes(c)
) {
mathChars++;
}
}
return mathChars / Math.max(text.length, 1);
}
/** Cheap validity probe: getDocument resolves only for parseable PDFs —
* no page walk, no text extraction (unlike pdfInfo). The reader's upfront
* unreadable-source gate; garbage files throw pdfjs's precise error. */
export async function pdfProbe(filePath: string): Promise<void> {
const { task } = await openDocument(filePath);
await task.destroy();
}
/** pdf_info: page count + per-page triage signals. ALWAYS run first. */
export async function pdfInfo(filePath: string): Promise<PdfInfo> {
const pdfjs = await loadPdfjs(); // OPS constant for the operator-list scan
const { doc } = await openDocument(filePath);
const pages: PdfPageInfo[] = [];
let totalText = 0;
for (let i = 1; i <= doc.numPages; i++) {
const page = await doc.getPage(i);
const content = await page.getTextContent();
const text = itemsToText(content.items as unknown[]);
// count page images: paintImageXObject ops in the operator list
const ops = await page.getOperatorList();
let imageCount = 0;
for (const fn of ops.fnArray) {
if (fn === pdfjs.OPS.paintImageXObject) imageCount++;
}
totalText += text.length;
pages.push({
page: i,
text_length: text.length,
image_count: imageCount,
math_density: Math.round(mathDensity(text) * 10000) / 10000,
});
}
return {
path: filePath,
page_count: doc.numPages,
mean_text_length: Math.round(totalText / Math.max(doc.numPages, 1)),
pages,
};
}
/** Parse 'all' | '1-5' | '1,3,7' | '3' into a sorted 1-indexed page list. */
export function parsePageSpec(spec: string, total: number): number[] {
const trimmed = spec.trim().toLowerCase();
if (trimmed === "all") return Array.from({ length: total }, (_, i) => i + 1);
const pages = new Set<number>();
for (const part of spec.split(",")) {
const p = part.trim();
if (!p) continue;
if (p.includes("-")) {
const [startRaw, endRaw] = p.split("-", 2) as [string, string];
const start = Math.max(1, parseInt(startRaw, 10));
const end = Math.min(total, parseInt(endRaw, 10));
for (let pg = start; pg <= end; pg++) pages.add(pg);
} else {
const pg = parseInt(p, 10);
if (pg >= 1 && pg <= total) pages.add(pg);
}
}
return [...pages].sort((a, b) => a - b);
}
/** pdf_extract: text of the requested pages, `--- Page N ---` headed. */
export async function pdfExtract(filePath: string, pageSpec: string): Promise<string> {
const { doc } = await openDocument(filePath);
const pages = parsePageSpec(pageSpec, doc.numPages);
if (pages.length === 0) return "No pages selected — page spec matched nothing.";
const out: string[] = [];
for (const i of pages) {
const page = await doc.getPage(i);
const content = await page.getTextContent();
out.push(`--- Page ${i} ---`);
out.push(itemsToText(content.items as unknown[]));
}
return out.join("\n");
}
export interface PdfSearchHit {
page: number;
line: number;
/** contextLines around the hit, `>>>` marking the hit line. */
context: string;
}
/** pdf_search: keyword/regex across all pages with context lines.
* pageTexts rides along so the agent can extract a hit's full page without
* a second pass (the skill pipelines `search → extract` the same way). */
export async function pdfSearch(
filePath: string,
query: string,
opts?: { contextLines?: number; literal?: boolean },
): Promise<{ hits: PdfSearchHit[]; pageTexts: Map<number, string> }> {
const contextLines = opts?.contextLines ?? 3;
const { doc } = await openDocument(filePath);
let pattern: RegExp;
if (opts?.literal) {
pattern = new RegExp(escapeRegex(query), "i");
} else {
try {
pattern = new RegExp(query, "i");
} catch (err) {
throw new Error(`Invalid regex: ${(err as Error).message}`);
}
}
const hits: PdfSearchHit[] = [];
const pageTexts = new Map<number, string>();
for (let i = 1; i <= doc.numPages; i++) {
const page = await doc.getPage(i);
const content = await page.getTextContent();
const text = itemsToText(content.items as unknown[]);
pageTexts.set(i, text);
const lines = text.split("\n");
for (let ln = 0; ln < lines.length; ln++) {
if (!pattern.test(lines[ln]!)) continue;
const start = Math.max(0, ln - contextLines);
const end = Math.min(lines.length, ln + contextLines + 1);
const context = lines
.slice(start, end)
.map((l, j) => `${start + j === ln ? ">>>" : " "} ${l}`)
.join("\n");
hits.push({ page: i, line: ln + 1, context });
}
}
return { hits, pageTexts };
}
function escapeRegex(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
/** Where rendered pages land (mirrors the skill's /tmp convention). */
export function renderDir(filePath: string): string {
// FNV-1a short hash of the absolute path — stable per file, no deps.
const abs = path.resolve(filePath);
let h = 0x811c9dc5;
for (let i = 0; i < abs.length; i++) {
h ^= abs.charCodeAt(i);
h = Math.imul(h, 0x01000193) >>> 0;
}
return path.join("/tmp", `lh-pdf-${h.toString(16).padStart(8, "0")}`);
}
+401
View File
@@ -0,0 +1,401 @@
import type { AssistantMessage, Message, ToolResultMessage, Usage } from "@earendil-works/pi-ai";
import { z } from "zod";
import { lhTool, piToolDeclarations, type LhTool } from "../lh-tool.ts";
import { piEngineFor, thinkingLevelFor, type PiEngine } from "../models.ts";
import type { PdfResult, PdfToolInput } from "../../core/contracts.ts";
import { getHeadless, type HeadlessApi } from "./headless.ts";
import { pdfExtract, pdfInfo, pdfProbe, pdfSearch, type PdfSearchHit } from "./pdf-ops.ts";
/**
* The pdf-reader sub-agent (sources v0, 2026-09): reads a learner-provided
* PDF on the teacher's behalf — a hand-rolled loop like the visual-maker's
* (image-in-the-loop can't ride the execute-tool plumbing: rendered pages
* are shown to THIS agent's eyes, never delivered onward — user-decided).
*
* System prompt = the pdf-reader skill's reading doctrine, verbatim in
* strategy: triage with pdf_info FIRST, then size-based strategy — short
* PDFs render all pages, medium render flagged (math/image-heavy) pages,
* long ones search for targeted lookups before reading anything at length.
*
* Its final text IS the brief (researcher idiom), with (p. N) citations;
* runPdfJob wraps it into PdfResult with pagesConsulted tracked by the loop.
*/
const INSTRUCTIONS = `You are a PDF reading specialist. You work for a teacher agent inside a learning
application: given a PDF path and a question, read the document on the teacher's behalf and
produce a focused brief that the teacher can teach from, with page citations.
You operate in an isolated context. The task gives you the PDF path and the question — that is
everything you know.
Reading doctrine — always in this order:
1. TRIAGE FIRST: call pdf_info. It returns page count plus per-page signals (text_length,
image_count, math_density). Never read or render anything before this.
2. Pick a strategy by document size:
- SHORT (≤ 10 pages): extract the full text; render every page that has meaningful
math_density or images (you read equations and diagrams visually, not as extracted text).
- MEDIUM (11-40 pages): extract promising pages; render only flagged pages (math- or
image-heavy, or where extraction looks thin).
- LONG (> 40 pages): pdf_search for the question's key terms first; extract and render only
the pages that hit. Never read a long document front-to-back.
3. When the extracted text of a page looks math- or diagram-heavy, or garbled, RENDER that
page and read it visually before drawing conclusions from it.
4. Render DPI guidance: 100 for text-dominant pages, 150 the default, 200 for dense small
print (math details, fine diagrams).
Tool outputs may be error strings (file unreadable, invalid regex, no hits). Adapt: narrow the
page spec, change the query, lower DPI. If the document genuinely cannot answer the question,
say so in the brief instead of inventing content.
Your FINAL assistant message is your entire deliverable — it must stand alone, in this format:
2-6 sentences (or short sections) answering the teacher's question from the document, each
claim cited as (p. N). End with a line "Pages consulted: N, N, …" listing every page you
extracted or rendered.`;
const MAX_ITERATIONS = 14;
/** Rendered pages per pdf_render call — the skill renders selectively; a
* batch cap keeps one over-eager call from flooding the vision context. */
const RENDER_BATCH_CAP = 6;
/** Extract output ceiling — narrow the spec instead of dumping a book. */
const EXTRACT_CAP = 24_000;
export interface PdfReaderOptions {
engine?: PiEngine;
abortSignal?: AbortSignal;
onUsage?: (usage: Usage) => void;
onActivity?: (line: string) => void;
headless?: HeadlessApi;
maxIterations?: number;
}
export interface PdfReaderInput extends PdfToolInput {}
export function createPdfTools(
pdfPath: string,
): {
pdf_info: LhTool<any, any>;
pdf_extract: LhTool<any, any>;
pdf_render: LhTool<any, any>;
pdf_search: LhTool<any, any>;
} {
return {
pdf_info: lhTool({
name: "pdf_info",
description: "Metadata + per-page triage signals for the PDF (page count, text length, image count, math density). ALWAYS call this first.",
schema: z.object({}),
execute: async () => {
const info = await pdfInfo(pdfPath);
const lines = [
`File: ${info.path}`,
`Pages: ${info.page_count} (mean text length/page: ${info.mean_text_length})`,
...info.pages.map(
(p) =>
`p${p.page}: len=${p.text_length} img=${p.image_count} math=${p.math_density.toFixed(2)}`,
),
];
return lines.join("\n");
},
}),
pdf_extract: lhTool({
name: "pdf_extract",
description: "Extract text from pages (spec: 'all', '1-5', '1,3,7', '3'). Use pdf_info first; narrow the spec for big documents.",
schema: z.object({ pages: z.string().describe("Page spec, e.g. 'all', '1-5', '1,3,7'") }),
execute: async ({ pages }) => {
const text = await pdfExtract(pdfPath, pages);
if (text.length > EXTRACT_CAP) {
return `${text.slice(0, EXTRACT_CAP)}\n\n[Output truncated at ${EXTRACT_CAP} chars — narrow the page spec and extract in smaller batches.]`;
}
return text;
},
}),
pdf_render: lhTool({
name: "pdf_render",
description:
"Render pages to images and show them to you for visual reading (equations, diagrams, layout). DPI: 100 text, 150 default, 200 dense small print. Max 6 pages per call.",
schema: z.object({
pages: z.string().describe("Page spec, e.g. '1-3' or '2,5,8'"),
dpi: z.number().int().min(72).max(300).default(150).describe("Render DPI (100 text / 150 default / 200 dense)"),
}),
execute: async () => {
// Actual rendering happens OUTSIDE execute — the loop appends the
// images as a user message after the tool results (the
// image-in-the-loop idiom). execute returns the plan the loop uses.
return "render requested";
},
}),
pdf_search: lhTool({
name: "pdf_search",
description: "Search the PDF's text by keyword or regex (case-insensitive); returns hits with context lines. Use for long documents before reading anything at length.",
schema: z.object({
query: z.string().describe("Keyword or regex"),
context_lines: z.number().int().min(0).max(8).default(3).describe("Context lines around each hit"),
literal: z.boolean().default(false).describe("true = treat the query literally (no regex)"),
}),
execute: async ({ query, context_lines, literal }) => {
const { hits } = await pdfSearch(pdfPath, query, { contextLines: context_lines, literal });
if (hits.length === 0) return "No hits.";
const cap = hits.slice(0, 40);
const body = cap.map((h: PdfSearchHit) => `=== Page ${h.page}, line ${h.line} ===\n${h.context}`).join("\n");
const more = hits.length > cap.length ? `\n\n[${hits.length - cap.length} more hits — narrow the query.]` : "";
return body + more;
},
}),
};
}
function toolText(toolCallId: string, toolName: string, value: string): ToolResultMessage {
return {
role: "toolResult",
toolCallId,
toolName,
content: [{ type: "text", text: value }],
isError: value.startsWith("ERROR:"),
timestamp: Date.now(),
};
}
/** Doctrine gate (the visual-maker's unlooked-finish rule, ported): never
* deliver a brief that consulted nothing — an unread PDF brief is made-up
* content, exactly the class of failure the gate exists for. */
const UNREAD_FINISH_MESSAGE =
"You finished without consulting any page of the PDF — your brief would be invention, not reading. Extract or render the relevant pages first, then write the brief from what you actually saw.";
export async function runPdfReaderAgent(
input: PdfReaderInput,
opts: PdfReaderOptions = {},
): Promise<PdfResult> {
const headless = opts.headless ?? getHeadless();
const maxIterations = opts.maxIterations ?? MAX_ITERATIONS;
const pdfPath = input.path;
// Deterministic upfront failure: unreadable / unparseable / no Chrome
// for renders — the brief never starts on a broken source.
const openError = await probeReadable(pdfPath);
if (openError) {
return { path: pdfPath, question: input.question, brief: `PDF failed: ${openError}`, pagesConsulted: [] };
}
const chromeReason = headless.unavailableReason();
// Chrome missing does NOT fail the run: text-side tools still work; the
// doctrine degrades to extract+search (render calls return an error note).
const tools = createPdfTools(pdfPath);
const toolDeclarations = piToolDeclarations(tools);
const engine = opts.engine ?? piEngineFor("pdfReader");
const reasoning = thinkingLevelFor("pdfReader");
const prompt = [
`PDF path: ${pdfPath}`,
`Question (what the teacher needs from this document): ${input.question || "(no question — give a structural overview)"}`,
"",
"Read the document per your doctrine, then produce your final brief in the required format.",
].join("\n");
let messages: Message[] = [{ role: "user", content: prompt, timestamp: Date.now() }];
/** Every page the reader extracted or rendered (the citation set). */
const consulted = new Set<number>();
let finalText = "";
let nudged = false;
try {
for (let iteration = 0; iteration < maxIterations; iteration++) {
const result = await streamOnce(engine, INSTRUCTIONS, messages, toolDeclarations, reasoning, opts.abortSignal);
// Money channel: each iteration is one model call.
opts.onUsage?.(result.usage);
messages = [...messages, result];
const toolCalls = result.content.filter(
(c): c is Extract<typeof c, { type: "toolCall" }> => c.type === "toolCall",
);
if (toolCalls.length === 0) {
finalText = result.content
.filter((c): c is { type: "text"; text: string } => c.type === "text")
.map((c) => c.text)
.join("")
.trim();
if (consulted.size === 0 && !nudged) {
nudged = true;
messages = [
...messages,
{ role: "user", content: UNREAD_FINISH_MESSAGE, timestamp: Date.now() },
];
continue;
}
return { path: pdfPath, question: input.question, brief: finalText || "(no brief text)", pagesConsulted: [...consulted].sort((a, b) => a - b) };
}
const toolResults: ToolResultMessage[] = [];
/** Rendered pages this batch, appended as images AFTER the tool
* results — the visual-maker's image-in-the-loop idiom. */
let renderedImages: { page: number; png: Buffer }[] | null = null;
for (const call of toolCalls) {
const args = (call.arguments ?? {}) as Record<string, unknown>;
try {
if (call.name === "pdf_info") {
opts.onActivity?.("pdf_info: triaging");
const r = await tools.pdf_info.execute!({}, { toolCallId: call.id });
toolResults.push(toolText(call.id, call.name, String(r)));
} else if (call.name === "pdf_extract") {
const spec = String(args.pages ?? "all");
opts.onActivity?.(`pdf_extract: ${spec}`);
const r = await tools.pdf_extract.execute!({ pages: spec }, { toolCallId: call.id });
toolResults.push(toolText(call.id, call.name, String(r)));
for (const p of specNumbers(spec, Infinity)) consulted.add(p);
} else if (call.name === "pdf_search") {
const query = String(args.query ?? "");
const contextLines = numOr(args.context_lines, 3);
const literal = Boolean(args.literal);
opts.onActivity?.(`pdf_search: "${query}"`);
// Direct call (not the tool's execute): the loop needs the hits'
// pages — a hit's context lines ARE consultation, so they join
// the citation set (the unread-finish gate counts them).
const { hits } = await pdfSearch(pdfPath, query, { contextLines, literal });
for (const h of hits) consulted.add(h.page);
const cap = hits.slice(0, 40);
const body =
cap.length === 0
? "No hits."
: cap.map((h) => `=== Page ${h.page}, line ${h.line} ===\n${h.context}`).join("\n") +
(hits.length > cap.length ? `\n\n[${hits.length - cap.length} more hits — narrow the query.]` : "");
toolResults.push(toolText(call.id, call.name, body));
} else if (call.name === "pdf_render") {
const spec = String(args.pages ?? "1");
const dpi = numOr(args.dpi, 150);
if (chromeReason) {
toolResults.push(
toolText(call.id, call.name, `ERROR: rendering unavailable — ${chromeReason} Read via pdf_extract instead.`),
);
} else {
const wanted = specNumbers(spec, RENDER_BATCH_CAP);
opts.onActivity?.(`pdf_render: ${spec} @${dpi}dpi`);
const pages = await headless.renderPdfPages(pdfPath, wanted, { dpi });
for (const p of pages) consulted.add(p.page);
if (pages.length === 0) {
toolResults.push(toolText(call.id, call.name, "ERROR: no pages rendered (spec matched nothing?)"));
} else {
renderedImages = pages;
toolResults.push(
toolText(
call.id,
call.name,
`Rendered ${pages.length} page(s) at ${dpi} DPI — the images follow below. Read them carefully (equations, diagrams, layout).`,
),
);
}
}
} else {
toolResults.push(toolText(call.id, call.name, `ERROR: unknown tool "${call.name}"`));
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
toolResults.push(toolText(call.id, call.name, `ERROR: ${msg}`));
}
}
messages = [...messages, ...toolResults];
if (renderedImages) {
messages = [
...messages,
{
role: "user",
content: [
{
type: "text",
text: `The rendered page(s), in order (${renderedImages.map((p) => `p${p.page}`).join(", ")}):`,
},
...renderedImages.map((p) => ({
type: "image" as const,
data: p.png.toString("base64"),
mimeType: "image/png",
})),
],
timestamp: Date.now(),
},
];
}
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return { path: pdfPath, question: input.question, brief: `PDF failed: the reader crashed — ${msg}`, pagesConsulted: [...consulted].sort((a, b) => a - b) };
}
// Iteration limit: deliver whatever was honestly consulted (never invention).
return {
path: pdfPath,
question: input.question,
brief: finalText || `PDF failed: iteration limit (${maxIterations}) reached without a brief.`,
pagesConsulted: [...consulted].sort((a, b) => a - b),
};
}
// ─── small helpers ───────────────────────────────────────────────────────────
/** Parse a page spec into concrete page numbers, capped. */
function specNumbers(spec: string, cap: number): number[] {
// total = Infinity: bounds come from the document at execution time; the
// cap alone protects the batch.
const parts = spec.trim().toLowerCase() === "all" ? [`${cap}`] : spec.split(",");
const out = new Set<number>();
for (const part of parts) {
const p = part.trim();
if (!p) continue;
if (p.includes("-")) {
const [s, e] = p.split("-", 2) as [string, string];
const start = Math.max(1, parseInt(s, 10) || 1);
const end = Math.min(parseInt(e, 10) || start, start + cap - 1);
for (let pg = start; pg <= end && out.size < cap; pg++) out.add(pg);
} else {
const pg = parseInt(p, 10);
if (Number.isFinite(pg) && pg >= 1 && out.size < cap) out.add(pg);
}
}
return [...out].sort((a, b) => a - b);
}
function numOr(v: unknown, dflt: number): number {
return typeof v === "number" && Number.isFinite(v) ? v : dflt;
}
/** Cheap upfront readability probe — real parsing happens inside the tools. */
async function probeReadable(pdfPath: string): Promise<string | null> {
try {
const fs = await import("node:fs/promises");
const stat = await fs.stat(pdfPath);
if (!stat.isFile()) return "not a regular file";
if (stat.size === 0) return "file is empty";
// parse probe: pdfProbe throws pdfjs's precise error for garbage files
await pdfProbe(pdfPath);
return null;
} catch (err) {
return err instanceof Error ? err.message : String(err);
}
}
/** One raw model call (no agent loop): the visual-maker's streamOnce,
* duplicated here because its module keeps it private. */
async function streamOnce(
engine: PiEngine,
systemPrompt: string,
messages: Message[],
tools: ReturnType<typeof piToolDeclarations>,
reasoning: ReturnType<typeof thinkingLevelFor>,
abortSignal?: AbortSignal,
): Promise<AssistantMessage> {
const stream = engine.streamFn(engine.model, { systemPrompt, messages, tools }, {
reasoning: reasoning === "off" ? undefined : reasoning,
apiKey: process.env.OPENROUTER_API_KEY || undefined,
signal: abortSignal,
} as never);
for await (const event of stream) {
if (event.type === "done") return event.message;
if (event.type === "error") {
if (event.reason === "aborted") {
const err = new Error("aborted");
const abortedErr = err as Error & { name: string };
abortedErr.name = "AbortError";
throw abortedErr;
}
throw new Error(event.error.errorMessage ?? "the model call failed");
}
}
throw new Error("the model stream ended without a result");
}
+54 -9
View File
@@ -9,6 +9,8 @@ import {
type JobKind,
type JobRecord,
type JobUsage,
type PdfResult,
type PdfToolInput,
type ResearchResult,
type ResearchToolInput,
type SubagentSteerMessage,
@@ -20,6 +22,7 @@ import { registerJobAbort, unregisterJobAbort } from "./aborts.ts";
import { MODELS } from "../../../config.ts";
import type { ChatStore, JobStore } from "../../core/ports.ts";
import { runDemoBuilder } from "./demo-builder.ts";
import { runPdfReaderAgent } from "./pdf-reader.ts";
import { runResearcherAgent } from "./researcher.ts";
import { runVisualMaker } from "./visual-maker.ts";
@@ -51,9 +54,10 @@ import { runVisualMaker } from "./visual-maker.ts";
export const researchModelOverride: { current?: PiEngine } = {};
export const demoModelOverride: { current?: PiEngine } = {};
export const visualModelOverride: { current?: PiEngine } = {};
export const pdfModelOverride: { current?: PiEngine } = {};
/** The dispatch-time tool input (from src/agent/tools/jobs.ts schemas). */
type JobToolInput = ResearchToolInput | DemoToolInput | VisualToolInput;
type JobToolInput = ResearchToolInput | DemoToolInput | VisualToolInput | PdfToolInput;
export interface JobRunnerDeps {
chatStore: ChatStore;
@@ -71,7 +75,7 @@ export interface JobRunner {
* The tool input rides along from dispatch (the spawn fires mid-stream,
* before the tool part is persisted — only crash-recovery re-spawns
* resolve the input from the transcript). */
runJob(jobId: string, input?: ResearchToolInput | DemoToolInput | VisualToolInput): Promise<void>;
runJob(jobId: string, input?: JobToolInput): Promise<void>;
}
/** One job loop's injected machinery: the kill switch (real abort, 2026-09
@@ -96,6 +100,7 @@ const KIND_MODEL: Record<JobKind, string> = {
research: MODELS.researcher.model,
demo: MODELS.demoBuilder.model,
visual: MODELS.visualMaker.model,
pdf: MODELS.pdfReader.model,
"graph-context": "",
};
@@ -145,6 +150,7 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
research: (jobId, input, loop) => runResearchJob(jobId, researchModelOverride.current, input, loop),
demo: (jobId, input, loop) => runDemoJob(jobId, demoModelOverride.current, input, loop),
visual: (jobId, input, loop) => runVisualJob(jobId, visualModelOverride.current, input, loop),
pdf: (jobId, input, loop) => runPdfJob(jobId, pdfModelOverride.current, input, loop),
};
async function runJob(jobId: string, input?: JobToolInput): Promise<void> {
@@ -185,15 +191,11 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
function resolveInput(
job: JobRecord,
input: JobToolInput | undefined,
): ResearchToolInput | DemoToolInput | VisualToolInput | undefined {
): JobToolInput | undefined {
if (input) return input;
const chat = chatStore.getChat(job.chatId);
if (!chat) return undefined;
return (findToolPart(chat.messages, job.toolCallId)?.input ?? undefined) as
| ResearchToolInput
| DemoToolInput
| VisualToolInput
| undefined;
return (findToolPart(chat.messages, job.toolCallId)?.input ?? undefined) as JobToolInput | undefined;
}
// ─── shared plumbing ────────────────────────────────────────────────────
@@ -213,7 +215,7 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
* actually resumed — until then the job reads "running" so the client's
* activity card keeps polling and re-syncs once, at the end.
*/
async function finishJob(jobId: string, result: ResearchResult | DemoResult | VisualResult): Promise<void> {
async function finishJob(jobId: string, result: ResearchResult | DemoResult | VisualResult | PdfResult): Promise<void> {
const job = jobStore.getJob(jobId);
if (!job) return;
// Aborted mid-run or mid-delivery (abortJob flips the record before
@@ -372,6 +374,44 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
}
}
// ─── pdf reading (sources v0) ────────────────────────────────────────────
async function runPdfJob(
jobId: string,
model: PiEngine | undefined,
passedInput: JobToolInput | undefined,
loop: LoopContext,
): Promise<void> {
const ctx = jobContext(jobId);
if (!ctx) return;
const { job } = ctx;
const input = (passedInput ?? resolveInput(job, passedInput)) as PdfToolInput | undefined;
const path = typeof input?.path === "string" && input.path.trim() ? input.path.trim() : "";
const question = typeof input?.question === "string" ? input.question : "";
try {
jobStore.updateJob(jobId, { activity: "reading pdf" });
const result = await runPdfReaderAgent(
{ path, question },
{
engine: model,
abortSignal: loop.abortSignal,
onUsage: (usage) => loop.onUsage(piUsageToStep(usage)),
onActivity: (line) => jobStore.updateJob(jobId, { activity: line }),
},
);
if (isPdfFailure(result) && stillRunning(jobId))
jobStore.updateJob(jobId, { error: result.brief.replace(/^PDF failed:\s*/i, "") });
await finishJob(jobId, result);
} catch (err) {
if (loop.abortSignal.aborted) return;
const reason = err instanceof Error ? err.message : String(err);
if (stillRunning(jobId)) jobStore.updateJob(jobId, { error: reason });
await finishJob(jobId, { path, question, brief: `PDF failed: ${reason}`, pagesConsulted: [] });
}
}
return { runJob };
}
@@ -386,6 +426,11 @@ function isVisualFailure(result: VisualResult): boolean {
return typeof result.svg !== "string" || result.svg.length === 0;
}
/** The pdf-reader's failure brief (mirrors the research failure shape). */
function isPdfFailure(result: PdfResult): boolean {
return /^PDF failed:/i.test(result.brief);
}
/** The activity line shown in the UI: `web_search: "query"` / `web_fetch: host`. */
function activityLine(toolName: string, input: { query?: string; url?: string } | undefined): string {
if (toolName === "web_search") return `web_search: "${input?.query ?? ""}"`;