fix(pdf): make pdfjs survive bundling + real-PDF font fidelity (live smoke)
Live dispatch smoke against the Turbopack dev server (master's ask, 2026-09-19) caught two things unit tests cannot - they run unbundled: - Bundling pdfjs-dist broke its node fake-worker: the chunk-relative pdf.worker.mjs import 404s inside .next/dev/server/chunks. Fix: serverExternalPackages + tracing includes for legacy/ + standard_fonts/ (next.config.ts - a clearly-marked seam, same precedent as bcdcd51; the standalone output now ships the worker and font files, verified). - Real-world PDFs warned about missing standardFontDataUrl (non-embedded standard fonts silently fall back) and flooded the log with JBIG2 warnings on scanned files. Fix: standardFontsDir() + verbosity 0 in both the node ops and the headless canvas shim. Verified live end-to-end on GEB (802pp, text layer): teacher dispatch, 12-step read with vision renders, brief with accurate page citations, wake turn folding the resolution (Chapter IX, pp. 268-270) into the lesson. Scanned/DRM Polya correctly refuses to fabricate citations.
This commit is contained in:
@@ -19,8 +19,16 @@ const nextConfig: NextConfig = {
|
|||||||
// pi-ai's provider catalog (39 JSON files) vanished from the trace and
|
// pi-ai's provider catalog (39 JSON files) vanished from the trace and
|
||||||
// the server 500s on module load. Include the whole data dirs of the
|
// the server 500s on module load. Include the whole data dirs of the
|
||||||
// runtime-loaded engine packages.
|
// runtime-loaded engine packages.
|
||||||
|
// pdfjs-dist (sources v0 live smoke, 2026-09-19): its legacy build loads
|
||||||
|
// the node fake worker (pdf.worker.mjs) and our standard_fonts dir via
|
||||||
|
// runtime-resolved paths — invisible to the tracer. Bundling (without
|
||||||
|
// serverExternalPackages below) additionally breaks the worker's
|
||||||
|
// import.meta.url resolution inside .next/dev chunks — the live smoke
|
||||||
|
// caught it; unit tests run unbundled and never see it.
|
||||||
outputFileTracingIncludes: {
|
outputFileTracingIncludes: {
|
||||||
"*": [
|
"*": [
|
||||||
|
"node_modules/pdfjs-dist/legacy/**/*",
|
||||||
|
"node_modules/pdfjs-dist/standard_fonts/**/*",
|
||||||
"node_modules/@earendil-works/pi-ai/dist/providers/data/**/*",
|
"node_modules/@earendil-works/pi-ai/dist/providers/data/**/*",
|
||||||
"node_modules/@earendil-works/pi-agent-core/dist/**/*.json",
|
"node_modules/@earendil-works/pi-agent-core/dist/**/*.json",
|
||||||
// onnxruntime-node: the .node binding was traced, but its
|
// onnxruntime-node: the .node binding was traced, but its
|
||||||
@@ -38,6 +46,7 @@ const nextConfig: NextConfig = {
|
|||||||
"@earendil-works/pi-ai",
|
"@earendil-works/pi-ai",
|
||||||
"@earendil-works/pi-agent-core",
|
"@earendil-works/pi-agent-core",
|
||||||
"@huggingface/transformers",
|
"@huggingface/transformers",
|
||||||
|
"pdfjs-dist", // sources v0: bundling breaks its node fake-worker path
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -326,8 +326,12 @@ async function renderPdfPages(
|
|||||||
const page = await browser.newPage();
|
const page = await browser.newPage();
|
||||||
await page.setViewport({ width: 1200, height: 1600, deviceScaleFactor: 1 });
|
await page.setViewport({ width: 1200, height: 1600, deviceScaleFactor: 1 });
|
||||||
await page.goto(pathToFileURL(file).href, { waitUntil: "load", timeout: 30_000 });
|
await page.goto(pathToFileURL(file).href, { waitUntil: "load", timeout: 30_000 });
|
||||||
|
// Standard-font data as a file URL (live-smoke finding 2026-09-19:
|
||||||
|
// without it pdfjs renders non-embedded standard fonts via fallback);
|
||||||
|
// verbosity 0 = errors only (JBIG2 noise on scanned PDFs).
|
||||||
|
const fontsUrl = pathToFileURL(path.join(PATHS_nodeModules(), "pdfjs-dist", "standard_fonts") + path.sep).href;
|
||||||
const results = await page.evaluate(
|
const results = await page.evaluate(
|
||||||
async (b64pdf, workerUrl, pageNos, dpi) => {
|
async (b64pdf, workerUrl, pageNos, dpi, fontsUrl) => {
|
||||||
// window-scope dynamic import (Chrome evaluates this string in the
|
// window-scope dynamic import (Chrome evaluates this string in the
|
||||||
// shim page, not under Node)
|
// shim page, not under Node)
|
||||||
const pdfjs = (await import(workerUrl)) as typeof import("pdfjs-dist");
|
const pdfjs = (await import(workerUrl)) as typeof import("pdfjs-dist");
|
||||||
@@ -335,7 +339,7 @@ async function renderPdfPages(
|
|||||||
const bin = atob(b64pdf);
|
const bin = atob(b64pdf);
|
||||||
const bytes = new Uint8Array(bin.length);
|
const bytes = new Uint8Array(bin.length);
|
||||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||||
const doc = await pdfjs.getDocument({ data: bytes }).promise;
|
const doc = await pdfjs.getDocument({ data: bytes, standardFontDataUrl: fontsUrl, verbosity: 0 }).promise;
|
||||||
const out: Array<{ page: number; dataUrl: string; w: number; h: number }> = [];
|
const out: Array<{ page: number; dataUrl: string; w: number; h: number }> = [];
|
||||||
for (const pageNo of pageNos) {
|
for (const pageNo of pageNos) {
|
||||||
if (pageNo < 1 || pageNo > doc.numPages) continue;
|
if (pageNo < 1 || pageNo > doc.numPages) continue;
|
||||||
@@ -355,6 +359,7 @@ async function renderPdfPages(
|
|||||||
pdfjsUrl,
|
pdfjsUrl,
|
||||||
pages,
|
pages,
|
||||||
dpi,
|
dpi,
|
||||||
|
fontsUrl,
|
||||||
);
|
);
|
||||||
return results.map((r) => ({
|
return results.map((r) => ({
|
||||||
page: r.page,
|
page: r.page,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import * as fs from "node:fs/promises";
|
import * as fs from "node:fs/promises";
|
||||||
import * as path from "node:path";
|
import * as path from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* PDF plumbing for the pdf-reader sub-agent — sources v0 (2026-09): Node
|
* PDF plumbing for the pdf-reader sub-agent — sources v0 (2026-09): Node
|
||||||
@@ -42,12 +43,22 @@ async function loadPdfjs() {
|
|||||||
return pdfjs;
|
return pdfjs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Standard-font data dir (live-smoke finding 2026-09-19: without it pdfjs
|
||||||
|
* warns and falls back on non-embedded standard fonts — Helvetica, Times…).
|
||||||
|
* The node side takes a plain filesystem path. Resolved from this module's
|
||||||
|
* location, the headless.ts PATHS_nodeModules idiom. */
|
||||||
|
export function standardFontsDir(): string {
|
||||||
|
return path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "node_modules", "pdfjs-dist", "standard_fonts") + path.sep;
|
||||||
|
}
|
||||||
|
|
||||||
async function openDocument(filePath: string) {
|
async function openDocument(filePath: string) {
|
||||||
const pdfjs = await loadPdfjs();
|
const pdfjs = await loadPdfjs();
|
||||||
const data = new Uint8Array(await fs.readFile(filePath));
|
const data = new Uint8Array(await fs.readFile(filePath));
|
||||||
// The legacy build in plain node runs on a main-thread fake worker (no
|
// The legacy build in plain node runs on a main-thread fake worker (no
|
||||||
// workerSrc needed); teardown is the loading task's destroy().
|
// workerSrc needed); teardown is the loading task's destroy().
|
||||||
const task = pdfjs.getDocument({ data });
|
// verbosity 0 = errors only — scanned PDFs (JBIG2 XObjects) otherwise
|
||||||
|
// flood the log with per-page warnings (live-smoke finding 2026-09-19).
|
||||||
|
const task = pdfjs.getDocument({ data, standardFontDataUrl: standardFontsDir(), verbosity: 0 });
|
||||||
return { doc: await task.promise, task };
|
return { doc: await task.promise, task };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user