fix(pdf): split pagesRead from pagesConsulted — the honest badge count
Live-smoke follow-up the user approved (2026-09-19): the GEB read showed
"78 pages consulted" when the reader genuinely READ ~12 - pdf_search hit
snippets inflated the count. PdfResult now carries both:
- pagesRead: extract/render output entered the loop - the honest count
the badge and brief footer show ("read N pages", "Pages read: ...").
- pagesConsulted: every page whose content entered the loop, search-hit
context lines included - unchanged, still the citation audit set (the
reader's instructions still say "Pages consulted: N, N, ..." and the
footer still lists it when it exceeds the read set).
The unread-finish gate deliberately stays on CONSULTED: search hits are
genuine consultation (anti-fabrication duty), so a search-only brief is
honest and must deliver - pagesRead is a reporting refinement, not a
delivery gate. Pinned by the search-only faux test (read [], consulted
[1], delivered).
UI tolerates pre-split persisted records via ?? fallbacks (old steers
lack pagesRead).
This commit is contained in:
@@ -44,6 +44,11 @@ export function PdfBriefCard({ output }: { output: unknown }) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
const failed = result.brief.startsWith("PDF failed");
|
const failed = result.brief.startsWith("PDF failed");
|
||||||
|
// Read = extract/render output entered the loop; consulted = every page
|
||||||
|
// incl. search hits (the full audit set — the brief's citations may point
|
||||||
|
// there). The ?? fallbacks tolerate pre-split persisted records.
|
||||||
|
const readList = result.pagesRead ?? result.pagesConsulted ?? [];
|
||||||
|
const consultedList = result.pagesConsulted ?? [];
|
||||||
return (
|
return (
|
||||||
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="pdf-brief-card">
|
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="pdf-brief-card">
|
||||||
<div className="mb-1 flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
<div className="mb-1 flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||||
@@ -53,9 +58,14 @@ export function PdfBriefCard({ output }: { output: unknown }) {
|
|||||||
<div className="text-sm [&_p:first-child]:mt-0 [&_p:last-child]:mb-0">
|
<div className="text-sm [&_p:first-child]:mt-0 [&_p:last-child]:mb-0">
|
||||||
<MessageResponse>{result.brief}</MessageResponse>
|
<MessageResponse>{result.brief}</MessageResponse>
|
||||||
</div>
|
</div>
|
||||||
{!failed && result.pagesConsulted && result.pagesConsulted.length > 0 && (
|
{!failed && readList.length > 0 && (
|
||||||
<div className="mt-3 border-t pt-2 text-xs text-muted-foreground">
|
<div className="mt-3 border-t pt-2 text-xs text-muted-foreground">
|
||||||
Pages consulted: {result.pagesConsulted.join(", ")}
|
Pages read: {readList.join(", ")}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{!failed && consultedList.length > readList.length && (
|
||||||
|
<div className="mt-3 border-t pt-2 text-xs text-muted-foreground">
|
||||||
|
Pages consulted: {consultedList.join(", ")}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -79,15 +79,18 @@ function AbortedBadge({ kind }: { kind: string }) {
|
|||||||
* the badge marks delivery, the collapsed details carries the full text
|
* the badge marks delivery, the collapsed details carries the full text
|
||||||
* + consulted pages. */
|
* + consulted pages. */
|
||||||
function PdfCompleteBadge({ result }: { result: PdfResult }) {
|
function PdfCompleteBadge({ result }: { result: PdfResult }) {
|
||||||
|
// The honest count is READ pages (extract/render — sources v1 split);
|
||||||
|
// the ?? is runtime tolerance for pre-split persisted steers.
|
||||||
|
const readPages = result.pagesRead ?? result.pagesConsulted;
|
||||||
return (
|
return (
|
||||||
<details data-testid="pdf-complete-badge" className="group w-full text-xs text-muted-foreground">
|
<details data-testid="pdf-complete-badge" className="group w-full text-xs text-muted-foreground">
|
||||||
<summary className="flex cursor-pointer select-none list-none items-center gap-2 [&::-webkit-details-marker]:hidden">
|
<summary className="flex cursor-pointer select-none list-none items-center gap-2 [&::-webkit-details-marker]:hidden">
|
||||||
<CheckIcon className="size-3.5" />
|
<CheckIcon className="size-3.5" />
|
||||||
<span className="font-medium uppercase tracking-wide">PDF</span>
|
<span className="font-medium uppercase tracking-wide">PDF</span>
|
||||||
<span>reading complete</span>
|
<span>reading complete</span>
|
||||||
{result.pagesConsulted.length > 0 && (
|
{readPages.length > 0 && (
|
||||||
<span>
|
<span>
|
||||||
· {result.pagesConsulted.length} page{result.pagesConsulted.length === 1 ? "" : "s"} consulted
|
· read {readPages.length} page{readPages.length === 1 ? "" : "s"}
|
||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
<span className="ml-1 text-xs text-muted-foreground/70 group-open:hidden">view brief</span>
|
<span className="ml-1 text-xs text-muted-foreground/70 group-open:hidden">view brief</span>
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ export async function runPdfReaderAgent(
|
|||||||
// for renders — the brief never starts on a broken source.
|
// for renders — the brief never starts on a broken source.
|
||||||
const openError = await probeReadable(pdfPath);
|
const openError = await probeReadable(pdfPath);
|
||||||
if (openError) {
|
if (openError) {
|
||||||
return { path: pdfPath, question: input.question, brief: `PDF failed: ${openError}`, pagesConsulted: [] };
|
return { path: pdfPath, question: input.question, brief: `PDF failed: ${openError}`, pagesRead: [], pagesConsulted: [] };
|
||||||
}
|
}
|
||||||
const chromeReason = headless.unavailableReason();
|
const chromeReason = headless.unavailableReason();
|
||||||
// Chrome missing does NOT fail the run: text-side tools still work; the
|
// Chrome missing does NOT fail the run: text-side tools still work; the
|
||||||
@@ -192,6 +192,10 @@ export async function runPdfReaderAgent(
|
|||||||
let messages: Message[] = [{ role: "user", content: prompt, timestamp: Date.now() }];
|
let messages: Message[] = [{ role: "user", content: prompt, timestamp: Date.now() }];
|
||||||
/** Every page the reader extracted or rendered (the citation set). */
|
/** Every page the reader extracted or rendered (the citation set). */
|
||||||
const consulted = new Set<number>();
|
const consulted = new Set<number>();
|
||||||
|
/** Pages genuinely READ — extract or render output entered the loop.
|
||||||
|
* Search-hit context lines consult but don't read (the "78 pages
|
||||||
|
* consulted" overstatement split, sources v1 2026-09-19). */
|
||||||
|
const read = new Set<number>();
|
||||||
let finalText = "";
|
let finalText = "";
|
||||||
let nudged = false;
|
let nudged = false;
|
||||||
|
|
||||||
@@ -211,6 +215,10 @@ export async function runPdfReaderAgent(
|
|||||||
.map((c) => c.text)
|
.map((c) => c.text)
|
||||||
.join("")
|
.join("")
|
||||||
.trim();
|
.trim();
|
||||||
|
// The gate stays on CONSULTED, not read: search-hit context
|
||||||
|
// lines are genuine consultation (anti-fabrication duty) — a
|
||||||
|
// search-only brief is honest. pagesRead is the UI's honest
|
||||||
|
// count, not a delivery gate.
|
||||||
if (consulted.size === 0 && !nudged) {
|
if (consulted.size === 0 && !nudged) {
|
||||||
nudged = true;
|
nudged = true;
|
||||||
messages = [
|
messages = [
|
||||||
@@ -219,7 +227,7 @@ export async function runPdfReaderAgent(
|
|||||||
];
|
];
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
return { path: pdfPath, question: input.question, brief: finalText || "(no brief text)", pagesConsulted: [...consulted].sort((a, b) => a - b) };
|
return { path: pdfPath, question: input.question, brief: finalText || "(no brief text)", pagesRead: [...read].sort((a, b) => a - b), pagesConsulted: [...consulted].sort((a, b) => a - b) };
|
||||||
}
|
}
|
||||||
|
|
||||||
const toolResults: ToolResultMessage[] = [];
|
const toolResults: ToolResultMessage[] = [];
|
||||||
@@ -239,7 +247,10 @@ export async function runPdfReaderAgent(
|
|||||||
opts.onActivity?.(`pdf_extract: ${spec}`);
|
opts.onActivity?.(`pdf_extract: ${spec}`);
|
||||||
const r = await tools.pdf_extract.execute!({ pages: spec }, { toolCallId: call.id });
|
const r = await tools.pdf_extract.execute!({ pages: spec }, { toolCallId: call.id });
|
||||||
toolResults.push(toolText(call.id, call.name, String(r)));
|
toolResults.push(toolText(call.id, call.name, String(r)));
|
||||||
for (const p of specNumbers(spec, Infinity)) consulted.add(p);
|
for (const p of specNumbers(spec, Infinity)) {
|
||||||
|
consulted.add(p);
|
||||||
|
read.add(p);
|
||||||
|
}
|
||||||
} else if (call.name === "pdf_search") {
|
} else if (call.name === "pdf_search") {
|
||||||
const query = String(args.query ?? "");
|
const query = String(args.query ?? "");
|
||||||
const contextLines = numOr(args.context_lines, 3);
|
const contextLines = numOr(args.context_lines, 3);
|
||||||
@@ -268,7 +279,10 @@ export async function runPdfReaderAgent(
|
|||||||
const wanted = specNumbers(spec, RENDER_BATCH_CAP);
|
const wanted = specNumbers(spec, RENDER_BATCH_CAP);
|
||||||
opts.onActivity?.(`pdf_render: ${spec} @${dpi}dpi`);
|
opts.onActivity?.(`pdf_render: ${spec} @${dpi}dpi`);
|
||||||
const pages = await headless.renderPdfPages(pdfPath, wanted, { dpi });
|
const pages = await headless.renderPdfPages(pdfPath, wanted, { dpi });
|
||||||
for (const p of pages) consulted.add(p.page);
|
for (const p of pages) {
|
||||||
|
consulted.add(p.page);
|
||||||
|
read.add(p.page);
|
||||||
|
}
|
||||||
if (pages.length === 0) {
|
if (pages.length === 0) {
|
||||||
toolResults.push(toolText(call.id, call.name, "ERROR: no pages rendered (spec matched nothing?)"));
|
toolResults.push(toolText(call.id, call.name, "ERROR: no pages rendered (spec matched nothing?)"));
|
||||||
} else {
|
} else {
|
||||||
@@ -315,7 +329,7 @@ export async function runPdfReaderAgent(
|
|||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : String(err);
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
return { path: pdfPath, question: input.question, brief: `PDF failed: the reader crashed — ${msg}`, pagesConsulted: [...consulted].sort((a, b) => a - b) };
|
return { path: pdfPath, question: input.question, brief: `PDF failed: the reader crashed — ${msg}`, pagesRead: [...read].sort((a, b) => a - b), pagesConsulted: [...consulted].sort((a, b) => a - b) };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Iteration limit: deliver whatever was honestly consulted (never invention).
|
// Iteration limit: deliver whatever was honestly consulted (never invention).
|
||||||
@@ -323,6 +337,7 @@ export async function runPdfReaderAgent(
|
|||||||
path: pdfPath,
|
path: pdfPath,
|
||||||
question: input.question,
|
question: input.question,
|
||||||
brief: finalText || `PDF failed: iteration limit (${maxIterations}) reached without a brief.`,
|
brief: finalText || `PDF failed: iteration limit (${maxIterations}) reached without a brief.`,
|
||||||
|
pagesRead: [...read].sort((a, b) => a - b),
|
||||||
pagesConsulted: [...consulted].sort((a, b) => a - b),
|
pagesConsulted: [...consulted].sort((a, b) => a - b),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -408,7 +408,7 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
|
|||||||
if (loop.abortSignal.aborted) return;
|
if (loop.abortSignal.aborted) return;
|
||||||
const reason = err instanceof Error ? err.message : String(err);
|
const reason = err instanceof Error ? err.message : String(err);
|
||||||
if (stillRunning(jobId)) jobStore.updateJob(jobId, { error: reason });
|
if (stillRunning(jobId)) jobStore.updateJob(jobId, { error: reason });
|
||||||
await finishJob(jobId, { path, question, brief: `PDF failed: ${reason}`, pagesConsulted: [] });
|
await finishJob(jobId, { path, question, brief: `PDF failed: ${reason}`, pagesRead: [], pagesConsulted: [] });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -231,7 +231,13 @@ export interface PdfResult {
|
|||||||
/** The distilled brief. Page citations inside the text point at
|
/** The distilled brief. Page citations inside the text point at
|
||||||
* pagesConsulted. Failure shape: /^PDF failed:/i (mirrors research). */
|
* pagesConsulted. Failure shape: /^PDF failed:/i (mirrors research). */
|
||||||
brief: string;
|
brief: string;
|
||||||
/** Pages the reader actually consulted. */
|
/** Pages the reader actually READ — extract or render output entered the
|
||||||
|
* loop (the honest count for the UI badge; search-hit snippets don't
|
||||||
|
* count — sources v1 split, 2026-09-19, after the "78 pages consulted"
|
||||||
|
* overstatement on a long-book search-first read). */
|
||||||
|
pagesRead: number[];
|
||||||
|
/** Every page whose content entered the loop, search-hit context lines
|
||||||
|
* included — the full audit set (the brief's citations may point here). */
|
||||||
pagesConsulted: number[];
|
pagesConsulted: number[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -422,13 +422,14 @@ describe("steer card rendering (the delivered artifact)", () => {
|
|||||||
path: "/Users/x/course/chapter-3.pdf",
|
path: "/Users/x/course/chapter-3.pdf",
|
||||||
question: "What is the integral of x dx?",
|
question: "What is the integral of x dx?",
|
||||||
brief: "The power rule appears on **page 1**: the integral of x dx is x^2/2 + C.",
|
brief: "The power rule appears on **page 1**: the integral of x dx is x^2/2 + C.",
|
||||||
|
pagesRead: [1],
|
||||||
pagesConsulted: [1, 2],
|
pagesConsulted: [1, 2],
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
const html = renderToStaticMarkup(<SubagentSteerCard message={steer as never} />);
|
const html = renderToStaticMarkup(<SubagentSteerCard message={steer as never} />);
|
||||||
assert.match(html, /PDF/);
|
assert.match(html, /PDF/);
|
||||||
assert.match(html, /reading complete/);
|
assert.match(html, /reading complete/);
|
||||||
assert.match(html, /2 pages consulted/);
|
assert.match(html, /read 1 page/, "the badge counts READ pages, not search hits");
|
||||||
assert.match(html, /<details[^>]*data-testid="pdf-complete-badge"/, "a real disclosure like the research badge");
|
assert.match(html, /<details[^>]*data-testid="pdf-complete-badge"/, "a real disclosure like the research badge");
|
||||||
assert.ok(!/<details[^>]*open/.test(html), "collapsed by default");
|
assert.ok(!/<details[^>]*open/.test(html), "collapsed by default");
|
||||||
assert.ok(html.includes("power rule"), "the brief rides inside the details");
|
assert.ok(html.includes("power rule"), "the brief rides inside the details");
|
||||||
@@ -439,7 +440,7 @@ describe("steer card rendering (the delivered artifact)", () => {
|
|||||||
it("renders the PDF failure brief with the failed badge when the reader FAILED", () => {
|
it("renders the PDF failure brief with the failed badge when the reader FAILED", () => {
|
||||||
const steer = buildSubagentSteer(
|
const steer = buildSubagentSteer(
|
||||||
{ id: "jpf2", kind: "pdf", toolCallId: "cpf2" },
|
{ id: "jpf2", kind: "pdf", toolCallId: "cpf2" },
|
||||||
{ path: "/tmp/doc.pdf", question: "q", brief: "PDF failed: the file could not be opened", pagesConsulted: [] },
|
{ path: "/tmp/doc.pdf", question: "q", brief: "PDF failed: the file could not be opened", pagesRead: [], pagesConsulted: [] },
|
||||||
);
|
);
|
||||||
const html = renderToStaticMarkup(<SubagentSteerCard message={steer as never} />);
|
const html = renderToStaticMarkup(<SubagentSteerCard message={steer as never} />);
|
||||||
assert.match(html, /pdf-brief-card/);
|
assert.match(html, /pdf-brief-card/);
|
||||||
|
|||||||
+3
-1
@@ -292,7 +292,7 @@ describe("steer helpers (contracts.ts)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("pdf steers (sources v0): brief + page citations on success, PDF failed: shape on failure, abort wording per kind", () => {
|
it("pdf steers (sources v0): brief + page citations on success, PDF failed: shape on failure, abort wording per kind", () => {
|
||||||
const pdf: PdfResult = { path: "/tmp/doc.pdf", question: "q?", brief: "The formula is on p. 2.", pagesConsulted: [1, 2] };
|
const pdf: PdfResult = { path: "/tmp/doc.pdf", question: "q?", brief: "The formula is on p. 2.", pagesRead: [2], pagesConsulted: [1, 2] };
|
||||||
const ok = buildSubagentSteer({ id: "jp", kind: "pdf", toolCallId: "cp" }, pdf);
|
const ok = buildSubagentSteer({ id: "jp", kind: "pdf", toolCallId: "cp" }, pdf);
|
||||||
assert.ok(ok.parts[0].type === "text" && ok.parts[0].text.includes("page citations"), "the teacher is told to cite pages");
|
assert.ok(ok.parts[0].type === "text" && ok.parts[0].text.includes("page citations"), "the teacher is told to cite pages");
|
||||||
assert.ok(ok.parts[0].type === "text" && ok.parts[0].text.includes("The formula is on p. 2."));
|
assert.ok(ok.parts[0].type === "text" && ok.parts[0].text.includes("The formula is on p. 2."));
|
||||||
@@ -622,6 +622,7 @@ describe("pdf runner (sources v0)", () => {
|
|||||||
const result = done.result as PdfResult;
|
const result = done.result as PdfResult;
|
||||||
assert.equal(result.path, PDF_FIXTURE);
|
assert.equal(result.path, PDF_FIXTURE);
|
||||||
assert.match(result.brief, /integral rule/);
|
assert.match(result.brief, /integral rule/);
|
||||||
|
assert.deepEqual(result.pagesRead, [1, 2], "extracted pages are READ pages");
|
||||||
assert.deepEqual(result.pagesConsulted, [1, 2]);
|
assert.deepEqual(result.pagesConsulted, [1, 2]);
|
||||||
|
|
||||||
assert.equal(delivery.steers.length, 1, "one job, one steer");
|
assert.equal(delivery.steers.length, 1, "one job, one steer");
|
||||||
@@ -651,6 +652,7 @@ describe("pdf runner (sources v0)", () => {
|
|||||||
assert.equal(done.status, "error", "the failure brief is a subagent failure");
|
assert.equal(done.status, "error", "the failure brief is a subagent failure");
|
||||||
const result = done.result as PdfResult;
|
const result = done.result as PdfResult;
|
||||||
assert.match(result.brief, /^PDF failed:/);
|
assert.match(result.brief, /^PDF failed:/);
|
||||||
|
assert.deepEqual(result.pagesRead, []);
|
||||||
assert.deepEqual(result.pagesConsulted, []);
|
assert.deepEqual(result.pagesConsulted, []);
|
||||||
|
|
||||||
const steer = delivery.steers[0]!;
|
const steer = delivery.steers[0]!;
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ describe("pdf-reader loop (faux engine)", () => {
|
|||||||
);
|
);
|
||||||
assert.equal(result.path, FIXTURE);
|
assert.equal(result.path, FIXTURE);
|
||||||
assert.match(result.brief, /integral rule/);
|
assert.match(result.brief, /integral rule/);
|
||||||
|
assert.deepEqual(result.pagesRead, [1, 2], "extracted pages are read pages");
|
||||||
assert.deepEqual(result.pagesConsulted, [1, 2]);
|
assert.deepEqual(result.pagesConsulted, [1, 2]);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -85,7 +86,9 @@ describe("pdf-reader loop (faux engine)", () => {
|
|||||||
{ engine, onActivity: (line) => void (line && (nudged ||= line.startsWith("pdf_search"))) },
|
{ engine, onActivity: (line) => void (line && (nudged ||= line.startsWith("pdf_search"))) },
|
||||||
);
|
);
|
||||||
assert.match(result.brief, /Found it on page 1/);
|
assert.match(result.brief, /Found it on page 1/);
|
||||||
|
assert.deepEqual(result.pagesRead, [], "search hits consult but do not read (the split)");
|
||||||
assert.deepEqual(result.pagesConsulted, [1]);
|
assert.deepEqual(result.pagesConsulted, [1]);
|
||||||
|
// and the gate lets it through: search hits ARE genuine consultation
|
||||||
// the first (invention) answer must NOT be the delivered brief
|
// the first (invention) answer must NOT be the delivered brief
|
||||||
assert.notEqual(result.brief, "The answer is 42, trust me.");
|
assert.notEqual(result.brief, "The answer is 42, trust me.");
|
||||||
});
|
});
|
||||||
@@ -102,6 +105,7 @@ describe("pdf-reader loop (faux engine)", () => {
|
|||||||
{ path: FIXTURE, question: "what is the force formula?" },
|
{ path: FIXTURE, question: "what is the force formula?" },
|
||||||
{ engine, headless },
|
{ engine, headless },
|
||||||
);
|
);
|
||||||
|
assert.deepEqual(result.pagesRead, [2, 3], "rendered pages are read pages");
|
||||||
assert.deepEqual(result.pagesConsulted, [2, 3]);
|
assert.deepEqual(result.pagesConsulted, [2, 3]);
|
||||||
assert.deepEqual(headless.calls, ["render:2,3"]);
|
assert.deepEqual(headless.calls, ["render:2,3"]);
|
||||||
});
|
});
|
||||||
@@ -137,6 +141,7 @@ describe("pdf-reader loop (faux engine)", () => {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
assert.match(result.brief, /Extracted instead/);
|
assert.match(result.brief, /Extracted instead/);
|
||||||
|
assert.deepEqual(result.pagesRead, [1]);
|
||||||
assert.deepEqual(result.pagesConsulted, [1]);
|
assert.deepEqual(result.pagesConsulted, [1]);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -149,6 +154,7 @@ describe("pdf-reader loop (faux engine)", () => {
|
|||||||
// had the model been consulted, the brief would be its text instead
|
// had the model been consulted, the brief would be its text instead
|
||||||
assert.match(result.brief, /^PDF failed:/);
|
assert.match(result.brief, /^PDF failed:/);
|
||||||
assert.ok(!result.brief.includes("never reached"));
|
assert.ok(!result.brief.includes("never reached"));
|
||||||
|
assert.deepEqual(result.pagesRead, []);
|
||||||
assert.deepEqual(result.pagesConsulted, []);
|
assert.deepEqual(result.pagesConsulted, []);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
Reference in New Issue
Block a user