fix: normalize model-visible images with Pi processing
This commit is contained in:
amosblomqvist
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Mario Zechner
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1 @@
|
||||
export function convertImageBytesToPng(bytes: Uint8Array): Promise<Uint8Array | null>;
|
||||
@@ -0,0 +1,25 @@
|
||||
// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-convert.js (MIT).
|
||||
// Only inline-image conversion is retained; terminal/Kitty registration is not used.
|
||||
import { applyExifOrientation } from "./exif-orientation.mjs";
|
||||
import { loadPhoton } from "./photon.mjs";
|
||||
|
||||
function encodePng(photon, bytes) {
|
||||
try {
|
||||
const rawImage = photon.PhotonImage.new_from_byteslice(bytes);
|
||||
const image = applyExifOrientation(photon, rawImage, bytes);
|
||||
if (image !== rawImage) rawImage.free();
|
||||
try {
|
||||
return new Uint8Array(image.get_bytes());
|
||||
} finally {
|
||||
image.free();
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function convertImageBytesToPng(bytes) {
|
||||
const photon = await loadPhoton();
|
||||
if (!photon) return null;
|
||||
return encodePng(photon, bytes);
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/exif-orientation.js (MIT).
|
||||
function readOrientationFromTiff(bytes, tiffStart) {
|
||||
if (tiffStart + 8 > bytes.length)
|
||||
return 1;
|
||||
const byteOrder = (bytes[tiffStart] << 8) | bytes[tiffStart + 1];
|
||||
const le = byteOrder === 0x4949;
|
||||
const read16 = (pos) => {
|
||||
if (le)
|
||||
return bytes[pos] | (bytes[pos + 1] << 8);
|
||||
return (bytes[pos] << 8) | bytes[pos + 1];
|
||||
};
|
||||
const read32 = (pos) => {
|
||||
if (le)
|
||||
return bytes[pos] | (bytes[pos + 1] << 8) | (bytes[pos + 2] << 16) | (bytes[pos + 3] << 24);
|
||||
return ((bytes[pos] << 24) | (bytes[pos + 1] << 16) | (bytes[pos + 2] << 8) | bytes[pos + 3]) >>> 0;
|
||||
};
|
||||
const ifdOffset = read32(tiffStart + 4);
|
||||
const ifdStart = tiffStart + ifdOffset;
|
||||
if (ifdStart + 2 > bytes.length)
|
||||
return 1;
|
||||
const entryCount = read16(ifdStart);
|
||||
for (let i = 0; i < entryCount; i++) {
|
||||
const entryPos = ifdStart + 2 + i * 12;
|
||||
if (entryPos + 12 > bytes.length)
|
||||
return 1;
|
||||
if (read16(entryPos) === 0x0112) {
|
||||
const value = read16(entryPos + 8);
|
||||
return value >= 1 && value <= 8 ? value : 1;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
function findJpegTiffOffset(bytes) {
|
||||
let offset = 2;
|
||||
while (offset < bytes.length - 1) {
|
||||
if (bytes[offset] !== 0xff)
|
||||
return -1;
|
||||
const marker = bytes[offset + 1];
|
||||
if (marker === 0xff) {
|
||||
offset++;
|
||||
continue;
|
||||
}
|
||||
if (marker === 0xe1) {
|
||||
if (offset + 4 >= bytes.length)
|
||||
return -1;
|
||||
const segmentStart = offset + 4;
|
||||
if (segmentStart + 6 > bytes.length)
|
||||
return -1;
|
||||
if (hasExifHeader(bytes, segmentStart))
|
||||
return segmentStart + 6;
|
||||
}
|
||||
if (offset + 4 > bytes.length)
|
||||
return -1;
|
||||
const length = (bytes[offset + 2] << 8) | bytes[offset + 3];
|
||||
offset += 2 + length;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
function findWebpTiffOffset(bytes) {
|
||||
let offset = 12;
|
||||
while (offset + 8 <= bytes.length) {
|
||||
const chunkId = String.fromCharCode(bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]);
|
||||
const chunkSize = bytes[offset + 4] | (bytes[offset + 5] << 8) | (bytes[offset + 6] << 16) | (bytes[offset + 7] << 24);
|
||||
const dataStart = offset + 8;
|
||||
if (chunkId === "EXIF") {
|
||||
if (dataStart + chunkSize > bytes.length)
|
||||
return -1;
|
||||
// Some WebP files have "Exif\0\0" prefix before the TIFF header
|
||||
const tiffStart = chunkSize >= 6 && hasExifHeader(bytes, dataStart) ? dataStart + 6 : dataStart;
|
||||
return tiffStart;
|
||||
}
|
||||
// RIFF chunks are padded to even size
|
||||
offset = dataStart + chunkSize + (chunkSize % 2);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
function hasExifHeader(bytes, offset) {
|
||||
return (bytes[offset] === 0x45 &&
|
||||
bytes[offset + 1] === 0x78 &&
|
||||
bytes[offset + 2] === 0x69 &&
|
||||
bytes[offset + 3] === 0x66 &&
|
||||
bytes[offset + 4] === 0x00 &&
|
||||
bytes[offset + 5] === 0x00);
|
||||
}
|
||||
function getExifOrientation(bytes) {
|
||||
let tiffOffset = -1;
|
||||
// JPEG: starts with FF D8
|
||||
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8) {
|
||||
tiffOffset = findJpegTiffOffset(bytes);
|
||||
}
|
||||
// WebP: starts with RIFF....WEBP
|
||||
else if (bytes.length >= 12 &&
|
||||
bytes[0] === 0x52 &&
|
||||
bytes[1] === 0x49 &&
|
||||
bytes[2] === 0x46 &&
|
||||
bytes[3] === 0x46 &&
|
||||
bytes[8] === 0x57 &&
|
||||
bytes[9] === 0x45 &&
|
||||
bytes[10] === 0x42 &&
|
||||
bytes[11] === 0x50) {
|
||||
tiffOffset = findWebpTiffOffset(bytes);
|
||||
}
|
||||
if (tiffOffset === -1)
|
||||
return 1;
|
||||
return readOrientationFromTiff(bytes, tiffOffset);
|
||||
}
|
||||
function rotate90(photon, image, dstIndex) {
|
||||
const w = image.get_width();
|
||||
const h = image.get_height();
|
||||
const src = image.get_raw_pixels();
|
||||
const dst = new Uint8Array(src.length);
|
||||
for (let y = 0; y < h; y++) {
|
||||
for (let x = 0; x < w; x++) {
|
||||
const srcIdx = (y * w + x) * 4;
|
||||
const dstIdx = dstIndex(x, y, w, h) * 4;
|
||||
dst[dstIdx] = src[srcIdx];
|
||||
dst[dstIdx + 1] = src[srcIdx + 1];
|
||||
dst[dstIdx + 2] = src[srcIdx + 2];
|
||||
dst[dstIdx + 3] = src[srcIdx + 3];
|
||||
}
|
||||
}
|
||||
return new photon.PhotonImage(dst, h, w);
|
||||
}
|
||||
// Flip orientations mutate in-place. Rotations return a new image (caller must free the old one if different).
|
||||
export function applyExifOrientation(photon, image, originalBytes) {
|
||||
const orientation = getExifOrientation(originalBytes);
|
||||
if (orientation === 1)
|
||||
return image;
|
||||
switch (orientation) {
|
||||
case 2:
|
||||
photon.fliph(image);
|
||||
return image;
|
||||
case 3:
|
||||
photon.fliph(image);
|
||||
photon.flipv(image);
|
||||
return image;
|
||||
case 4:
|
||||
photon.flipv(image);
|
||||
return image;
|
||||
case 5: {
|
||||
const rotated = rotate90(photon, image, (x, y, _w, h) => x * h + (h - 1 - y));
|
||||
photon.fliph(rotated);
|
||||
return rotated;
|
||||
}
|
||||
case 6:
|
||||
return rotate90(photon, image, (x, y, _w, h) => x * h + (h - 1 - y));
|
||||
case 7: {
|
||||
const rotated = rotate90(photon, image, (x, y, w, h) => (w - 1 - x) * h + y);
|
||||
photon.fliph(rotated);
|
||||
return rotated;
|
||||
}
|
||||
case 8:
|
||||
return rotate90(photon, image, (x, y, w, h) => (w - 1 - x) * h + y);
|
||||
default:
|
||||
return image;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-process.js (MIT).
|
||||
import { convertImageBytesToPng } from "./convert.mjs";
|
||||
import { formatDimensionNote, resizeImage } from "./resize.ts";
|
||||
import type { ProcessImageOptions, ProcessImageResult } from "./types.ts";
|
||||
|
||||
export { detectSupportedImageMimeType, detectSupportedImageMimeTypeFromFile } from "./mime.mjs";
|
||||
export type { ModelImageResizeOptions, ProcessImageOptions, ProcessImageResult, ResizedImage, SupportedImageMimeType } from "./types.ts";
|
||||
|
||||
function baseMimeType(mimeType: string): string {
|
||||
return mimeType.split(";")[0]?.trim().toLowerCase() ?? mimeType.toLowerCase();
|
||||
}
|
||||
|
||||
function normalizeSupportedImageMimeType(mimeType: string): string | null {
|
||||
switch (baseMimeType(mimeType)) {
|
||||
case "image/png": return "image/png";
|
||||
case "image/jpeg":
|
||||
case "image/jpg": return "image/jpeg";
|
||||
case "image/gif": return "image/gif";
|
||||
case "image/webp": return "image/webp";
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function normalizeImage(bytes: Uint8Array, mimeType: string): Promise<{ bytes: Uint8Array; mimeType: string; convertedFrom?: string } | null> {
|
||||
const normalizedMimeType = normalizeSupportedImageMimeType(mimeType);
|
||||
if (normalizedMimeType) return { bytes, mimeType: normalizedMimeType };
|
||||
const pngBytes = await convertImageBytesToPng(bytes);
|
||||
if (!pngBytes) return null;
|
||||
return { bytes: pngBytes, mimeType: "image/png", convertedFrom: baseMimeType(mimeType) };
|
||||
}
|
||||
|
||||
function conversionHint(from: string | undefined, to: string): string | undefined {
|
||||
if (!from || from === to) return undefined;
|
||||
return `[Image converted from ${from} to ${to}.]`;
|
||||
}
|
||||
|
||||
export async function processImage(bytes: Uint8Array, mimeType: string, options?: ProcessImageOptions): Promise<ProcessImageResult> {
|
||||
const autoResizeImages = options?.autoResizeImages ?? true;
|
||||
const normalized = await normalizeImage(bytes, mimeType);
|
||||
if (!normalized) {
|
||||
return { ok: false, message: "[Image omitted: could not be converted to a supported inline image format.]" };
|
||||
}
|
||||
if (autoResizeImages) {
|
||||
const resized = await resizeImage(normalized.bytes, normalized.mimeType, options?.resizeOptions);
|
||||
if (!resized) {
|
||||
return { ok: false, message: "[Image omitted: could not be resized below the inline image size limit.]" };
|
||||
}
|
||||
const hints: string[] = [];
|
||||
const convertedHint = conversionHint(normalized.convertedFrom, resized.mimeType);
|
||||
if (convertedHint) hints.push(convertedHint);
|
||||
const dimensionNote = formatDimensionNote(resized);
|
||||
if (dimensionNote) hints.push(dimensionNote);
|
||||
return { ok: true, data: resized.data, mimeType: resized.mimeType, hints };
|
||||
}
|
||||
const hints: string[] = [];
|
||||
const convertedHint = conversionHint(normalized.convertedFrom, normalized.mimeType);
|
||||
if (convertedHint) hints.push(convertedHint);
|
||||
return { ok: true, data: Buffer.from(normalized.bytes).toString("base64"), mimeType: normalized.mimeType, hints };
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
import type { SupportedImageMimeType } from "./types.ts";
|
||||
export function detectSupportedImageMimeType(buffer: Uint8Array): SupportedImageMimeType | null;
|
||||
export function detectSupportedImageMimeTypeFromFile(filePath: string): Promise<SupportedImageMimeType | null>;
|
||||
@@ -0,0 +1,110 @@
|
||||
// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/mime.js (MIT).
|
||||
import { open } from "node:fs/promises";
|
||||
const IMAGE_TYPE_SNIFF_BYTES = 4100;
|
||||
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
||||
export function detectSupportedImageMimeType(buffer) {
|
||||
if (startsWith(buffer, [0xff, 0xd8, 0xff])) {
|
||||
return buffer[3] === 0xf7 ? null : "image/jpeg";
|
||||
}
|
||||
if (startsWith(buffer, PNG_SIGNATURE)) {
|
||||
return isPng(buffer) && !isAnimatedPng(buffer) ? "image/png" : null;
|
||||
}
|
||||
if (startsWithAscii(buffer, 0, "GIF87a") || startsWithAscii(buffer, 0, "GIF89a")) {
|
||||
return "image/gif";
|
||||
}
|
||||
if (startsWithAscii(buffer, 0, "RIFF") && startsWithAscii(buffer, 8, "WEBP")) {
|
||||
return "image/webp";
|
||||
}
|
||||
if (startsWithAscii(buffer, 0, "BM") && isBmp(buffer)) {
|
||||
return "image/bmp";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
export async function detectSupportedImageMimeTypeFromFile(filePath) {
|
||||
const fileHandle = await open(filePath, "r");
|
||||
try {
|
||||
const buffer = Buffer.alloc(IMAGE_TYPE_SNIFF_BYTES);
|
||||
const { bytesRead } = await fileHandle.read(buffer, 0, IMAGE_TYPE_SNIFF_BYTES, 0);
|
||||
return detectSupportedImageMimeType(buffer.subarray(0, bytesRead));
|
||||
}
|
||||
finally {
|
||||
await fileHandle.close();
|
||||
}
|
||||
}
|
||||
function isPng(buffer) {
|
||||
return (buffer.length >= 16 && readUint32BE(buffer, PNG_SIGNATURE.length) === 13 && startsWithAscii(buffer, 12, "IHDR"));
|
||||
}
|
||||
function isAnimatedPng(buffer) {
|
||||
let offset = PNG_SIGNATURE.length;
|
||||
while (offset + 8 <= buffer.length) {
|
||||
const chunkLength = readUint32BE(buffer, offset);
|
||||
const chunkTypeOffset = offset + 4;
|
||||
if (startsWithAscii(buffer, chunkTypeOffset, "acTL"))
|
||||
return true;
|
||||
if (startsWithAscii(buffer, chunkTypeOffset, "IDAT"))
|
||||
return false;
|
||||
const nextOffset = offset + 8 + chunkLength + 4;
|
||||
if (nextOffset <= offset || nextOffset > buffer.length)
|
||||
return false;
|
||||
offset = nextOffset;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
function isBmp(buffer) {
|
||||
if (buffer.length < 26)
|
||||
return false;
|
||||
const declaredFileSize = readUint32LE(buffer, 2);
|
||||
const pixelDataOffset = readUint32LE(buffer, 10);
|
||||
const dibHeaderSize = readUint32LE(buffer, 14);
|
||||
if (declaredFileSize !== 0 && declaredFileSize < 26)
|
||||
return false;
|
||||
if (pixelDataOffset < 14 + dibHeaderSize)
|
||||
return false;
|
||||
if (declaredFileSize !== 0 && pixelDataOffset >= declaredFileSize)
|
||||
return false;
|
||||
let colorPlanes;
|
||||
let bitsPerPixel;
|
||||
if (dibHeaderSize === 12) {
|
||||
colorPlanes = readUint16LE(buffer, 22);
|
||||
bitsPerPixel = readUint16LE(buffer, 24);
|
||||
}
|
||||
else if (dibHeaderSize >= 40 && dibHeaderSize <= 124) {
|
||||
if (buffer.length < 30)
|
||||
return false;
|
||||
colorPlanes = readUint16LE(buffer, 26);
|
||||
bitsPerPixel = readUint16LE(buffer, 28);
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
}
|
||||
return colorPlanes === 1 && [1, 4, 8, 16, 24, 32].includes(bitsPerPixel);
|
||||
}
|
||||
function readUint16LE(buffer, offset) {
|
||||
return (buffer[offset] ?? 0) + ((buffer[offset + 1] ?? 0) << 8);
|
||||
}
|
||||
function readUint32BE(buffer, offset) {
|
||||
return ((buffer[offset] ?? 0) * 0x1000000 +
|
||||
((buffer[offset + 1] ?? 0) << 16) +
|
||||
((buffer[offset + 2] ?? 0) << 8) +
|
||||
(buffer[offset + 3] ?? 0));
|
||||
}
|
||||
function readUint32LE(buffer, offset) {
|
||||
return ((buffer[offset] ?? 0) +
|
||||
((buffer[offset + 1] ?? 0) << 8) +
|
||||
((buffer[offset + 2] ?? 0) << 16) +
|
||||
(buffer[offset + 3] ?? 0) * 0x1000000);
|
||||
}
|
||||
function startsWith(buffer, bytes) {
|
||||
if (buffer.length < bytes.length)
|
||||
return false;
|
||||
return bytes.every((byte, index) => buffer[index] === byte);
|
||||
}
|
||||
function startsWithAscii(buffer, offset, text) {
|
||||
if (buffer.length < offset + text.length)
|
||||
return false;
|
||||
for (let index = 0; index < text.length; index++) {
|
||||
if (buffer[offset + index] !== text.charCodeAt(index))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// Adapted from @earendil-works/pi-coding-agent 1.1.0, dist/utils/photon.js (MIT).
|
||||
// Source-vs-runtime adaptation: this app runs in Node/Electron, not a Bun
|
||||
// compiled executable. Photon loads its colocated WASM normally; no global fs
|
||||
// monkeypatch or executable-relative WASM fallback is needed.
|
||||
let photonModule = null;
|
||||
let loadPromise = null;
|
||||
|
||||
export async function loadPhoton() {
|
||||
if (photonModule) return photonModule;
|
||||
if (loadPromise) return loadPromise;
|
||||
loadPromise = (async () => {
|
||||
try {
|
||||
photonModule = await import("@silvia-odwyer/photon-node");
|
||||
return photonModule;
|
||||
} catch {
|
||||
photonModule = null;
|
||||
return photonModule;
|
||||
}
|
||||
})();
|
||||
return loadPromise;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import type { ModelImageResizeOptions, ResizedImage } from "./types.ts";
|
||||
export const IMAGE_RESIZE_WORKER_RESPONSE_TYPE: "pi:image-resize-response";
|
||||
export function resizeImageInProcess(inputBytes: Uint8Array, mimeType: string, options?: ModelImageResizeOptions): Promise<ResizedImage | null>;
|
||||
export type ResizeImageWorkerResponse = {
|
||||
type: typeof IMAGE_RESIZE_WORKER_RESPONSE_TYPE;
|
||||
result: ResizedImage | null;
|
||||
} | {
|
||||
type: typeof IMAGE_RESIZE_WORKER_RESPONSE_TYPE;
|
||||
error: string;
|
||||
};
|
||||
@@ -0,0 +1,130 @@
|
||||
// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-resize-core.js (MIT).
|
||||
// Runtime .mjs resources stay unbundled for Next's Node/Electron worker packaging.
|
||||
import { applyExifOrientation } from "./exif-orientation.mjs";
|
||||
import { loadPhoton } from "./photon.mjs";
|
||||
/**
|
||||
* Tag on image resize worker replies. Node can post its own messages on the worker channel
|
||||
* (for example `{ "watch:require": [...] }` under `node --watch`), so replies must be identifiable.
|
||||
*/
|
||||
export const IMAGE_RESIZE_WORKER_RESPONSE_TYPE = "pi:image-resize-response";
|
||||
// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit.
|
||||
const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;
|
||||
const DEFAULT_OPTIONS = {
|
||||
maxWidth: 2000,
|
||||
maxHeight: 2000,
|
||||
maxBytes: DEFAULT_MAX_BYTES,
|
||||
jpegQuality: 80,
|
||||
};
|
||||
function encodeCandidate(buffer, mimeType) {
|
||||
const data = Buffer.from(buffer).toString("base64");
|
||||
return {
|
||||
data,
|
||||
encodedSize: Buffer.byteLength(data, "utf-8"),
|
||||
mimeType,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Resize an image to fit within the specified max dimensions and encoded file size.
|
||||
* Returns null if the image cannot be resized below maxBytes.
|
||||
*
|
||||
* Uses Photon (Rust/WASM) for image processing. If Photon is not available,
|
||||
* returns null.
|
||||
*
|
||||
* Strategy for staying under maxBytes:
|
||||
* 1. First resize to maxWidth/maxHeight
|
||||
* 2. Try PNG first, then JPEG qualities in insertion order (first under cap wins)
|
||||
* 3. If still too large, try the remaining JPEG qualities
|
||||
* 4. If still too large, progressively reduce dimensions until 1x1
|
||||
*/
|
||||
export async function resizeImageInProcess(inputBytes, mimeType, options) {
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4;
|
||||
const photon = await loadPhoton();
|
||||
if (!photon) {
|
||||
return null;
|
||||
}
|
||||
let image;
|
||||
try {
|
||||
const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes);
|
||||
image = applyExifOrientation(photon, rawImage, inputBytes);
|
||||
if (image !== rawImage)
|
||||
rawImage.free();
|
||||
const originalWidth = image.get_width();
|
||||
const originalHeight = image.get_height();
|
||||
const format = mimeType.split("/")[1] ?? "png";
|
||||
// Check if already within all limits (dimensions AND encoded size)
|
||||
if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) {
|
||||
return {
|
||||
data: Buffer.from(inputBytes).toString("base64"),
|
||||
mimeType: mimeType || `image/${format}`,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
width: originalWidth,
|
||||
height: originalHeight,
|
||||
wasResized: false,
|
||||
};
|
||||
}
|
||||
// Calculate initial dimensions respecting max limits
|
||||
let targetWidth = originalWidth;
|
||||
let targetHeight = originalHeight;
|
||||
if (targetWidth > opts.maxWidth) {
|
||||
targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);
|
||||
targetWidth = opts.maxWidth;
|
||||
}
|
||||
if (targetHeight > opts.maxHeight) {
|
||||
targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);
|
||||
targetHeight = opts.maxHeight;
|
||||
}
|
||||
function tryEncodings(width, height, jpegQualities) {
|
||||
const resized = photon.resize(image, width, height, photon.SamplingFilter.Lanczos3);
|
||||
try {
|
||||
const candidates = [encodeCandidate(resized.get_bytes(), "image/png")];
|
||||
for (const quality of jpegQualities) {
|
||||
candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg"));
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
finally {
|
||||
resized.free();
|
||||
}
|
||||
}
|
||||
const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40]));
|
||||
let currentWidth = targetWidth;
|
||||
let currentHeight = targetHeight;
|
||||
while (true) {
|
||||
const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps);
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.encodedSize < opts.maxBytes) {
|
||||
return {
|
||||
data: candidate.data,
|
||||
mimeType: candidate.mimeType,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
width: currentWidth,
|
||||
height: currentHeight,
|
||||
wasResized: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (currentWidth === 1 && currentHeight === 1) {
|
||||
break;
|
||||
}
|
||||
const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75));
|
||||
const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75));
|
||||
if (nextWidth === currentWidth && nextHeight === currentHeight) {
|
||||
break;
|
||||
}
|
||||
currentWidth = nextWidth;
|
||||
currentHeight = nextHeight;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
finally {
|
||||
if (image) {
|
||||
image.free();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-resize-worker.js (MIT).
|
||||
import { parentPort } from "node:worker_threads";
|
||||
import { IMAGE_RESIZE_WORKER_RESPONSE_TYPE, resizeImageInProcess, } from "./resize-core.mjs";
|
||||
function isResizeImageWorkerRequest(value) {
|
||||
if (!value || typeof value !== "object")
|
||||
return false;
|
||||
const record = value;
|
||||
return record.inputBytes instanceof Uint8Array && typeof record.mimeType === "string";
|
||||
}
|
||||
const port = parentPort;
|
||||
if (!port) {
|
||||
throw new Error("image resize worker requires parentPort");
|
||||
}
|
||||
port.once("message", (message) => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (!isResizeImageWorkerRequest(message)) {
|
||||
throw new Error("Invalid image resize worker request");
|
||||
}
|
||||
const result = await resizeImageInProcess(message.inputBytes, message.mimeType, message.options);
|
||||
const response = { type: IMAGE_RESIZE_WORKER_RESPONSE_TYPE, result };
|
||||
port.postMessage(response);
|
||||
}
|
||||
catch (error) {
|
||||
const response = {
|
||||
type: IMAGE_RESIZE_WORKER_RESPONSE_TYPE,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
port.postMessage(response);
|
||||
}
|
||||
})();
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-resize.js (MIT).
|
||||
import path from "node:path";
|
||||
import { Worker } from "node:worker_threads";
|
||||
import {
|
||||
IMAGE_RESIZE_WORKER_RESPONSE_TYPE,
|
||||
resizeImageInProcess,
|
||||
type ResizeImageWorkerResponse,
|
||||
} from "./resize-core.mjs";
|
||||
import type { ModelImageResizeOptions, ResizedImage } from "./types.ts";
|
||||
|
||||
function isResizeImageWorkerResponse(value: unknown): value is ResizeImageWorkerResponse {
|
||||
return value !== null && typeof value === "object" && "type" in value && value.type === IMAGE_RESIZE_WORKER_RESPONSE_TYPE;
|
||||
}
|
||||
|
||||
async function resizeImageInWorker(workerPath: string, inputBytes: Uint8Array, mimeType: string, options?: ModelImageResizeOptions): Promise<ResizedImage | null> {
|
||||
const worker = new Worker(workerPath);
|
||||
try {
|
||||
// Transfer a worker-owned copy so the caller's bytes remain intact.
|
||||
const inputBytesForWorker = new Uint8Array(inputBytes);
|
||||
return await new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
const settle = (result: ResizedImage | null) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(result);
|
||||
};
|
||||
const fail = (error: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
reject(error);
|
||||
};
|
||||
worker.on("message", (message: unknown) => {
|
||||
// Ignore Node's own worker-channel messages (e.g. node --watch).
|
||||
if (!isResizeImageWorkerResponse(message)) return;
|
||||
if ("error" in message) fail(new Error(message.error));
|
||||
else settle(message.result);
|
||||
});
|
||||
worker.once("error", fail);
|
||||
worker.once("exit", (code) => {
|
||||
if (!settled) fail(new Error(`Image resize worker exited with code ${code}`));
|
||||
});
|
||||
worker.postMessage({ inputBytes: inputBytesForWorker, mimeType, options }, [inputBytesForWorker.buffer]);
|
||||
});
|
||||
} finally {
|
||||
void worker.terminate().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export async function resizeImage(inputBytes: Uint8Array, mimeType: string, options?: ModelImageResizeOptions): Promise<ResizedImage | null> {
|
||||
// Source-vs-runtime adaptation: Next bundles the TS orchestration, but traces
|
||||
// unbundled .mjs worker/core/EXIF/Photon resources at this cwd-relative path.
|
||||
// The packaged server's cwd is the standalone root. No Bun path is needed.
|
||||
const workerPath = path.join(process.cwd(), "src/agent/image-processing/resize-worker.mjs");
|
||||
try {
|
||||
return await resizeImageInWorker(workerPath, inputBytes, mimeType, options);
|
||||
} catch {
|
||||
return resizeImageInProcess(inputBytes, mimeType, options);
|
||||
}
|
||||
}
|
||||
|
||||
export function formatDimensionNote(result: ResizedImage): string | undefined {
|
||||
if (!result.wasResized) return undefined;
|
||||
const scale = result.originalWidth / result.width;
|
||||
return `[Image: original ${result.originalWidth}x${result.originalHeight}, displayed at ${result.width}x${result.height}. Multiply coordinates by ${scale.toFixed(2)} to map to original image.]`;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Port baseline: @earendil-works/pi-coding-agent 1.1.0 image utilities (MIT).
|
||||
/** Native resize overrides, supplied by the caller for the selected model. */
|
||||
export interface ModelImageResizeOptions {
|
||||
maxWidth?: number;
|
||||
maxHeight?: number;
|
||||
maxBytes?: number;
|
||||
jpegQuality?: number;
|
||||
}
|
||||
|
||||
export interface ResizedImage {
|
||||
data: string;
|
||||
mimeType: string;
|
||||
originalWidth: number;
|
||||
originalHeight: number;
|
||||
width: number;
|
||||
height: number;
|
||||
wasResized: boolean;
|
||||
}
|
||||
|
||||
export interface ProcessImageOptions {
|
||||
autoResizeImages?: boolean;
|
||||
resizeOptions?: ModelImageResizeOptions;
|
||||
}
|
||||
|
||||
export type ProcessImageResult = {
|
||||
ok: true;
|
||||
data: string;
|
||||
mimeType: string;
|
||||
hints: string[];
|
||||
} | {
|
||||
ok: false;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type SupportedImageMimeType = "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp";
|
||||
@@ -461,6 +461,54 @@ async function renderSvg(svg: string, opts: RenderSvgOptions = {}): Promise<Buff
|
||||
|
||||
// ─── PDF page rasterization (pdfjs paints, Skia hosts) ─────────────────
|
||||
|
||||
/** Decimal megapixels: at four bytes per pixel this is about 61 MiB per
|
||||
* page canvas, before pdfjs/Skia's other working memory. */
|
||||
export const MAX_PDF_CANVAS_PIXELS = 16_000_000;
|
||||
export const MAX_PDF_CANVAS_SIDE = 8192;
|
||||
|
||||
/** Plan the allocation from the scale-1 viewport (including PDF rotation).
|
||||
* Keep ordinary DPI/ceil arithmetic unchanged; oversized pages lose scale,
|
||||
* never gain it. Exported as a pure seam so extreme geometry needs no canvas. */
|
||||
export function pdfCanvasGeometry(baseWidth: number, baseHeight: number, dpi: number): {
|
||||
scale: number;
|
||||
width: number;
|
||||
height: number;
|
||||
} {
|
||||
if (![baseWidth, baseHeight].every((n) => Number.isFinite(n) && n > 0)) {
|
||||
throw new Error("PDF page dimensions must be finite and positive");
|
||||
}
|
||||
if (!Number.isFinite(dpi) || dpi <= 0 || dpi / 72 === 0) {
|
||||
throw new Error("PDF render DPI must be finite and positive");
|
||||
}
|
||||
// Limit sides first, so the area calculation cannot overflow even when
|
||||
// the unscaled page dimensions or requested DPI are enormous.
|
||||
let scale = Math.min(dpi / 72, MAX_PDF_CANVAS_SIDE / baseWidth, MAX_PDF_CANVAS_SIDE / baseHeight);
|
||||
const area = (baseWidth * scale) * (baseHeight * scale);
|
||||
if (area > MAX_PDF_CANVAS_PIXELS) scale *= Math.sqrt(MAX_PDF_CANVAS_PIXELS / area);
|
||||
const dimensions = (s: number) => ({
|
||||
width: Math.max(1, Math.ceil(baseWidth * s)),
|
||||
height: Math.max(1, Math.ceil(baseHeight * s)),
|
||||
});
|
||||
const fits = ({ width, height }: { width: number; height: number }) =>
|
||||
width <= MAX_PDF_CANVAS_SIDE && height <= MAX_PDF_CANVAS_SIDE && width * height <= MAX_PDF_CANVAS_PIXELS;
|
||||
// Ceil can put an otherwise bounded floating-point viewport over budget.
|
||||
// Find a safe scale rather than clipping the viewport or allocating first.
|
||||
if (!fits(dimensions(scale))) {
|
||||
let low = 0;
|
||||
let high = scale;
|
||||
for (let i = 0; i < 64; i++) {
|
||||
const mid = low + (high - low) / 2;
|
||||
if (fits(dimensions(mid))) low = mid;
|
||||
else high = mid;
|
||||
}
|
||||
scale = low;
|
||||
}
|
||||
if (scale <= 0 || baseWidth * scale === 0 || baseHeight * scale === 0) {
|
||||
throw new Error("PDF page dimensions cannot be rendered at a positive scale");
|
||||
}
|
||||
return { scale, ...dimensions(scale) };
|
||||
}
|
||||
|
||||
/** Render PDF pages to PNGs — the sources-v0 pdf-reader's eyes.
|
||||
*
|
||||
* History: pdfjs ALWAYS painted the pages (scale = dpi/72, the skill's
|
||||
@@ -475,6 +523,9 @@ async function renderPdfPages(
|
||||
opts: RenderPdfOptions = {},
|
||||
): Promise<RenderedPdfPage[]> {
|
||||
const dpi = opts.dpi ?? 150;
|
||||
if (!Number.isFinite(dpi) || dpi <= 0 || dpi / 72 === 0) {
|
||||
throw new Error("PDF render DPI must be finite and positive");
|
||||
}
|
||||
// NATIVE CANVAS (Wave E, owner 2026-10-10): pdfjs paints into
|
||||
// @napi-rs/canvas in this process — no browser involved. pdfjs-dist v6's
|
||||
// own Node integration IS @napi-rs/canvas (its optionalDependency: the
|
||||
@@ -495,8 +546,10 @@ async function renderPdfPages(
|
||||
for (const pageNo of pages) {
|
||||
if (pageNo < 1 || pageNo > doc.numPages) continue;
|
||||
const pg = await doc.getPage(pageNo);
|
||||
const vp = pg.getViewport({ scale: dpi / 72 });
|
||||
const canvas = createCanvas(Math.ceil(vp.width), Math.ceil(vp.height));
|
||||
const base = pg.getViewport({ scale: 1 });
|
||||
const geometry = pdfCanvasGeometry(base.width, base.height, dpi);
|
||||
const vp = pg.getViewport({ scale: geometry.scale });
|
||||
const canvas = createCanvas(geometry.width, geometry.height);
|
||||
const ctx = canvas.getContext("2d");
|
||||
// pdfjs's render types speak the DOM context; @napi-rs/canvas's
|
||||
// context is the same 2D API implemented over Skia.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { AssistantMessage, Message, ToolResultMessage, Usage, Tool } from "@earendil-works/pi-ai";
|
||||
import { normalizeContext } from "@earendil-works/pi-ai";
|
||||
import { processImage, type ProcessImageResult } from "../image-processing/index.ts";
|
||||
import { Type } from "typebox";
|
||||
import { resolveSourcePath } from "../tools/fs-shared.ts";
|
||||
import { piEngineFor, type PiEngine, type ThinkingLevel } from "../models.ts";
|
||||
@@ -246,7 +247,7 @@ export async function runPdfReaderAgent(
|
||||
const toolResults: ToolResultMessage[] = [];
|
||||
/** Rendered pages this batch, appended as images AFTER the tool
|
||||
* results — the visual-maker's image-in-the-loop idiom. */
|
||||
let renderedImages: { page: number; png: Buffer }[] | null = null;
|
||||
let renderedImages: { page: number; image: ProcessImageResult }[] | null = null;
|
||||
|
||||
for (const call of toolCalls) {
|
||||
const args = (call.arguments ?? {}) as Record<string, unknown>;
|
||||
@@ -287,19 +288,26 @@ export async function runPdfReaderAgent(
|
||||
const wanted = specNumbers(spec, RENDER_BATCH_CAP);
|
||||
opts.onActivity?.(`pdf_render: ${spec} @${dpi}dpi`);
|
||||
const pages = await headless.renderPdfPages(pdfPath, wanted, { dpi });
|
||||
const processed: { page: number; image: ProcessImageResult }[] = [];
|
||||
for (const p of pages) {
|
||||
consulted.add(p.page);
|
||||
read.add(p.page);
|
||||
const image = await processImage(p.png, "image/png", { resizeOptions: engine.model.inputLimits?.images?.resize });
|
||||
opts.abortSignal?.throwIfAborted();
|
||||
processed.push({ page: p.page, image });
|
||||
// Only images actually delivered to the model count as read.
|
||||
if (image.ok) {
|
||||
consulted.add(p.page);
|
||||
read.add(p.page);
|
||||
}
|
||||
}
|
||||
if (pages.length === 0) {
|
||||
toolResults.push(toolText(call.id, call.name, "ERROR: no pages rendered (spec matched nothing?)"));
|
||||
} else {
|
||||
renderedImages = pages;
|
||||
renderedImages = [...(renderedImages ?? []), ...processed];
|
||||
toolResults.push(
|
||||
toolText(
|
||||
call.id,
|
||||
call.name,
|
||||
`Rendered ${pages.length} page(s) at ${dpi} DPI — the images follow below. Read them carefully (equations, diagrams, layout).`,
|
||||
`Rendered ${pages.length} page(s) at ${dpi} DPI — processed images or explicit omission notes follow below. Read the images carefully (equations, diagrams, layout).`,
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -323,11 +331,12 @@ export async function runPdfReaderAgent(
|
||||
type: "text",
|
||||
text: `The rendered page(s), in order (${renderedImages.map((p) => `p${p.page}`).join(", ")}):`,
|
||||
},
|
||||
...renderedImages.map((p) => ({
|
||||
type: "image" as const,
|
||||
data: p.png.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
})),
|
||||
...renderedImages.flatMap(({ page, image }) => image.ok
|
||||
? [
|
||||
{ type: "text" as const, text: [`Page ${page}`, ...image.hints].join("\n") },
|
||||
{ type: "image" as const, data: image.data, mimeType: image.mimeType },
|
||||
]
|
||||
: [{ type: "text" as const, text: `Page ${page}: ${image.message}` }]),
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { AssistantMessage, Message, Tool, ToolResultMessage, Usage } from "@earendil-works/pi-ai";
|
||||
import { normalizeContext } from "@earendil-works/pi-ai";
|
||||
import { processImage, type ProcessImageResult } from "../image-processing/index.ts";
|
||||
import { Type } from "typebox";
|
||||
import { piEngineFor, type PiEngine, type ThinkingLevel } from "../models.ts";
|
||||
import { providerEventObserver, type ProviderEventObserver } from "../provider-events.ts";
|
||||
@@ -193,6 +194,9 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return { svg: "", notes: `FAILED: rendering the final SVG failed — ${msg}` };
|
||||
}
|
||||
const image = await processImage(png, "image/png", { resizeOptions: engine.model.inputLimits?.images?.resize });
|
||||
opts.abortSignal?.throwIfAborted();
|
||||
if (!image.ok) return { svg: "", notes: `FAILED: inspecting the final SVG failed — ${image.message}` };
|
||||
opts.onActivity?.("render_svg: inspecting final render");
|
||||
renderedSvg = currentSvg;
|
||||
messages = [
|
||||
@@ -200,8 +204,8 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: UNLOOKED_FINISH_MESSAGE },
|
||||
{ type: "image", data: png.toString("base64"), mimeType: "image/png" },
|
||||
{ type: "text", text: [UNLOOKED_FINISH_MESSAGE, ...image.hints].join("\n") },
|
||||
{ type: "image", data: image.data, mimeType: image.mimeType },
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
@@ -212,7 +216,7 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO
|
||||
}
|
||||
|
||||
const toolResults: ToolResultMessage[] = [];
|
||||
let renderPng: Buffer | null = null; // appended as the image message AFTER the tool results
|
||||
let renderImage: Extract<ProcessImageResult, { ok: true }> | null = null; // appended AFTER the tool results
|
||||
for (const call of toolCalls) {
|
||||
if (call.name === "write_svg") {
|
||||
const svg = (call.arguments as { svg?: unknown }).svg;
|
||||
@@ -260,7 +264,10 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO
|
||||
}
|
||||
try {
|
||||
const png = await headless.renderSvg(currentSvg);
|
||||
renderPng = png;
|
||||
const image = await processImage(png, "image/png", { resizeOptions: engine.model.inputLimits?.images?.resize });
|
||||
opts.abortSignal?.throwIfAborted();
|
||||
if (!image.ok) throw new Error(image.message);
|
||||
renderImage = image;
|
||||
renderedSvg = currentSvg;
|
||||
opts.onActivity?.("render_svg: inspecting render");
|
||||
toolResults.push(toolText(call.id, call.name, "Rendered — the PNG follows in the next message. Look at it."));
|
||||
@@ -275,14 +282,14 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO
|
||||
}
|
||||
// Message order: assistant tool-calls → tool results → render image.
|
||||
messages = [...messages, ...toolResults];
|
||||
if (renderPng) {
|
||||
if (renderImage) {
|
||||
messages = [
|
||||
...messages,
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: LOOK_MESSAGE },
|
||||
{ type: "image", data: renderPng.toString("base64"), mimeType: "image/png" },
|
||||
{ type: "text", text: [LOOK_MESSAGE, ...renderImage.hints].join("\n") },
|
||||
{ type: "image", data: renderImage.data, mimeType: renderImage.mimeType },
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
import { practiceTools, type PracticeToolContext } from "./tools/practice.ts";
|
||||
import { healToolRecord, withArgumentHeal } from "./tools/heal.ts";
|
||||
import { sendToUserTool } from "./tools/send-to-user.ts";
|
||||
import { createReadTool, readTool } from "./tools/read.ts";
|
||||
import { quizInputObject, quizInputProblems } from "../core/question-core.ts";
|
||||
import { QUESTION_TOOLS, type EnabledTools } from "../core/pending.ts";
|
||||
import { toLlmMessages, nativeInputCause, PRACTICE_PROBLEM_TOOL } from "../core/native-messages.ts";
|
||||
@@ -201,7 +202,9 @@ function baseAgentOptions(kind: AgentKind, opts: AgentBuildOptions, tools: Agent
|
||||
// Match Claude across providers (including OpenRouter), not the API
|
||||
// transport. Explicit test registries remain caller-owned.
|
||||
tools: [
|
||||
...tools,
|
||||
...tools.map((tool) => tool.name === "read" && tool.execute === readTool.execute
|
||||
? { ...tool, execute: createReadTool(engine.model).execute as AgentTool<any>["execute"] }
|
||||
: tool),
|
||||
...(opts.tools === undefined && engine.model.id.toLowerCase().includes("claude")
|
||||
? [withArgumentHeal(sendToUserTool)]
|
||||
: []),
|
||||
|
||||
+38
-35
@@ -1,9 +1,10 @@
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as path from "node:path";
|
||||
import { Type } from "typebox";
|
||||
import type { Api, Model } from "@earendil-works/pi-ai";
|
||||
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import { serverExecute, type ImageFileResult } from "../tool-execute.ts";
|
||||
import { resolveSourcePath } from "./fs-shared.ts";
|
||||
import { detectSupportedImageMimeTypeFromFile, processImage } from "../image-processing/index.ts";
|
||||
|
||||
/**
|
||||
* The read tool — sources v0 (2026-09): a Node port of pi's read semantics
|
||||
@@ -31,17 +32,6 @@ import { resolveSourcePath } from "./fs-shared.ts";
|
||||
const MAX_LINES = 2000;
|
||||
const MAX_BYTES = 50 * 1024;
|
||||
|
||||
/** Extension map for v0 (pi sniffs magic bytes; extensions suffice here —
|
||||
* learner files are pasted paths, not adversarial uploads). */
|
||||
const IMAGE_MIME: Record<string, string> = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".bmp": "image/bmp",
|
||||
};
|
||||
|
||||
const readInput = Type.Object({
|
||||
path: Type.String({ minLength: 1, description: "Path to the file to read (relative or absolute)" }),
|
||||
offset: Type.Optional(Type.Integer({ minimum: 1, description: "Line number to start reading from (1-indexed)" })),
|
||||
@@ -95,26 +85,39 @@ function readTextSlice(text: string, offset: number | undefined, limit: number |
|
||||
* fragment (prompts/shared/). Relative paths resolve against the
|
||||
* learner's home (fs-shared.ts), the harness's anchor.
|
||||
*/
|
||||
export const readTool: AgentTool<typeof readInput> = {
|
||||
name: "read",
|
||||
label: "read",
|
||||
description: `Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50.0KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`,
|
||||
parameters: readInput,
|
||||
execute: serverExecute("read", readInput, async ({ path: rawPath, offset, limit }): Promise<string | ImageFileResult> => {
|
||||
const absolute = resolveSourcePath(rawPath);
|
||||
const ext = path.extname(absolute).toLowerCase();
|
||||
const mime = IMAGE_MIME[ext];
|
||||
const buffer = await fs.readFile(absolute); // ENOENT/EACCES → error result the model sees
|
||||
if (mime) {
|
||||
// NOTE v0: no resize — raw bytes as base64. Learner files are study
|
||||
// material (screenshots, figures), not adversarial payloads; a resize
|
||||
// step (sharp) can land later if large images prove wasteful.
|
||||
return {
|
||||
text: `Read image file [${mime}] — ${path.basename(absolute)} (${formatSize(buffer.byteLength)}), attached as an image.`,
|
||||
images: [{ data: buffer.toString("base64"), mimeType: mime }],
|
||||
};
|
||||
}
|
||||
const text = buffer.toString("utf-8");
|
||||
return readTextSlice(text, offset, limit);
|
||||
}),
|
||||
};
|
||||
export type ReadModel = Pick<Model<Api>, "input" | "inputLimits">;
|
||||
|
||||
/** One implementation for every learner-file reader. Bind the active model
|
||||
* per agent, rather than sharing mutable model state between conversations.
|
||||
* Home-relative paths, text slicing and the app's abort guard stay unchanged;
|
||||
* image detection/processing and its notes follow native Pi read behavior. */
|
||||
export function createReadTool(model?: ReadModel): AgentTool<typeof readInput> {
|
||||
return {
|
||||
name: "read",
|
||||
label: "read",
|
||||
description: `Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50.0KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`,
|
||||
parameters: readInput,
|
||||
execute: serverExecute("read", readInput, async ({ path: rawPath, offset, limit }, { signal }): Promise<string | ImageFileResult> => {
|
||||
const absolute = resolveSourcePath(rawPath);
|
||||
const mime = await detectSupportedImageMimeTypeFromFile(absolute);
|
||||
const buffer = await fs.readFile(absolute, { signal });
|
||||
if (mime) {
|
||||
const processed = await processImage(buffer, mime, { resizeOptions: model?.inputLimits?.images?.resize });
|
||||
signal?.throwIfAborted();
|
||||
const note = processed.ok
|
||||
? [`Read image file [${processed.mimeType}]`, ...processed.hints]
|
||||
: [`Read image file [${mime}]`, processed.message];
|
||||
if (model && !model.input.includes("image")) {
|
||||
note.push("[Current model does not support images. The image will be omitted from this request.]");
|
||||
}
|
||||
return {
|
||||
text: note.join("\n"),
|
||||
...(processed.ok ? { images: [{ data: processed.data, mimeType: processed.mimeType }] } : {}),
|
||||
};
|
||||
}
|
||||
return readTextSlice(buffer.toString("utf-8"), offset, limit);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export const readTool = createReadTool();
|
||||
|
||||
Reference in New Issue
Block a user