W2: async job infra + researcher sub-agent
- src/store/jobs.ts: per-job JSON store in data/jobs (create/get/update/list, dedup by toolCallId) - src/agent/tools/jobs.ts: research tool — client-side (no execute), turn ends at the call; input per contracts ResearchToolInput - src/agent/tools/web.ts: server-side web_search (Google CSE) + web_fetch (Readability/Turndown/linkedom, Jina fallback) — errors return as strings, never thrown into the agent loop. Deps: @mozilla/readability, linkedom, turndown (+@types/turndown dev) - src/agent/subagents/researcher.ts: isolated ToolLoopAgent (web tools only, 12-step cap), researcher prompt ported from pi - src/agent/subagents/runner.ts: spawnPendingJobs hook + runResearchJob — fire-and-forget, activity line per tool call, fulfills the part in the LATEST chat record (output-available) before flipping job status; failure still fulfills the part so the teacher learns the outcome - src/server/chat-handler.ts: onFinish spawn hook after saveMessages - app/api/jobs/route.ts: GET ?chatId= polling endpoint - components/artifacts/job-card.tsx + registry entry: polling activity card (2s) → syncFromServer on finish; research-brief card (markdown + sources) - src/agent/prompts/20-jobs.md: research doctrine for the teacher - tests/jobs.test.ts: store CRUD, spawn hook via scripted mock teacher, runner completion/failure with fake model + stubbed fetch
This commit is contained in:
Amos
@@ -0,0 +1,27 @@
|
||||
## Background research
|
||||
|
||||
You have a `research` tool that dispatches a background web-research agent. Use it to
|
||||
outsource what you cannot reliably know yourself:
|
||||
|
||||
- **Use it for:** facts you are uncertain about, current or fast-moving data (library versions,
|
||||
release dates, recent events), niche specifics outside your training, and anything where
|
||||
being wrong would poison the lesson.
|
||||
- **Never use it for:** anything you know cold, internal reasoning, or math. Do not offload
|
||||
thinking — only facts you genuinely need verified.
|
||||
|
||||
**The call is asynchronous and your turn ends at it.** The discipline:
|
||||
|
||||
1. **Say your piece first.** Explain the concept, set up what you're doing and why, and tell
|
||||
the learner roughly what you're checking — the research brief arrives later as the tool's
|
||||
output, and the lesson resumes from there. A bare tool call with no surrounding text is a
|
||||
failed turn.
|
||||
2. **Call with a single, specific question.** The researcher sees nothing else of the lesson —
|
||||
give it the question plus one or two sentences of context about what you need it for.
|
||||
3. **Resume from the brief.** When the result returns, weave it into the lesson. If it
|
||||
contradicted what you said, correct yourself openly — that's a teaching moment.
|
||||
4. **Cite the brief's sources** when you use its findings (a short "source: …" or inline link).
|
||||
Never present researched facts as your own certainty.
|
||||
|
||||
At most one research call per turn — don't fan out; the researcher searches multiple angles on
|
||||
its own. If research fails or is unconfigured, you'll get a failure brief: say so plainly and
|
||||
continue with what you can establish, flagging the uncertainty.
|
||||
@@ -0,0 +1,71 @@
|
||||
import { ToolLoopAgent, stepCountIs, type LanguageModel } from "ai";
|
||||
import { openrouter } from "@openrouter/ai-sdk-provider";
|
||||
import { webTools } from "../tools/web.ts";
|
||||
|
||||
/**
|
||||
* The researcher sub-agent (vision §2.5): a SEPARATE ToolLoopAgent instance
|
||||
* from the teacher — an isolated loop with only web_search + web_fetch, so
|
||||
* raw search/fetch churn never touches the teacher's context window.
|
||||
*
|
||||
* System prompt adapted from the pi researcher agent
|
||||
* (~/.pi/agent/extensions/interactive-subagents/agents/researcher.md).
|
||||
* The model is injectable so tests run the loop with a MockLanguageModelV4.
|
||||
*/
|
||||
|
||||
const DEFAULT_RESEARCHER_MODEL =
|
||||
process.env.LH_RESEARCH_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
|
||||
|
||||
const INSTRUCTIONS = `You are a research specialist. You work for a teacher agent inside a learning
|
||||
application: given a research question, conduct thorough web research and produce a focused,
|
||||
well-sourced brief that the teacher can teach from.
|
||||
|
||||
You operate in an isolated context with no knowledge of the lesson or any prior conversation.
|
||||
All necessary context is in the task description.
|
||||
|
||||
Process:
|
||||
1. Break the question into 2-4 searchable facets.
|
||||
2. Search with web_search using varied angles:
|
||||
- the direct answer query (the obvious one)
|
||||
- an authoritative-source query (official docs, specs, primary sources)
|
||||
- a practical-experience query (case studies, benchmarks, real-world usage)
|
||||
- a recent-developments query (only if the topic is time-sensitive)
|
||||
3. Read the results. Identify what's well-covered and what has gaps. If the first round
|
||||
doesn't fully answer the question, search again with refined queries targeting the gaps.
|
||||
4. For the 2-3 most promising source URLs, use web_fetch to get the full page content.
|
||||
5. Synthesize everything into a brief that directly answers the question.
|
||||
|
||||
Evaluation — what to keep vs drop:
|
||||
- Official docs and primary sources outweigh blog posts and forum threads.
|
||||
- Recent sources outweigh stale ones.
|
||||
- Sources that directly address the question outweigh tangentially related ones.
|
||||
- Drop SEO filler, outdated info, and beginner tutorials (unless that's the audience).
|
||||
|
||||
Tool outputs may be error strings (e.g. search unconfigured, a page that failed to fetch).
|
||||
Adapt: try another URL or another search angle; if research is genuinely impossible, say so
|
||||
in the brief instead of inventing facts.
|
||||
|
||||
Your FINAL assistant message is your entire deliverable — it must stand alone, using this format:
|
||||
|
||||
## Summary
|
||||
2-3 sentence direct answer.
|
||||
|
||||
## Findings
|
||||
Numbered findings with inline source citations:
|
||||
1. **Finding** — explanation. [Source](url)
|
||||
|
||||
## Sources
|
||||
- Source Title (url) — why it's relevant
|
||||
|
||||
## Gaps
|
||||
What couldn't be answered. Suggested next steps.`;
|
||||
|
||||
export function createResearcherAgent(model?: LanguageModel) {
|
||||
return new ToolLoopAgent({
|
||||
model: model ?? openrouter(DEFAULT_RESEARCHER_MODEL),
|
||||
instructions: INSTRUCTIONS,
|
||||
tools: webTools,
|
||||
stopWhen: stepCountIs(12),
|
||||
});
|
||||
}
|
||||
|
||||
export type ResearcherAgent = ReturnType<typeof createResearcherAgent>;
|
||||
@@ -0,0 +1,182 @@
|
||||
import { isToolUIPart, type LanguageModel, type UIMessage } from "ai";
|
||||
import type { ResearchResult, ResearchToolInput } from "../../core/contracts.ts";
|
||||
import { JOB_TOOLS } from "../../core/pending.ts";
|
||||
import { getChat, saveMessages } from "../../store/chats.ts";
|
||||
import { createJob, findJobByToolCallId, getJob, updateJob } from "../../store/jobs.ts";
|
||||
import { createResearcherAgent } from "./researcher.ts";
|
||||
|
||||
/**
|
||||
* The background job runner + the chat-handler spawn hook (contracts.ts
|
||||
* "THE ASYNC PROTOCOL").
|
||||
*
|
||||
* spawnPendingJobs — called from onFinish AFTER saveMessages: finds
|
||||
* input-available JOB_TOOLS parts in the last assistant message with no
|
||||
* existing job and fires runResearchJob WITHOUT awaiting (the HTTP
|
||||
* response is long gone).
|
||||
*
|
||||
* runResearchJob — runs the researcher agent loop in the background,
|
||||
* updating the job's activity line on every tool call, then rewrites the
|
||||
* tool part in the LATEST chat record to output-available (read-modify-
|
||||
* write; no awaits between the read and the write). The part is fulfilled
|
||||
* BEFORE the job status flips, so a client that polls the job and then
|
||||
* re-syncs always sees the finished part.
|
||||
*
|
||||
* On failure the part is still fulfilled ({brief: "Research failed: …"}) —
|
||||
* the teacher must learn the outcome — and the job is marked error.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Test seam: the spawn hook creates runners with no injection point, so
|
||||
* tests swap in a MockLanguageModelV4 here to keep job tests off the
|
||||
* network. Undefined in production.
|
||||
*/
|
||||
export const researchModelOverride: { current?: LanguageModel } = {};
|
||||
|
||||
export interface RunResearchJobOptions {
|
||||
model?: LanguageModel;
|
||||
}
|
||||
|
||||
export function spawnPendingJobs(chatId: string, messages: UIMessage[]): void {
|
||||
const last = [...messages].reverse().find((m) => m.role === "assistant");
|
||||
if (!last) return;
|
||||
for (const part of last.parts) {
|
||||
if (!isToolUIPart(part) || part.state !== "input-available") continue;
|
||||
const tool = part.type.slice("tool-".length);
|
||||
if (!(JOB_TOOLS as readonly string[]).includes(tool)) continue;
|
||||
if (tool !== "research") continue; // demo/visual runners arrive in W6
|
||||
if (findJobByToolCallId(chatId, part.toolCallId)) continue; // already has a job
|
||||
const job = createJob({ kind: "research", chatId, toolCallId: part.toolCallId });
|
||||
void runResearchJob(job.id, { model: researchModelOverride.current }).catch((err) => {
|
||||
console.error(`[job ${job.id}] research runner crashed:`, err);
|
||||
updateJob(job.id, { status: "error", error: String(err), activity: "failed" });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function runResearchJob(jobId: string, opts: RunResearchJobOptions = {}): Promise<void> {
|
||||
const job = getJob(jobId);
|
||||
if (!job || job.status !== "running") return;
|
||||
|
||||
const chat = getChat(job.chatId);
|
||||
if (!chat) {
|
||||
updateJob(jobId, { status: "error", error: `Chat not found: ${job.chatId}`, activity: "failed" });
|
||||
return;
|
||||
}
|
||||
const input = findToolPart(chat.messages, job.toolCallId)?.input as ResearchToolInput | undefined;
|
||||
const question = typeof input?.question === "string" && input.question.trim() ? input.question : "(no question)";
|
||||
const context = typeof input?.context === "string" && input.context.trim() ? input.context : undefined;
|
||||
|
||||
const prompt = [
|
||||
`Research question: ${question}`,
|
||||
context ? `Context (what this is needed for): ${context}` : null,
|
||||
"Produce your final brief in the required format.",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
|
||||
const fetchedUrls: string[] = [];
|
||||
const agent = createResearcherAgent(opts.model);
|
||||
|
||||
try {
|
||||
updateJob(jobId, { activity: "researching" });
|
||||
const result = await agent.generate({
|
||||
prompt,
|
||||
onToolExecutionStart: ({ toolCall }) => {
|
||||
const input = toolCall.input as { query?: string; url?: string } | undefined;
|
||||
if (toolCall.toolName === "web_fetch" && typeof input?.url === "string") {
|
||||
fetchedUrls.push(input.url);
|
||||
}
|
||||
updateJob(jobId, { activity: activityLine(toolCall.toolName, input) });
|
||||
},
|
||||
});
|
||||
|
||||
const brief = result.text.trim();
|
||||
if (!brief) throw new Error("the researcher produced no text");
|
||||
const research: ResearchResult = { brief, sources: parseSources(brief, fetchedUrls) };
|
||||
|
||||
fulfillPart(job.chatId, job.toolCallId, research);
|
||||
updateJob(jobId, { status: "done", result: research }); // activity line kept: the last tool call is the record
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
fulfillPart(job.chatId, job.toolCallId, { brief: `Research failed: ${reason}`, sources: [] });
|
||||
updateJob(jobId, { status: "error", activity: "failed", error: reason });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function findToolPart(messages: UIMessage[], toolCallId: string) {
|
||||
for (const m of messages) {
|
||||
for (const part of m.parts) {
|
||||
if (isToolUIPart(part) && part.toolCallId === toolCallId) return part;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The activity line shown in the UI: `web_search: "query"` / `web_fetch: host`. */
|
||||
function activityLine(toolName: string, input: { query?: string; url?: string } | undefined): string {
|
||||
if (toolName === "web_search") return `web_search: "${input?.query ?? ""}"`;
|
||||
if (toolName === "web_fetch") return `web_fetch: ${hostOf(input?.url)}`;
|
||||
return toolName;
|
||||
}
|
||||
|
||||
function hostOf(url: string | undefined): string {
|
||||
if (!url) return "(no url)";
|
||||
try {
|
||||
return new URL(url).hostname;
|
||||
} catch {
|
||||
return url.slice(0, 40);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite the tool part in the LATEST chat record to output-available.
|
||||
* Synchronous read-modify-write (no awaits) — atomic within the event loop.
|
||||
*/
|
||||
function fulfillPart(chatId: string, toolCallId: string, output: ResearchResult): void {
|
||||
const chat = getChat(chatId);
|
||||
if (!chat) return;
|
||||
let found = false;
|
||||
const messages = chat.messages.map((m) =>
|
||||
m.role !== "assistant"
|
||||
? m
|
||||
: {
|
||||
...m,
|
||||
parts: m.parts.map((part) => {
|
||||
if (isToolUIPart(part) && part.toolCallId === toolCallId && part.state === "input-available") {
|
||||
found = true;
|
||||
return { ...part, state: "output-available" as const, output };
|
||||
}
|
||||
return part;
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!found) return;
|
||||
try {
|
||||
saveMessages(chatId, messages);
|
||||
} catch (err) {
|
||||
console.error(`[chat ${chatId}] fulfilling job part ${toolCallId} failed:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Sources for the result: markdown links cited in the brief, else fetched URLs. */
|
||||
function parseSources(brief: string, fetchedUrls: string[]): ResearchResult["sources"] {
|
||||
const sources: ResearchResult["sources"] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const match of brief.matchAll(/\[([^\]]+)\]\((https?:\/\/[^)\s]+)\)/g)) {
|
||||
const title = (match[1] ?? "").trim();
|
||||
const url = match[2]!;
|
||||
if (!title || seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
sources.push({ title, url });
|
||||
}
|
||||
if (sources.length === 0) {
|
||||
for (const url of fetchedUrls) {
|
||||
if (seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
sources.push({ title: hostOf(url), url });
|
||||
}
|
||||
}
|
||||
return sources.slice(0, 10);
|
||||
}
|
||||
+39
-6
@@ -1,13 +1,46 @@
|
||||
import type { ToolSet } from "ai";
|
||||
import { tool, type ToolSet } from "ai";
|
||||
import { z } from "zod";
|
||||
import type { TeacherToolContext } from "../tools.ts";
|
||||
|
||||
/**
|
||||
* Async sub-agent job tools — W2 workstream (research; W6 adds demo + visual).
|
||||
* CLIENT-SIDE semantics (no execute): the turn ends with the part pending, a
|
||||
* background runner fulfills it, and the client continuation resumes the
|
||||
* teacher. See src/core/contracts.ts "THE ASYNC PROTOCOL" — read it first.
|
||||
* Async sub-agent job tools (W2: research; W6 adds demo + visual).
|
||||
* CLIENT-SIDE semantics (no execute) — exactly like quiz/ask: the tool call
|
||||
* streams to the browser in `input-available` state and the server turn ends
|
||||
* there. chat-handler's onFinish spawns a background runner that fulfills
|
||||
* the part server-side; the client polls /api/jobs and the continuation
|
||||
* resumes the teacher with the result as a normal tool output.
|
||||
* See src/core/contracts.ts "THE ASYNC PROTOCOL".
|
||||
*/
|
||||
|
||||
export const researchInputSchema = z.object({
|
||||
question: z
|
||||
.string()
|
||||
.describe("The single research question to answer. Make it specific — the researcher sees nothing else of the lesson."),
|
||||
context: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Why you need this / what the lesson needs it for — focuses the brief. One or two sentences."),
|
||||
});
|
||||
|
||||
export const researchTool = tool({
|
||||
description:
|
||||
"Dispatch a background web-research agent: fact verification, current data, or niche specifics you are not certain about. " +
|
||||
"The call is ASYNC — your turn ENDS here, so say everything you want to say first; the research brief arrives later as " +
|
||||
"this tool's output and the lesson resumes from it. Cite the brief's sources when you use them. " +
|
||||
"NEVER use it for things you know cold, internal reasoning, or math.",
|
||||
inputSchema: researchInputSchema,
|
||||
});
|
||||
|
||||
/**
|
||||
* Per-request job tool registry. Context-free for now (the runner resolves
|
||||
* everything from the chat record by toolCallId) — the ctx parameter keeps
|
||||
* the createTeacherTools factory shape stable for W6's demo/visual tools.
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
export function jobTools(ctx: TeacherToolContext): ToolSet {
|
||||
return {};
|
||||
return {
|
||||
research: researchTool,
|
||||
};
|
||||
}
|
||||
|
||||
export type ResearchInput = z.infer<typeof researchInputSchema>;
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
import { tool } from "ai";
|
||||
import { z } from "zod";
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { parseHTML } from "linkedom";
|
||||
import TurndownService from "turndown";
|
||||
|
||||
/**
|
||||
* Server-side web tools for the researcher sub-agent — a minimal port of the
|
||||
* pi extensions (~/.pi/agent/extensions/web-search + web-fetch):
|
||||
*
|
||||
* web_search(query, count?) → {title, url, snippet}[] (Google Custom Search)
|
||||
* web_fetch(url) → readable markdown text (Readability + Turndown,
|
||||
* Jina Reader fallback for
|
||||
* JS-rendered pages)
|
||||
*
|
||||
* Credentials come from env (the pi extension used auth.json):
|
||||
* GOOGLE_SEARCH_API_KEY, GOOGLE_CSE_ID
|
||||
*
|
||||
* Tool contract: NEVER throw into the agent loop — missing credentials and
|
||||
* every failure path return a clear error STRING as the tool output, so the
|
||||
* researcher can adapt (or report) instead of crashing.
|
||||
*/
|
||||
|
||||
// ── web_search ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface WebSearchResult {
|
||||
title: string;
|
||||
url: string;
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
function searchCredentials(): { apiKey: string; cseId: string } | null {
|
||||
const apiKey = process.env.GOOGLE_SEARCH_API_KEY;
|
||||
const cseId = process.env.GOOGLE_CSE_ID;
|
||||
return apiKey && cseId ? { apiKey, cseId } : null;
|
||||
}
|
||||
|
||||
async function googleSearch(query: string, count: number, creds: { apiKey: string; cseId: string }): Promise<WebSearchResult[]> {
|
||||
const url = new URL("https://www.googleapis.com/customsearch/v1");
|
||||
url.searchParams.set("key", creds.apiKey);
|
||||
url.searchParams.set("cx", creds.cseId);
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("num", String(Math.min(count, 10)));
|
||||
|
||||
const resp = await fetch(url, { signal: AbortSignal.timeout(30_000) });
|
||||
if (!resp.ok) {
|
||||
const body = await resp.text();
|
||||
throw new Error(`Google Custom Search API ${resp.status}: ${body.slice(0, 200)}`);
|
||||
}
|
||||
const data = (await resp.json()) as { items?: Array<{ title: string; link: string; snippet?: string }> };
|
||||
return (data.items ?? []).map((item) => ({
|
||||
title: item.title,
|
||||
url: item.link,
|
||||
snippet: item.snippet?.replace(/\n/g, " ") ?? "",
|
||||
}));
|
||||
}
|
||||
|
||||
// ── web_fetch ────────────────────────────────────────────────────────────────
|
||||
|
||||
const USER_AGENT =
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
||||
const FETCH_TIMEOUT_MS = 30_000;
|
||||
const MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
|
||||
/** Keep the researcher's context sane — long pages truncate with a marker. */
|
||||
const MAX_CONTENT_CHARS = 40_000;
|
||||
const MIN_USEFUL_CONTENT = 200;
|
||||
|
||||
const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
|
||||
|
||||
function truncate(text: string): string {
|
||||
if (text.length <= MAX_CONTENT_CHARS) return text;
|
||||
return `${text.slice(0, MAX_CONTENT_CHARS)}\n\n[Truncated at ${MAX_CONTENT_CHARS} characters]`;
|
||||
}
|
||||
|
||||
/** Jina Reader fallback for JS-rendered pages (no deps — plain fetch). */
|
||||
async function fetchViaJina(url: string): Promise<string | null> {
|
||||
try {
|
||||
const res = await fetch(`https://r.jina.ai/${url}`, {
|
||||
headers: { Accept: "text/markdown", "X-No-Cache": "true" },
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const content = await res.text();
|
||||
const marker = "Markdown Content:";
|
||||
const start = content.indexOf(marker);
|
||||
if (start < 0) return null;
|
||||
const markdown = content.slice(start + marker.length).trim();
|
||||
if (markdown.length < MIN_USEFUL_CONTENT || markdown.startsWith("Loading...")) return null;
|
||||
return markdown;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function htmlToMarkdown(html: string): { title: string; content: string } | null {
|
||||
const { document } = parseHTML(html);
|
||||
const article = new Readability(document as unknown as Document).parse();
|
||||
if (!article?.content) return null;
|
||||
const markdown = turndown.turndown(article.content);
|
||||
if (markdown.trim().length < MIN_USEFUL_CONTENT) return null;
|
||||
return { title: article.title ?? "", content: markdown };
|
||||
}
|
||||
|
||||
async function fetchAndExtract(url: string): Promise<string> {
|
||||
const resp = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": USER_AGENT,
|
||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
|
||||
|
||||
const contentType = resp.headers.get("content-type") ?? "";
|
||||
if (contentType.includes("application/pdf")) throw new Error("PDF documents are not supported");
|
||||
if (/image\/|audio\/|video\/|zip|octet-stream/.test(contentType)) {
|
||||
throw new Error(`Unsupported content type: ${contentType.split(";")[0]}`);
|
||||
}
|
||||
const length = Number(resp.headers.get("content-length") ?? 0);
|
||||
if (length > MAX_RESPONSE_BYTES) throw new Error("Response too large");
|
||||
|
||||
const text = await resp.text();
|
||||
if (!contentType.includes("html")) return truncate(text); // plain text / markdown / code
|
||||
|
||||
const extracted = htmlToMarkdown(text);
|
||||
if (extracted) {
|
||||
const header = extracted.title ? `# ${extracted.title}\n\nSource: ${url}\n\n---\n\n` : "";
|
||||
return header + truncate(extracted.content);
|
||||
}
|
||||
// Readability failed — likely JS-rendered. Try Jina Reader.
|
||||
const jina = await fetchViaJina(url);
|
||||
if (jina) return truncate(jina);
|
||||
throw new Error("Could not extract readable content (the page may be JavaScript-rendered)");
|
||||
}
|
||||
|
||||
// ── tool registry ────────────────────────────────────────────────────────────
|
||||
|
||||
export const webTools = {
|
||||
web_search: tool({
|
||||
description:
|
||||
"Search the web via Google Custom Search. One search per call — use one call per search angle. " +
|
||||
"Returns an array of results with title, url, and snippet.",
|
||||
inputSchema: z.object({
|
||||
query: z.string().describe("The search query (a normal search string, phrases included)"),
|
||||
count: z.number().int().min(1).max(10).optional().describe("Number of results (default 5, max 10)"),
|
||||
}),
|
||||
execute: async ({ query, count }): Promise<WebSearchResult[] | string> => {
|
||||
const creds = searchCredentials();
|
||||
if (!creds) {
|
||||
return (
|
||||
"web_search is not configured on this server: GOOGLE_SEARCH_API_KEY and/or GOOGLE_CSE_ID " +
|
||||
"are missing. Report in your brief that web search is unavailable."
|
||||
);
|
||||
}
|
||||
try {
|
||||
return await googleSearch(query, count ?? 5, creds);
|
||||
} catch (err) {
|
||||
return `web_search failed: ${err instanceof Error ? err.message : String(err)}`;
|
||||
}
|
||||
},
|
||||
}),
|
||||
|
||||
web_fetch: tool({
|
||||
description:
|
||||
"Fetch a web page and extract its readable content as markdown (Readability). " +
|
||||
"Handles HTML pages and plain text; JS-rendered pages fall back to a reader service.",
|
||||
inputSchema: z.object({
|
||||
url: z.string().describe("The URL to fetch"),
|
||||
}),
|
||||
execute: async ({ url }): Promise<string> => {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error("Only http(s) URLs are supported");
|
||||
} catch {
|
||||
return `web_fetch failed: invalid URL "${url}"`;
|
||||
}
|
||||
try {
|
||||
return await fetchAndExtract(url);
|
||||
} catch (err) {
|
||||
return `web_fetch failed for ${url}: ${err instanceof Error ? err.message : String(err)}`;
|
||||
}
|
||||
},
|
||||
}),
|
||||
};
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createAgentUIStreamResponse, type UIMessage } from "ai";
|
||||
import { createTeacherAgent, type TeacherAgent } from "../agent/teacher-agent.ts";
|
||||
import { createTeacherTools } from "../agent/tools.ts";
|
||||
import { spawnPendingJobs } from "../agent/subagents/runner.ts";
|
||||
import { getChat, saveMessages } from "../store/chats.ts";
|
||||
|
||||
/**
|
||||
@@ -38,6 +39,14 @@ export async function handleChatRequest(req: Request, agent?: TeacherAgent): Pro
|
||||
} catch (err) {
|
||||
console.error(`[chat ${chatId}] persist failed:`, err, { isAborted });
|
||||
}
|
||||
// Async sub-agent protocol (contracts.ts): scan the last assistant
|
||||
// message for pending job tool parts and fire the background runners —
|
||||
// fire-and-forget, never awaited; the HTTP response is already gone.
|
||||
try {
|
||||
spawnPendingJobs(chatId, messages as UIMessage[]);
|
||||
} catch (err) {
|
||||
console.error(`[chat ${chatId}] job spawn failed:`, err);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { PATHS } from "../../config.ts";
|
||||
import type { JobKind, JobRecord } from "../core/contracts.ts";
|
||||
|
||||
/**
|
||||
* Job store — one JSON file per background sub-agent job in data/jobs/
|
||||
* (contracts.ts "THE ASYNC PROTOCOL"). Keyed by toolCallId for dedup: the
|
||||
* chat-handler spawn hook never creates two jobs for the same tool part.
|
||||
*
|
||||
* Follows src/store/chats.ts idioms: sync fs, id-guarded filenames, JSON
|
||||
* with tab indentation. Writes are small and rare (activity line updates);
|
||||
* the read-modify-write sections in the runner contain no awaits, so they
|
||||
* are atomic within the single-threaded event loop.
|
||||
*/
|
||||
|
||||
/** Background jobs (data/jobs). Not in config.ts PATHS (orchestrator-owned). */
|
||||
const JOBS_DIR = path.join(PATHS.data, "jobs");
|
||||
|
||||
function jobFile(id: string): string {
|
||||
if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error(`Invalid job id: ${id}`);
|
||||
return path.join(JOBS_DIR, `${id}.json`);
|
||||
}
|
||||
|
||||
export interface CreateJobInit {
|
||||
kind: JobKind;
|
||||
chatId: string;
|
||||
toolCallId: string;
|
||||
/** First activity line (defaults to "starting"). */
|
||||
activity?: string;
|
||||
}
|
||||
|
||||
export function createJob(init: CreateJobInit): JobRecord {
|
||||
const now = new Date().toISOString();
|
||||
const record: JobRecord = {
|
||||
id: `job_${Date.now().toString(36)}_${crypto.randomUUID().slice(0, 8)}`,
|
||||
kind: init.kind,
|
||||
chatId: init.chatId,
|
||||
toolCallId: init.toolCallId,
|
||||
status: "running",
|
||||
activity: init.activity ?? "starting",
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
fs.mkdirSync(JOBS_DIR, { recursive: true });
|
||||
fs.writeFileSync(jobFile(record.id), JSON.stringify(record, null, "\t"), "utf-8");
|
||||
return record;
|
||||
}
|
||||
|
||||
export function getJob(id: string): JobRecord | null {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(jobFile(id), "utf-8")) as JobRecord;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Merge a partial patch into the record (identity fields are immutable). */
|
||||
export function updateJob(
|
||||
id: string,
|
||||
patch: Partial<Omit<JobRecord, "id" | "kind" | "chatId" | "toolCallId" | "createdAt">>,
|
||||
): JobRecord | null {
|
||||
const record = getJob(id);
|
||||
if (!record) return null;
|
||||
const next: JobRecord = { ...record, ...patch, updatedAt: new Date().toISOString() };
|
||||
fs.writeFileSync(jobFile(id), JSON.stringify(next, null, "\t"), "utf-8");
|
||||
return next;
|
||||
}
|
||||
|
||||
/** All jobs for a chat, newest activity first. */
|
||||
export function listJobs(chatId: string): JobRecord[] {
|
||||
try {
|
||||
fs.mkdirSync(JOBS_DIR, { recursive: true });
|
||||
return fs
|
||||
.readdirSync(JOBS_DIR)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.map((f) => getJob(f.slice(0, -5)))
|
||||
.filter((j): j is JobRecord => j !== null && j.chatId === chatId)
|
||||
.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** The job fulfilling a given tool part, if any (spawn-hook dedup). */
|
||||
export function findJobByToolCallId(chatId: string, toolCallId: string): JobRecord | null {
|
||||
return listJobs(chatId).find((j) => j.toolCallId === toolCallId) ?? null;
|
||||
}
|
||||
Reference in New Issue
Block a user