sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

W2: async job infra + researcher sub-agent

Browse Source 
- src/store/jobs.ts: per-job JSON store in data/jobs (create/get/update/list,
  dedup by toolCallId)
- src/agent/tools/jobs.ts: research tool — client-side (no execute), turn ends
  at the call; input per contracts ResearchToolInput
- src/agent/tools/web.ts: server-side web_search (Google CSE) + web_fetch
  (Readability/Turndown/linkedom, Jina fallback) — errors return as strings,
  never thrown into the agent loop. Deps: @mozilla/readability, linkedom,
  turndown (+@types/turndown dev)
- src/agent/subagents/researcher.ts: isolated ToolLoopAgent (web tools only,
  12-step cap), researcher prompt ported from pi
- src/agent/subagents/runner.ts: spawnPendingJobs hook + runResearchJob —
  fire-and-forget, activity line per tool call, fulfills the part in the
  LATEST chat record (output-available) before flipping job status; failure
  still fulfills the part so the teacher learns the outcome
- src/server/chat-handler.ts: onFinish spawn hook after saveMessages
- app/api/jobs/route.ts: GET ?chatId= polling endpoint
- components/artifacts/job-card.tsx + registry entry: polling activity card
  (2s) → syncFromServer on finish; research-brief card (markdown + sources)
- src/agent/prompts/20-jobs.md: research doctrine for the teacher
- tests/jobs.test.ts: store CRUD, spawn hook via scripted mock teacher,
  runner completion/failure with fake model + stubbed fetch
This commit is contained in:
Amos
2026-08-22 09:36:15 +03:00
parent 92fb8e56d8
commit df16750560
13 changed files with 1456 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/jobs/route.ts
+18
View File
@@ -0,0 +1,18 @@
import { listJobs } from "@/src/store/jobs";
/**
* Job status polling endpoint (contracts.ts "THE ASYNC PROTOCOL" §4): the
* client polls GET /api/jobs?chatId=… while any job tool part is pending,
* finds the job by toolCallId, renders its activity line, and re-syncs the
* chat when the job flips to done/error.
*/
export async function GET(req: Request) {
const chatId = new URL(req.url).searchParams.get("chatId");
if (!chatId) {
return Response.json({ error: "chatId query parameter is required" }, { status: 400 });
}
if (!/^[A-Za-z0-9_-]+$/.test(chatId)) {
return Response.json({ error: "Invalid chatId" }, { status: 400 });
}
return Response.json({ jobs: listJobs(chatId) });
}
components/artifacts/index.ts
+2 -1
View File
@@ -1,5 +1,6 @@
import type { ComponentType } from "react";
import type { ArtifactPartProps } from "@/src/core/contracts";
import { JobCard } from "@/components/artifacts/job-card";
/**
* Artifact renderer registry — the extension point for tool-part UI.
@@ -14,6 +15,6 @@ import type { ArtifactPartProps } from "@/src/core/contracts";
* artifact itself from part.output.
*/
export const artifactRenderers: Record<string, ComponentType<ArtifactPartProps>> = {
// W2: "research" → job activity card
research: JobCard,
// W6: "demo" → sandboxed iframe card, "visual" → svg card
};
components/artifacts/job-card.tsx
+133
View File
@@ -0,0 +1,133 @@
"use client";
import { useContext, useEffect, useRef, useState } from "react";
import { MessageResponse } from "@/components/ai-elements/message";
import { ChatActionsContext } from "@/components/chat";
import type { ArtifactPartProps, JobRecord, ResearchResult } from "@/src/core/contracts";
/**
* The research job card (contracts.ts "THE ASYNC PROTOCOL" §4).
*
* While the part is input-available: an activity card — poll /api/jobs every
* 2s, find the job by toolCallId, show a pulsing dot + "Researching…" + the
* job's live activity line. When the job flips to done/error, call
* syncFromServer() ONCE (fresh messages replace local state; the part
* re-renders as the brief card and the sendAutomaticallyWhen continuation
* resumes the teacher) and stop polling.
*
* When output-available: the research brief — markdown body + a small source
* link list, in the visual language of answered-card.tsx.
*/
export function JobCard({ part }: ArtifactPartProps) {
if (part.state === "output-available") {
return <ResearchBriefCard output={part.output as ResearchResult} />;
}
return <ResearchActivityCard toolCallId={part.toolCallId} />;
}
function ResearchActivityCard({ toolCallId }: { toolCallId: string }) {
const actions = useContext(ChatActionsContext);
const [job, setJob] = useState<JobRecord | null>(null);
const finishedRef = useRef(false);
useEffect(() => {
if (!actions) return;
let cancelled = false;
const poll = async () => {
if (finishedRef.current) return true;
try {
const res = await fetch(`/api/jobs?chatId=${encodeURIComponent(actions.chatId)}`);
if (!res.ok) return false;
const { jobs } = (await res.json()) as { jobs?: JobRecord[] };
const match = jobs?.find((j) => j.toolCallId === toolCallId) ?? null;
if (cancelled) return false;
setJob(match);
if (match && match.status !== "running") {
// The part has (or is about to be) fulfilled server-side — pull the
// fresh messages once; the continuation resumes the teacher.
finishedRef.current = true;
void actions.syncFromServer();
return true;
}
} catch {
/* transient fetch failure — keep polling */
}
return false;
};
const timer = setInterval(() => {
if (finishedRef.current) {
clearInterval(timer);
return;
}
void poll().then((stop) => {
if (stop) clearInterval(timer);
});
}, 2000);
void poll().then((stop) => {
if (stop) clearInterval(timer);
});
return () => {
cancelled = true;
clearInterval(timer);
};
}, [actions, toolCallId]);
return (
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="research-activity-card">
<div className="flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
<span className="inline-block size-2 animate-pulse rounded-full bg-primary" />
Researching…
</div>
{job?.activity && (
<div className="mt-1 truncate text-sm text-muted-foreground" data-testid="research-activity-line">
{job.activity}
</div>
)}
</div>
);
}
function ResearchBriefCard({ output }: { output: unknown }) {
const result = output as ResearchResult | undefined;
if (!result || typeof result.brief !== "string") {
return (
<div className="rounded-xl border bg-card p-4 text-sm text-muted-foreground">
Research completed, but the result could not be displayed.
</div>
);
}
const failed = result.brief.startsWith("Research failed");
return (
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="research-brief-card">
<div className="mb-1 flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
Research brief
{failed && <span className="rounded bg-red-500/15 px-1.5 py-0.5 text-red-500">failed</span>}
</div>
<div className="text-sm [&_p:first-child]:mt-0 [&_p:last-child]:mb-0">
<MessageResponse>{result.brief}</MessageResponse>
</div>
{result.sources.length > 0 && (
<div className="mt-3 border-t pt-2">
<div className="mb-1 text-xs font-medium uppercase tracking-wide text-muted-foreground">Sources</div>
<ul className="flex flex-col gap-0.5 text-xs">
{result.sources.map((s) => (
<li key={s.url} className="truncate">
<a
href={s.url}
target="_blank"
rel="noopener noreferrer"
className="text-primary underline-offset-2 hover:underline"
>
{s.title}
</a>
</li>
))}
</ul>
</div>
)}
</div>
);
}
package-lock.json
Generated
+351
View File
@@ -9,6 +9,7 @@
"version": "0.2.0",
"dependencies": {
"@ai-sdk/react": "^4.0.79",
"@mozilla/readability": "^0.6.0",
"@openrouter/ai-sdk-provider": "^3.0.0",
"@radix-ui/react-use-controllable-state": "^1.2.6",
"@rive-app/react-webgl2": "^4.32.1",
@@ -24,8 +25,10 @@
"cmdk": "^1.1.1",
"embla-carousel-react": "^8.6.0",
"katex": "^0.16.47",
"linkedom": "^0.18.13",
"lucide-react": "^1.33.0",
"media-chrome": "^4.19.2",
"mermaid": "^11.17.0",
"motion": "^13.1.1",
"nanoid": "^6.0.1",
"next": "^16.3.2",
@@ -34,9 +37,11 @@
"react": "^19.2.8",
"react-dom": "^19.2.8",
"react-jsx-parser": "^2.4.1",
"remark-breaks": "^4.0.0",
"streamdown": "^2.5.0",
"tailwind-merge": "^3.6.0",
"tokenlens": "^1.3.1",
"turndown": "^7.2.4",
"use-stick-to-bottom": "^1.1.6",
"zod": "^4.4.3"
},
@@ -45,6 +50,7 @@
"@types/node": "^24.0.0",
"@types/react": "^19.2.18",
"@types/react-dom": "^19.2.4",
"@types/turndown": "^5.0.6",
"tailwindcss": "^4.3.3",
"tsx": "^4.19.0",
"typescript": "^5.6.0"
@@ -1240,6 +1246,21 @@
"@chevrotain/types": "~11.1.2"
}
},
"node_modules/@mixmark-io/domino": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
"license": "BSD-2-Clause"
},
"node_modules/@mozilla/readability": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz",
"integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@next/env": {
"version": "16.3.2",
"resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.2.tgz",
@@ -3713,6 +3734,13 @@
"license": "MIT",
"optional": true
},
"node_modules/@types/turndown": {
"version": "5.0.6",
"resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.6.tgz",
"integrity": "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/unist": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
@@ -3885,6 +3913,19 @@
"node": ">=6.0.0"
}
},
"node_modules/boolbase": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-2.0.0.tgz",
"integrity": "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA==",
"license": "ISC",
"engines": {
"node": ">=20.19.0"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/caniuse-lite": {
"version": "1.0.30001809",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001809.tgz",
@@ -4041,6 +4082,45 @@
"layout-base": "^1.0.0"
}
},
"node_modules/css-select": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-7.0.0.tgz",
"integrity": "sha512-snmjEVXy+1LnwXdxhYvTMj1d9tOh4HxkA1YmoayVBeeyR2C14Pum7fcxJIm4SswYspVy866eYNwlH6xC3/VH5g==",
"license": "BSD-2-Clause",
"dependencies": {
"boolbase": "^2.0.0",
"css-what": "^8.0.0",
"domhandler": "^6.0.1",
"domutils": "^4.0.2",
"nth-check": "^3.0.1"
},
"engines": {
"node": ">=20.19.0"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/css-what": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/css-what/-/css-what-8.0.0.tgz",
"integrity": "sha512-DH0Bqq3DNp5tdOReuNyAA+Ev4Y2GS5FMbZpeTLP6C4CDi0h5nL0BmUPChXw3o/qbHLDWHl49sbNqQVY7bMSDdw==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=20.19.0"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/cssom": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
"integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==",
"license": "MIT"
},
"node_modules/csstype": {
"version": "3.2.3",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
@@ -4639,6 +4719,67 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/dom-serializer": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-3.1.1.tgz",
"integrity": "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw==",
"license": "MIT",
"dependencies": {
"domelementtype": "^3.0.0",
"domhandler": "^6.0.0",
"entities": "^8.0.0"
},
"engines": {
"node": ">=20.19.0"
},
"funding": {
"type": "github",
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
}
},
"node_modules/dom-serializer/node_modules/entities": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz",
"integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=20.19.0"
},
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/domelementtype": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-3.0.0.tgz",
"integrity": "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
],
"license": "BSD-2-Clause",
"engines": {
"node": ">=20.19.0"
}
},
"node_modules/domhandler": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-6.0.1.tgz",
"integrity": "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg==",
"license": "BSD-2-Clause",
"dependencies": {
"domelementtype": "^3.0.0"
},
"engines": {
"node": ">=20.19.0"
},
"funding": {
"type": "github",
"url": "https://github.com/fb55/domhandler?sponsor=1"
}
},
"node_modules/dompurify": {
"version": "3.4.14",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.14.tgz",
@@ -4648,6 +4789,24 @@
"@types/trusted-types": "^2.0.7"
}
},
"node_modules/domutils": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-4.0.2.tgz",
"integrity": "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA==",
"license": "BSD-2-Clause",
"dependencies": {
"dom-serializer": "^3.0.0",
"domelementtype": "^3.0.0",
"domhandler": "^6.0.0"
},
"engines": {
"node": ">=20.19.0"
},
"funding": {
"type": "github",
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/embla-carousel": {
"version": "8.6.0",
"resolved": "https://registry.npmjs.org/embla-carousel/-/embla-carousel-8.6.0.tgz",
@@ -5130,6 +5289,12 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/html-escaper": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
"integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==",
"license": "MIT"
},
"node_modules/html-url-attributes": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz",
@@ -5150,6 +5315,104 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/htmlparser2": {
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
"integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
"funding": [
"https://github.com/fb55/htmlparser2?sponsor=1",
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
],
"license": "MIT",
"dependencies": {
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3",
"domutils": "^3.2.2",
"entities": "^7.0.1"
}
},
"node_modules/htmlparser2/node_modules/dom-serializer": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
"license": "MIT",
"dependencies": {
"domelementtype": "^2.3.0",
"domhandler": "^5.0.2",
"entities": "^4.2.0"
},
"funding": {
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
}
},
"node_modules/htmlparser2/node_modules/dom-serializer/node_modules/entities": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=0.12"
},
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/htmlparser2/node_modules/domelementtype": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
],
"license": "BSD-2-Clause"
},
"node_modules/htmlparser2/node_modules/domhandler": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
"license": "BSD-2-Clause",
"dependencies": {
"domelementtype": "^2.3.0"
},
"engines": {
"node": ">= 4"
},
"funding": {
"url": "https://github.com/fb55/domhandler?sponsor=1"
}
},
"node_modules/htmlparser2/node_modules/domutils": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
"integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
"license": "BSD-2-Clause",
"dependencies": {
"dom-serializer": "^2.0.0",
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3"
},
"funding": {
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/htmlparser2/node_modules/entities": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
"integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=0.12"
},
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/iconv-lite": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
@@ -5547,6 +5810,30 @@
"url": "https://opencollective.com/parcel"
}
},
"node_modules/linkedom": {
"version": "0.18.13",
"resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.13.tgz",
"integrity": "sha512-ES/o9qotMpzpN2MHs+Iq/JcVoOj8Fa5wiQYrTdFpvAnwXL0g66XHHUc9WUMk6nAlBtGsFQ24ne+SYnvnaQ2FSw==",
"license": "ISC",
"dependencies": {
"css-select": "^7.0.0",
"cssom": "^0.5.0",
"html-escaper": "^3.0.3",
"htmlparser2": "^10.1.0",
"uhyphen": "^0.2.0"
},
"engines": {
"node": ">=16"
},
"peerDependencies": {
"canvas": ">= 2"
},
"peerDependenciesMeta": {
"canvas": {
"optional": true
}
}
},
"node_modules/linkify-it": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz",
@@ -5833,6 +6120,20 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-newline-to-break": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/mdast-util-newline-to-break/-/mdast-util-newline-to-break-2.0.0.tgz",
"integrity": "sha512-MbgeFca0hLYIEx/2zGsszCSEJJ1JSCdiY5xQxRcLDDGa8EPvlLPupJ4DSajbMPAnC0je8jfb9TiUATnxxrHUog==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"mdast-util-find-and-replace": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-phrasing": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz",
@@ -6809,6 +7110,22 @@
"node": "^10 || ^12 || >=14"
}
},
"node_modules/nth-check": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-3.0.1.tgz",
"integrity": "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ==",
"license": "BSD-2-Clause",
"dependencies": {
"boolbase": "^2.0.0"
},
"engines": {
"node": ">=20.19.0"
},
"funding": {
"type": "github",
"url": "https://github.com/fb55/nth-check?sponsor=1"
}
},
"node_modules/oniguruma-parser": {
"version": "0.12.2",
"resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz",
@@ -7254,6 +7571,21 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/remark-breaks": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/remark-breaks/-/remark-breaks-4.0.0.tgz",
"integrity": "sha512-IjEjJOkH4FuJvHZVIW0QCDWxcG96kCq7An/KVH2NfJe6rKZU2AsHeB3OEjPNRxi4QC34Xdx7I2KGYn6IpT7gxQ==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"mdast-util-newline-to-break": "^2.0.0",
"unified": "^11.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/remark-cjk-friendly": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/remark-cjk-friendly/-/remark-cjk-friendly-2.3.1.tgz",
@@ -7742,6 +8074,19 @@
"fsevents": "~2.3.3"
}
},
"node_modules/turndown": {
"version": "7.2.4",
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz",
"integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==",
"license": "MIT",
"dependencies": {
"@mixmark-io/domino": "^2.2.0"
},
"engines": {
"node": ">=18",
"npm": ">=9"
}
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
@@ -7762,6 +8107,12 @@
"integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==",
"license": "MIT"
},
"node_modules/uhyphen": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz",
"integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==",
"license": "ISC"
},
"node_modules/undici": {
"version": "7.29.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
package.json
+6
View File
@@ -16,6 +16,7 @@
},
"dependencies": {
"@ai-sdk/react": "^4.0.79",
"@mozilla/readability": "^0.6.0",
"@openrouter/ai-sdk-provider": "^3.0.0",
"@radix-ui/react-use-controllable-state": "^1.2.6",
"@rive-app/react-webgl2": "^4.32.1",
@@ -31,8 +32,10 @@
"cmdk": "^1.1.1",
"embla-carousel-react": "^8.6.0",
"katex": "^0.16.47",
"linkedom": "^0.18.13",
"lucide-react": "^1.33.0",
"media-chrome": "^4.19.2",
"mermaid": "^11.17.0",
"motion": "^13.1.1",
"nanoid": "^6.0.1",
"next": "^16.3.2",
@@ -41,9 +44,11 @@
"react": "^19.2.8",
"react-dom": "^19.2.8",
"react-jsx-parser": "^2.4.1",
"remark-breaks": "^4.0.0",
"streamdown": "^2.5.0",
"tailwind-merge": "^3.6.0",
"tokenlens": "^1.3.1",
"turndown": "^7.2.4",
"use-stick-to-bottom": "^1.1.6",
"zod": "^4.4.3"
},
@@ -52,6 +57,7 @@
"@types/node": "^24.0.0",
"@types/react": "^19.2.18",
"@types/react-dom": "^19.2.4",
"@types/turndown": "^5.0.6",
"tailwindcss": "^4.3.3",
"tsx": "^4.19.0",
"typescript": "^5.6.0"
src/agent/prompts/20-jobs.md
+27
View File
@@ -0,0 +1,27 @@
## Background research
You have a `research` tool that dispatches a background web-research agent. Use it to
outsource what you cannot reliably know yourself:
- **Use it for:** facts you are uncertain about, current or fast-moving data (library versions,
release dates, recent events), niche specifics outside your training, and anything where
being wrong would poison the lesson.
- **Never use it for:** anything you know cold, internal reasoning, or math. Do not offload
thinking — only facts you genuinely need verified.
**The call is asynchronous and your turn ends at it.** The discipline:
1. **Say your piece first.** Explain the concept, set up what you're doing and why, and tell
the learner roughly what you're checking — the research brief arrives later as the tool's
output, and the lesson resumes from there. A bare tool call with no surrounding text is a
failed turn.
2. **Call with a single, specific question.** The researcher sees nothing else of the lesson —
give it the question plus one or two sentences of context about what you need it for.
3. **Resume from the brief.** When the result returns, weave it into the lesson. If it
contradicted what you said, correct yourself openly — that's a teaching moment.
4. **Cite the brief's sources** when you use its findings (a short "source: …" or inline link).
Never present researched facts as your own certainty.
At most one research call per turn — don't fan out; the researcher searches multiple angles on
its own. If research fails or is unconfigured, you'll get a failure brief: say so plainly and
continue with what you can establish, flagging the uncertainty.
src/agent/subagents/researcher.ts
+71
View File
@@ -0,0 +1,71 @@
import { ToolLoopAgent, stepCountIs, type LanguageModel } from "ai";
import { openrouter } from "@openrouter/ai-sdk-provider";
import { webTools } from "../tools/web.ts";
/**
* The researcher sub-agent (vision §2.5): a SEPARATE ToolLoopAgent instance
* from the teacher — an isolated loop with only web_search + web_fetch, so
* raw search/fetch churn never touches the teacher's context window.
*
* System prompt adapted from the pi researcher agent
* (~/.pi/agent/extensions/interactive-subagents/agents/researcher.md).
* The model is injectable so tests run the loop with a MockLanguageModelV4.
*/
const DEFAULT_RESEARCHER_MODEL =
process.env.LH_RESEARCH_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
const INSTRUCTIONS = `You are a research specialist. You work for a teacher agent inside a learning
application: given a research question, conduct thorough web research and produce a focused,
well-sourced brief that the teacher can teach from.
You operate in an isolated context with no knowledge of the lesson or any prior conversation.
All necessary context is in the task description.
Process:
1. Break the question into 2-4 searchable facets.
2. Search with web_search using varied angles:
- the direct answer query (the obvious one)
- an authoritative-source query (official docs, specs, primary sources)
- a practical-experience query (case studies, benchmarks, real-world usage)
- a recent-developments query (only if the topic is time-sensitive)
3. Read the results. Identify what's well-covered and what has gaps. If the first round
doesn't fully answer the question, search again with refined queries targeting the gaps.
4. For the 2-3 most promising source URLs, use web_fetch to get the full page content.
5. Synthesize everything into a brief that directly answers the question.
Evaluation — what to keep vs drop:
- Official docs and primary sources outweigh blog posts and forum threads.
- Recent sources outweigh stale ones.
- Sources that directly address the question outweigh tangentially related ones.
- Drop SEO filler, outdated info, and beginner tutorials (unless that's the audience).
Tool outputs may be error strings (e.g. search unconfigured, a page that failed to fetch).
Adapt: try another URL or another search angle; if research is genuinely impossible, say so
in the brief instead of inventing facts.
Your FINAL assistant message is your entire deliverable — it must stand alone, using this format:
## Summary
2-3 sentence direct answer.
## Findings
Numbered findings with inline source citations:
1. **Finding** — explanation. [Source](url)
## Sources
- Source Title (url) — why it's relevant
## Gaps
What couldn't be answered. Suggested next steps.`;
export function createResearcherAgent(model?: LanguageModel) {
return new ToolLoopAgent({
model: model ?? openrouter(DEFAULT_RESEARCHER_MODEL),
instructions: INSTRUCTIONS,
tools: webTools,
stopWhen: stepCountIs(12),
});
}
export type ResearcherAgent = ReturnType<typeof createResearcherAgent>;
src/agent/subagents/runner.ts
+182
View File
@@ -0,0 +1,182 @@
import { isToolUIPart, type LanguageModel, type UIMessage } from "ai";
import type { ResearchResult, ResearchToolInput } from "../../core/contracts.ts";
import { JOB_TOOLS } from "../../core/pending.ts";
import { getChat, saveMessages } from "../../store/chats.ts";
import { createJob, findJobByToolCallId, getJob, updateJob } from "../../store/jobs.ts";
import { createResearcherAgent } from "./researcher.ts";
/**
* The background job runner + the chat-handler spawn hook (contracts.ts
* "THE ASYNC PROTOCOL").
*
* spawnPendingJobs — called from onFinish AFTER saveMessages: finds
* input-available JOB_TOOLS parts in the last assistant message with no
* existing job and fires runResearchJob WITHOUT awaiting (the HTTP
* response is long gone).
*
* runResearchJob — runs the researcher agent loop in the background,
* updating the job's activity line on every tool call, then rewrites the
* tool part in the LATEST chat record to output-available (read-modify-
* write; no awaits between the read and the write). The part is fulfilled
* BEFORE the job status flips, so a client that polls the job and then
* re-syncs always sees the finished part.
*
* On failure the part is still fulfilled ({brief: "Research failed: …"}) —
* the teacher must learn the outcome — and the job is marked error.
*/
/**
* Test seam: the spawn hook creates runners with no injection point, so
* tests swap in a MockLanguageModelV4 here to keep job tests off the
* network. Undefined in production.
*/
export const researchModelOverride: { current?: LanguageModel } = {};
export interface RunResearchJobOptions {
model?: LanguageModel;
}
export function spawnPendingJobs(chatId: string, messages: UIMessage[]): void {
const last = [...messages].reverse().find((m) => m.role === "assistant");
if (!last) return;
for (const part of last.parts) {
if (!isToolUIPart(part) || part.state !== "input-available") continue;
const tool = part.type.slice("tool-".length);
if (!(JOB_TOOLS as readonly string[]).includes(tool)) continue;
if (tool !== "research") continue; // demo/visual runners arrive in W6
if (findJobByToolCallId(chatId, part.toolCallId)) continue; // already has a job
const job = createJob({ kind: "research", chatId, toolCallId: part.toolCallId });
void runResearchJob(job.id, { model: researchModelOverride.current }).catch((err) => {
console.error(`[job ${job.id}] research runner crashed:`, err);
updateJob(job.id, { status: "error", error: String(err), activity: "failed" });
});
}
}
export async function runResearchJob(jobId: string, opts: RunResearchJobOptions = {}): Promise<void> {
const job = getJob(jobId);
if (!job || job.status !== "running") return;
const chat = getChat(job.chatId);
if (!chat) {
updateJob(jobId, { status: "error", error: `Chat not found: ${job.chatId}`, activity: "failed" });
return;
}
const input = findToolPart(chat.messages, job.toolCallId)?.input as ResearchToolInput | undefined;
const question = typeof input?.question === "string" && input.question.trim() ? input.question : "(no question)";
const context = typeof input?.context === "string" && input.context.trim() ? input.context : undefined;
const prompt = [
`Research question: ${question}`,
context ? `Context (what this is needed for): ${context}` : null,
"Produce your final brief in the required format.",
]
.filter(Boolean)
.join("\n\n");
const fetchedUrls: string[] = [];
const agent = createResearcherAgent(opts.model);
try {
updateJob(jobId, { activity: "researching" });
const result = await agent.generate({
prompt,
onToolExecutionStart: ({ toolCall }) => {
const input = toolCall.input as { query?: string; url?: string } | undefined;
if (toolCall.toolName === "web_fetch" && typeof input?.url === "string") {
fetchedUrls.push(input.url);
}
updateJob(jobId, { activity: activityLine(toolCall.toolName, input) });
},
});
const brief = result.text.trim();
if (!brief) throw new Error("the researcher produced no text");
const research: ResearchResult = { brief, sources: parseSources(brief, fetchedUrls) };
fulfillPart(job.chatId, job.toolCallId, research);
updateJob(jobId, { status: "done", result: research }); // activity line kept: the last tool call is the record
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
fulfillPart(job.chatId, job.toolCallId, { brief: `Research failed: ${reason}`, sources: [] });
updateJob(jobId, { status: "error", activity: "failed", error: reason });
}
}
// ─── helpers ─────────────────────────────────────────────────────────────────
function findToolPart(messages: UIMessage[], toolCallId: string) {
for (const m of messages) {
for (const part of m.parts) {
if (isToolUIPart(part) && part.toolCallId === toolCallId) return part;
}
}
return null;
}
/** The activity line shown in the UI: `web_search: "query"` / `web_fetch: host`. */
function activityLine(toolName: string, input: { query?: string; url?: string } | undefined): string {
if (toolName === "web_search") return `web_search: "${input?.query ?? ""}"`;
if (toolName === "web_fetch") return `web_fetch: ${hostOf(input?.url)}`;
return toolName;
}
function hostOf(url: string | undefined): string {
if (!url) return "(no url)";
try {
return new URL(url).hostname;
} catch {
return url.slice(0, 40);
}
}
/**
* Rewrite the tool part in the LATEST chat record to output-available.
* Synchronous read-modify-write (no awaits) — atomic within the event loop.
*/
function fulfillPart(chatId: string, toolCallId: string, output: ResearchResult): void {
const chat = getChat(chatId);
if (!chat) return;
let found = false;
const messages = chat.messages.map((m) =>
m.role !== "assistant"
? m
: {
...m,
parts: m.parts.map((part) => {
if (isToolUIPart(part) && part.toolCallId === toolCallId && part.state === "input-available") {
found = true;
return { ...part, state: "output-available" as const, output };
}
return part;
}),
},
);
if (!found) return;
try {
saveMessages(chatId, messages);
} catch (err) {
console.error(`[chat ${chatId}] fulfilling job part ${toolCallId} failed:`, err);
}
}
/** Sources for the result: markdown links cited in the brief, else fetched URLs. */
function parseSources(brief: string, fetchedUrls: string[]): ResearchResult["sources"] {
const sources: ResearchResult["sources"] = [];
const seen = new Set<string>();
for (const match of brief.matchAll(/\[([^\]]+)\]\((https?:\/\/[^)\s]+)\)/g)) {
const title = (match[1] ?? "").trim();
const url = match[2]!;
if (!title || seen.has(url)) continue;
seen.add(url);
sources.push({ title, url });
}
if (sources.length === 0) {
for (const url of fetchedUrls) {
if (seen.has(url)) continue;
seen.add(url);
sources.push({ title: hostOf(url), url });
}
}
return sources.slice(0, 10);
}
src/agent/tools/jobs.ts
+39 -6
View File
@@ -1,13 +1,46 @@
import type { ToolSet } from "ai";
import { tool, type ToolSet } from "ai";
import { z } from "zod";
import type { TeacherToolContext } from "../tools.ts";
/**
* Async sub-agent job tools — W2 workstream (research; W6 adds demo + visual).
* CLIENT-SIDE semantics (no execute): the turn ends with the part pending, a
* background runner fulfills it, and the client continuation resumes the
* teacher. See src/core/contracts.ts "THE ASYNC PROTOCOL" — read it first.
* Async sub-agent job tools (W2: research; W6 adds demo + visual).
* CLIENT-SIDE semantics (no execute) — exactly like quiz/ask: the tool call
* streams to the browser in `input-available` state and the server turn ends
* there. chat-handler's onFinish spawns a background runner that fulfills
* the part server-side; the client polls /api/jobs and the continuation
* resumes the teacher with the result as a normal tool output.
* See src/core/contracts.ts "THE ASYNC PROTOCOL".
*/
export const researchInputSchema = z.object({
question: z
.string()
.describe("The single research question to answer. Make it specific — the researcher sees nothing else of the lesson."),
context: z
.string()
.optional()
.describe("Why you need this / what the lesson needs it for — focuses the brief. One or two sentences."),
});
export const researchTool = tool({
description:
"Dispatch a background web-research agent: fact verification, current data, or niche specifics you are not certain about. " +
"The call is ASYNC — your turn ENDS here, so say everything you want to say first; the research brief arrives later as " +
"this tool's output and the lesson resumes from it. Cite the brief's sources when you use them. " +
"NEVER use it for things you know cold, internal reasoning, or math.",
inputSchema: researchInputSchema,
});
/**
* Per-request job tool registry. Context-free for now (the runner resolves
* everything from the chat record by toolCallId) — the ctx parameter keeps
* the createTeacherTools factory shape stable for W6's demo/visual tools.
*/
// eslint-disable-next-line @typescript-eslint/no-unused-vars
export function jobTools(ctx: TeacherToolContext): ToolSet {
return {};
return {
research: researchTool,
};
}
export type ResearchInput = z.infer<typeof researchInputSchema>;
src/agent/tools/web.ts
+185
View File
@@ -0,0 +1,185 @@
import { tool } from "ai";
import { z } from "zod";
import { Readability } from "@mozilla/readability";
import { parseHTML } from "linkedom";
import TurndownService from "turndown";
/**
* Server-side web tools for the researcher sub-agent — a minimal port of the
* pi extensions (~/.pi/agent/extensions/web-search + web-fetch):
*
* web_search(query, count?) → {title, url, snippet}[] (Google Custom Search)
* web_fetch(url) → readable markdown text (Readability + Turndown,
* Jina Reader fallback for
* JS-rendered pages)
*
* Credentials come from env (the pi extension used auth.json):
* GOOGLE_SEARCH_API_KEY, GOOGLE_CSE_ID
*
* Tool contract: NEVER throw into the agent loop — missing credentials and
* every failure path return a clear error STRING as the tool output, so the
* researcher can adapt (or report) instead of crashing.
*/
// ── web_search ───────────────────────────────────────────────────────────────
export interface WebSearchResult {
title: string;
url: string;
snippet: string;
}
function searchCredentials(): { apiKey: string; cseId: string } | null {
const apiKey = process.env.GOOGLE_SEARCH_API_KEY;
const cseId = process.env.GOOGLE_CSE_ID;
return apiKey && cseId ? { apiKey, cseId } : null;
}
async function googleSearch(query: string, count: number, creds: { apiKey: string; cseId: string }): Promise<WebSearchResult[]> {
const url = new URL("https://www.googleapis.com/customsearch/v1");
url.searchParams.set("key", creds.apiKey);
url.searchParams.set("cx", creds.cseId);
url.searchParams.set("q", query);
url.searchParams.set("num", String(Math.min(count, 10)));
const resp = await fetch(url, { signal: AbortSignal.timeout(30_000) });
if (!resp.ok) {
const body = await resp.text();
throw new Error(`Google Custom Search API ${resp.status}: ${body.slice(0, 200)}`);
}
const data = (await resp.json()) as { items?: Array<{ title: string; link: string; snippet?: string }> };
return (data.items ?? []).map((item) => ({
title: item.title,
url: item.link,
snippet: item.snippet?.replace(/\n/g, " ") ?? "",
}));
}
// ── web_fetch ────────────────────────────────────────────────────────────────
const USER_AGENT =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
const FETCH_TIMEOUT_MS = 30_000;
const MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
/** Keep the researcher's context sane — long pages truncate with a marker. */
const MAX_CONTENT_CHARS = 40_000;
const MIN_USEFUL_CONTENT = 200;
const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
function truncate(text: string): string {
if (text.length <= MAX_CONTENT_CHARS) return text;
return `${text.slice(0, MAX_CONTENT_CHARS)}\n\n[Truncated at ${MAX_CONTENT_CHARS} characters]`;
}
/** Jina Reader fallback for JS-rendered pages (no deps — plain fetch). */
async function fetchViaJina(url: string): Promise<string | null> {
try {
const res = await fetch(`https://r.jina.ai/${url}`, {
headers: { Accept: "text/markdown", "X-No-Cache": "true" },
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (!res.ok) return null;
const content = await res.text();
const marker = "Markdown Content:";
const start = content.indexOf(marker);
if (start < 0) return null;
const markdown = content.slice(start + marker.length).trim();
if (markdown.length < MIN_USEFUL_CONTENT || markdown.startsWith("Loading...")) return null;
return markdown;
} catch {
return null;
}
}
function htmlToMarkdown(html: string): { title: string; content: string } | null {
const { document } = parseHTML(html);
const article = new Readability(document as unknown as Document).parse();
if (!article?.content) return null;
const markdown = turndown.turndown(article.content);
if (markdown.trim().length < MIN_USEFUL_CONTENT) return null;
return { title: article.title ?? "", content: markdown };
}
async function fetchAndExtract(url: string): Promise<string> {
const resp = await fetch(url, {
headers: {
"User-Agent": USER_AGENT,
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
},
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (!resp.ok) throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
const contentType = resp.headers.get("content-type") ?? "";
if (contentType.includes("application/pdf")) throw new Error("PDF documents are not supported");
if (/image\/|audio\/|video\/|zip|octet-stream/.test(contentType)) {
throw new Error(`Unsupported content type: ${contentType.split(";")[0]}`);
}
const length = Number(resp.headers.get("content-length") ?? 0);
if (length > MAX_RESPONSE_BYTES) throw new Error("Response too large");
const text = await resp.text();
if (!contentType.includes("html")) return truncate(text); // plain text / markdown / code
const extracted = htmlToMarkdown(text);
if (extracted) {
const header = extracted.title ? `# ${extracted.title}\n\nSource: ${url}\n\n---\n\n` : "";
return header + truncate(extracted.content);
}
// Readability failed — likely JS-rendered. Try Jina Reader.
const jina = await fetchViaJina(url);
if (jina) return truncate(jina);
throw new Error("Could not extract readable content (the page may be JavaScript-rendered)");
}
// ── tool registry ────────────────────────────────────────────────────────────
export const webTools = {
web_search: tool({
description:
"Search the web via Google Custom Search. One search per call — use one call per search angle. " +
"Returns an array of results with title, url, and snippet.",
inputSchema: z.object({
query: z.string().describe("The search query (a normal search string, phrases included)"),
count: z.number().int().min(1).max(10).optional().describe("Number of results (default 5, max 10)"),
}),
execute: async ({ query, count }): Promise<WebSearchResult[] | string> => {
const creds = searchCredentials();
if (!creds) {
return (
"web_search is not configured on this server: GOOGLE_SEARCH_API_KEY and/or GOOGLE_CSE_ID " +
"are missing. Report in your brief that web search is unavailable."
);
}
try {
return await googleSearch(query, count ?? 5, creds);
} catch (err) {
return `web_search failed: ${err instanceof Error ? err.message : String(err)}`;
}
},
}),
web_fetch: tool({
description:
"Fetch a web page and extract its readable content as markdown (Readability). " +
"Handles HTML pages and plain text; JS-rendered pages fall back to a reader service.",
inputSchema: z.object({
url: z.string().describe("The URL to fetch"),
}),
execute: async ({ url }): Promise<string> => {
try {
const parsed = new URL(url);
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error("Only http(s) URLs are supported");
} catch {
return `web_fetch failed: invalid URL "${url}"`;
}
try {
return await fetchAndExtract(url);
} catch (err) {
return `web_fetch failed for ${url}: ${err instanceof Error ? err.message : String(err)}`;
}
},
}),
};
src/server/chat-handler.ts
+9
View File
@@ -1,6 +1,7 @@
import { createAgentUIStreamResponse, type UIMessage } from "ai";
import { createTeacherAgent, type TeacherAgent } from "../agent/teacher-agent.ts";
import { createTeacherTools } from "../agent/tools.ts";
import { spawnPendingJobs } from "../agent/subagents/runner.ts";
import { getChat, saveMessages } from "../store/chats.ts";
/**
@@ -38,6 +39,14 @@ export async function handleChatRequest(req: Request, agent?: TeacherAgent): Pro
} catch (err) {
console.error(`[chat ${chatId}] persist failed:`, err, { isAborted });
}
// Async sub-agent protocol (contracts.ts): scan the last assistant
// message for pending job tool parts and fire the background runners —
// fire-and-forget, never awaited; the HTTP response is already gone.
try {
spawnPendingJobs(chatId, messages as UIMessage[]);
} catch (err) {
console.error(`[chat ${chatId}] job spawn failed:`, err);
}
},
});
}
src/store/jobs.ts
+88
View File
@@ -0,0 +1,88 @@
import * as fs from "node:fs";
import * as path from "node:path";
import { PATHS } from "../../config.ts";
import type { JobKind, JobRecord } from "../core/contracts.ts";
/**
* Job store — one JSON file per background sub-agent job in data/jobs/
* (contracts.ts "THE ASYNC PROTOCOL"). Keyed by toolCallId for dedup: the
* chat-handler spawn hook never creates two jobs for the same tool part.
*
* Follows src/store/chats.ts idioms: sync fs, id-guarded filenames, JSON
* with tab indentation. Writes are small and rare (activity line updates);
* the read-modify-write sections in the runner contain no awaits, so they
* are atomic within the single-threaded event loop.
*/
/** Background jobs (data/jobs). Not in config.ts PATHS (orchestrator-owned). */
const JOBS_DIR = path.join(PATHS.data, "jobs");
function jobFile(id: string): string {
if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error(`Invalid job id: ${id}`);
return path.join(JOBS_DIR, `${id}.json`);
}
export interface CreateJobInit {
kind: JobKind;
chatId: string;
toolCallId: string;
/** First activity line (defaults to "starting"). */
activity?: string;
}
export function createJob(init: CreateJobInit): JobRecord {
const now = new Date().toISOString();
const record: JobRecord = {
id: `job_${Date.now().toString(36)}_${crypto.randomUUID().slice(0, 8)}`,
kind: init.kind,
chatId: init.chatId,
toolCallId: init.toolCallId,
status: "running",
activity: init.activity ?? "starting",
createdAt: now,
updatedAt: now,
};
fs.mkdirSync(JOBS_DIR, { recursive: true });
fs.writeFileSync(jobFile(record.id), JSON.stringify(record, null, "\t"), "utf-8");
return record;
}
export function getJob(id: string): JobRecord | null {
try {
return JSON.parse(fs.readFileSync(jobFile(id), "utf-8")) as JobRecord;
} catch {
return null;
}
}
/** Merge a partial patch into the record (identity fields are immutable). */
export function updateJob(
id: string,
patch: Partial<Omit<JobRecord, "id" | "kind" | "chatId" | "toolCallId" | "createdAt">>,
): JobRecord | null {
const record = getJob(id);
if (!record) return null;
const next: JobRecord = { ...record, ...patch, updatedAt: new Date().toISOString() };
fs.writeFileSync(jobFile(id), JSON.stringify(next, null, "\t"), "utf-8");
return next;
}
/** All jobs for a chat, newest activity first. */
export function listJobs(chatId: string): JobRecord[] {
try {
fs.mkdirSync(JOBS_DIR, { recursive: true });
return fs
.readdirSync(JOBS_DIR)
.filter((f) => f.endsWith(".json"))
.map((f) => getJob(f.slice(0, -5)))
.filter((j): j is JobRecord => j !== null && j.chatId === chatId)
.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
} catch {
return [];
}
}
/** The job fulfilling a given tool part, if any (spawn-hook dedup). */
export function findJobByToolCallId(chatId: string, toolCallId: string): JobRecord | null {
return listJobs(chatId).find((j) => j.toolCallId === toolCallId) ?? null;
}
tests/jobs.test.ts
+345
View File
@@ -0,0 +1,345 @@
/**
* Job infrastructure tests — deterministic, zero network:
*
* 1. job store CRUD (create → get → update → list → dedup by toolCallId)
* 2. the spawn hook: drive handleChatRequest with a scripted mock teacher
* that emits a research tool call → a job record appears (runner fires
* with the mock override, so no network)
* 3. runner completion: a fake researcher model (web_search call → final
* text brief, globalThis.fetch stubbed) → the stored chat's part flips
* to output-available with a ResearchResult-shaped output, the job is
* done with the activity recorded
* 4. runner failure: a model that throws → the part is still fulfilled
* with a failure brief and the job is marked error
*
* Cleanup follows tests/chat-flow.test.ts: created chat/job files removed in
* finally blocks / the after() hook.
*/
import assert from "node:assert/strict";
import * as fs from "node:fs";
import * as path from "node:path";
import { after, describe, it } from "node:test";
import { MockLanguageModelV4, simulateReadableStream } from "ai/test";
import type { LanguageModelV4StreamPart } from "@ai-sdk/provider";
import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
import { handleChatRequest } from "../src/server/chat-handler.ts";
import { createChat, getChat } from "../src/store/chats.ts";
import { createJob, findJobByToolCallId, getJob, listJobs, updateJob } from "../src/store/jobs.ts";
import { researchModelOverride, runResearchJob, spawnPendingJobs } from "../src/agent/subagents/runner.ts";
import { createResearcherAgent } from "../src/agent/subagents/researcher.ts";
import { PATHS } from "../config.ts";
const JOBS_DIR = path.join(PATHS.data, "jobs");
const createdChats: string[] = [];
const createdJobs: string[] = [];
function trackChat(id: string) {
createdChats.push(id);
}
function trackJob(id: string) {
createdJobs.push(id);
}
after(() => {
for (const id of createdChats) {
try {
fs.unlinkSync(path.join(PATHS.chats, `${id}.json`));
} catch {
/* already gone */
}
}
for (const id of createdJobs) {
try {
fs.unlinkSync(path.join(JOBS_DIR, `${id}.json`));
} catch {
/* already gone */
}
}
});
function usage() {
return {
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
outputTokens: { total: 5, text: 5, reasoning: 0 },
};
}
async function drain(res: Response): Promise<string> {
if (res.status !== 200) {
assert.fail(`expected 200, got ${res.status}: ${await res.text()}`);
}
return res.text();
}
function post(chatId: string, messages: unknown[]): Request {
return new Request("http://test/api/chat", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ chatId, messages }),
});
}
// ─── 1. job store CRUD ───────────────────────────────────────────────────────
describe("job store", () => {
it("create → get → update → list round-trips", () => {
const job = createJob({ kind: "research", chatId: "chat_storetest", toolCallId: "call-store" });
trackJob(job.id);
assert.match(job.id, /^job_/);
assert.equal(job.status, "running");
assert.equal(job.activity, "starting");
const loaded = getJob(job.id)!;
assert.equal(loaded.toolCallId, "call-store");
assert.equal(loaded.kind, "research");
const updated = updateJob(job.id, { activity: 'web_search: "test"', status: "done", result: { brief: "b" } })!;
assert.equal(updated.activity, 'web_search: "test"');
assert.equal(updated.status, "done");
assert.ok(updated.updatedAt >= job.updatedAt);
// immutable identity fields survive a patch
assert.equal(updated.toolCallId, "call-store");
assert.equal(getJob(updated.id)!.toolCallId, "call-store");
const listed = listJobs("chat_storetest");
assert.ok(listed.some((j) => j.id === job.id));
assert.equal(listJobs("chat_other").some((j) => j.id === job.id), false, "jobs are scoped to their chat");
const byTool = findJobByToolCallId("chat_storetest", "call-store");
assert.equal(byTool?.id, job.id);
assert.equal(findJobByToolCallId("chat_storetest", "call-missing"), null);
});
});
// ─── scripted teacher: emits a research tool call ────────────────────────────
const RESEARCH_INPUT = {
question: "What is the latest stable sigma.js version?",
context: "Choosing a graph rendering library for the lesson.",
};
/** A teacher that calls research on turn 1; plain text thereafter. */
function researchCallingTeacher(): MockLanguageModelV4 {
let turn = 0;
return new MockLanguageModelV4({
doStream: async () => {
turn++;
const chunks: LanguageModelV4StreamPart[] =
turn === 1
? [
{ type: "stream-start", warnings: [] },
{ type: "text-start", id: "t1" },
{ type: "text-delta", id: "t1", delta: "Let me check the current state of the library." },
{ type: "text-end", id: "t1" },
{ type: "tool-input-start", id: "call-r1", toolName: "research" },
{ type: "tool-input-delta", id: "call-r1", delta: JSON.stringify(RESEARCH_INPUT) },
{ type: "tool-input-end", id: "call-r1" },
{ type: "tool-call", toolCallId: "call-r1", toolName: "research", input: JSON.stringify(RESEARCH_INPUT) },
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
]
: [
{ type: "stream-start", warnings: [] },
{ type: "text-start", id: "t2" },
{ type: "text-delta", id: "t2", delta: "Back with the brief." },
{ type: "text-end", id: "t2" },
{ type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: usage() },
];
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
},
});
}
describe("spawn hook (async protocol §2)", () => {
it("a pending research part in the final assistant message spawns exactly one job", async () => {
const chat = createChat("Test: research spawn");
trackChat(chat.id);
const agent = createTeacherAgent(researchCallingTeacher());
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me graph rendering" }] };
await drain(await handleChatRequest(post(chat.id, [firstUser]), agent));
// The research part is pending (input-available) in the stored chat.
const saved = getChat(chat.id)!;
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-research");
assert.ok(part && part.type === "tool-research", "research tool part persisted");
assert.equal(part.state, "input-available", "job tool part stays pending until the runner fulfills it");
// A job record exists for it.
const job = findJobByToolCallId(chat.id, "call-r1");
assert.ok(job, "spawn hook created a job record");
assert.equal(job!.kind, "research");
assert.equal(job!.status, "running");
trackJob(job!.id);
// Re-running the spawn hook (idempotency) does not create a second job.
spawnPendingJobs(chat.id, saved.messages);
assert.equal(listJobs(chat.id).length, 1, "no duplicate job for the same tool part");
});
});
// ─── the researcher loop with a fake model + stubbed fetch ──────────────────
const BRIEF = `## Summary
sigma.js v3 is current.
## Findings
1. **v3 is the latest major.** [sigma.js docs](https://www.sigmajs.org/docs)
## Sources
- sigma.js docs (https://www.sigmajs.org/docs)`;
/**
* The researcher loop runs via agent.generate (non-streaming) — the mock
* therefore scripts doGenerate. Turn 1: a web_search tool call; turn 2: the
* final text brief.
*/
function researcherModel(): MockLanguageModelV4 {
let turn = 0;
return new MockLanguageModelV4({
doGenerate: async () => {
turn++;
return turn === 1
? {
content: [
{
type: "tool-call" as const,
toolCallId: "ws1",
toolName: "web_search",
input: JSON.stringify({ query: "sigma.js latest version" }),
},
],
finishReason: { unified: "tool-calls" as const, raw: "tool-calls" },
usage: usage(),
warnings: [],
}
: {
content: [{ type: "text" as const, text: BRIEF }],
finishReason: { unified: "stop" as const, raw: "stop" },
usage: usage(),
warnings: [],
};
},
});
}
function stubFetch() {
const original = globalThis.fetch;
globalThis.fetch = (async (input: RequestInfo | URL) => {
const url = String(input);
if (url.startsWith("https://www.googleapis.com/")) {
return new Response(
JSON.stringify({
items: [{ title: "sigma.js", link: "https://www.sigmajs.org/", snippet: "v3 released" }],
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
);
}
return new Response("unexpected fetch: " + url, { status: 500 });
}) as typeof fetch;
return () => {
globalThis.fetch = original;
};
}
describe("research runner (async protocol §3)", () => {
it("fulfills the part with a ResearchResult and marks the job done with activity recorded", async () => {
const restoreFetch = stubFetch();
try {
const chat = createChat("Test: runner completion");
trackChat(chat.id);
// A pending research part, as the spawn hook would find it.
const messages = [
{ id: "u1", role: "user", parts: [{ type: "text", text: "which sigma version?" }] },
{
id: "a1",
role: "assistant",
parts: [
{ type: "text", text: "Checking." },
{ type: "tool-research", toolCallId: "call-run1", state: "input-available", input: RESEARCH_INPUT },
],
},
] as never;
// saveMessages requires the chat to exist; use the store directly.
const { saveMessages } = await import("../src/store/chats.ts");
saveMessages(chat.id, messages);
const job = createJob({ kind: "research", chatId: chat.id, toolCallId: "call-run1" });
trackJob(job.id);
await runResearchJob(job.id, { model: researcherModel() });
// The job is done, with the activity line from the tool call.
const done = getJob(job.id)!;
assert.equal(done.status, "done");
assert.equal(done.activity, 'web_search: "sigma.js latest version"', "activity line recorded");
assert.ok(done.result, "result stored on the job");
// The stored chat's part flipped to output-available with a
// ResearchResult-shaped output.
const saved = getChat(chat.id)!;
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-research");
assert.ok(part && part.type === "tool-research");
assert.equal(part.state, "output-available");
const output = part.output as { brief: string; sources: { title: string; url: string }[] };
assert.equal(output.brief, BRIEF);
assert.ok(Array.isArray(output.sources) && output.sources.length > 0, "sources parsed from the brief");
assert.equal(output.sources[0]!.url, "https://www.sigmajs.org/docs");
} finally {
restoreFetch();
}
});
it("a failing runner still fulfills the part (the teacher learns the outcome) and marks the job error", async () => {
const failing = new MockLanguageModelV4({
doGenerate: async () => {
throw new Error("model exploded");
},
});
const chat = createChat("Test: runner failure");
trackChat(chat.id);
const { saveMessages } = await import("../src/store/chats.ts");
saveMessages(chat.id, [
{ id: "u1", role: "user", parts: [{ type: "text", text: "hi" }] },
{
id: "a1",
role: "assistant",
parts: [{ type: "tool-research", toolCallId: "call-err1", state: "input-available", input: RESEARCH_INPUT }],
},
] as never);
const job = createJob({ kind: "research", chatId: chat.id, toolCallId: "call-err1" });
trackJob(job.id);
await runResearchJob(job.id, { model: failing });
const errored = getJob(job.id)!;
assert.equal(errored.status, "error");
assert.match(errored.error ?? "", /model exploded/);
const saved = getChat(chat.id)!;
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-research");
assert.ok(part && part.type === "tool-research");
assert.equal(part.state, "output-available", "failure still fulfills the part");
const output = part.output as { brief: string; sources: unknown[] };
assert.match(output.brief, /^Research failed: /);
assert.deepEqual(output.sources, []);
});
});
// ─── the researcher agent construction (model resolution seam) ──────────────
describe("researcher agent", () => {
it("builds with the web tool set (no network touched)", () => {
const agent = createResearcherAgent(researcherModel());
assert.ok(agent.tools.web_search, "web_search available");
assert.ok(agent.tools.web_fetch, "web_fetch available");
assert.equal(Object.keys(agent.tools).length, 2, "ONLY the web tools — no question/plan tools");
// reset any accidental global override
researchModelOverride.current = undefined;
});
});