W6: demo + visual artifacts, verifying builders
This commit is contained in:
Amos
@@ -1,10 +1,257 @@
|
||||
import type { ArtifactPartProps } from "@/src/core/contracts";
|
||||
"use client";
|
||||
|
||||
import { useContext, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { ChatActionsContext } from "@/components/chat";
|
||||
import type { ArtifactPartProps, DemoResult, JobRecord } from "@/src/core/contracts";
|
||||
|
||||
/**
|
||||
* Demo artifact card (W6): sandboxed iframe rendering DemoResult.html with
|
||||
* the auto-injected resize/error bridge (contracts.ts "Demo iframe bridge").
|
||||
* While input-available: job activity card (poll /api/jobs).
|
||||
* Demo artifact card (W6): while the part is input-available, a job activity
|
||||
* card (poll /api/jobs — the job-card idiom); when output-available, the demo
|
||||
* runs in a SANDBOXED iframe (sandbox="allow-scripts", no allow-same-origin →
|
||||
* no access to app storage/cookies/DOM).
|
||||
*
|
||||
* The bridge script (contracts.ts "Demo iframe bridge") is auto-injected into
|
||||
* every demo by buildDemoSrcDoc — never the model's job: ResizeObserver →
|
||||
* postMessage {type:"demo-resize",height} (parent sizes the iframe) and
|
||||
* window.onerror → {type:"demo-error",message} (error chip on the card, with
|
||||
* a "Send to teacher" button that feeds the error back into the lesson — the
|
||||
* fix loop).
|
||||
*
|
||||
* Bridge message handling is a pure function (reduceDemoBridgeState) so
|
||||
* tests cover it without a DOM.
|
||||
*/
|
||||
export function DemoCard(_props: ArtifactPartProps) {
|
||||
return null;
|
||||
|
||||
// ─── bridge injection + message handling (pure, testable) ───────────────────
|
||||
|
||||
export const DEMO_BRIDGE_SCRIPT = `<script>(function(){
|
||||
var send=function(m){try{parent.postMessage(m,"*")}catch(e){}};
|
||||
var post=function(){send({type:"demo-resize",height:document.documentElement.scrollHeight})};
|
||||
if(typeof ResizeObserver!=="undefined"){new ResizeObserver(post).observe(document.documentElement)}
|
||||
window.addEventListener("resize",post);
|
||||
window.addEventListener("load",post);
|
||||
window.addEventListener("error",function(e){send({type:"demo-error",message:e.message||String(e)})});
|
||||
setTimeout(post,50);
|
||||
setTimeout(post,400);
|
||||
})();</script>`;
|
||||
|
||||
/** Inject the bridge before </body> (or at the end of a body-less document). */
|
||||
export function buildDemoSrcDoc(html: string): string {
|
||||
if (/<\/body>/i.test(html)) return html.replace(/<\/body>/i, `${DEMO_BRIDGE_SCRIPT}</body>`);
|
||||
return `${html}${DEMO_BRIDGE_SCRIPT}`;
|
||||
}
|
||||
|
||||
export interface DemoBridgeState {
|
||||
height: number | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export const INITIAL_DEMO_BRIDGE_STATE: DemoBridgeState = { height: null, error: null };
|
||||
|
||||
const MIN_HEIGHT = 80;
|
||||
const MAX_HEIGHT = 2400;
|
||||
|
||||
/** Pure reducer over incoming postMessage data (contracts.ts DemoBridgeMessage). */
|
||||
export function reduceDemoBridgeState(state: DemoBridgeState, data: unknown): DemoBridgeState {
|
||||
if (typeof data !== "object" || data === null) return state;
|
||||
const message = data as { type?: unknown; height?: unknown; message?: unknown };
|
||||
if (message.type === "demo-resize") {
|
||||
if (typeof message.height !== "number" || !Number.isFinite(message.height) || message.height <= 0) return state;
|
||||
const height = Math.min(Math.max(Math.round(message.height), MIN_HEIGHT), MAX_HEIGHT);
|
||||
return height === state.height ? state : { ...state, height };
|
||||
}
|
||||
if (message.type === "demo-error") {
|
||||
const text = typeof message.message === "string" && message.message.trim() ? message.message : "unknown error";
|
||||
return text === state.error ? state : { ...state, error: text };
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
// ─── shared job activity card (input-available state) ───────────────────────
|
||||
|
||||
/** Live activity card for pending job parts — the job-card.tsx idiom,
|
||||
* parameterized by label. Exported for the visual card (also W6-owned). */
|
||||
export function JobActivityCard({
|
||||
toolCallId,
|
||||
label,
|
||||
testId,
|
||||
}: {
|
||||
toolCallId: string;
|
||||
label: string;
|
||||
testId?: string;
|
||||
}) {
|
||||
const actions = useContext(ChatActionsContext);
|
||||
const [job, setJob] = useState<JobRecord | null>(null);
|
||||
const finishedRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!actions) return;
|
||||
let cancelled = false;
|
||||
|
||||
const poll = async () => {
|
||||
if (finishedRef.current) return true;
|
||||
try {
|
||||
const res = await fetch(`/api/jobs?chatId=${encodeURIComponent(actions.chatId)}`);
|
||||
if (!res.ok) return false;
|
||||
const { jobs } = (await res.json()) as { jobs?: JobRecord[] };
|
||||
const match = jobs?.find((j) => j.toolCallId === toolCallId) ?? null;
|
||||
if (cancelled) return false;
|
||||
setJob(match);
|
||||
if (match && match.status !== "running") {
|
||||
// The part has (or is about to be) fulfilled server-side — pull the
|
||||
// fresh messages once; the continuation resumes the teacher.
|
||||
finishedRef.current = true;
|
||||
void actions.syncFromServer();
|
||||
return true;
|
||||
}
|
||||
} catch {
|
||||
/* transient fetch failure — keep polling */
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const timer = setInterval(() => {
|
||||
if (finishedRef.current) {
|
||||
clearInterval(timer);
|
||||
return;
|
||||
}
|
||||
void poll().then((stop) => {
|
||||
if (stop) clearInterval(timer);
|
||||
});
|
||||
}, 2000);
|
||||
void poll().then((stop) => {
|
||||
if (stop) clearInterval(timer);
|
||||
});
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(timer);
|
||||
};
|
||||
}, [actions, toolCallId]);
|
||||
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid={testId ?? "job-activity-card"}>
|
||||
<div className="flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||
<span className="inline-block size-2 animate-pulse rounded-full bg-primary" />
|
||||
{label}
|
||||
</div>
|
||||
{job?.activity && (
|
||||
<div className="mt-1 truncate text-sm text-muted-foreground" data-testid="job-activity-line">
|
||||
{job.activity}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ─── the demo card ──────────────────────────────────────────────────────────
|
||||
|
||||
export function DemoCard({ part }: ArtifactPartProps) {
|
||||
if (part.state === "output-available") {
|
||||
return <DemoOutputCard output={part.output} />;
|
||||
}
|
||||
return <JobActivityCard toolCallId={part.toolCallId} label="Building demo…" testId="demo-activity-card" />;
|
||||
}
|
||||
|
||||
function DemoOutputCard({ output }: { output: unknown }) {
|
||||
const result = output as DemoResult | undefined;
|
||||
if (!result || typeof result.html !== "string" || typeof result.title !== "string") {
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-4 text-sm text-muted-foreground">
|
||||
Demo completed, but the result could not be displayed.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
const failed = result.html.length === 0;
|
||||
if (failed) {
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="demo-card-failed">
|
||||
<div className="flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||
Interactive demo
|
||||
<span className="rounded bg-red-500/15 px-1.5 py-0.5 text-red-500">failed</span>
|
||||
</div>
|
||||
<p className="mt-2 text-sm text-muted-foreground">{result.notes ?? "The demo could not be built."}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return <DemoFrame title={result.title} html={result.html} notes={result.notes} />;
|
||||
}
|
||||
|
||||
function DemoFrame({ title, html, notes }: { title: string; html: string; notes?: string }) {
|
||||
const actions = useContext(ChatActionsContext);
|
||||
const iframeRef = useRef<HTMLIFrameElement | null>(null);
|
||||
const [state, setState] = useState<DemoBridgeState>(INITIAL_DEMO_BRIDGE_STATE);
|
||||
const srcDoc = useMemo(() => buildDemoSrcDoc(html), [html]);
|
||||
|
||||
useEffect(() => {
|
||||
const onMessage = (event: MessageEvent) => {
|
||||
// Only messages from THIS demo's iframe (the sandbox gives it an
|
||||
// opaque origin; the window reference still matches).
|
||||
if (!iframeRef.current || event.source !== iframeRef.current.contentWindow) return;
|
||||
setState((prev) => reduceDemoBridgeState(prev, event.data));
|
||||
};
|
||||
window.addEventListener("message", onMessage);
|
||||
return () => window.removeEventListener("message", onMessage);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="demo-card">
|
||||
<div className="flex items-baseline justify-between gap-2">
|
||||
<div className="font-medium">{title}</div>
|
||||
<div className="shrink-0 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||
interactive demo
|
||||
</div>
|
||||
</div>
|
||||
{state.error && (
|
||||
<DemoErrorChip
|
||||
title={title}
|
||||
message={state.error}
|
||||
onSend={actions ? () => actions.sendText(`The demo "${title}" threw an error: ${state.error}`) : undefined}
|
||||
/>
|
||||
)}
|
||||
<iframe
|
||||
ref={iframeRef}
|
||||
title={title}
|
||||
// allow-scripts ONLY — no allow-same-origin: the demo can run code
|
||||
// but never touch the app's storage, cookies, or DOM.
|
||||
sandbox="allow-scripts"
|
||||
srcDoc={srcDoc}
|
||||
className="mt-2 w-full rounded-lg border bg-white"
|
||||
style={{ height: state.height ?? 360 }}
|
||||
data-testid="demo-iframe"
|
||||
/>
|
||||
{notes && <p className="mt-2 text-sm text-muted-foreground">{notes}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function DemoErrorChip({
|
||||
title,
|
||||
message,
|
||||
onSend,
|
||||
}: {
|
||||
title: string;
|
||||
message: string;
|
||||
onSend?: () => void;
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
className="mt-2 flex flex-wrap items-center gap-2 rounded-lg bg-red-500/10 px-3 py-2 text-sm text-red-500"
|
||||
data-testid="demo-error-chip"
|
||||
>
|
||||
<span className="font-medium">This demo threw an error</span>
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-xs" title={message}>
|
||||
{message}
|
||||
</span>
|
||||
{onSend && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onSend}
|
||||
className="rounded-md border border-red-500/40 px-2 py-1 text-xs font-medium hover:bg-red-500/15"
|
||||
data-testid="demo-error-send"
|
||||
>
|
||||
Send to teacher
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,64 @@
|
||||
import type { ArtifactPartProps } from "@/src/core/contracts";
|
||||
"use client";
|
||||
|
||||
import type { ArtifactPartProps, VisualResult } from "@/src/core/contracts";
|
||||
import { JobActivityCard } from "@/components/artifacts/demo-card";
|
||||
|
||||
/**
|
||||
* Visual artifact card (W6): renders VisualResult.svg inline.
|
||||
* While input-available: job activity card (poll /api/jobs).
|
||||
* Visual artifact card (W6): while the part is input-available, a job activity
|
||||
* card; when output-available, VisualResult.svg renders INLINE. The SVG is
|
||||
* model-authored for the learner's own machine (their own teacher wrote it,
|
||||
* verified by looking) — dangerouslySetInnerHTML is acceptable here, with a
|
||||
* cheap hygiene pass (sanitizeSvg: strip <script> blocks, on* handlers, and
|
||||
* javascript:/data:text/html URLs) since it lands in the app's DOM.
|
||||
*/
|
||||
export function VisualCard(_props: ArtifactPartProps) {
|
||||
return null;
|
||||
|
||||
/** Cheap SVG hygiene — not a full sanitizer, just removing the obvious exec vectors. */
|
||||
export function sanitizeSvg(svg: string): string {
|
||||
return svg
|
||||
.replace(/<script[\s\S]*?<\/script\s*>/gi, "")
|
||||
.replace(/<script[^>]*\/?>/gi, "")
|
||||
.replace(/\son[a-z]+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, "")
|
||||
.replace(/(href|xlink:href)\s*=\s*(?:"\s*(?:javascript|data:text\/html)[^"]*"|'\s*(?:javascript|data:text\/html)[^']*'|(?:javascript|data:text\/html)[^\s>]*)/gi, "");
|
||||
}
|
||||
|
||||
export function VisualCard({ part }: ArtifactPartProps) {
|
||||
if (part.state === "output-available") {
|
||||
return <VisualOutputCard output={part.output} />;
|
||||
}
|
||||
return <JobActivityCard toolCallId={part.toolCallId} label="Drawing figure…" testId="visual-activity-card" />;
|
||||
}
|
||||
|
||||
function VisualOutputCard({ output }: { output: unknown }) {
|
||||
const result = output as VisualResult | undefined;
|
||||
if (!result || typeof result.svg !== "string") {
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-4 text-sm text-muted-foreground">
|
||||
Figure completed, but the result could not be displayed.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
const failed = result.svg.length === 0;
|
||||
if (failed) {
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="visual-card-failed">
|
||||
<div className="flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||
Figure
|
||||
<span className="rounded bg-red-500/15 px-1.5 py-0.5 text-red-500">failed</span>
|
||||
</div>
|
||||
<p className="mt-2 text-sm text-muted-foreground">{result.notes ?? "The figure could not be drawn."}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="visual-card">
|
||||
<div className="text-xs font-medium uppercase tracking-wide text-muted-foreground">Figure</div>
|
||||
{/* max-w-full + h-auto keeps any oversized model-authored svg inside the column */}
|
||||
<div
|
||||
className="mt-2 max-w-full overflow-hidden rounded-lg bg-white [&_svg]:h-auto [&_svg]:max-w-full"
|
||||
dangerouslySetInnerHTML={{ __html: sanitizeSvg(result.svg) }}
|
||||
data-testid="visual-svg"
|
||||
/>
|
||||
{result.notes && <p className="mt-2 text-sm text-muted-foreground">{result.notes}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
## Interactive demos and precise figures — build the instrument, don't describe it
|
||||
|
||||
If the concept is DYNAMIC — a parameter to sweep, a process to animate, a distribution to sample, a
|
||||
system to simulate — build the learner the instrument rather than describing its behavior in prose.
|
||||
A paragraph about how frequency response changes with damping teaches less than ten seconds of
|
||||
dragging a slider. You have two artifact tools for this, both asynchronous (same discipline as
|
||||
research):
|
||||
|
||||
### `demo` — interactive teaching instruments
|
||||
|
||||
A background sub-agent builds a self-contained interactive demo (single-file HTML: parameter sweeps,
|
||||
simulations, animations) and **headlessly tests that it actually runs** before it reaches the lesson.
|
||||
Nothing untested ever appears.
|
||||
|
||||
- Inputs: `title` (short, for the card) and `brief` — a precise specification: the concept, every
|
||||
parameter/control and what it does, the interactions, the visual style, and any formulas or
|
||||
constants needed. **The builder never sees the lesson** — the brief is its entire world. Include
|
||||
everything; a vague brief yields a vague demo.
|
||||
- Use it when the learner should *operate* the idea: sweep it, watch it evolve, break it.
|
||||
|
||||
### `visual` — precise static figures
|
||||
|
||||
A background sub-agent hand-authors one SVG, renders it, **looks at the render**, and iterates until
|
||||
the picture is correct. Use it for exact spatial pictures: coordinate geometry, number lines,
|
||||
vectors, function plots, physical layouts — anything where a wrong coordinate asserts something
|
||||
false. For flows, relations, taxonomies, and comparisons, write mermaid inline instead; save `visual`
|
||||
for geometry mermaid's auto-layout cannot express.
|
||||
|
||||
- Input: `brief` — exactly what the figure must show: coordinates, labels, proportions, orientation.
|
||||
One idea per figure. The maker sees nothing else of the lesson.
|
||||
|
||||
### The async discipline
|
||||
|
||||
The call is asynchronous and **your turn ends at it**, exactly like research:
|
||||
|
||||
1. **Say your piece first.** Set up the concept, explain what the learner is about to see and what to
|
||||
watch for — the artifact arrives later as the tool's output and the lesson resumes from there. A
|
||||
bare tool call with no surrounding text is a failed turn.
|
||||
2. **At most one demo or visual per turn.**
|
||||
3. **When the artifact arrives, TEACH WITH IT.** Reference its controls and elements by name: "drag
|
||||
the damping slider all the way left and watch the overshoot", "notice the vector at t=2s". An
|
||||
artifact you never point at is decoration.
|
||||
4. **Failures arrive as notes** ("FAILED: …") with no artifact — say so plainly, fall back to
|
||||
explaining in prose, and if the brief was the problem, re-call with a tighter one. Re-calling a
|
||||
tool produces a new version of the artifact; use that to fix a demo that misbehaves in the field.
|
||||
@@ -0,0 +1,238 @@
|
||||
import { ToolLoopAgent, stepCountIs, tool, type LanguageModel, type ToolSet } from "ai";
|
||||
import { openrouter } from "@openrouter/ai-sdk-provider";
|
||||
import { z } from "zod";
|
||||
import type { DemoResult } from "../../core/contracts.ts";
|
||||
import { getHeadless, type DemoRunResult, type HeadlessApi } from "./headless.ts";
|
||||
|
||||
/**
|
||||
* The demo-builder sub-agent (vision §2 interactive demos + §2.5 "sub-agents
|
||||
* verify; they never one-shot"): given a brief it writes a self-contained
|
||||
* single-file HTML/JS demo, TESTS IT HEADLESSLY (run_demo), and iterates
|
||||
* fix → re-run until clean — before anything reaches the lesson.
|
||||
*
|
||||
* Shape: a ToolLoopAgent with two executable tools (write_demo / run_demo) —
|
||||
* the model is injectable (tests use MockLanguageModelV4) and the headless
|
||||
* API is injectable (tests use a fake; production uses system Chrome).
|
||||
*
|
||||
* Failure semantics (decided in the W6 brief): `tested: true` means "the
|
||||
* verify loop ran"; on failure the result is { title, html: "", tested: true,
|
||||
* notes: "FAILED: <reason>" } — the card renders the failure note, no iframe.
|
||||
* The job wrapper treats html === "" as failure and marks the job error.
|
||||
*/
|
||||
|
||||
const DEFAULT_DEMO_MODEL =
|
||||
process.env.LH_DEMO_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
|
||||
|
||||
export interface DemoBuilderInput {
|
||||
title: string;
|
||||
brief: string;
|
||||
}
|
||||
|
||||
export interface DemoBuilderOptions {
|
||||
model?: LanguageModel;
|
||||
headless?: HeadlessApi;
|
||||
/** Max agent steps (each write/run/fix is one step). Default 24 ≈ 6 cycles. */
|
||||
maxSteps?: number;
|
||||
/** Live activity-line callback (updates the job record). */
|
||||
onActivity?: (line: string) => void;
|
||||
}
|
||||
|
||||
const INSTRUCTIONS = `You are a demo builder — a specialist sub-agent inside a learning application. You
|
||||
receive a brief for ONE interactive teaching instrument and deliver a single self-contained HTML document
|
||||
that runs it. You never see the lesson; the brief is everything you know.
|
||||
|
||||
Runtime contract (the document runs in a sandboxed iframe, sandbox="allow-scripts", opaque origin):
|
||||
- Single file. Vanilla JS + Canvas 2D or SVG. CDN imports (e.g. a plotting or math library) are allowed
|
||||
when they genuinely help — but prefer dependency-free; you own the failure if a CDN import fails.
|
||||
- NO localStorage, sessionStorage, or document.cookie — the sandbox makes them throw.
|
||||
- The parent sizes the iframe to the document height automatically; just build normal flowing layout.
|
||||
Make the demo fill the width and size its own height sensibly.
|
||||
|
||||
Build quality:
|
||||
- The demo must directly serve the brief: every control named in the brief must exist and do the thing.
|
||||
- Controls need labels; sliders/inputs need current-value readouts. A learner should operate it unguided.
|
||||
- Smooth enough interaction; requestAnimationFrame for animation, not setInterval when it matters.
|
||||
- Dark-mode neutral: pick a clean light background for the demo area so it reads on any card.
|
||||
|
||||
THE HOUSE RULE — never deliver untested:
|
||||
1. Call write_demo with the complete HTML document.
|
||||
2. Call run_demo. It loads the file in headless Chrome, waits ~2s, and reports console errors, uncaught
|
||||
errors (window.onerror), and failed requests. Optionally pass a probe: a JS expression evaluated in
|
||||
the page (e.g. "typeof window.simulate === 'function'" or a state assertion) to check deeper.
|
||||
3. If run_demo reports problems, FIX the document and call write_demo again, then run_demo again.
|
||||
Iterate until run_demo reports PASSED.
|
||||
4. Only then finish. Your final message is 1-3 sentences of NOTES for the teacher (what the demo shows,
|
||||
its controls, what to try) — NEVER the HTML itself.
|
||||
|
||||
Never claim success you did not see: a PASSED run_demo is the only definition of done.`;
|
||||
|
||||
interface BuilderState {
|
||||
html: string | null;
|
||||
/** The html that the last run_demo tested (null = never tested). */
|
||||
testedHtml: string | null;
|
||||
lastResult: DemoRunResult | null;
|
||||
}
|
||||
|
||||
export function createDemoBuilderTools(
|
||||
state: BuilderState,
|
||||
headless: HeadlessApi,
|
||||
onActivity?: (line: string) => void,
|
||||
): ToolSet {
|
||||
return {
|
||||
write_demo: tool({
|
||||
description:
|
||||
"Write the complete self-contained HTML document for the demo (overwrites any previous version). " +
|
||||
"Always follow with run_demo to test it headlessly.",
|
||||
inputSchema: z.object({
|
||||
html: z.string().describe("The complete HTML document: <!doctype html>…</html>, all JS inline."),
|
||||
}),
|
||||
execute: async ({ html }) => {
|
||||
if (!html || html.trim().length === 0) return "ERROR: empty document — write the full HTML.";
|
||||
state.html = html;
|
||||
onActivity?.(`write_demo: ${(html.length / 1024).toFixed(1)} KB`);
|
||||
return `Saved (${html.length} bytes). Now call run_demo to test it — never deliver untested.`;
|
||||
},
|
||||
}),
|
||||
run_demo: tool({
|
||||
description:
|
||||
"Headlessly execute the current demo in Chrome and report console errors, uncaught errors, and failed requests. " +
|
||||
"Optionally pass a probe expression to assert on page state.",
|
||||
inputSchema: z.object({
|
||||
probe: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Optional JS expression evaluated in the page after settling, e.g. \"document.querySelectorAll('input').length\"."),
|
||||
}),
|
||||
execute: async ({ probe }) => {
|
||||
if (!state.html) return "ERROR: nothing to run — call write_demo first.";
|
||||
const reason = headless.unavailableReason();
|
||||
if (reason) return `HEADLESS UNAVAILABLE: ${reason}`;
|
||||
let result: DemoRunResult;
|
||||
try {
|
||||
result = await headless.runDemo(state.html, probe ? { probe } : undefined);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
onActivity?.("run_demo: browser error");
|
||||
return `HEADLESS ERROR (retrying may help): ${msg}`;
|
||||
}
|
||||
state.testedHtml = state.html;
|
||||
state.lastResult = result;
|
||||
onActivity?.(
|
||||
result.ok ? "run_demo: clean" : `run_demo: ${summarizeCounts(result)}`,
|
||||
);
|
||||
return formatRunResult(result);
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeCounts(result: DemoRunResult): string {
|
||||
const counts = [
|
||||
result.consoleErrors.length > 0 ? `${result.consoleErrors.length} console error${result.consoleErrors.length === 1 ? "" : "s"}` : null,
|
||||
result.pageErrors.length > 0 ? `${result.pageErrors.length} page error${result.pageErrors.length === 1 ? "" : "s"}` : null,
|
||||
result.failedRequests.length > 0 ? `${result.failedRequests.length} failed request${result.failedRequests.length === 1 ? "" : "s"}` : null,
|
||||
].filter(Boolean);
|
||||
return counts.length > 0 ? counts.join(", ") : "ok";
|
||||
}
|
||||
|
||||
function formatRunResult(result: DemoRunResult): string {
|
||||
if (result.ok) {
|
||||
const probe =
|
||||
result.probeResult !== undefined
|
||||
? ` Probe result: ${JSON.stringify(result.probeResult) ?? "undefined"}.`
|
||||
: "";
|
||||
return `PASSED: no console errors, no uncaught errors, no failed requests.${probe} If you are satisfied, finish with your notes; otherwise keep improving and re-test.`;
|
||||
}
|
||||
const sections: string[] = [`FAILED: ${summarizeCounts(result)}.`];
|
||||
if (result.consoleErrors.length > 0)
|
||||
sections.push(`Console errors:\n${result.consoleErrors.slice(0, 10).map((e) => `- ${e}`).join("\n")}`);
|
||||
if (result.pageErrors.length > 0)
|
||||
sections.push(`Uncaught errors:\n${result.pageErrors.slice(0, 10).map((e) => `- ${e}`).join("\n")}`);
|
||||
if (result.failedRequests.length > 0)
|
||||
sections.push(`Failed requests:\n${result.failedRequests.slice(0, 10).map((r) => `- ${r}`).join("\n")}`);
|
||||
sections.push("Fix the document (write_demo) and re-test (run_demo).");
|
||||
return sections.join("\n");
|
||||
}
|
||||
|
||||
/** Short summary of a run for failure notes. */
|
||||
function failureSummary(result: DemoRunResult | null): string {
|
||||
if (!result) return "never tested";
|
||||
const details = [...result.consoleErrors, ...result.pageErrors, ...result.failedRequests].slice(0, 5);
|
||||
return details.length > 0 ? `${summarizeCounts(result)} — ${details.join(" | ")}` : summarizeCounts(result);
|
||||
}
|
||||
|
||||
export async function runDemoBuilder(input: DemoBuilderInput, opts: DemoBuilderOptions = {}): Promise<DemoResult> {
|
||||
const headless = opts.headless ?? getHeadless();
|
||||
const title = input.title.trim() || "Interactive demo";
|
||||
const brief = input.brief.trim() || "(no brief)";
|
||||
const notes = (text: string) => (text.trim() ? text.trim() : undefined);
|
||||
|
||||
const reason = headless.unavailableReason();
|
||||
if (reason) {
|
||||
return { title, html: "", tested: true, notes: `FAILED: ${reason}` };
|
||||
}
|
||||
|
||||
const state: BuilderState = { html: null, testedHtml: null, lastResult: null };
|
||||
const agent = new ToolLoopAgent({
|
||||
model: opts.model ?? openrouter(DEFAULT_DEMO_MODEL),
|
||||
instructions: INSTRUCTIONS,
|
||||
tools: createDemoBuilderTools(state, headless, opts.onActivity),
|
||||
stopWhen: stepCountIs(opts.maxSteps ?? 24),
|
||||
});
|
||||
|
||||
const prompt = [
|
||||
`Demo title: ${title}`,
|
||||
`Brief (all you know about the lesson — this comes from the teacher, verbatim):`,
|
||||
brief,
|
||||
"",
|
||||
"Build the demo, test it with run_demo, iterate until PASSED, then finish with your notes.",
|
||||
].join("\n");
|
||||
|
||||
let finalText = "";
|
||||
try {
|
||||
const result = await agent.generate({
|
||||
prompt,
|
||||
onToolExecutionStart: ({ toolCall }) => {
|
||||
opts.onActivity?.(toolCall.toolName);
|
||||
},
|
||||
});
|
||||
finalText = result.text;
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return { title, html: "", tested: true, notes: `FAILED: the builder crashed — ${msg}` };
|
||||
}
|
||||
|
||||
// The model produced nothing.
|
||||
if (!state.html) {
|
||||
return {
|
||||
title,
|
||||
html: "",
|
||||
tested: true,
|
||||
notes: `FAILED: the builder never wrote a demo.${finalText ? ` (${finalText.slice(0, 300)})` : ""}`,
|
||||
};
|
||||
}
|
||||
|
||||
// The verify gate: the delivered html must have a clean headless run —
|
||||
// even when the model claims success or forgets the final run_demo.
|
||||
if (state.testedHtml !== state.html || !state.lastResult?.ok) {
|
||||
opts.onActivity?.("final verification");
|
||||
let verify: DemoRunResult;
|
||||
try {
|
||||
verify = await headless.runDemo(state.html);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return { title, html: "", tested: true, notes: `FAILED: final verification crashed — ${msg}` };
|
||||
}
|
||||
state.lastResult = verify;
|
||||
if (!verify.ok) {
|
||||
return {
|
||||
title,
|
||||
html: "",
|
||||
tested: true,
|
||||
notes: `FAILED: the demo never passed headless verification (${failureSummary(verify)}).`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { title, html: state.html, tested: true, notes: notes(finalText) };
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
/**
|
||||
* Shared headless-browser helper for the verifying sub-agents (W6):
|
||||
* the demo-builder executes demos and captures errors; the visual-maker
|
||||
* rasterizes SVGs so it can look at them. Both use the SAME system Chrome
|
||||
* via puppeteer-core — NO bundled chromium is downloaded.
|
||||
*
|
||||
* Chrome resolution: process.env.CHROME_PATH first, then the common
|
||||
* macOS/Linux install paths. If none exists we fall back to puppeteer's
|
||||
* `channel: "chrome"` at launch time; `available()` is cheap and only knows
|
||||
* about the explicit candidates, so a channel-only setup reports unavailable
|
||||
* and the builders refuse to run (the house rule: never skip testing).
|
||||
*
|
||||
* Test seam: `headlessOverride.current` replaces the whole API (tests inject
|
||||
* a fake; nothing here touches a real browser). Builders call `getHeadless()`.
|
||||
*/
|
||||
|
||||
export interface DemoRunResult {
|
||||
/** console.error / console messages of type "error". */
|
||||
consoleErrors: string[];
|
||||
/** Uncaught exceptions (window.onerror / unhandled rejections). */
|
||||
pageErrors: string[];
|
||||
/** Network requests that failed (CDN scripts that didn't load, …). */
|
||||
failedRequests: string[];
|
||||
/** Value of the optional probe expression, when requested and successful. */
|
||||
probeResult?: unknown;
|
||||
/** True when nothing above recorded anything. */
|
||||
ok: boolean;
|
||||
}
|
||||
|
||||
export interface RunDemoOptions {
|
||||
/** How long to let the page settle (animations, async CDN loads) — default 2s. */
|
||||
settleMs?: number;
|
||||
/** Optional JS expression evaluated in the page after settling; its value
|
||||
* is returned to the builder so it can assert on internal state. */
|
||||
probe?: string;
|
||||
}
|
||||
|
||||
export interface RenderSvgOptions {
|
||||
/** Viewport size — defaults to the SVG's width/height attrs, else 900×700. */
|
||||
width?: number;
|
||||
height?: number;
|
||||
}
|
||||
|
||||
export interface HeadlessApi {
|
||||
/** True when a Chrome executable is resolvable (cheap, no launch). */
|
||||
available(): boolean;
|
||||
/** Why not — for error messages. */
|
||||
unavailableReason(): string | null;
|
||||
/** Load the HTML in headless Chrome, capture errors, optionally probe. */
|
||||
runDemo(html: string, opts?: RunDemoOptions): Promise<DemoRunResult>;
|
||||
/** Rasterize an SVG to a PNG buffer (for verify-by-looking). */
|
||||
renderSvg(svg: string, opts?: RenderSvgOptions): Promise<Buffer>;
|
||||
}
|
||||
|
||||
/** Test seam — undefined in production. */
|
||||
export const headlessOverride: { current?: HeadlessApi } = {};
|
||||
|
||||
export function getHeadless(): HeadlessApi {
|
||||
return headlessOverride.current ?? realHeadless;
|
||||
}
|
||||
|
||||
// ─── Chrome resolution ───────────────────────────────────────────────────────
|
||||
|
||||
const CHROME_CANDIDATES = [
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
||||
"/usr/bin/google-chrome",
|
||||
"/usr/bin/google-chrome-stable",
|
||||
"/usr/bin/chromium",
|
||||
"/usr/bin/chromium-browser",
|
||||
];
|
||||
|
||||
export function resolveChromeExecutablePath(): string | null {
|
||||
const candidates = [process.env.CHROME_PATH, ...CHROME_CANDIDATES].filter(
|
||||
(p): p is string => typeof p === "string" && p.length > 0,
|
||||
);
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
if (fs.existsSync(candidate)) return candidate;
|
||||
} catch {
|
||||
/* not readable — try the next */
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const NO_CHROME_MESSAGE =
|
||||
"No Chrome/Chromium found for headless testing. Install Google Chrome or set CHROME_PATH to a Chrome executable. " +
|
||||
"(Verification is mandatory — demos and visuals are never delivered untested.)";
|
||||
|
||||
async function launchBrowser(): Promise<import("puppeteer-core").Browser> {
|
||||
const puppeteer = (await import("puppeteer-core")).default;
|
||||
const executablePath = resolveChromeExecutablePath();
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
// Explicit path when we found one; otherwise let puppeteer resolve the
|
||||
// system Chrome install itself via the "chrome" channel.
|
||||
...(executablePath ? { executablePath } : { channel: "chrome" as const }),
|
||||
args: ["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"],
|
||||
});
|
||||
return browser;
|
||||
}
|
||||
|
||||
/** A temp dir per invocation; removed in finally. */
|
||||
function tempDir(prefix: string): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
// ─── Demo execution ──────────────────────────────────────────────────────────
|
||||
|
||||
async function runDemo(html: string, opts: RunDemoOptions = {}): Promise<DemoRunResult> {
|
||||
const settleMs = opts.settleMs ?? 2000;
|
||||
const dir = tempDir("lh-demo-");
|
||||
const file = path.join(dir, "demo.html");
|
||||
fs.writeFileSync(file, html, "utf-8");
|
||||
|
||||
const consoleErrors: string[] = [];
|
||||
const pageErrors: string[] = [];
|
||||
const failedRequests: string[] = [];
|
||||
let probeResult: unknown;
|
||||
let probed = false;
|
||||
|
||||
const browser = await launchBrowser();
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
page.on("console", (msg) => {
|
||||
if (msg.type() === "error") consoleErrors.push(msg.text());
|
||||
});
|
||||
page.on("pageerror", (err) => pageErrors.push(err instanceof Error ? err.message : String(err)));
|
||||
page.on("requestfailed", (req) => failedRequests.push(`${req.url()} (${req.failure()?.errorText ?? "failed"})`));
|
||||
|
||||
try {
|
||||
await page.goto(pathToFileURL(file).href, { waitUntil: "load", timeout: 30_000 });
|
||||
await new Promise((resolve) => setTimeout(resolve, settleMs));
|
||||
if (opts.probe) {
|
||||
try {
|
||||
probeResult = await page.evaluate(opts.probe);
|
||||
} catch (err) {
|
||||
pageErrors.push(`probe failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
probed = true;
|
||||
}
|
||||
} catch (err) {
|
||||
pageErrors.push(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
} finally {
|
||||
await browser.close().catch(() => {});
|
||||
try {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort cleanup */
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
consoleErrors,
|
||||
pageErrors,
|
||||
failedRequests,
|
||||
...(probed ? { probeResult } : {}),
|
||||
ok: consoleErrors.length === 0 && pageErrors.length === 0 && failedRequests.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── SVG rasterization ───────────────────────────────────────────────────────
|
||||
|
||||
function svgViewport(svg: string, opts: RenderSvgOptions): { width: number; height: number } {
|
||||
if (opts.width && opts.height) return { width: opts.width, height: opts.height };
|
||||
const width = Number(svg.match(/\bwidth\s*=\s*"?(\d+(?:\.\d+)?)/)?.[1]);
|
||||
const height = Number(svg.match(/\bheight\s*=\s*"?(\d+(?:\.\d+)?)/)?.[1]);
|
||||
return {
|
||||
width: Math.ceil(Number.isFinite(width) && width > 0 ? width : 900),
|
||||
height: Math.ceil(Number.isFinite(height) && height > 0 ? height : 700),
|
||||
};
|
||||
}
|
||||
|
||||
async function renderSvg(svg: string, opts: RenderSvgOptions = {}): Promise<Buffer> {
|
||||
const dir = tempDir("lh-visual-");
|
||||
const file = path.join(dir, "figure.html");
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
`<!doctype html><html><head><meta charset="utf-8"><style>html,body{margin:0;padding:0;background:#fff}svg{display:block}</style></head><body>${svg}</body></html>`,
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const browser = await launchBrowser();
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
const { width, height } = svgViewport(svg, opts);
|
||||
await page.setViewport({ width, height });
|
||||
await page.goto(pathToFileURL(file).href, { waitUntil: "load", timeout: 30_000 });
|
||||
const png = (await page.screenshot({ fullPage: true, type: "png" })) as Buffer;
|
||||
return png;
|
||||
} finally {
|
||||
await browser.close().catch(() => {});
|
||||
try {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort cleanup */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── the real API ────────────────────────────────────────────────────────────
|
||||
|
||||
export const realHeadless: HeadlessApi = {
|
||||
available() {
|
||||
return resolveChromeExecutablePath() !== null;
|
||||
},
|
||||
unavailableReason() {
|
||||
return resolveChromeExecutablePath() === null ? NO_CHROME_MESSAGE : null;
|
||||
},
|
||||
runDemo,
|
||||
renderSvg,
|
||||
};
|
||||
+122
-16
@@ -1,9 +1,11 @@
|
||||
import { isToolUIPart, type LanguageModel, type UIMessage } from "ai";
|
||||
import type { ResearchResult, ResearchToolInput } from "../../core/contracts.ts";
|
||||
import type { DemoResult, DemoToolInput, JobKind, ResearchResult, ResearchToolInput, VisualResult, VisualToolInput } from "../../core/contracts.ts";
|
||||
import { JOB_TOOLS } from "../../core/pending.ts";
|
||||
import { getChat, saveMessages } from "../../store/chats.ts";
|
||||
import { createJob, findJobByToolCallId, getJob, updateJob } from "../../store/jobs.ts";
|
||||
import { runDemoBuilder } from "./demo-builder.ts";
|
||||
import { createResearcherAgent } from "./researcher.ts";
|
||||
import { runVisualMaker } from "./visual-maker.ts";
|
||||
|
||||
/**
|
||||
* The background job runner + the chat-handler spawn hook (contracts.ts
|
||||
@@ -11,18 +13,22 @@ import { createResearcherAgent } from "./researcher.ts";
|
||||
*
|
||||
* spawnPendingJobs — called from onFinish AFTER saveMessages: finds
|
||||
* input-available JOB_TOOLS parts in the last assistant message with no
|
||||
* existing job and fires runResearchJob WITHOUT awaiting (the HTTP
|
||||
* response is long gone).
|
||||
* existing job and fires the kind's runner WITHOUT awaiting (the HTTP
|
||||
* response is long gone). Kind dispatch (research/demo/visual) is a
|
||||
* self-contained map here, so chat-handler needs no changes for new
|
||||
* job kinds — anything listed in JOB_TOOLS routes automatically.
|
||||
*
|
||||
* runResearchJob — runs the researcher agent loop in the background,
|
||||
* updating the job's activity line on every tool call, then rewrites the
|
||||
* tool part in the LATEST chat record to output-available (read-modify-
|
||||
* write; no awaits between the read and the write). The part is fulfilled
|
||||
* BEFORE the job status flips, so a client that polls the job and then
|
||||
* re-syncs always sees the finished part.
|
||||
* runResearchJob / runDemoJob / runVisualJob — run the sub-agent loop in
|
||||
* the background, updating the job's activity line on every iteration,
|
||||
* then rewrite the tool part in the LATEST chat record to
|
||||
* output-available (read-modify-write; no awaits between the read and
|
||||
* the write). The part is fulfilled BEFORE the job status flips, so a
|
||||
* client that polls the job and then re-syncs always sees the finished
|
||||
* part.
|
||||
*
|
||||
* On failure the part is still fulfilled ({brief: "Research failed: …"}) —
|
||||
* the teacher must learn the outcome — and the job is marked error.
|
||||
* On failure the part is still fulfilled (DemoResult/VisualResult failure
|
||||
* shape: empty html/svg + "FAILED: …" notes; research: failure brief) — the
|
||||
* teacher must learn the outcome — and the job is marked error.
|
||||
*/
|
||||
|
||||
/**
|
||||
@@ -31,11 +37,20 @@ import { createResearcherAgent } from "./researcher.ts";
|
||||
* network. Undefined in production.
|
||||
*/
|
||||
export const researchModelOverride: { current?: LanguageModel } = {};
|
||||
export const demoModelOverride: { current?: LanguageModel } = {};
|
||||
export const visualModelOverride: { current?: LanguageModel } = {};
|
||||
|
||||
export interface RunResearchJobOptions {
|
||||
model?: LanguageModel;
|
||||
}
|
||||
|
||||
/** The kind→runner map: everything in JOB_TOOLS routes through here. */
|
||||
const KIND_RUNNERS: Record<JobKind, (jobId: string) => Promise<void>> = {
|
||||
research: (jobId) => runResearchJob(jobId, { model: researchModelOverride.current }),
|
||||
demo: (jobId) => runDemoJob(jobId, { model: demoModelOverride.current }),
|
||||
visual: (jobId) => runVisualJob(jobId, { model: visualModelOverride.current }),
|
||||
};
|
||||
|
||||
export function spawnPendingJobs(chatId: string, messages: UIMessage[]): void {
|
||||
const last = [...messages].reverse().find((m) => m.role === "assistant");
|
||||
if (!last) return;
|
||||
@@ -43,11 +58,12 @@ export function spawnPendingJobs(chatId: string, messages: UIMessage[]): void {
|
||||
if (!isToolUIPart(part) || part.state !== "input-available") continue;
|
||||
const tool = part.type.slice("tool-".length);
|
||||
if (!(JOB_TOOLS as readonly string[]).includes(tool)) continue;
|
||||
if (tool !== "research") continue; // demo/visual runners arrive in W6
|
||||
const runner = KIND_RUNNERS[tool as JobKind];
|
||||
if (!runner) continue;
|
||||
if (findJobByToolCallId(chatId, part.toolCallId)) continue; // already has a job
|
||||
const job = createJob({ kind: "research", chatId, toolCallId: part.toolCallId });
|
||||
void runResearchJob(job.id, { model: researchModelOverride.current }).catch((err) => {
|
||||
console.error(`[job ${job.id}] research runner crashed:`, err);
|
||||
const job = createJob({ kind: tool as JobKind, chatId, toolCallId: part.toolCallId });
|
||||
void runner(job.id).catch((err) => {
|
||||
console.error(`[job ${job.id}] ${tool} runner crashed:`, err);
|
||||
updateJob(job.id, { status: "error", error: String(err), activity: "failed" });
|
||||
});
|
||||
}
|
||||
@@ -103,6 +119,96 @@ export async function runResearchJob(jobId: string, opts: RunResearchJobOptions
|
||||
}
|
||||
}
|
||||
|
||||
// ─── demo + visual job wrappers ───────────────────────────────────────────
|
||||
|
||||
export interface RunDemoJobOptions {
|
||||
model?: LanguageModel;
|
||||
}
|
||||
|
||||
/** Failure shape (W6 brief): html === "" with "FAILED: …" notes. */
|
||||
function isDemoFailure(result: DemoResult): boolean {
|
||||
return typeof result.html !== "string" || result.html.length === 0;
|
||||
}
|
||||
|
||||
function isVisualFailure(result: VisualResult): boolean {
|
||||
return typeof result.svg !== "string" || result.svg.length === 0;
|
||||
}
|
||||
|
||||
export async function runDemoJob(jobId: string, opts: RunDemoJobOptions = {}): Promise<void> {
|
||||
const job = getJob(jobId);
|
||||
if (!job || job.status !== "running") return;
|
||||
|
||||
const chat = getChat(job.chatId);
|
||||
if (!chat) {
|
||||
updateJob(jobId, { status: "error", error: `Chat not found: ${job.chatId}`, activity: "failed" });
|
||||
return;
|
||||
}
|
||||
const input = findToolPart(chat.messages, job.toolCallId)?.input as DemoToolInput | undefined;
|
||||
const title = typeof input?.title === "string" && input.title.trim() ? input.title : "Interactive demo";
|
||||
const brief = typeof input?.brief === "string" ? input.brief : "";
|
||||
|
||||
try {
|
||||
updateJob(jobId, { activity: "building demo" });
|
||||
const result = await runDemoBuilder(
|
||||
{ title, brief },
|
||||
{
|
||||
model: opts.model,
|
||||
onActivity: (line) => updateJob(jobId, { activity: line }),
|
||||
},
|
||||
);
|
||||
fulfillPart(job.chatId, job.toolCallId, result);
|
||||
if (isDemoFailure(result)) {
|
||||
updateJob(jobId, { status: "error", error: result.notes ?? "demo build failed", activity: "failed" });
|
||||
} else {
|
||||
updateJob(jobId, { status: "done", result });
|
||||
}
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
const failure: DemoResult = { title, html: "", tested: true, notes: `FAILED: ${reason}` };
|
||||
fulfillPart(job.chatId, job.toolCallId, failure);
|
||||
updateJob(jobId, { status: "error", error: reason, activity: "failed" });
|
||||
}
|
||||
}
|
||||
|
||||
export interface RunVisualJobOptions {
|
||||
model?: LanguageModel;
|
||||
}
|
||||
|
||||
export async function runVisualJob(jobId: string, opts: RunVisualJobOptions = {}): Promise<void> {
|
||||
const job = getJob(jobId);
|
||||
if (!job || job.status !== "running") return;
|
||||
|
||||
const chat = getChat(job.chatId);
|
||||
if (!chat) {
|
||||
updateJob(jobId, { status: "error", error: `Chat not found: ${job.chatId}`, activity: "failed" });
|
||||
return;
|
||||
}
|
||||
const input = findToolPart(chat.messages, job.toolCallId)?.input as VisualToolInput | undefined;
|
||||
const brief = typeof input?.brief === "string" ? input.brief : "";
|
||||
|
||||
try {
|
||||
updateJob(jobId, { activity: "drawing figure" });
|
||||
const result = await runVisualMaker(
|
||||
{ brief },
|
||||
{
|
||||
model: opts.model,
|
||||
onActivity: (line) => updateJob(jobId, { activity: line }),
|
||||
},
|
||||
);
|
||||
fulfillPart(job.chatId, job.toolCallId, result);
|
||||
if (isVisualFailure(result)) {
|
||||
updateJob(jobId, { status: "error", error: result.notes ?? "visual build failed", activity: "failed" });
|
||||
} else {
|
||||
updateJob(jobId, { status: "done", result });
|
||||
}
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
const failure: VisualResult = { svg: "", notes: `FAILED: ${reason}` };
|
||||
fulfillPart(job.chatId, job.toolCallId, failure);
|
||||
updateJob(jobId, { status: "error", error: reason, activity: "failed" });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function findToolPart(messages: UIMessage[], toolCallId: string) {
|
||||
@@ -134,7 +240,7 @@ function hostOf(url: string | undefined): string {
|
||||
* Rewrite the tool part in the LATEST chat record to output-available.
|
||||
* Synchronous read-modify-write (no awaits) — atomic within the event loop.
|
||||
*/
|
||||
function fulfillPart(chatId: string, toolCallId: string, output: ResearchResult): void {
|
||||
function fulfillPart(chatId: string, toolCallId: string, output: ResearchResult | DemoResult | VisualResult): void {
|
||||
const chat = getChat(chatId);
|
||||
if (!chat) return;
|
||||
let found = false;
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
import { generateText, tool, type LanguageModel, type ModelMessage, type ToolResultPart } from "ai";
|
||||
import { openrouter } from "@openrouter/ai-sdk-provider";
|
||||
import { z } from "zod";
|
||||
import type { VisualResult } from "../../core/contracts.ts";
|
||||
import { getHeadless, type HeadlessApi } from "./headless.ts";
|
||||
|
||||
/**
|
||||
* The visual-maker sub-agent (vision §2.5): authors ONE precise SVG from a
|
||||
* brief, renders it to a PNG in the SAME headless Chrome, LOOKS at the
|
||||
* result, and iterates until correct — a direct port of the vault POC's
|
||||
* svg-maker doctrine ("the one rule that matters: verify by looking").
|
||||
*
|
||||
* LOOP SHAPE (decided in the W6 brief): hand-rolled with generateText + a
|
||||
* messages array, because the loop must feed an IMAGE back to the model after
|
||||
* each render. write_svg / render_svg are declared WITHOUT execute — when the
|
||||
* model calls them generateText stops with the tool calls, and this loop
|
||||
* appends the tool results itself, followed by a user message containing the
|
||||
* rendered PNG as a file part ("Here is the current render"). ToolLoopAgent's
|
||||
* execute-tool plumbing can't express that image-in-the-loop shape cleanly.
|
||||
*
|
||||
* Doctrine enforcement in code: if the model tries to finish while the
|
||||
* current SVG has never been rendered (or changed since its last render), the
|
||||
* loop renders it anyway, shows the image, and asks the model to look before
|
||||
* finishing. Nothing is delivered without having been looked at.
|
||||
*
|
||||
* Failure semantics mirror the demo-builder: { svg: "", notes: "FAILED: …" }.
|
||||
*/
|
||||
|
||||
const DEFAULT_VISUAL_MODEL =
|
||||
process.env.LH_VISUAL_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
|
||||
|
||||
export interface VisualMakerInput {
|
||||
brief: string;
|
||||
}
|
||||
|
||||
export interface VisualMakerOptions {
|
||||
model?: LanguageModel;
|
||||
headless?: HeadlessApi;
|
||||
/** Max model rounds (write/render cycles + final answer). Default 8. */
|
||||
maxIterations?: number;
|
||||
/** Live activity-line callback (updates the job record). */
|
||||
onActivity?: (line: string) => void;
|
||||
}
|
||||
|
||||
const INSTRUCTIONS = `You are a visual maker — a diagram author + renderer for precise, spatial figures. You
|
||||
receive a brief describing ONE idea that needs exact placement (coordinate geometry, number lines,
|
||||
vectors, function plots, physical layouts — things auto-layout diagrams cannot express), and you return
|
||||
ONE clean, correct SVG. You never see the lesson; the brief is everything you know.
|
||||
|
||||
You do NOT decide what idea to show — the teacher already decided. Your job is faithful, precise
|
||||
composition, and above everything CORRECTNESS: the picture must not assert anything false. A right-angle
|
||||
mark on the wrong corner, a vector pointing the wrong way, a point plotted at the wrong coordinate is a
|
||||
failure even if it renders cleanly.
|
||||
|
||||
Your two tools:
|
||||
- write_svg({ svg }) — write the complete <svg>…</svg> source (overwrites the previous version).
|
||||
- render_svg({}) — rasterize the current SVG in headless Chrome. The render arrives as an image in the
|
||||
message right after the tool result.
|
||||
|
||||
THE ONE RULE THAT MATTERS MOST: verify by looking. You are done only when you have LOOKED at the
|
||||
rendered PNG and confirmed it is true to the brief. Rendering success only proves the SVG parsed; it
|
||||
says nothing about whether the geometry is right or the picture is readable. When you look:
|
||||
|
||||
- Is every coordinate, angle, direction, and proportion actually correct? Re-derive the geometry if unsure.
|
||||
- Are labels placed clearly, not overlapping lines or each other?
|
||||
- Is anything clipped by the viewBox, too small to read, or cramped?
|
||||
- Would a learner instantly read the intended idea from this picture alone?
|
||||
|
||||
Workflow: plan the coordinate space (viewBox + margins, nothing touching the edges) → write_svg →
|
||||
render_svg → LOOK critically → fix (write_svg) and re-render until correct and clean. Never finish
|
||||
without having looked at a render of your FINAL SVG.
|
||||
|
||||
SVG requirements:
|
||||
- A complete <svg> with explicit width/height AND a matching viewBox.
|
||||
- White background (<rect> or style), dark strokes, sans-serif labels at generous sizes.
|
||||
- One idea, fewest elements — sparse and large beats busy and tiny.
|
||||
- Draw only what the brief specifies; never invent data, values, or shapes to fill space.
|
||||
- Plain, clean styling: at most one accent color. An explanatory diagram, not art.
|
||||
|
||||
Your FINAL message: 1-2 sentences of notes for the teacher (what the figure shows, any caveats) —
|
||||
NEVER the SVG source itself. If you genuinely cannot make a correct picture of the brief, say exactly
|
||||
that in your final message and why.`;
|
||||
|
||||
export function createVisualTools() {
|
||||
return {
|
||||
write_svg: tool({
|
||||
description: "Write the complete SVG source for the figure (overwrites any previous version).",
|
||||
inputSchema: z.object({
|
||||
svg: z.string().describe('Complete <svg xmlns="…">…</svg> markup.'),
|
||||
}),
|
||||
}),
|
||||
render_svg: tool({
|
||||
description: "Render the current SVG to a PNG and show it to you for inspection. No arguments.",
|
||||
inputSchema: z.object({}),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const LOOK_MESSAGE =
|
||||
"Here is the current render of your SVG — inspect it critically (geometry, labels, clipping, readability):";
|
||||
const UNLOOKED_FINISH_MESSAGE =
|
||||
"You finished without looking at a render of your final SVG. Above is its render. If it is correct and clean, reply with just your notes; otherwise call write_svg with fixes and render again.";
|
||||
|
||||
export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerOptions = {}): Promise<VisualResult> {
|
||||
const headless = opts.headless ?? getHeadless();
|
||||
const brief = input.brief.trim() || "(no brief)";
|
||||
const maxIterations = opts.maxIterations ?? 8;
|
||||
|
||||
const reason = headless.unavailableReason();
|
||||
if (reason) {
|
||||
return { svg: "", notes: `FAILED: ${reason}` };
|
||||
}
|
||||
|
||||
const tools = createVisualTools();
|
||||
const prompt = [
|
||||
"Brief for ONE precise figure (from the teacher, verbatim — it is everything you know):",
|
||||
brief,
|
||||
"",
|
||||
"Author the SVG, render it, look at it, iterate until correct, then finish with your notes.",
|
||||
].join("\n");
|
||||
|
||||
let messages: ModelMessage[] = [{ role: "user", content: prompt }];
|
||||
let currentSvg: string | null = null;
|
||||
/** The svg shown in the most recent render (null = never rendered). */
|
||||
let renderedSvg: string | null = null;
|
||||
let finalText = "";
|
||||
|
||||
try {
|
||||
for (let iteration = 0; iteration < maxIterations; iteration++) {
|
||||
const result = await generateText({ model: opts.model ?? openrouter(DEFAULT_VISUAL_MODEL), system: INSTRUCTIONS, messages, tools });
|
||||
messages = [...messages, ...result.response.messages];
|
||||
|
||||
const toolCalls = result.toolCalls;
|
||||
if (toolCalls.length === 0) {
|
||||
finalText = result.text.trim();
|
||||
// Doctrine gate: never deliver an unlooked SVG. If the current SVG
|
||||
// changed since its last render (or was never rendered), show it now
|
||||
// and make the model look before accepting the finish.
|
||||
if (currentSvg && renderedSvg !== currentSvg) {
|
||||
let png: Buffer;
|
||||
try {
|
||||
png = await headless.renderSvg(currentSvg);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return { svg: "", notes: `FAILED: rendering the final SVG failed — ${msg}` };
|
||||
}
|
||||
opts.onActivity?.("render_svg: inspecting final render");
|
||||
renderedSvg = currentSvg;
|
||||
messages = [
|
||||
...messages,
|
||||
{ role: "user", content: [{ type: "file", data: png, mediaType: "image/png" }, { type: "text", text: UNLOOKED_FINISH_MESSAGE }] },
|
||||
];
|
||||
continue;
|
||||
}
|
||||
return finish(currentSvg, finalText);
|
||||
}
|
||||
|
||||
const toolResults: ToolResultPart[] = [];
|
||||
let renderPng: Buffer | null = null; // appended as the image message AFTER the tool results
|
||||
for (const call of toolCalls) {
|
||||
if (call.toolName === "write_svg") {
|
||||
const svg = (call.input as { svg?: unknown }).svg;
|
||||
if (typeof svg !== "string" || !svg.includes("<svg")) {
|
||||
toolResults.push(toolText(call.toolCallId, call.toolName, "ERROR: expected a complete <svg>…</svg> document in the `svg` field."));
|
||||
continue;
|
||||
}
|
||||
currentSvg = svg;
|
||||
opts.onActivity?.(`write_svg: ${(svg.length / 1024).toFixed(1)} KB`);
|
||||
toolResults.push(toolText(call.toolCallId, call.toolName, `Saved (${svg.length} bytes). Call render_svg to look at it.`));
|
||||
} else if (call.toolName === "render_svg") {
|
||||
if (!currentSvg) {
|
||||
toolResults.push(toolText(call.toolCallId, call.toolName, "ERROR: nothing to render — call write_svg first."));
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const png = await headless.renderSvg(currentSvg);
|
||||
renderPng = png;
|
||||
renderedSvg = currentSvg;
|
||||
opts.onActivity?.("render_svg: inspecting render");
|
||||
toolResults.push(toolText(call.toolCallId, call.toolName, "Rendered — the PNG follows in the next message. Look at it."));
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
opts.onActivity?.("render_svg: render error");
|
||||
toolResults.push(toolText(call.toolCallId, call.toolName, `RENDER ERROR (the SVG may be malformed): ${msg}. Fix it with write_svg and render again.`));
|
||||
}
|
||||
} else {
|
||||
toolResults.push(toolText(call.toolCallId, call.toolName, `ERROR: unknown tool "${call.toolName}". Use write_svg or render_svg.`));
|
||||
}
|
||||
}
|
||||
// Message order: assistant tool-calls → tool results → render image.
|
||||
messages = [...messages, { role: "tool", content: toolResults }];
|
||||
if (renderPng) {
|
||||
messages = [
|
||||
...messages,
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: LOOK_MESSAGE },
|
||||
{ type: "file", data: renderPng, mediaType: "image/png" },
|
||||
],
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return { svg: "", notes: `FAILED: the maker crashed — ${msg}` };
|
||||
}
|
||||
|
||||
// Iteration limit: deliver the last written SVG only if it was looked at.
|
||||
if (currentSvg && renderedSvg === currentSvg) {
|
||||
return { svg: currentSvg, notes: finalText || "Iteration limit reached — delivering the last verified SVG." };
|
||||
}
|
||||
return { svg: "", notes: `FAILED: iteration limit (${maxIterations}) reached without a verified SVG.` };
|
||||
}
|
||||
|
||||
function toolText(toolCallId: string, toolName: string, value: string): ToolResultPart {
|
||||
return { type: "tool-result", toolCallId, toolName, output: { type: "text", value } };
|
||||
}
|
||||
|
||||
function finish(svg: string | null, finalText: string): VisualResult {
|
||||
if (!svg) {
|
||||
return {
|
||||
svg: "",
|
||||
notes: `FAILED: the maker never wrote an SVG.${finalText ? ` (${finalText.slice(0, 300)})` : ""}`,
|
||||
};
|
||||
}
|
||||
return { svg, notes: finalText || undefined };
|
||||
}
|
||||
+45
-2
@@ -31,16 +31,59 @@ export const researchTool = tool({
|
||||
inputSchema: researchInputSchema,
|
||||
});
|
||||
|
||||
export const demoInputSchema = z.object({
|
||||
title: z.string().describe("Short title for the demo card, e.g. 'Damped harmonic oscillator'."),
|
||||
brief: z
|
||||
.string()
|
||||
.describe(
|
||||
"Precise brief for the builder: the concept, the parameters/controls and what each does, the interactions, and the visual style. " +
|
||||
"The builder sees NOTHING else of the lesson — include everything it needs, including necessary formulas and constants.",
|
||||
),
|
||||
});
|
||||
|
||||
export const demoTool = tool({
|
||||
description:
|
||||
"Dispatch a background sub-agent that BUILDS and headlessly TESTS an interactive teaching instrument — a single-file HTML " +
|
||||
"demo (parameter sweep, simulation, animation, sampling) rendered live in the lesson. Use it when the concept is DYNAMIC: " +
|
||||
"something to sweep, animate, or sample is better experienced than described. " +
|
||||
"The call is ASYNC — your turn ENDS here, so say everything you want to say first; the tested demo arrives later as this " +
|
||||
"tool's output and the lesson resumes with it. Write a precise brief: concept, controls, interactions, style.",
|
||||
inputSchema: demoInputSchema,
|
||||
});
|
||||
|
||||
export const visualInputSchema = z.object({
|
||||
brief: z
|
||||
.string()
|
||||
.describe(
|
||||
"Precise brief for ONE static figure: exactly what it must show — coordinates, labels, proportions, orientation. " +
|
||||
"The maker sees NOTHING else of the lesson; a wrong or vague brief produces a wrong figure.",
|
||||
),
|
||||
});
|
||||
|
||||
export const visualTool = tool({
|
||||
description:
|
||||
"Dispatch a background sub-agent that hand-authors ONE precise SVG figure, renders it, LOOKS at the render, and iterates " +
|
||||
"until correct. Use it for exact spatial pictures auto-layout diagrams can't do: coordinate geometry, number lines, vectors, " +
|
||||
"function plots, physical layouts. For flows/relations/taxonomies write mermaid inline instead. " +
|
||||
"The call is ASYNC — your turn ENDS here, so say everything you want to say first; the verified figure arrives later as this " +
|
||||
"tool's output and the lesson resumes with it.",
|
||||
inputSchema: visualInputSchema,
|
||||
});
|
||||
|
||||
/**
|
||||
* Per-request job tool registry. Context-free for now (the runner resolves
|
||||
* Per-request job tool registry. Context-free (the runner resolves
|
||||
* everything from the chat record by toolCallId) — the ctx parameter keeps
|
||||
* the createTeacherTools factory shape stable for W6's demo/visual tools.
|
||||
* the createTeacherTools factory shape stable.
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
export function jobTools(ctx: TeacherToolContext): ToolSet {
|
||||
return {
|
||||
research: researchTool,
|
||||
demo: demoTool,
|
||||
visual: visualTool,
|
||||
};
|
||||
}
|
||||
|
||||
export type ResearchInput = z.infer<typeof researchInputSchema>;
|
||||
export type DemoInput = z.infer<typeof demoInputSchema>;
|
||||
export type VisualInput = z.infer<typeof visualInputSchema>;
|
||||
|
||||
@@ -0,0 +1,619 @@
|
||||
/**
|
||||
* Demo + visual artifact tests (W6) — deterministic, zero network, no real
|
||||
* Chrome:
|
||||
*
|
||||
* 1. bridge-script injection (buildDemoSrcDoc) + the pure bridge-message
|
||||
* reducer (reduceDemoBridgeState)
|
||||
* 2. card rendering (static markup): sandboxed iframe attrs, error chip,
|
||||
* failure notes, inline sanitized svg
|
||||
* 3. the demo-builder loop with a FAKE model + FAKE headless (success with
|
||||
* a fix iteration, untested-final verification, never-clean failure,
|
||||
* no-Chrome failure)
|
||||
* 4. the visual-maker loop (fake model + fake headless): render-and-look
|
||||
* success, forced look on premature finish, never-writes failure
|
||||
* 5. job wrappers + spawn-hook kind dispatch (demo part → demo job →
|
||||
* part fulfilled; visual failure path)
|
||||
* 6. tool registry wiring (client-side demo/visual tools, schemas)
|
||||
*
|
||||
* Cleanup follows tests/jobs.test.ts idioms (created chat/job files removed
|
||||
* in the after() hook; overrides reset in finally blocks).
|
||||
*/
|
||||
|
||||
import React from "react";
|
||||
(globalThis as Record<string, unknown>).React = React; // tsx classic-JSX shim (see render-cards.test.tsx)
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { MockLanguageModelV4, simulateReadableStream } from "ai/test";
|
||||
import type { LanguageModelV4StreamPart } from "@ai-sdk/provider";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { PATHS } from "../config.ts";
|
||||
import {
|
||||
buildDemoSrcDoc,
|
||||
DemoCard,
|
||||
DemoErrorChip,
|
||||
INITIAL_DEMO_BRIDGE_STATE,
|
||||
reduceDemoBridgeState,
|
||||
} from "../components/artifacts/demo-card.tsx";
|
||||
import { sanitizeSvg, VisualCard } from "../components/artifacts/visual-card.tsx";
|
||||
import { demoInputSchema, jobTools, visualInputSchema } from "../src/agent/tools/jobs.ts";
|
||||
import { runDemoBuilder } from "../src/agent/subagents/demo-builder.ts";
|
||||
import { headlessOverride, type DemoRunResult, type HeadlessApi } from "../src/agent/subagents/headless.ts";
|
||||
import { demoModelOverride, runDemoJob, runVisualJob, spawnPendingJobs, visualModelOverride } from "../src/agent/subagents/runner.ts";
|
||||
import { runVisualMaker } from "../src/agent/subagents/visual-maker.ts";
|
||||
import { createChat, getChat, saveMessages } from "../src/store/chats.ts";
|
||||
import { createJob, findJobByToolCallId, getJob, listJobs } from "../src/store/jobs.ts";
|
||||
import { handleChatRequest } from "../src/server/chat-handler.ts";
|
||||
import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
|
||||
import { artifactRenderers } from "../components/artifacts/index.ts";
|
||||
|
||||
const JOBS_DIR = path.join(PATHS.data, "jobs");
|
||||
const createdChats: string[] = [];
|
||||
const createdJobs: string[] = [];
|
||||
|
||||
after(() => {
|
||||
for (const id of createdChats) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(PATHS.chats, `${id}.json`));
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
}
|
||||
for (const id of createdJobs) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(JOBS_DIR, `${id}.json`));
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
}
|
||||
headlessOverride.current = undefined;
|
||||
demoModelOverride.current = undefined;
|
||||
visualModelOverride.current = undefined;
|
||||
});
|
||||
|
||||
function usage() {
|
||||
return {
|
||||
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
|
||||
outputTokens: { total: 5, text: 5, reasoning: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
/** A model that answers a scripted sequence of doGenerate turns (tool calls or final text). */
|
||||
function scriptedModel(turns: Array<{ toolCalls?: Array<{ toolCallId: string; toolName: string; input: unknown }>; text?: string }>): MockLanguageModelV4 {
|
||||
let turn = -1;
|
||||
return new MockLanguageModelV4({
|
||||
doGenerate: async () => {
|
||||
turn++;
|
||||
const step = turns[turn];
|
||||
if (!step) throw new Error(`scripted model exhausted after ${turns.length} turns`);
|
||||
if (step.toolCalls) {
|
||||
return {
|
||||
content: step.toolCalls.map((c) => ({ type: "tool-call" as const, toolCallId: c.toolCallId, toolName: c.toolName, input: JSON.stringify(c.input) })),
|
||||
finishReason: { unified: "tool-calls" as const, raw: "tool-calls" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
return {
|
||||
content: [{ type: "text" as const, text: step.text ?? "" }],
|
||||
finishReason: { unified: "stop" as const, raw: "stop" },
|
||||
usage: usage(),
|
||||
warnings: [],
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ─── fake headless ───────────────────────────────────────────────────────────
|
||||
|
||||
const CLEAN_RUN: DemoRunResult = { consoleErrors: [], pageErrors: [], failedRequests: [], ok: true };
|
||||
const BROKEN_RUN: DemoRunResult = {
|
||||
consoleErrors: ["Uncaught ReferenceError: foo is not defined"],
|
||||
pageErrors: [],
|
||||
failedRequests: [],
|
||||
ok: false,
|
||||
};
|
||||
|
||||
function fakeHeadless(opts: {
|
||||
/** Return BROKEN_RUN for html matching this substring, else CLEAN_RUN. */
|
||||
brokenWhenIncludes?: string;
|
||||
/** Throw for the first N renderSvg calls (then succeed). */
|
||||
renderFailures?: number;
|
||||
} = {}): HeadlessApi & { demoRuns: string[]; svgRenders: string[] } {
|
||||
const api = {
|
||||
demoRuns: [] as string[],
|
||||
svgRenders: [] as string[],
|
||||
available() {
|
||||
return true;
|
||||
},
|
||||
unavailableReason() {
|
||||
return null;
|
||||
},
|
||||
async runDemo(html: string): Promise<DemoRunResult> {
|
||||
api.demoRuns.push(html);
|
||||
return opts.brokenWhenIncludes && html.includes(opts.brokenWhenIncludes) ? BROKEN_RUN : CLEAN_RUN;
|
||||
},
|
||||
async renderSvg(svg: string): Promise<Buffer> {
|
||||
api.svgRenders.push(svg);
|
||||
if (opts.renderFailures && api.svgRenders.length <= opts.renderFailures) {
|
||||
throw new Error("XML parse error");
|
||||
}
|
||||
return Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); // PNG magic
|
||||
},
|
||||
};
|
||||
return api as HeadlessApi & { demoRuns: string[]; svgRenders: string[] };
|
||||
}
|
||||
|
||||
function noChromeHeadless(): HeadlessApi {
|
||||
return {
|
||||
available: () => false,
|
||||
unavailableReason: () => "No Chrome/Chromium found for headless testing. Install Google Chrome or set CHROME_PATH.",
|
||||
runDemo: async () => {
|
||||
throw new Error("should not run");
|
||||
},
|
||||
renderSvg: async () => {
|
||||
throw new Error("should not render");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ─── 1. bridge injection + reducer ───────────────────────────────────────────
|
||||
|
||||
describe("demo bridge", () => {
|
||||
it("injects the resize + error hooks before </body>", () => {
|
||||
const srcDoc = buildDemoSrcDoc("<!doctype html><html><body><canvas id=\"c\"></canvas></body></html>");
|
||||
assert.match(srcDoc, /demo-resize/);
|
||||
assert.match(srcDoc, /demo-error/);
|
||||
assert.match(srcDoc, /ResizeObserver/);
|
||||
assert.match(srcDoc, /window\.addEventListener\("error"/);
|
||||
// the bridge lands inside the document, and the original content survives
|
||||
assert.ok(srcDoc.indexOf("ResizeObserver") < srcDoc.indexOf("</body>"));
|
||||
assert.match(srcDoc, /<canvas id="c">/);
|
||||
});
|
||||
|
||||
it("appends the bridge when the document has no </body>", () => {
|
||||
const srcDoc = buildDemoSrcDoc("<div>fragment demo</div>");
|
||||
assert.match(srcDoc, /demo-resize/);
|
||||
assert.ok(srcDoc.endsWith("</script>"));
|
||||
assert.match(srcDoc, /fragment demo/);
|
||||
});
|
||||
|
||||
it("reduceDemoBridgeState handles resize, error, and garbage", () => {
|
||||
let state = INITIAL_DEMO_BRIDGE_STATE;
|
||||
state = reduceDemoBridgeState(state, { type: "demo-resize", height: 421.7 });
|
||||
assert.equal(state.height, 422);
|
||||
assert.equal(state.error, null);
|
||||
// clamped
|
||||
state = reduceDemoBridgeState(state, { type: "demo-resize", height: 99999 });
|
||||
assert.equal(state.height, 2400);
|
||||
state = reduceDemoBridgeState(state, { type: "demo-resize", height: 5 });
|
||||
assert.equal(state.height, 80);
|
||||
// invalid payloads ignored
|
||||
assert.equal(reduceDemoBridgeState(state, { type: "demo-resize", height: "tall" }).height, 80);
|
||||
assert.equal(reduceDemoBridgeState(state, { type: "demo-resize" }).height, 80);
|
||||
// errors recorded, with a fallback message
|
||||
state = reduceDemoBridgeState(state, { type: "demo-error", message: "x is not defined" });
|
||||
assert.equal(state.error, "x is not defined");
|
||||
assert.equal(state.height, 80, "error does not clobber height");
|
||||
state = reduceDemoBridgeState(state, { type: "demo-error" });
|
||||
assert.equal(state.error, "unknown error");
|
||||
// unknown types and non-objects ignored
|
||||
assert.deepEqual(reduceDemoBridgeState(state, { type: "something-else" }), state);
|
||||
assert.deepEqual(reduceDemoBridgeState(state, "demo-resize"), state);
|
||||
assert.deepEqual(reduceDemoBridgeState(state, null), state);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── 2. card rendering ───────────────────────────────────────────────────────
|
||||
|
||||
const DEMO_HTML = "<!doctype html><html><body><h1>oscillator</h1><script>let t=0;</script></body></html>";
|
||||
|
||||
function part(state: string, output?: unknown) {
|
||||
return { type: "tool-demo", state, toolCallId: "call-1", input: { title: "Oscillator", brief: "b" }, output };
|
||||
}
|
||||
|
||||
describe("demo card rendering", () => {
|
||||
it("renders a sandboxed iframe (allow-scripts, NO allow-same-origin) with the bridge injected", () => {
|
||||
const html = renderToStaticMarkup(<DemoCard part={part("output-available", { title: "Damped oscillator", html: DEMO_HTML, tested: true }) as never} />);
|
||||
assert.match(html, /sandbox="allow-scripts"/);
|
||||
assert.ok(!html.includes("allow-same-origin"), "never allow-same-origin");
|
||||
assert.match(html, /demo-resize/);
|
||||
assert.match(html, /demo-error/);
|
||||
assert.match(html, /Damped oscillator/);
|
||||
assert.match(html, /interactive demo/i);
|
||||
});
|
||||
|
||||
it("renders the failure note with no iframe when the build FAILED", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<DemoCard part={part("output-available", { title: "T", html: "", tested: true, notes: "FAILED: never passed verification" }) as never} />,
|
||||
);
|
||||
assert.match(html, /FAILED: never passed verification/);
|
||||
assert.ok(!html.includes("<iframe"), "no iframe for a failed build");
|
||||
});
|
||||
|
||||
it("renders notes in muted small text", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<DemoCard part={part("output-available", { title: "T", html: DEMO_HTML, tested: true, notes: "Drag the damping slider." }) as never} />,
|
||||
);
|
||||
assert.match(html, /Drag the damping slider\./);
|
||||
});
|
||||
|
||||
it("renders the activity card while the job runs", () => {
|
||||
const html = renderToStaticMarkup(<DemoCard part={part("input-available") as never} />);
|
||||
assert.match(html, /Building demo…/);
|
||||
assert.ok(!html.includes("<iframe"));
|
||||
});
|
||||
|
||||
it("renders the error chip with a Send to teacher button", () => {
|
||||
const html = renderToStaticMarkup(<DemoErrorChip title="T" message="boom is not defined" onSend={() => {}} />);
|
||||
assert.match(html, /This demo threw an error/);
|
||||
assert.match(html, /boom is not defined/);
|
||||
assert.match(html, /Send to teacher/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("visual card rendering", () => {
|
||||
it("renders the verified svg inline (sanitized) with notes", () => {
|
||||
const svg = '<svg xmlns="http://www.w3.org/2000/svg" width="200" height="100"><circle cx="50" cy="50" r="20"/></svg>';
|
||||
const html = renderToStaticMarkup(
|
||||
<VisualCard part={{ type: "tool-visual", state: "output-available", toolCallId: "c", output: { svg, notes: "The unit circle." } } as never} />,
|
||||
);
|
||||
assert.match(html, /The unit circle\./);
|
||||
assert.ok(html.includes("circle"), "svg content rendered inline");
|
||||
assert.match(html, /Figure/);
|
||||
});
|
||||
|
||||
it("renders the failure note with no svg when the build FAILED", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<VisualCard part={{ type: "tool-visual", state: "output-available", toolCallId: "c", output: { svg: "", notes: "FAILED: iteration limit" } } as never} />,
|
||||
);
|
||||
assert.match(html, /FAILED: iteration limit/);
|
||||
assert.ok(!html.includes("circle") && !html.includes("<svg"), "no svg for a failed build");
|
||||
});
|
||||
|
||||
it("renders the activity card while the job runs", () => {
|
||||
const html = renderToStaticMarkup(<VisualCard part={{ type: "tool-visual", state: "input-available", toolCallId: "c" } as never} />);
|
||||
assert.match(html, /Drawing figure…/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sanitizeSvg", () => {
|
||||
it("strips script blocks, event handlers, and javascript: urls, keeps geometry", () => {
|
||||
const dirty = '<svg width="10" height="10" onload="alert(1)"><script>alert(2)</script><circle r="1" onclick="pwn()"/><a href="javascript:alert(3)"><rect width="2" height="2"/></a></svg>';
|
||||
const clean = sanitizeSvg(dirty);
|
||||
assert.ok(!/<script/i.test(clean), "script tags removed");
|
||||
assert.ok(!/\sonload/i.test(clean), "onload removed");
|
||||
assert.ok(!/\sonclick/i.test(clean), "onclick removed");
|
||||
assert.ok(!/javascript:/i.test(clean), "javascript: urls removed");
|
||||
assert.ok(clean.includes("circle") && clean.includes("rect"), "geometry preserved");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── 3. the demo-builder loop ────────────────────────────────────────────────
|
||||
|
||||
const BROKEN_HTML = "<!doctype html><html><body>BROKEN<script>foo();</script></body></html>";
|
||||
const GOOD_HTML = "<!doctype html><html><body><canvas id=\"c\"></canvas><script>requestAnimationFrame(function step(){});</script></body></html>";
|
||||
|
||||
describe("demo builder (fake model + fake headless)", () => {
|
||||
it("iterates write → run → fix → run → clean → delivers with notes", async () => {
|
||||
const headless = fakeHeadless({ brokenWhenIncludes: "BROKEN" });
|
||||
const activities: string[] = [];
|
||||
const model = scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: BROKEN_HTML } }] },
|
||||
{ toolCalls: [{ toolCallId: "r1", toolName: "run_demo", input: {} }] },
|
||||
{ toolCalls: [{ toolCallId: "w2", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||
{ toolCalls: [{ toolCallId: "r2", toolName: "run_demo", input: { probe: "typeof step === 'function'" } }] },
|
||||
{ text: "Drag the damping slider to see the oscillation decay." },
|
||||
]);
|
||||
|
||||
const result = await runDemoBuilder(
|
||||
{ title: "Damped oscillator", brief: "A mass on a spring with a damping slider." },
|
||||
{ model, headless, onActivity: (line) => activities.push(line) },
|
||||
);
|
||||
|
||||
assert.equal(result.title, "Damped oscillator");
|
||||
assert.equal(result.html, GOOD_HTML, "delivers the FIXED html");
|
||||
assert.equal(result.tested, true);
|
||||
assert.equal(result.notes, "Drag the damping slider to see the oscillation decay.");
|
||||
assert.equal(headless.demoRuns.length, 2, "ran the broken demo, then the fixed one — no extra verification needed");
|
||||
assert.ok(activities.some((a) => a.startsWith("run_demo: 1 console error")), `activity recorded per iteration: ${activities.join(" | ")}`);
|
||||
assert.ok(activities.includes("run_demo: clean"), "clean run recorded in activity");
|
||||
});
|
||||
|
||||
it("verifies headlessly even when the model forgets run_demo before finishing", async () => {
|
||||
const headless = fakeHeadless();
|
||||
const model = scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||
{ text: "Done." },
|
||||
]);
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless });
|
||||
assert.equal(result.html, GOOD_HTML);
|
||||
assert.equal(result.tested, true, "tested by the final verification gate");
|
||||
assert.equal(headless.demoRuns.length, 1, "the gate ran the demo once itself");
|
||||
});
|
||||
|
||||
it("fails (empty html + FAILED notes) when the demo never passes verification", async () => {
|
||||
const headless = fakeHeadless({ brokenWhenIncludes: "BROKEN" });
|
||||
const model = scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: BROKEN_HTML } }] },
|
||||
{ text: "It works, trust me." },
|
||||
]);
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless });
|
||||
assert.equal(result.html, "");
|
||||
assert.equal(result.tested, true, "the loop ran");
|
||||
assert.match(result.notes ?? "", /^FAILED:/);
|
||||
assert.match(result.notes ?? "", /ReferenceError/);
|
||||
});
|
||||
|
||||
it("fails fast with a clear message when no Chrome is available (never skips testing)", async () => {
|
||||
const model = scriptedModel([{ text: "should never be reached" }]);
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless: noChromeHeadless() });
|
||||
assert.equal(result.html, "");
|
||||
assert.match(result.notes ?? "", /^FAILED: .*Chrome/);
|
||||
});
|
||||
|
||||
it("fails when the model never writes a demo", async () => {
|
||||
const model = scriptedModel([{ text: "I cannot build this." }]);
|
||||
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless: fakeHeadless() });
|
||||
assert.equal(result.html, "");
|
||||
assert.match(result.notes ?? "", /^FAILED: the builder never wrote a demo/);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── 4. the visual-maker loop ────────────────────────────────────────────────
|
||||
|
||||
const SVG_1 = '<svg xmlns="http://www.w3.org/2000/svg" width="300" height="100"><line x1="10" y1="50" x2="290" y2="50"/></svg>';
|
||||
|
||||
describe("visual maker (fake model + fake headless)", () => {
|
||||
it("write → render → look → finish delivers the svg with notes", async () => {
|
||||
const headless = fakeHeadless();
|
||||
const model = scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_svg", input: { svg: SVG_1 } }] },
|
||||
{ toolCalls: [{ toolCallId: "r1", toolName: "render_svg", input: {} }] },
|
||||
{ text: "A number line from -3 to 3 with the root marked at 0." },
|
||||
]);
|
||||
const activities: string[] = [];
|
||||
const result = await runVisualMaker({ brief: "Number line, root at 0." }, { model, headless, onActivity: (l) => activities.push(l) });
|
||||
|
||||
assert.equal(result.svg, SVG_1);
|
||||
assert.equal(result.notes, "A number line from -3 to 3 with the root marked at 0.");
|
||||
assert.equal(headless.svgRenders.length, 1);
|
||||
assert.ok(activities.some((a) => a.startsWith("render_svg")), "render activity recorded");
|
||||
});
|
||||
|
||||
it("forces a look when the model finishes without rendering the final svg", async () => {
|
||||
const headless = fakeHeadless();
|
||||
const model = scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_svg", input: { svg: SVG_1 } }] },
|
||||
{ text: "Done." }, // premature finish — no render yet
|
||||
{ text: "Looked at it — correct." }, // after the forced look
|
||||
]);
|
||||
const result = await runVisualMaker({ brief: "b" }, { model, headless });
|
||||
assert.equal(result.svg, SVG_1, "delivered after the forced look");
|
||||
assert.equal(result.notes, "Looked at it — correct.");
|
||||
assert.equal(headless.svgRenders.length, 1, "the gate rendered the svg once itself");
|
||||
});
|
||||
|
||||
it("feeds render errors back to the model instead of crashing", async () => {
|
||||
const headless = fakeHeadless({ renderFailures: 1 });
|
||||
const model = scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_svg", input: { svg: "<svg>broken" } }] },
|
||||
{ toolCalls: [{ toolCallId: "r1", toolName: "render_svg", input: {} }] },
|
||||
{ toolCalls: [{ toolCallId: "w2", toolName: "write_svg", input: { svg: SVG_1 } }] },
|
||||
{ toolCalls: [{ toolCallId: "r2", toolName: "render_svg", input: {} }] },
|
||||
{ text: "Fixed and verified." },
|
||||
]);
|
||||
const result = await runVisualMaker({ brief: "b" }, { model, headless });
|
||||
assert.equal(result.svg, SVG_1);
|
||||
assert.equal(result.notes, "Fixed and verified.");
|
||||
});
|
||||
|
||||
it("fails when the model never writes an svg", async () => {
|
||||
const model = scriptedModel([{ text: "This idea is purely relational — belongs in mermaid." }]);
|
||||
const result = await runVisualMaker({ brief: "b" }, { model, headless: fakeHeadless() });
|
||||
assert.equal(result.svg, "");
|
||||
assert.match(result.notes ?? "", /^FAILED: the maker never wrote an SVG/);
|
||||
});
|
||||
|
||||
it("fails with a clear message when no Chrome is available", async () => {
|
||||
const result = await runVisualMaker({ brief: "b" }, { model: scriptedModel([{ text: "x" }]), headless: noChromeHeadless() });
|
||||
assert.equal(result.svg, "");
|
||||
assert.match(result.notes ?? "", /^FAILED: .*Chrome/);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── 5. job wrappers + spawn dispatch ────────────────────────────────────────
|
||||
|
||||
const DEMO_INPUT = { title: "Compound interest", brief: "Slider for rate, curve of growth over 30 years." };
|
||||
|
||||
function waitFor(cond: () => boolean, timeoutMs = 5000, what = "condition"): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const started = Date.now();
|
||||
const tick = () => {
|
||||
if (cond()) return resolve();
|
||||
if (Date.now() - started > timeoutMs) return reject(new Error(`timeout waiting for ${what}`));
|
||||
setTimeout(tick, 25);
|
||||
};
|
||||
tick();
|
||||
});
|
||||
}
|
||||
|
||||
async function drain(res: Response): Promise<string> {
|
||||
if (res.status !== 200) assert.fail(`expected 200, got ${res.status}: ${await res.text()}`);
|
||||
return res.text();
|
||||
}
|
||||
|
||||
describe("demo/visual job runners", () => {
|
||||
it("runDemoJob fulfills the part with a tested DemoResult and marks the job done", async () => {
|
||||
const chat = createChat("Test: demo runner");
|
||||
createdChats.push(chat.id);
|
||||
saveMessages(chat.id, [
|
||||
{ id: "u1", role: "user", parts: [{ type: "text", text: "show me compounding" }] },
|
||||
{
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "tool-demo", toolCallId: "call-d1", state: "input-available", input: DEMO_INPUT }],
|
||||
},
|
||||
] as never);
|
||||
|
||||
const job = createJob({ kind: "demo", chatId: chat.id, toolCallId: "call-d1" });
|
||||
createdJobs.push(job.id);
|
||||
|
||||
// runDemoJob has no headless injection point — the module override is the seam.
|
||||
headlessOverride.current = fakeHeadless();
|
||||
try {
|
||||
await runDemoJob(job.id, {
|
||||
model: scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||
{ toolCalls: [{ toolCallId: "r1", toolName: "run_demo", input: {} }] },
|
||||
{ text: "Sweep the rate slider." },
|
||||
]),
|
||||
});
|
||||
} finally {
|
||||
headlessOverride.current = undefined;
|
||||
}
|
||||
|
||||
const done = getJob(job.id)!;
|
||||
assert.equal(done.status, "done");
|
||||
assert.equal(done.activity, "run_demo: clean", "final activity line recorded");
|
||||
const result = done.result as { title: string; html: string; tested: boolean };
|
||||
assert.equal(result.title, "Compound interest");
|
||||
assert.equal(result.tested, true);
|
||||
|
||||
const saved = getChat(chat.id)!;
|
||||
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-demo");
|
||||
assert.ok(part && part.type === "tool-demo", "demo tool part persisted");
|
||||
assert.equal(part.state, "output-available", "part fulfilled");
|
||||
assert.equal((part as { output?: { html?: string } }).output?.html, GOOD_HTML);
|
||||
});
|
||||
|
||||
it("runVisualJob fulfills the part even on failure and marks the job error", async () => {
|
||||
const chat = createChat("Test: visual runner failure");
|
||||
createdChats.push(chat.id);
|
||||
saveMessages(chat.id, [
|
||||
{ id: "u1", role: "user", parts: [{ type: "text", text: "draw it" }] },
|
||||
{
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "tool-visual", toolCallId: "call-v1", state: "input-available", input: { brief: "b" } }],
|
||||
},
|
||||
] as never);
|
||||
|
||||
const job = createJob({ kind: "visual", chatId: chat.id, toolCallId: "call-v1" });
|
||||
createdJobs.push(job.id);
|
||||
|
||||
await runVisualJob(job.id, { model: scriptedModel([{ text: "cannot" }]) });
|
||||
|
||||
const errored = getJob(job.id)!;
|
||||
assert.equal(errored.status, "error");
|
||||
assert.match(errored.error ?? "", /never wrote an SVG/);
|
||||
|
||||
const saved = getChat(chat.id)!;
|
||||
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-visual");
|
||||
assert.ok(part && part.type === "tool-visual", "visual tool part persisted");
|
||||
assert.equal(part.state, "output-available", "failure still fulfills the part");
|
||||
const output = (part as { output?: { svg?: string; notes?: string } }).output!;
|
||||
assert.equal(output.svg, "");
|
||||
assert.match(output.notes ?? "", /^FAILED:/);
|
||||
});
|
||||
|
||||
it("the spawn hook dispatches a demo part to the demo runner (kind→runner map)", async () => {
|
||||
const chat = createChat("Test: demo spawn dispatch");
|
||||
createdChats.push(chat.id);
|
||||
|
||||
// A teacher that calls demo on turn 1 (scripted stream, like jobs.test.ts).
|
||||
let turn = 0;
|
||||
const teacher = new MockLanguageModelV4({
|
||||
doStream: async () => {
|
||||
turn++;
|
||||
const chunks: LanguageModelV4StreamPart[] =
|
||||
turn === 1
|
||||
? [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "tool-input-start", id: "call-d2", toolName: "demo" },
|
||||
{ type: "tool-input-delta", id: "call-d2", delta: JSON.stringify(DEMO_INPUT) },
|
||||
{ type: "tool-input-end", id: "call-d2" },
|
||||
{ type: "tool-call", toolCallId: "call-d2", toolName: "demo", input: JSON.stringify(DEMO_INPUT) },
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
]
|
||||
: [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "text-start", id: "t2" },
|
||||
{ type: "text-delta", id: "t2", delta: "Here it is." },
|
||||
{ type: "text-end", id: "t2" },
|
||||
{ type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: usage() },
|
||||
];
|
||||
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
|
||||
},
|
||||
});
|
||||
|
||||
demoModelOverride.current = scriptedModel([
|
||||
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||
{ toolCalls: [{ toolCallId: "r1", toolName: "run_demo", input: {} }] },
|
||||
{ text: "Sweep it." },
|
||||
]);
|
||||
headlessOverride.current = fakeHeadless();
|
||||
|
||||
try {
|
||||
await drain(
|
||||
await handleChatRequest(
|
||||
new Request("http://test/api/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
chatId: chat.id,
|
||||
messages: [{ id: "u1", role: "user", parts: [{ type: "text", text: "teach me compounding" }] }],
|
||||
}),
|
||||
}),
|
||||
createTeacherAgent(teacher),
|
||||
),
|
||||
);
|
||||
|
||||
const job = findJobByToolCallId(chat.id, "call-d2");
|
||||
assert.ok(job, "spawn hook created a job for the demo part");
|
||||
assert.equal(job!.kind, "demo");
|
||||
createdJobs.push(job!.id);
|
||||
|
||||
// The background runner (fire-and-forget) completes with the overrides.
|
||||
await waitFor(() => getJob(job!.id)!.status !== "running", 5000, "demo job completion");
|
||||
const done = getJob(job!.id)!;
|
||||
assert.equal(done.status, "done");
|
||||
assert.equal(done.activity, "run_demo: clean");
|
||||
|
||||
const saved = getChat(chat.id)!;
|
||||
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-demo");
|
||||
assert.ok(part && part.type === "tool-demo", "demo tool part persisted");
|
||||
assert.equal(part.state, "output-available", "demo part fulfilled by the background runner");
|
||||
|
||||
// Idempotency: re-running the spawn hook creates no second job.
|
||||
spawnPendingJobs(chat.id, saved.messages);
|
||||
assert.equal(listJobs(chat.id).length, 1, "no duplicate job for the same tool part");
|
||||
} finally {
|
||||
demoModelOverride.current = undefined;
|
||||
headlessOverride.current = undefined;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── 6. registry wiring ──────────────────────────────────────────────────────
|
||||
|
||||
describe("tool + renderer registry wiring", () => {
|
||||
it("demo and visual are CLIENT-SIDE tools with the contract schemas", () => {
|
||||
const tools = jobTools({ chatId: "c" });
|
||||
assert.deepEqual(Object.keys(tools).sort(), ["demo", "research", "visual"]);
|
||||
for (const name of ["demo", "visual"] as const) {
|
||||
assert.equal(typeof (tools[name] as { execute?: unknown }).execute, "undefined", `${name} has no execute — client-side like quiz/ask`);
|
||||
}
|
||||
assert.equal(demoInputSchema.safeParse({ title: "T", brief: "b" }).success, true);
|
||||
assert.equal(demoInputSchema.safeParse({ brief: "b" }).success, false, "demo requires a title");
|
||||
assert.equal(visualInputSchema.safeParse({ brief: "b" }).success, true);
|
||||
assert.equal(visualInputSchema.safeParse({}).success, false, "visual requires a brief");
|
||||
});
|
||||
|
||||
it("the artifact renderer registry maps demo/visual to the W6 cards", () => {
|
||||
assert.equal(artifactRenderers.demo, DemoCard);
|
||||
assert.equal(artifactRenderers.visual, VisualCard);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user