W6: demo + visual artifacts, verifying builders
This commit is contained in:
Amos
@@ -1,10 +1,257 @@
|
|||||||
import type { ArtifactPartProps } from "@/src/core/contracts";
|
"use client";
|
||||||
|
|
||||||
|
import { useContext, useEffect, useMemo, useRef, useState } from "react";
|
||||||
|
import { ChatActionsContext } from "@/components/chat";
|
||||||
|
import type { ArtifactPartProps, DemoResult, JobRecord } from "@/src/core/contracts";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Demo artifact card (W6): sandboxed iframe rendering DemoResult.html with
|
* Demo artifact card (W6): while the part is input-available, a job activity
|
||||||
* the auto-injected resize/error bridge (contracts.ts "Demo iframe bridge").
|
* card (poll /api/jobs — the job-card idiom); when output-available, the demo
|
||||||
* While input-available: job activity card (poll /api/jobs).
|
* runs in a SANDBOXED iframe (sandbox="allow-scripts", no allow-same-origin →
|
||||||
|
* no access to app storage/cookies/DOM).
|
||||||
|
*
|
||||||
|
* The bridge script (contracts.ts "Demo iframe bridge") is auto-injected into
|
||||||
|
* every demo by buildDemoSrcDoc — never the model's job: ResizeObserver →
|
||||||
|
* postMessage {type:"demo-resize",height} (parent sizes the iframe) and
|
||||||
|
* window.onerror → {type:"demo-error",message} (error chip on the card, with
|
||||||
|
* a "Send to teacher" button that feeds the error back into the lesson — the
|
||||||
|
* fix loop).
|
||||||
|
*
|
||||||
|
* Bridge message handling is a pure function (reduceDemoBridgeState) so
|
||||||
|
* tests cover it without a DOM.
|
||||||
*/
|
*/
|
||||||
export function DemoCard(_props: ArtifactPartProps) {
|
|
||||||
return null;
|
// ─── bridge injection + message handling (pure, testable) ───────────────────
|
||||||
|
|
||||||
|
export const DEMO_BRIDGE_SCRIPT = `<script>(function(){
|
||||||
|
var send=function(m){try{parent.postMessage(m,"*")}catch(e){}};
|
||||||
|
var post=function(){send({type:"demo-resize",height:document.documentElement.scrollHeight})};
|
||||||
|
if(typeof ResizeObserver!=="undefined"){new ResizeObserver(post).observe(document.documentElement)}
|
||||||
|
window.addEventListener("resize",post);
|
||||||
|
window.addEventListener("load",post);
|
||||||
|
window.addEventListener("error",function(e){send({type:"demo-error",message:e.message||String(e)})});
|
||||||
|
setTimeout(post,50);
|
||||||
|
setTimeout(post,400);
|
||||||
|
})();</script>`;
|
||||||
|
|
||||||
|
/** Inject the bridge before </body> (or at the end of a body-less document). */
|
||||||
|
export function buildDemoSrcDoc(html: string): string {
|
||||||
|
if (/<\/body>/i.test(html)) return html.replace(/<\/body>/i, `${DEMO_BRIDGE_SCRIPT}</body>`);
|
||||||
|
return `${html}${DEMO_BRIDGE_SCRIPT}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DemoBridgeState {
|
||||||
|
height: number | null;
|
||||||
|
error: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const INITIAL_DEMO_BRIDGE_STATE: DemoBridgeState = { height: null, error: null };
|
||||||
|
|
||||||
|
const MIN_HEIGHT = 80;
|
||||||
|
const MAX_HEIGHT = 2400;
|
||||||
|
|
||||||
|
/** Pure reducer over incoming postMessage data (contracts.ts DemoBridgeMessage). */
|
||||||
|
export function reduceDemoBridgeState(state: DemoBridgeState, data: unknown): DemoBridgeState {
|
||||||
|
if (typeof data !== "object" || data === null) return state;
|
||||||
|
const message = data as { type?: unknown; height?: unknown; message?: unknown };
|
||||||
|
if (message.type === "demo-resize") {
|
||||||
|
if (typeof message.height !== "number" || !Number.isFinite(message.height) || message.height <= 0) return state;
|
||||||
|
const height = Math.min(Math.max(Math.round(message.height), MIN_HEIGHT), MAX_HEIGHT);
|
||||||
|
return height === state.height ? state : { ...state, height };
|
||||||
|
}
|
||||||
|
if (message.type === "demo-error") {
|
||||||
|
const text = typeof message.message === "string" && message.message.trim() ? message.message : "unknown error";
|
||||||
|
return text === state.error ? state : { ...state, error: text };
|
||||||
|
}
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── shared job activity card (input-available state) ───────────────────────
|
||||||
|
|
||||||
|
/** Live activity card for pending job parts — the job-card.tsx idiom,
|
||||||
|
* parameterized by label. Exported for the visual card (also W6-owned). */
|
||||||
|
export function JobActivityCard({
|
||||||
|
toolCallId,
|
||||||
|
label,
|
||||||
|
testId,
|
||||||
|
}: {
|
||||||
|
toolCallId: string;
|
||||||
|
label: string;
|
||||||
|
testId?: string;
|
||||||
|
}) {
|
||||||
|
const actions = useContext(ChatActionsContext);
|
||||||
|
const [job, setJob] = useState<JobRecord | null>(null);
|
||||||
|
const finishedRef = useRef(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!actions) return;
|
||||||
|
let cancelled = false;
|
||||||
|
|
||||||
|
const poll = async () => {
|
||||||
|
if (finishedRef.current) return true;
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/jobs?chatId=${encodeURIComponent(actions.chatId)}`);
|
||||||
|
if (!res.ok) return false;
|
||||||
|
const { jobs } = (await res.json()) as { jobs?: JobRecord[] };
|
||||||
|
const match = jobs?.find((j) => j.toolCallId === toolCallId) ?? null;
|
||||||
|
if (cancelled) return false;
|
||||||
|
setJob(match);
|
||||||
|
if (match && match.status !== "running") {
|
||||||
|
// The part has (or is about to be) fulfilled server-side — pull the
|
||||||
|
// fresh messages once; the continuation resumes the teacher.
|
||||||
|
finishedRef.current = true;
|
||||||
|
void actions.syncFromServer();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* transient fetch failure — keep polling */
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
const timer = setInterval(() => {
|
||||||
|
if (finishedRef.current) {
|
||||||
|
clearInterval(timer);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
void poll().then((stop) => {
|
||||||
|
if (stop) clearInterval(timer);
|
||||||
|
});
|
||||||
|
}, 2000);
|
||||||
|
void poll().then((stop) => {
|
||||||
|
if (stop) clearInterval(timer);
|
||||||
|
});
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
clearInterval(timer);
|
||||||
|
};
|
||||||
|
}, [actions, toolCallId]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid={testId ?? "job-activity-card"}>
|
||||||
|
<div className="flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||||
|
<span className="inline-block size-2 animate-pulse rounded-full bg-primary" />
|
||||||
|
{label}
|
||||||
|
</div>
|
||||||
|
{job?.activity && (
|
||||||
|
<div className="mt-1 truncate text-sm text-muted-foreground" data-testid="job-activity-line">
|
||||||
|
{job.activity}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── the demo card ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export function DemoCard({ part }: ArtifactPartProps) {
|
||||||
|
if (part.state === "output-available") {
|
||||||
|
return <DemoOutputCard output={part.output} />;
|
||||||
|
}
|
||||||
|
return <JobActivityCard toolCallId={part.toolCallId} label="Building demo…" testId="demo-activity-card" />;
|
||||||
|
}
|
||||||
|
|
||||||
|
function DemoOutputCard({ output }: { output: unknown }) {
|
||||||
|
const result = output as DemoResult | undefined;
|
||||||
|
if (!result || typeof result.html !== "string" || typeof result.title !== "string") {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border bg-card p-4 text-sm text-muted-foreground">
|
||||||
|
Demo completed, but the result could not be displayed.
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const failed = result.html.length === 0;
|
||||||
|
if (failed) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="demo-card-failed">
|
||||||
|
<div className="flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||||
|
Interactive demo
|
||||||
|
<span className="rounded bg-red-500/15 px-1.5 py-0.5 text-red-500">failed</span>
|
||||||
|
</div>
|
||||||
|
<p className="mt-2 text-sm text-muted-foreground">{result.notes ?? "The demo could not be built."}</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return <DemoFrame title={result.title} html={result.html} notes={result.notes} />;
|
||||||
|
}
|
||||||
|
|
||||||
|
function DemoFrame({ title, html, notes }: { title: string; html: string; notes?: string }) {
|
||||||
|
const actions = useContext(ChatActionsContext);
|
||||||
|
const iframeRef = useRef<HTMLIFrameElement | null>(null);
|
||||||
|
const [state, setState] = useState<DemoBridgeState>(INITIAL_DEMO_BRIDGE_STATE);
|
||||||
|
const srcDoc = useMemo(() => buildDemoSrcDoc(html), [html]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const onMessage = (event: MessageEvent) => {
|
||||||
|
// Only messages from THIS demo's iframe (the sandbox gives it an
|
||||||
|
// opaque origin; the window reference still matches).
|
||||||
|
if (!iframeRef.current || event.source !== iframeRef.current.contentWindow) return;
|
||||||
|
setState((prev) => reduceDemoBridgeState(prev, event.data));
|
||||||
|
};
|
||||||
|
window.addEventListener("message", onMessage);
|
||||||
|
return () => window.removeEventListener("message", onMessage);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="demo-card">
|
||||||
|
<div className="flex items-baseline justify-between gap-2">
|
||||||
|
<div className="font-medium">{title}</div>
|
||||||
|
<div className="shrink-0 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||||
|
interactive demo
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{state.error && (
|
||||||
|
<DemoErrorChip
|
||||||
|
title={title}
|
||||||
|
message={state.error}
|
||||||
|
onSend={actions ? () => actions.sendText(`The demo "${title}" threw an error: ${state.error}`) : undefined}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
<iframe
|
||||||
|
ref={iframeRef}
|
||||||
|
title={title}
|
||||||
|
// allow-scripts ONLY — no allow-same-origin: the demo can run code
|
||||||
|
// but never touch the app's storage, cookies, or DOM.
|
||||||
|
sandbox="allow-scripts"
|
||||||
|
srcDoc={srcDoc}
|
||||||
|
className="mt-2 w-full rounded-lg border bg-white"
|
||||||
|
style={{ height: state.height ?? 360 }}
|
||||||
|
data-testid="demo-iframe"
|
||||||
|
/>
|
||||||
|
{notes && <p className="mt-2 text-sm text-muted-foreground">{notes}</p>}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function DemoErrorChip({
|
||||||
|
title,
|
||||||
|
message,
|
||||||
|
onSend,
|
||||||
|
}: {
|
||||||
|
title: string;
|
||||||
|
message: string;
|
||||||
|
onSend?: () => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="mt-2 flex flex-wrap items-center gap-2 rounded-lg bg-red-500/10 px-3 py-2 text-sm text-red-500"
|
||||||
|
data-testid="demo-error-chip"
|
||||||
|
>
|
||||||
|
<span className="font-medium">This demo threw an error</span>
|
||||||
|
<span className="min-w-0 flex-1 truncate font-mono text-xs" title={message}>
|
||||||
|
{message}
|
||||||
|
</span>
|
||||||
|
{onSend && (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={onSend}
|
||||||
|
className="rounded-md border border-red-500/40 px-2 py-1 text-xs font-medium hover:bg-red-500/15"
|
||||||
|
data-testid="demo-error-send"
|
||||||
|
>
|
||||||
|
Send to teacher
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,64 @@
|
|||||||
import type { ArtifactPartProps } from "@/src/core/contracts";
|
"use client";
|
||||||
|
|
||||||
|
import type { ArtifactPartProps, VisualResult } from "@/src/core/contracts";
|
||||||
|
import { JobActivityCard } from "@/components/artifacts/demo-card";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Visual artifact card (W6): renders VisualResult.svg inline.
|
* Visual artifact card (W6): while the part is input-available, a job activity
|
||||||
* While input-available: job activity card (poll /api/jobs).
|
* card; when output-available, VisualResult.svg renders INLINE. The SVG is
|
||||||
|
* model-authored for the learner's own machine (their own teacher wrote it,
|
||||||
|
* verified by looking) — dangerouslySetInnerHTML is acceptable here, with a
|
||||||
|
* cheap hygiene pass (sanitizeSvg: strip <script> blocks, on* handlers, and
|
||||||
|
* javascript:/data:text/html URLs) since it lands in the app's DOM.
|
||||||
*/
|
*/
|
||||||
export function VisualCard(_props: ArtifactPartProps) {
|
|
||||||
return null;
|
/** Cheap SVG hygiene — not a full sanitizer, just removing the obvious exec vectors. */
|
||||||
|
export function sanitizeSvg(svg: string): string {
|
||||||
|
return svg
|
||||||
|
.replace(/<script[\s\S]*?<\/script\s*>/gi, "")
|
||||||
|
.replace(/<script[^>]*\/?>/gi, "")
|
||||||
|
.replace(/\son[a-z]+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, "")
|
||||||
|
.replace(/(href|xlink:href)\s*=\s*(?:"\s*(?:javascript|data:text\/html)[^"]*"|'\s*(?:javascript|data:text\/html)[^']*'|(?:javascript|data:text\/html)[^\s>]*)/gi, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function VisualCard({ part }: ArtifactPartProps) {
|
||||||
|
if (part.state === "output-available") {
|
||||||
|
return <VisualOutputCard output={part.output} />;
|
||||||
|
}
|
||||||
|
return <JobActivityCard toolCallId={part.toolCallId} label="Drawing figure…" testId="visual-activity-card" />;
|
||||||
|
}
|
||||||
|
|
||||||
|
function VisualOutputCard({ output }: { output: unknown }) {
|
||||||
|
const result = output as VisualResult | undefined;
|
||||||
|
if (!result || typeof result.svg !== "string") {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border bg-card p-4 text-sm text-muted-foreground">
|
||||||
|
Figure completed, but the result could not be displayed.
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const failed = result.svg.length === 0;
|
||||||
|
if (failed) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="visual-card-failed">
|
||||||
|
<div className="flex items-center gap-2 text-xs font-medium uppercase tracking-wide text-muted-foreground">
|
||||||
|
Figure
|
||||||
|
<span className="rounded bg-red-500/15 px-1.5 py-0.5 text-red-500">failed</span>
|
||||||
|
</div>
|
||||||
|
<p className="mt-2 text-sm text-muted-foreground">{result.notes ?? "The figure could not be drawn."}</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<div className="rounded-xl border bg-card p-4 text-card-foreground" data-testid="visual-card">
|
||||||
|
<div className="text-xs font-medium uppercase tracking-wide text-muted-foreground">Figure</div>
|
||||||
|
{/* max-w-full + h-auto keeps any oversized model-authored svg inside the column */}
|
||||||
|
<div
|
||||||
|
className="mt-2 max-w-full overflow-hidden rounded-lg bg-white [&_svg]:h-auto [&_svg]:max-w-full"
|
||||||
|
dangerouslySetInnerHTML={{ __html: sanitizeSvg(result.svg) }}
|
||||||
|
data-testid="visual-svg"
|
||||||
|
/>
|
||||||
|
{result.notes && <p className="mt-2 text-sm text-muted-foreground">{result.notes}</p>}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
## Interactive demos and precise figures — build the instrument, don't describe it
|
||||||
|
|
||||||
|
If the concept is DYNAMIC — a parameter to sweep, a process to animate, a distribution to sample, a
|
||||||
|
system to simulate — build the learner the instrument rather than describing its behavior in prose.
|
||||||
|
A paragraph about how frequency response changes with damping teaches less than ten seconds of
|
||||||
|
dragging a slider. You have two artifact tools for this, both asynchronous (same discipline as
|
||||||
|
research):
|
||||||
|
|
||||||
|
### `demo` — interactive teaching instruments
|
||||||
|
|
||||||
|
A background sub-agent builds a self-contained interactive demo (single-file HTML: parameter sweeps,
|
||||||
|
simulations, animations) and **headlessly tests that it actually runs** before it reaches the lesson.
|
||||||
|
Nothing untested ever appears.
|
||||||
|
|
||||||
|
- Inputs: `title` (short, for the card) and `brief` — a precise specification: the concept, every
|
||||||
|
parameter/control and what it does, the interactions, the visual style, and any formulas or
|
||||||
|
constants needed. **The builder never sees the lesson** — the brief is its entire world. Include
|
||||||
|
everything; a vague brief yields a vague demo.
|
||||||
|
- Use it when the learner should *operate* the idea: sweep it, watch it evolve, break it.
|
||||||
|
|
||||||
|
### `visual` — precise static figures
|
||||||
|
|
||||||
|
A background sub-agent hand-authors one SVG, renders it, **looks at the render**, and iterates until
|
||||||
|
the picture is correct. Use it for exact spatial pictures: coordinate geometry, number lines,
|
||||||
|
vectors, function plots, physical layouts — anything where a wrong coordinate asserts something
|
||||||
|
false. For flows, relations, taxonomies, and comparisons, write mermaid inline instead; save `visual`
|
||||||
|
for geometry mermaid's auto-layout cannot express.
|
||||||
|
|
||||||
|
- Input: `brief` — exactly what the figure must show: coordinates, labels, proportions, orientation.
|
||||||
|
One idea per figure. The maker sees nothing else of the lesson.
|
||||||
|
|
||||||
|
### The async discipline
|
||||||
|
|
||||||
|
The call is asynchronous and **your turn ends at it**, exactly like research:
|
||||||
|
|
||||||
|
1. **Say your piece first.** Set up the concept, explain what the learner is about to see and what to
|
||||||
|
watch for — the artifact arrives later as the tool's output and the lesson resumes from there. A
|
||||||
|
bare tool call with no surrounding text is a failed turn.
|
||||||
|
2. **At most one demo or visual per turn.**
|
||||||
|
3. **When the artifact arrives, TEACH WITH IT.** Reference its controls and elements by name: "drag
|
||||||
|
the damping slider all the way left and watch the overshoot", "notice the vector at t=2s". An
|
||||||
|
artifact you never point at is decoration.
|
||||||
|
4. **Failures arrive as notes** ("FAILED: …") with no artifact — say so plainly, fall back to
|
||||||
|
explaining in prose, and if the brief was the problem, re-call with a tighter one. Re-calling a
|
||||||
|
tool produces a new version of the artifact; use that to fix a demo that misbehaves in the field.
|
||||||
@@ -0,0 +1,238 @@
|
|||||||
|
import { ToolLoopAgent, stepCountIs, tool, type LanguageModel, type ToolSet } from "ai";
|
||||||
|
import { openrouter } from "@openrouter/ai-sdk-provider";
|
||||||
|
import { z } from "zod";
|
||||||
|
import type { DemoResult } from "../../core/contracts.ts";
|
||||||
|
import { getHeadless, type DemoRunResult, type HeadlessApi } from "./headless.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The demo-builder sub-agent (vision §2 interactive demos + §2.5 "sub-agents
|
||||||
|
* verify; they never one-shot"): given a brief it writes a self-contained
|
||||||
|
* single-file HTML/JS demo, TESTS IT HEADLESSLY (run_demo), and iterates
|
||||||
|
* fix → re-run until clean — before anything reaches the lesson.
|
||||||
|
*
|
||||||
|
* Shape: a ToolLoopAgent with two executable tools (write_demo / run_demo) —
|
||||||
|
* the model is injectable (tests use MockLanguageModelV4) and the headless
|
||||||
|
* API is injectable (tests use a fake; production uses system Chrome).
|
||||||
|
*
|
||||||
|
* Failure semantics (decided in the W6 brief): `tested: true` means "the
|
||||||
|
* verify loop ran"; on failure the result is { title, html: "", tested: true,
|
||||||
|
* notes: "FAILED: <reason>" } — the card renders the failure note, no iframe.
|
||||||
|
* The job wrapper treats html === "" as failure and marks the job error.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const DEFAULT_DEMO_MODEL =
|
||||||
|
process.env.LH_DEMO_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
|
||||||
|
|
||||||
|
export interface DemoBuilderInput {
|
||||||
|
title: string;
|
||||||
|
brief: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DemoBuilderOptions {
|
||||||
|
model?: LanguageModel;
|
||||||
|
headless?: HeadlessApi;
|
||||||
|
/** Max agent steps (each write/run/fix is one step). Default 24 ≈ 6 cycles. */
|
||||||
|
maxSteps?: number;
|
||||||
|
/** Live activity-line callback (updates the job record). */
|
||||||
|
onActivity?: (line: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const INSTRUCTIONS = `You are a demo builder — a specialist sub-agent inside a learning application. You
|
||||||
|
receive a brief for ONE interactive teaching instrument and deliver a single self-contained HTML document
|
||||||
|
that runs it. You never see the lesson; the brief is everything you know.
|
||||||
|
|
||||||
|
Runtime contract (the document runs in a sandboxed iframe, sandbox="allow-scripts", opaque origin):
|
||||||
|
- Single file. Vanilla JS + Canvas 2D or SVG. CDN imports (e.g. a plotting or math library) are allowed
|
||||||
|
when they genuinely help — but prefer dependency-free; you own the failure if a CDN import fails.
|
||||||
|
- NO localStorage, sessionStorage, or document.cookie — the sandbox makes them throw.
|
||||||
|
- The parent sizes the iframe to the document height automatically; just build normal flowing layout.
|
||||||
|
Make the demo fill the width and size its own height sensibly.
|
||||||
|
|
||||||
|
Build quality:
|
||||||
|
- The demo must directly serve the brief: every control named in the brief must exist and do the thing.
|
||||||
|
- Controls need labels; sliders/inputs need current-value readouts. A learner should operate it unguided.
|
||||||
|
- Smooth enough interaction; requestAnimationFrame for animation, not setInterval when it matters.
|
||||||
|
- Dark-mode neutral: pick a clean light background for the demo area so it reads on any card.
|
||||||
|
|
||||||
|
THE HOUSE RULE — never deliver untested:
|
||||||
|
1. Call write_demo with the complete HTML document.
|
||||||
|
2. Call run_demo. It loads the file in headless Chrome, waits ~2s, and reports console errors, uncaught
|
||||||
|
errors (window.onerror), and failed requests. Optionally pass a probe: a JS expression evaluated in
|
||||||
|
the page (e.g. "typeof window.simulate === 'function'" or a state assertion) to check deeper.
|
||||||
|
3. If run_demo reports problems, FIX the document and call write_demo again, then run_demo again.
|
||||||
|
Iterate until run_demo reports PASSED.
|
||||||
|
4. Only then finish. Your final message is 1-3 sentences of NOTES for the teacher (what the demo shows,
|
||||||
|
its controls, what to try) — NEVER the HTML itself.
|
||||||
|
|
||||||
|
Never claim success you did not see: a PASSED run_demo is the only definition of done.`;
|
||||||
|
|
||||||
|
interface BuilderState {
|
||||||
|
html: string | null;
|
||||||
|
/** The html that the last run_demo tested (null = never tested). */
|
||||||
|
testedHtml: string | null;
|
||||||
|
lastResult: DemoRunResult | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createDemoBuilderTools(
|
||||||
|
state: BuilderState,
|
||||||
|
headless: HeadlessApi,
|
||||||
|
onActivity?: (line: string) => void,
|
||||||
|
): ToolSet {
|
||||||
|
return {
|
||||||
|
write_demo: tool({
|
||||||
|
description:
|
||||||
|
"Write the complete self-contained HTML document for the demo (overwrites any previous version). " +
|
||||||
|
"Always follow with run_demo to test it headlessly.",
|
||||||
|
inputSchema: z.object({
|
||||||
|
html: z.string().describe("The complete HTML document: <!doctype html>…</html>, all JS inline."),
|
||||||
|
}),
|
||||||
|
execute: async ({ html }) => {
|
||||||
|
if (!html || html.trim().length === 0) return "ERROR: empty document — write the full HTML.";
|
||||||
|
state.html = html;
|
||||||
|
onActivity?.(`write_demo: ${(html.length / 1024).toFixed(1)} KB`);
|
||||||
|
return `Saved (${html.length} bytes). Now call run_demo to test it — never deliver untested.`;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
run_demo: tool({
|
||||||
|
description:
|
||||||
|
"Headlessly execute the current demo in Chrome and report console errors, uncaught errors, and failed requests. " +
|
||||||
|
"Optionally pass a probe expression to assert on page state.",
|
||||||
|
inputSchema: z.object({
|
||||||
|
probe: z
|
||||||
|
.string()
|
||||||
|
.optional()
|
||||||
|
.describe("Optional JS expression evaluated in the page after settling, e.g. \"document.querySelectorAll('input').length\"."),
|
||||||
|
}),
|
||||||
|
execute: async ({ probe }) => {
|
||||||
|
if (!state.html) return "ERROR: nothing to run — call write_demo first.";
|
||||||
|
const reason = headless.unavailableReason();
|
||||||
|
if (reason) return `HEADLESS UNAVAILABLE: ${reason}`;
|
||||||
|
let result: DemoRunResult;
|
||||||
|
try {
|
||||||
|
result = await headless.runDemo(state.html, probe ? { probe } : undefined);
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
onActivity?.("run_demo: browser error");
|
||||||
|
return `HEADLESS ERROR (retrying may help): ${msg}`;
|
||||||
|
}
|
||||||
|
state.testedHtml = state.html;
|
||||||
|
state.lastResult = result;
|
||||||
|
onActivity?.(
|
||||||
|
result.ok ? "run_demo: clean" : `run_demo: ${summarizeCounts(result)}`,
|
||||||
|
);
|
||||||
|
return formatRunResult(result);
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function summarizeCounts(result: DemoRunResult): string {
|
||||||
|
const counts = [
|
||||||
|
result.consoleErrors.length > 0 ? `${result.consoleErrors.length} console error${result.consoleErrors.length === 1 ? "" : "s"}` : null,
|
||||||
|
result.pageErrors.length > 0 ? `${result.pageErrors.length} page error${result.pageErrors.length === 1 ? "" : "s"}` : null,
|
||||||
|
result.failedRequests.length > 0 ? `${result.failedRequests.length} failed request${result.failedRequests.length === 1 ? "" : "s"}` : null,
|
||||||
|
].filter(Boolean);
|
||||||
|
return counts.length > 0 ? counts.join(", ") : "ok";
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatRunResult(result: DemoRunResult): string {
|
||||||
|
if (result.ok) {
|
||||||
|
const probe =
|
||||||
|
result.probeResult !== undefined
|
||||||
|
? ` Probe result: ${JSON.stringify(result.probeResult) ?? "undefined"}.`
|
||||||
|
: "";
|
||||||
|
return `PASSED: no console errors, no uncaught errors, no failed requests.${probe} If you are satisfied, finish with your notes; otherwise keep improving and re-test.`;
|
||||||
|
}
|
||||||
|
const sections: string[] = [`FAILED: ${summarizeCounts(result)}.`];
|
||||||
|
if (result.consoleErrors.length > 0)
|
||||||
|
sections.push(`Console errors:\n${result.consoleErrors.slice(0, 10).map((e) => `- ${e}`).join("\n")}`);
|
||||||
|
if (result.pageErrors.length > 0)
|
||||||
|
sections.push(`Uncaught errors:\n${result.pageErrors.slice(0, 10).map((e) => `- ${e}`).join("\n")}`);
|
||||||
|
if (result.failedRequests.length > 0)
|
||||||
|
sections.push(`Failed requests:\n${result.failedRequests.slice(0, 10).map((r) => `- ${r}`).join("\n")}`);
|
||||||
|
sections.push("Fix the document (write_demo) and re-test (run_demo).");
|
||||||
|
return sections.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Short summary of a run for failure notes. */
|
||||||
|
function failureSummary(result: DemoRunResult | null): string {
|
||||||
|
if (!result) return "never tested";
|
||||||
|
const details = [...result.consoleErrors, ...result.pageErrors, ...result.failedRequests].slice(0, 5);
|
||||||
|
return details.length > 0 ? `${summarizeCounts(result)} — ${details.join(" | ")}` : summarizeCounts(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runDemoBuilder(input: DemoBuilderInput, opts: DemoBuilderOptions = {}): Promise<DemoResult> {
|
||||||
|
const headless = opts.headless ?? getHeadless();
|
||||||
|
const title = input.title.trim() || "Interactive demo";
|
||||||
|
const brief = input.brief.trim() || "(no brief)";
|
||||||
|
const notes = (text: string) => (text.trim() ? text.trim() : undefined);
|
||||||
|
|
||||||
|
const reason = headless.unavailableReason();
|
||||||
|
if (reason) {
|
||||||
|
return { title, html: "", tested: true, notes: `FAILED: ${reason}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
const state: BuilderState = { html: null, testedHtml: null, lastResult: null };
|
||||||
|
const agent = new ToolLoopAgent({
|
||||||
|
model: opts.model ?? openrouter(DEFAULT_DEMO_MODEL),
|
||||||
|
instructions: INSTRUCTIONS,
|
||||||
|
tools: createDemoBuilderTools(state, headless, opts.onActivity),
|
||||||
|
stopWhen: stepCountIs(opts.maxSteps ?? 24),
|
||||||
|
});
|
||||||
|
|
||||||
|
const prompt = [
|
||||||
|
`Demo title: ${title}`,
|
||||||
|
`Brief (all you know about the lesson — this comes from the teacher, verbatim):`,
|
||||||
|
brief,
|
||||||
|
"",
|
||||||
|
"Build the demo, test it with run_demo, iterate until PASSED, then finish with your notes.",
|
||||||
|
].join("\n");
|
||||||
|
|
||||||
|
let finalText = "";
|
||||||
|
try {
|
||||||
|
const result = await agent.generate({
|
||||||
|
prompt,
|
||||||
|
onToolExecutionStart: ({ toolCall }) => {
|
||||||
|
opts.onActivity?.(toolCall.toolName);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
finalText = result.text;
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
return { title, html: "", tested: true, notes: `FAILED: the builder crashed — ${msg}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
// The model produced nothing.
|
||||||
|
if (!state.html) {
|
||||||
|
return {
|
||||||
|
title,
|
||||||
|
html: "",
|
||||||
|
tested: true,
|
||||||
|
notes: `FAILED: the builder never wrote a demo.${finalText ? ` (${finalText.slice(0, 300)})` : ""}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// The verify gate: the delivered html must have a clean headless run —
|
||||||
|
// even when the model claims success or forgets the final run_demo.
|
||||||
|
if (state.testedHtml !== state.html || !state.lastResult?.ok) {
|
||||||
|
opts.onActivity?.("final verification");
|
||||||
|
let verify: DemoRunResult;
|
||||||
|
try {
|
||||||
|
verify = await headless.runDemo(state.html);
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
return { title, html: "", tested: true, notes: `FAILED: final verification crashed — ${msg}` };
|
||||||
|
}
|
||||||
|
state.lastResult = verify;
|
||||||
|
if (!verify.ok) {
|
||||||
|
return {
|
||||||
|
title,
|
||||||
|
html: "",
|
||||||
|
tested: true,
|
||||||
|
notes: `FAILED: the demo never passed headless verification (${failureSummary(verify)}).`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { title, html: state.html, tested: true, notes: notes(finalText) };
|
||||||
|
}
|
||||||
@@ -0,0 +1,220 @@
|
|||||||
|
import * as fs from "node:fs";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import { pathToFileURL } from "node:url";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shared headless-browser helper for the verifying sub-agents (W6):
|
||||||
|
* the demo-builder executes demos and captures errors; the visual-maker
|
||||||
|
* rasterizes SVGs so it can look at them. Both use the SAME system Chrome
|
||||||
|
* via puppeteer-core — NO bundled chromium is downloaded.
|
||||||
|
*
|
||||||
|
* Chrome resolution: process.env.CHROME_PATH first, then the common
|
||||||
|
* macOS/Linux install paths. If none exists we fall back to puppeteer's
|
||||||
|
* `channel: "chrome"` at launch time; `available()` is cheap and only knows
|
||||||
|
* about the explicit candidates, so a channel-only setup reports unavailable
|
||||||
|
* and the builders refuse to run (the house rule: never skip testing).
|
||||||
|
*
|
||||||
|
* Test seam: `headlessOverride.current` replaces the whole API (tests inject
|
||||||
|
* a fake; nothing here touches a real browser). Builders call `getHeadless()`.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface DemoRunResult {
|
||||||
|
/** console.error / console messages of type "error". */
|
||||||
|
consoleErrors: string[];
|
||||||
|
/** Uncaught exceptions (window.onerror / unhandled rejections). */
|
||||||
|
pageErrors: string[];
|
||||||
|
/** Network requests that failed (CDN scripts that didn't load, …). */
|
||||||
|
failedRequests: string[];
|
||||||
|
/** Value of the optional probe expression, when requested and successful. */
|
||||||
|
probeResult?: unknown;
|
||||||
|
/** True when nothing above recorded anything. */
|
||||||
|
ok: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RunDemoOptions {
|
||||||
|
/** How long to let the page settle (animations, async CDN loads) — default 2s. */
|
||||||
|
settleMs?: number;
|
||||||
|
/** Optional JS expression evaluated in the page after settling; its value
|
||||||
|
* is returned to the builder so it can assert on internal state. */
|
||||||
|
probe?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RenderSvgOptions {
|
||||||
|
/** Viewport size — defaults to the SVG's width/height attrs, else 900×700. */
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HeadlessApi {
|
||||||
|
/** True when a Chrome executable is resolvable (cheap, no launch). */
|
||||||
|
available(): boolean;
|
||||||
|
/** Why not — for error messages. */
|
||||||
|
unavailableReason(): string | null;
|
||||||
|
/** Load the HTML in headless Chrome, capture errors, optionally probe. */
|
||||||
|
runDemo(html: string, opts?: RunDemoOptions): Promise<DemoRunResult>;
|
||||||
|
/** Rasterize an SVG to a PNG buffer (for verify-by-looking). */
|
||||||
|
renderSvg(svg: string, opts?: RenderSvgOptions): Promise<Buffer>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test seam — undefined in production. */
|
||||||
|
export const headlessOverride: { current?: HeadlessApi } = {};
|
||||||
|
|
||||||
|
export function getHeadless(): HeadlessApi {
|
||||||
|
return headlessOverride.current ?? realHeadless;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Chrome resolution ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const CHROME_CANDIDATES = [
|
||||||
|
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||||
|
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||||
|
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
||||||
|
"/usr/bin/google-chrome",
|
||||||
|
"/usr/bin/google-chrome-stable",
|
||||||
|
"/usr/bin/chromium",
|
||||||
|
"/usr/bin/chromium-browser",
|
||||||
|
];
|
||||||
|
|
||||||
|
export function resolveChromeExecutablePath(): string | null {
|
||||||
|
const candidates = [process.env.CHROME_PATH, ...CHROME_CANDIDATES].filter(
|
||||||
|
(p): p is string => typeof p === "string" && p.length > 0,
|
||||||
|
);
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(candidate)) return candidate;
|
||||||
|
} catch {
|
||||||
|
/* not readable — try the next */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const NO_CHROME_MESSAGE =
|
||||||
|
"No Chrome/Chromium found for headless testing. Install Google Chrome or set CHROME_PATH to a Chrome executable. " +
|
||||||
|
"(Verification is mandatory — demos and visuals are never delivered untested.)";
|
||||||
|
|
||||||
|
async function launchBrowser(): Promise<import("puppeteer-core").Browser> {
|
||||||
|
const puppeteer = (await import("puppeteer-core")).default;
|
||||||
|
const executablePath = resolveChromeExecutablePath();
|
||||||
|
const browser = await puppeteer.launch({
|
||||||
|
headless: true,
|
||||||
|
// Explicit path when we found one; otherwise let puppeteer resolve the
|
||||||
|
// system Chrome install itself via the "chrome" channel.
|
||||||
|
...(executablePath ? { executablePath } : { channel: "chrome" as const }),
|
||||||
|
args: ["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"],
|
||||||
|
});
|
||||||
|
return browser;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A temp dir per invocation; removed in finally. */
|
||||||
|
function tempDir(prefix: string): string {
|
||||||
|
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Demo execution ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async function runDemo(html: string, opts: RunDemoOptions = {}): Promise<DemoRunResult> {
|
||||||
|
const settleMs = opts.settleMs ?? 2000;
|
||||||
|
const dir = tempDir("lh-demo-");
|
||||||
|
const file = path.join(dir, "demo.html");
|
||||||
|
fs.writeFileSync(file, html, "utf-8");
|
||||||
|
|
||||||
|
const consoleErrors: string[] = [];
|
||||||
|
const pageErrors: string[] = [];
|
||||||
|
const failedRequests: string[] = [];
|
||||||
|
let probeResult: unknown;
|
||||||
|
let probed = false;
|
||||||
|
|
||||||
|
const browser = await launchBrowser();
|
||||||
|
try {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
page.on("console", (msg) => {
|
||||||
|
if (msg.type() === "error") consoleErrors.push(msg.text());
|
||||||
|
});
|
||||||
|
page.on("pageerror", (err) => pageErrors.push(err instanceof Error ? err.message : String(err)));
|
||||||
|
page.on("requestfailed", (req) => failedRequests.push(`${req.url()} (${req.failure()?.errorText ?? "failed"})`));
|
||||||
|
|
||||||
|
try {
|
||||||
|
await page.goto(pathToFileURL(file).href, { waitUntil: "load", timeout: 30_000 });
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, settleMs));
|
||||||
|
if (opts.probe) {
|
||||||
|
try {
|
||||||
|
probeResult = await page.evaluate(opts.probe);
|
||||||
|
} catch (err) {
|
||||||
|
pageErrors.push(`probe failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||||
|
}
|
||||||
|
probed = true;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
pageErrors.push(err instanceof Error ? err.message : String(err));
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await browser.close().catch(() => {});
|
||||||
|
try {
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
} catch {
|
||||||
|
/* best-effort cleanup */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
consoleErrors,
|
||||||
|
pageErrors,
|
||||||
|
failedRequests,
|
||||||
|
...(probed ? { probeResult } : {}),
|
||||||
|
ok: consoleErrors.length === 0 && pageErrors.length === 0 && failedRequests.length === 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── SVG rasterization ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function svgViewport(svg: string, opts: RenderSvgOptions): { width: number; height: number } {
|
||||||
|
if (opts.width && opts.height) return { width: opts.width, height: opts.height };
|
||||||
|
const width = Number(svg.match(/\bwidth\s*=\s*"?(\d+(?:\.\d+)?)/)?.[1]);
|
||||||
|
const height = Number(svg.match(/\bheight\s*=\s*"?(\d+(?:\.\d+)?)/)?.[1]);
|
||||||
|
return {
|
||||||
|
width: Math.ceil(Number.isFinite(width) && width > 0 ? width : 900),
|
||||||
|
height: Math.ceil(Number.isFinite(height) && height > 0 ? height : 700),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renderSvg(svg: string, opts: RenderSvgOptions = {}): Promise<Buffer> {
|
||||||
|
const dir = tempDir("lh-visual-");
|
||||||
|
const file = path.join(dir, "figure.html");
|
||||||
|
fs.writeFileSync(
|
||||||
|
file,
|
||||||
|
`<!doctype html><html><head><meta charset="utf-8"><style>html,body{margin:0;padding:0;background:#fff}svg{display:block}</style></head><body>${svg}</body></html>`,
|
||||||
|
"utf-8",
|
||||||
|
);
|
||||||
|
|
||||||
|
const browser = await launchBrowser();
|
||||||
|
try {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
const { width, height } = svgViewport(svg, opts);
|
||||||
|
await page.setViewport({ width, height });
|
||||||
|
await page.goto(pathToFileURL(file).href, { waitUntil: "load", timeout: 30_000 });
|
||||||
|
const png = (await page.screenshot({ fullPage: true, type: "png" })) as Buffer;
|
||||||
|
return png;
|
||||||
|
} finally {
|
||||||
|
await browser.close().catch(() => {});
|
||||||
|
try {
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
} catch {
|
||||||
|
/* best-effort cleanup */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── the real API ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export const realHeadless: HeadlessApi = {
|
||||||
|
available() {
|
||||||
|
return resolveChromeExecutablePath() !== null;
|
||||||
|
},
|
||||||
|
unavailableReason() {
|
||||||
|
return resolveChromeExecutablePath() === null ? NO_CHROME_MESSAGE : null;
|
||||||
|
},
|
||||||
|
runDemo,
|
||||||
|
renderSvg,
|
||||||
|
};
|
||||||
+122
-16
@@ -1,9 +1,11 @@
|
|||||||
import { isToolUIPart, type LanguageModel, type UIMessage } from "ai";
|
import { isToolUIPart, type LanguageModel, type UIMessage } from "ai";
|
||||||
import type { ResearchResult, ResearchToolInput } from "../../core/contracts.ts";
|
import type { DemoResult, DemoToolInput, JobKind, ResearchResult, ResearchToolInput, VisualResult, VisualToolInput } from "../../core/contracts.ts";
|
||||||
import { JOB_TOOLS } from "../../core/pending.ts";
|
import { JOB_TOOLS } from "../../core/pending.ts";
|
||||||
import { getChat, saveMessages } from "../../store/chats.ts";
|
import { getChat, saveMessages } from "../../store/chats.ts";
|
||||||
import { createJob, findJobByToolCallId, getJob, updateJob } from "../../store/jobs.ts";
|
import { createJob, findJobByToolCallId, getJob, updateJob } from "../../store/jobs.ts";
|
||||||
|
import { runDemoBuilder } from "./demo-builder.ts";
|
||||||
import { createResearcherAgent } from "./researcher.ts";
|
import { createResearcherAgent } from "./researcher.ts";
|
||||||
|
import { runVisualMaker } from "./visual-maker.ts";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The background job runner + the chat-handler spawn hook (contracts.ts
|
* The background job runner + the chat-handler spawn hook (contracts.ts
|
||||||
@@ -11,18 +13,22 @@ import { createResearcherAgent } from "./researcher.ts";
|
|||||||
*
|
*
|
||||||
* spawnPendingJobs — called from onFinish AFTER saveMessages: finds
|
* spawnPendingJobs — called from onFinish AFTER saveMessages: finds
|
||||||
* input-available JOB_TOOLS parts in the last assistant message with no
|
* input-available JOB_TOOLS parts in the last assistant message with no
|
||||||
* existing job and fires runResearchJob WITHOUT awaiting (the HTTP
|
* existing job and fires the kind's runner WITHOUT awaiting (the HTTP
|
||||||
* response is long gone).
|
* response is long gone). Kind dispatch (research/demo/visual) is a
|
||||||
|
* self-contained map here, so chat-handler needs no changes for new
|
||||||
|
* job kinds — anything listed in JOB_TOOLS routes automatically.
|
||||||
*
|
*
|
||||||
* runResearchJob — runs the researcher agent loop in the background,
|
* runResearchJob / runDemoJob / runVisualJob — run the sub-agent loop in
|
||||||
* updating the job's activity line on every tool call, then rewrites the
|
* the background, updating the job's activity line on every iteration,
|
||||||
* tool part in the LATEST chat record to output-available (read-modify-
|
* then rewrite the tool part in the LATEST chat record to
|
||||||
* write; no awaits between the read and the write). The part is fulfilled
|
* output-available (read-modify-write; no awaits between the read and
|
||||||
* BEFORE the job status flips, so a client that polls the job and then
|
* the write). The part is fulfilled BEFORE the job status flips, so a
|
||||||
* re-syncs always sees the finished part.
|
* client that polls the job and then re-syncs always sees the finished
|
||||||
|
* part.
|
||||||
*
|
*
|
||||||
* On failure the part is still fulfilled ({brief: "Research failed: …"}) —
|
* On failure the part is still fulfilled (DemoResult/VisualResult failure
|
||||||
* the teacher must learn the outcome — and the job is marked error.
|
* shape: empty html/svg + "FAILED: …" notes; research: failure brief) — the
|
||||||
|
* teacher must learn the outcome — and the job is marked error.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -31,11 +37,20 @@ import { createResearcherAgent } from "./researcher.ts";
|
|||||||
* network. Undefined in production.
|
* network. Undefined in production.
|
||||||
*/
|
*/
|
||||||
export const researchModelOverride: { current?: LanguageModel } = {};
|
export const researchModelOverride: { current?: LanguageModel } = {};
|
||||||
|
export const demoModelOverride: { current?: LanguageModel } = {};
|
||||||
|
export const visualModelOverride: { current?: LanguageModel } = {};
|
||||||
|
|
||||||
export interface RunResearchJobOptions {
|
export interface RunResearchJobOptions {
|
||||||
model?: LanguageModel;
|
model?: LanguageModel;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The kind→runner map: everything in JOB_TOOLS routes through here. */
|
||||||
|
const KIND_RUNNERS: Record<JobKind, (jobId: string) => Promise<void>> = {
|
||||||
|
research: (jobId) => runResearchJob(jobId, { model: researchModelOverride.current }),
|
||||||
|
demo: (jobId) => runDemoJob(jobId, { model: demoModelOverride.current }),
|
||||||
|
visual: (jobId) => runVisualJob(jobId, { model: visualModelOverride.current }),
|
||||||
|
};
|
||||||
|
|
||||||
export function spawnPendingJobs(chatId: string, messages: UIMessage[]): void {
|
export function spawnPendingJobs(chatId: string, messages: UIMessage[]): void {
|
||||||
const last = [...messages].reverse().find((m) => m.role === "assistant");
|
const last = [...messages].reverse().find((m) => m.role === "assistant");
|
||||||
if (!last) return;
|
if (!last) return;
|
||||||
@@ -43,11 +58,12 @@ export function spawnPendingJobs(chatId: string, messages: UIMessage[]): void {
|
|||||||
if (!isToolUIPart(part) || part.state !== "input-available") continue;
|
if (!isToolUIPart(part) || part.state !== "input-available") continue;
|
||||||
const tool = part.type.slice("tool-".length);
|
const tool = part.type.slice("tool-".length);
|
||||||
if (!(JOB_TOOLS as readonly string[]).includes(tool)) continue;
|
if (!(JOB_TOOLS as readonly string[]).includes(tool)) continue;
|
||||||
if (tool !== "research") continue; // demo/visual runners arrive in W6
|
const runner = KIND_RUNNERS[tool as JobKind];
|
||||||
|
if (!runner) continue;
|
||||||
if (findJobByToolCallId(chatId, part.toolCallId)) continue; // already has a job
|
if (findJobByToolCallId(chatId, part.toolCallId)) continue; // already has a job
|
||||||
const job = createJob({ kind: "research", chatId, toolCallId: part.toolCallId });
|
const job = createJob({ kind: tool as JobKind, chatId, toolCallId: part.toolCallId });
|
||||||
void runResearchJob(job.id, { model: researchModelOverride.current }).catch((err) => {
|
void runner(job.id).catch((err) => {
|
||||||
console.error(`[job ${job.id}] research runner crashed:`, err);
|
console.error(`[job ${job.id}] ${tool} runner crashed:`, err);
|
||||||
updateJob(job.id, { status: "error", error: String(err), activity: "failed" });
|
updateJob(job.id, { status: "error", error: String(err), activity: "failed" });
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -103,6 +119,96 @@ export async function runResearchJob(jobId: string, opts: RunResearchJobOptions
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── demo + visual job wrappers ───────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface RunDemoJobOptions {
|
||||||
|
model?: LanguageModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Failure shape (W6 brief): html === "" with "FAILED: …" notes. */
|
||||||
|
function isDemoFailure(result: DemoResult): boolean {
|
||||||
|
return typeof result.html !== "string" || result.html.length === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isVisualFailure(result: VisualResult): boolean {
|
||||||
|
return typeof result.svg !== "string" || result.svg.length === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runDemoJob(jobId: string, opts: RunDemoJobOptions = {}): Promise<void> {
|
||||||
|
const job = getJob(jobId);
|
||||||
|
if (!job || job.status !== "running") return;
|
||||||
|
|
||||||
|
const chat = getChat(job.chatId);
|
||||||
|
if (!chat) {
|
||||||
|
updateJob(jobId, { status: "error", error: `Chat not found: ${job.chatId}`, activity: "failed" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const input = findToolPart(chat.messages, job.toolCallId)?.input as DemoToolInput | undefined;
|
||||||
|
const title = typeof input?.title === "string" && input.title.trim() ? input.title : "Interactive demo";
|
||||||
|
const brief = typeof input?.brief === "string" ? input.brief : "";
|
||||||
|
|
||||||
|
try {
|
||||||
|
updateJob(jobId, { activity: "building demo" });
|
||||||
|
const result = await runDemoBuilder(
|
||||||
|
{ title, brief },
|
||||||
|
{
|
||||||
|
model: opts.model,
|
||||||
|
onActivity: (line) => updateJob(jobId, { activity: line }),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
fulfillPart(job.chatId, job.toolCallId, result);
|
||||||
|
if (isDemoFailure(result)) {
|
||||||
|
updateJob(jobId, { status: "error", error: result.notes ?? "demo build failed", activity: "failed" });
|
||||||
|
} else {
|
||||||
|
updateJob(jobId, { status: "done", result });
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
const reason = err instanceof Error ? err.message : String(err);
|
||||||
|
const failure: DemoResult = { title, html: "", tested: true, notes: `FAILED: ${reason}` };
|
||||||
|
fulfillPart(job.chatId, job.toolCallId, failure);
|
||||||
|
updateJob(jobId, { status: "error", error: reason, activity: "failed" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RunVisualJobOptions {
|
||||||
|
model?: LanguageModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runVisualJob(jobId: string, opts: RunVisualJobOptions = {}): Promise<void> {
|
||||||
|
const job = getJob(jobId);
|
||||||
|
if (!job || job.status !== "running") return;
|
||||||
|
|
||||||
|
const chat = getChat(job.chatId);
|
||||||
|
if (!chat) {
|
||||||
|
updateJob(jobId, { status: "error", error: `Chat not found: ${job.chatId}`, activity: "failed" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const input = findToolPart(chat.messages, job.toolCallId)?.input as VisualToolInput | undefined;
|
||||||
|
const brief = typeof input?.brief === "string" ? input.brief : "";
|
||||||
|
|
||||||
|
try {
|
||||||
|
updateJob(jobId, { activity: "drawing figure" });
|
||||||
|
const result = await runVisualMaker(
|
||||||
|
{ brief },
|
||||||
|
{
|
||||||
|
model: opts.model,
|
||||||
|
onActivity: (line) => updateJob(jobId, { activity: line }),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
fulfillPart(job.chatId, job.toolCallId, result);
|
||||||
|
if (isVisualFailure(result)) {
|
||||||
|
updateJob(jobId, { status: "error", error: result.notes ?? "visual build failed", activity: "failed" });
|
||||||
|
} else {
|
||||||
|
updateJob(jobId, { status: "done", result });
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
const reason = err instanceof Error ? err.message : String(err);
|
||||||
|
const failure: VisualResult = { svg: "", notes: `FAILED: ${reason}` };
|
||||||
|
fulfillPart(job.chatId, job.toolCallId, failure);
|
||||||
|
updateJob(jobId, { status: "error", error: reason, activity: "failed" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ─── helpers ─────────────────────────────────────────────────────────────────
|
// ─── helpers ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
function findToolPart(messages: UIMessage[], toolCallId: string) {
|
function findToolPart(messages: UIMessage[], toolCallId: string) {
|
||||||
@@ -134,7 +240,7 @@ function hostOf(url: string | undefined): string {
|
|||||||
* Rewrite the tool part in the LATEST chat record to output-available.
|
* Rewrite the tool part in the LATEST chat record to output-available.
|
||||||
* Synchronous read-modify-write (no awaits) — atomic within the event loop.
|
* Synchronous read-modify-write (no awaits) — atomic within the event loop.
|
||||||
*/
|
*/
|
||||||
function fulfillPart(chatId: string, toolCallId: string, output: ResearchResult): void {
|
function fulfillPart(chatId: string, toolCallId: string, output: ResearchResult | DemoResult | VisualResult): void {
|
||||||
const chat = getChat(chatId);
|
const chat = getChat(chatId);
|
||||||
if (!chat) return;
|
if (!chat) return;
|
||||||
let found = false;
|
let found = false;
|
||||||
|
|||||||
@@ -0,0 +1,229 @@
|
|||||||
|
import { generateText, tool, type LanguageModel, type ModelMessage, type ToolResultPart } from "ai";
|
||||||
|
import { openrouter } from "@openrouter/ai-sdk-provider";
|
||||||
|
import { z } from "zod";
|
||||||
|
import type { VisualResult } from "../../core/contracts.ts";
|
||||||
|
import { getHeadless, type HeadlessApi } from "./headless.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The visual-maker sub-agent (vision §2.5): authors ONE precise SVG from a
|
||||||
|
* brief, renders it to a PNG in the SAME headless Chrome, LOOKS at the
|
||||||
|
* result, and iterates until correct — a direct port of the vault POC's
|
||||||
|
* svg-maker doctrine ("the one rule that matters: verify by looking").
|
||||||
|
*
|
||||||
|
* LOOP SHAPE (decided in the W6 brief): hand-rolled with generateText + a
|
||||||
|
* messages array, because the loop must feed an IMAGE back to the model after
|
||||||
|
* each render. write_svg / render_svg are declared WITHOUT execute — when the
|
||||||
|
* model calls them generateText stops with the tool calls, and this loop
|
||||||
|
* appends the tool results itself, followed by a user message containing the
|
||||||
|
* rendered PNG as a file part ("Here is the current render"). ToolLoopAgent's
|
||||||
|
* execute-tool plumbing can't express that image-in-the-loop shape cleanly.
|
||||||
|
*
|
||||||
|
* Doctrine enforcement in code: if the model tries to finish while the
|
||||||
|
* current SVG has never been rendered (or changed since its last render), the
|
||||||
|
* loop renders it anyway, shows the image, and asks the model to look before
|
||||||
|
* finishing. Nothing is delivered without having been looked at.
|
||||||
|
*
|
||||||
|
* Failure semantics mirror the demo-builder: { svg: "", notes: "FAILED: …" }.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const DEFAULT_VISUAL_MODEL =
|
||||||
|
process.env.LH_VISUAL_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
|
||||||
|
|
||||||
|
export interface VisualMakerInput {
|
||||||
|
brief: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VisualMakerOptions {
|
||||||
|
model?: LanguageModel;
|
||||||
|
headless?: HeadlessApi;
|
||||||
|
/** Max model rounds (write/render cycles + final answer). Default 8. */
|
||||||
|
maxIterations?: number;
|
||||||
|
/** Live activity-line callback (updates the job record). */
|
||||||
|
onActivity?: (line: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const INSTRUCTIONS = `You are a visual maker — a diagram author + renderer for precise, spatial figures. You
|
||||||
|
receive a brief describing ONE idea that needs exact placement (coordinate geometry, number lines,
|
||||||
|
vectors, function plots, physical layouts — things auto-layout diagrams cannot express), and you return
|
||||||
|
ONE clean, correct SVG. You never see the lesson; the brief is everything you know.
|
||||||
|
|
||||||
|
You do NOT decide what idea to show — the teacher already decided. Your job is faithful, precise
|
||||||
|
composition, and above everything CORRECTNESS: the picture must not assert anything false. A right-angle
|
||||||
|
mark on the wrong corner, a vector pointing the wrong way, a point plotted at the wrong coordinate is a
|
||||||
|
failure even if it renders cleanly.
|
||||||
|
|
||||||
|
Your two tools:
|
||||||
|
- write_svg({ svg }) — write the complete <svg>…</svg> source (overwrites the previous version).
|
||||||
|
- render_svg({}) — rasterize the current SVG in headless Chrome. The render arrives as an image in the
|
||||||
|
message right after the tool result.
|
||||||
|
|
||||||
|
THE ONE RULE THAT MATTERS MOST: verify by looking. You are done only when you have LOOKED at the
|
||||||
|
rendered PNG and confirmed it is true to the brief. Rendering success only proves the SVG parsed; it
|
||||||
|
says nothing about whether the geometry is right or the picture is readable. When you look:
|
||||||
|
|
||||||
|
- Is every coordinate, angle, direction, and proportion actually correct? Re-derive the geometry if unsure.
|
||||||
|
- Are labels placed clearly, not overlapping lines or each other?
|
||||||
|
- Is anything clipped by the viewBox, too small to read, or cramped?
|
||||||
|
- Would a learner instantly read the intended idea from this picture alone?
|
||||||
|
|
||||||
|
Workflow: plan the coordinate space (viewBox + margins, nothing touching the edges) → write_svg →
|
||||||
|
render_svg → LOOK critically → fix (write_svg) and re-render until correct and clean. Never finish
|
||||||
|
without having looked at a render of your FINAL SVG.
|
||||||
|
|
||||||
|
SVG requirements:
|
||||||
|
- A complete <svg> with explicit width/height AND a matching viewBox.
|
||||||
|
- White background (<rect> or style), dark strokes, sans-serif labels at generous sizes.
|
||||||
|
- One idea, fewest elements — sparse and large beats busy and tiny.
|
||||||
|
- Draw only what the brief specifies; never invent data, values, or shapes to fill space.
|
||||||
|
- Plain, clean styling: at most one accent color. An explanatory diagram, not art.
|
||||||
|
|
||||||
|
Your FINAL message: 1-2 sentences of notes for the teacher (what the figure shows, any caveats) —
|
||||||
|
NEVER the SVG source itself. If you genuinely cannot make a correct picture of the brief, say exactly
|
||||||
|
that in your final message and why.`;
|
||||||
|
|
||||||
|
export function createVisualTools() {
|
||||||
|
return {
|
||||||
|
write_svg: tool({
|
||||||
|
description: "Write the complete SVG source for the figure (overwrites any previous version).",
|
||||||
|
inputSchema: z.object({
|
||||||
|
svg: z.string().describe('Complete <svg xmlns="…">…</svg> markup.'),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
render_svg: tool({
|
||||||
|
description: "Render the current SVG to a PNG and show it to you for inspection. No arguments.",
|
||||||
|
inputSchema: z.object({}),
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const LOOK_MESSAGE =
|
||||||
|
"Here is the current render of your SVG — inspect it critically (geometry, labels, clipping, readability):";
|
||||||
|
const UNLOOKED_FINISH_MESSAGE =
|
||||||
|
"You finished without looking at a render of your final SVG. Above is its render. If it is correct and clean, reply with just your notes; otherwise call write_svg with fixes and render again.";
|
||||||
|
|
||||||
|
export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerOptions = {}): Promise<VisualResult> {
|
||||||
|
const headless = opts.headless ?? getHeadless();
|
||||||
|
const brief = input.brief.trim() || "(no brief)";
|
||||||
|
const maxIterations = opts.maxIterations ?? 8;
|
||||||
|
|
||||||
|
const reason = headless.unavailableReason();
|
||||||
|
if (reason) {
|
||||||
|
return { svg: "", notes: `FAILED: ${reason}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
const tools = createVisualTools();
|
||||||
|
const prompt = [
|
||||||
|
"Brief for ONE precise figure (from the teacher, verbatim — it is everything you know):",
|
||||||
|
brief,
|
||||||
|
"",
|
||||||
|
"Author the SVG, render it, look at it, iterate until correct, then finish with your notes.",
|
||||||
|
].join("\n");
|
||||||
|
|
||||||
|
let messages: ModelMessage[] = [{ role: "user", content: prompt }];
|
||||||
|
let currentSvg: string | null = null;
|
||||||
|
/** The svg shown in the most recent render (null = never rendered). */
|
||||||
|
let renderedSvg: string | null = null;
|
||||||
|
let finalText = "";
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (let iteration = 0; iteration < maxIterations; iteration++) {
|
||||||
|
const result = await generateText({ model: opts.model ?? openrouter(DEFAULT_VISUAL_MODEL), system: INSTRUCTIONS, messages, tools });
|
||||||
|
messages = [...messages, ...result.response.messages];
|
||||||
|
|
||||||
|
const toolCalls = result.toolCalls;
|
||||||
|
if (toolCalls.length === 0) {
|
||||||
|
finalText = result.text.trim();
|
||||||
|
// Doctrine gate: never deliver an unlooked SVG. If the current SVG
|
||||||
|
// changed since its last render (or was never rendered), show it now
|
||||||
|
// and make the model look before accepting the finish.
|
||||||
|
if (currentSvg && renderedSvg !== currentSvg) {
|
||||||
|
let png: Buffer;
|
||||||
|
try {
|
||||||
|
png = await headless.renderSvg(currentSvg);
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
return { svg: "", notes: `FAILED: rendering the final SVG failed — ${msg}` };
|
||||||
|
}
|
||||||
|
opts.onActivity?.("render_svg: inspecting final render");
|
||||||
|
renderedSvg = currentSvg;
|
||||||
|
messages = [
|
||||||
|
...messages,
|
||||||
|
{ role: "user", content: [{ type: "file", data: png, mediaType: "image/png" }, { type: "text", text: UNLOOKED_FINISH_MESSAGE }] },
|
||||||
|
];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return finish(currentSvg, finalText);
|
||||||
|
}
|
||||||
|
|
||||||
|
const toolResults: ToolResultPart[] = [];
|
||||||
|
let renderPng: Buffer | null = null; // appended as the image message AFTER the tool results
|
||||||
|
for (const call of toolCalls) {
|
||||||
|
if (call.toolName === "write_svg") {
|
||||||
|
const svg = (call.input as { svg?: unknown }).svg;
|
||||||
|
if (typeof svg !== "string" || !svg.includes("<svg")) {
|
||||||
|
toolResults.push(toolText(call.toolCallId, call.toolName, "ERROR: expected a complete <svg>…</svg> document in the `svg` field."));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
currentSvg = svg;
|
||||||
|
opts.onActivity?.(`write_svg: ${(svg.length / 1024).toFixed(1)} KB`);
|
||||||
|
toolResults.push(toolText(call.toolCallId, call.toolName, `Saved (${svg.length} bytes). Call render_svg to look at it.`));
|
||||||
|
} else if (call.toolName === "render_svg") {
|
||||||
|
if (!currentSvg) {
|
||||||
|
toolResults.push(toolText(call.toolCallId, call.toolName, "ERROR: nothing to render — call write_svg first."));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const png = await headless.renderSvg(currentSvg);
|
||||||
|
renderPng = png;
|
||||||
|
renderedSvg = currentSvg;
|
||||||
|
opts.onActivity?.("render_svg: inspecting render");
|
||||||
|
toolResults.push(toolText(call.toolCallId, call.toolName, "Rendered — the PNG follows in the next message. Look at it."));
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
opts.onActivity?.("render_svg: render error");
|
||||||
|
toolResults.push(toolText(call.toolCallId, call.toolName, `RENDER ERROR (the SVG may be malformed): ${msg}. Fix it with write_svg and render again.`));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
toolResults.push(toolText(call.toolCallId, call.toolName, `ERROR: unknown tool "${call.toolName}". Use write_svg or render_svg.`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Message order: assistant tool-calls → tool results → render image.
|
||||||
|
messages = [...messages, { role: "tool", content: toolResults }];
|
||||||
|
if (renderPng) {
|
||||||
|
messages = [
|
||||||
|
...messages,
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
{ type: "text", text: LOOK_MESSAGE },
|
||||||
|
{ type: "file", data: renderPng, mediaType: "image/png" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
return { svg: "", notes: `FAILED: the maker crashed — ${msg}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Iteration limit: deliver the last written SVG only if it was looked at.
|
||||||
|
if (currentSvg && renderedSvg === currentSvg) {
|
||||||
|
return { svg: currentSvg, notes: finalText || "Iteration limit reached — delivering the last verified SVG." };
|
||||||
|
}
|
||||||
|
return { svg: "", notes: `FAILED: iteration limit (${maxIterations}) reached without a verified SVG.` };
|
||||||
|
}
|
||||||
|
|
||||||
|
function toolText(toolCallId: string, toolName: string, value: string): ToolResultPart {
|
||||||
|
return { type: "tool-result", toolCallId, toolName, output: { type: "text", value } };
|
||||||
|
}
|
||||||
|
|
||||||
|
function finish(svg: string | null, finalText: string): VisualResult {
|
||||||
|
if (!svg) {
|
||||||
|
return {
|
||||||
|
svg: "",
|
||||||
|
notes: `FAILED: the maker never wrote an SVG.${finalText ? ` (${finalText.slice(0, 300)})` : ""}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { svg, notes: finalText || undefined };
|
||||||
|
}
|
||||||
+45
-2
@@ -31,16 +31,59 @@ export const researchTool = tool({
|
|||||||
inputSchema: researchInputSchema,
|
inputSchema: researchInputSchema,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const demoInputSchema = z.object({
|
||||||
|
title: z.string().describe("Short title for the demo card, e.g. 'Damped harmonic oscillator'."),
|
||||||
|
brief: z
|
||||||
|
.string()
|
||||||
|
.describe(
|
||||||
|
"Precise brief for the builder: the concept, the parameters/controls and what each does, the interactions, and the visual style. " +
|
||||||
|
"The builder sees NOTHING else of the lesson — include everything it needs, including necessary formulas and constants.",
|
||||||
|
),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const demoTool = tool({
|
||||||
|
description:
|
||||||
|
"Dispatch a background sub-agent that BUILDS and headlessly TESTS an interactive teaching instrument — a single-file HTML " +
|
||||||
|
"demo (parameter sweep, simulation, animation, sampling) rendered live in the lesson. Use it when the concept is DYNAMIC: " +
|
||||||
|
"something to sweep, animate, or sample is better experienced than described. " +
|
||||||
|
"The call is ASYNC — your turn ENDS here, so say everything you want to say first; the tested demo arrives later as this " +
|
||||||
|
"tool's output and the lesson resumes with it. Write a precise brief: concept, controls, interactions, style.",
|
||||||
|
inputSchema: demoInputSchema,
|
||||||
|
});
|
||||||
|
|
||||||
|
export const visualInputSchema = z.object({
|
||||||
|
brief: z
|
||||||
|
.string()
|
||||||
|
.describe(
|
||||||
|
"Precise brief for ONE static figure: exactly what it must show — coordinates, labels, proportions, orientation. " +
|
||||||
|
"The maker sees NOTHING else of the lesson; a wrong or vague brief produces a wrong figure.",
|
||||||
|
),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const visualTool = tool({
|
||||||
|
description:
|
||||||
|
"Dispatch a background sub-agent that hand-authors ONE precise SVG figure, renders it, LOOKS at the render, and iterates " +
|
||||||
|
"until correct. Use it for exact spatial pictures auto-layout diagrams can't do: coordinate geometry, number lines, vectors, " +
|
||||||
|
"function plots, physical layouts. For flows/relations/taxonomies write mermaid inline instead. " +
|
||||||
|
"The call is ASYNC — your turn ENDS here, so say everything you want to say first; the verified figure arrives later as this " +
|
||||||
|
"tool's output and the lesson resumes with it.",
|
||||||
|
inputSchema: visualInputSchema,
|
||||||
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Per-request job tool registry. Context-free for now (the runner resolves
|
* Per-request job tool registry. Context-free (the runner resolves
|
||||||
* everything from the chat record by toolCallId) — the ctx parameter keeps
|
* everything from the chat record by toolCallId) — the ctx parameter keeps
|
||||||
* the createTeacherTools factory shape stable for W6's demo/visual tools.
|
* the createTeacherTools factory shape stable.
|
||||||
*/
|
*/
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||||
export function jobTools(ctx: TeacherToolContext): ToolSet {
|
export function jobTools(ctx: TeacherToolContext): ToolSet {
|
||||||
return {
|
return {
|
||||||
research: researchTool,
|
research: researchTool,
|
||||||
|
demo: demoTool,
|
||||||
|
visual: visualTool,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ResearchInput = z.infer<typeof researchInputSchema>;
|
export type ResearchInput = z.infer<typeof researchInputSchema>;
|
||||||
|
export type DemoInput = z.infer<typeof demoInputSchema>;
|
||||||
|
export type VisualInput = z.infer<typeof visualInputSchema>;
|
||||||
|
|||||||
@@ -0,0 +1,619 @@
|
|||||||
|
/**
|
||||||
|
* Demo + visual artifact tests (W6) — deterministic, zero network, no real
|
||||||
|
* Chrome:
|
||||||
|
*
|
||||||
|
* 1. bridge-script injection (buildDemoSrcDoc) + the pure bridge-message
|
||||||
|
* reducer (reduceDemoBridgeState)
|
||||||
|
* 2. card rendering (static markup): sandboxed iframe attrs, error chip,
|
||||||
|
* failure notes, inline sanitized svg
|
||||||
|
* 3. the demo-builder loop with a FAKE model + FAKE headless (success with
|
||||||
|
* a fix iteration, untested-final verification, never-clean failure,
|
||||||
|
* no-Chrome failure)
|
||||||
|
* 4. the visual-maker loop (fake model + fake headless): render-and-look
|
||||||
|
* success, forced look on premature finish, never-writes failure
|
||||||
|
* 5. job wrappers + spawn-hook kind dispatch (demo part → demo job →
|
||||||
|
* part fulfilled; visual failure path)
|
||||||
|
* 6. tool registry wiring (client-side demo/visual tools, schemas)
|
||||||
|
*
|
||||||
|
* Cleanup follows tests/jobs.test.ts idioms (created chat/job files removed
|
||||||
|
* in the after() hook; overrides reset in finally blocks).
|
||||||
|
*/
|
||||||
|
|
||||||
|
import React from "react";
|
||||||
|
(globalThis as Record<string, unknown>).React = React; // tsx classic-JSX shim (see render-cards.test.tsx)
|
||||||
|
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import * as fs from "node:fs";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import { after, describe, it } from "node:test";
|
||||||
|
import { MockLanguageModelV4, simulateReadableStream } from "ai/test";
|
||||||
|
import type { LanguageModelV4StreamPart } from "@ai-sdk/provider";
|
||||||
|
import { renderToStaticMarkup } from "react-dom/server";
|
||||||
|
import { PATHS } from "../config.ts";
|
||||||
|
import {
|
||||||
|
buildDemoSrcDoc,
|
||||||
|
DemoCard,
|
||||||
|
DemoErrorChip,
|
||||||
|
INITIAL_DEMO_BRIDGE_STATE,
|
||||||
|
reduceDemoBridgeState,
|
||||||
|
} from "../components/artifacts/demo-card.tsx";
|
||||||
|
import { sanitizeSvg, VisualCard } from "../components/artifacts/visual-card.tsx";
|
||||||
|
import { demoInputSchema, jobTools, visualInputSchema } from "../src/agent/tools/jobs.ts";
|
||||||
|
import { runDemoBuilder } from "../src/agent/subagents/demo-builder.ts";
|
||||||
|
import { headlessOverride, type DemoRunResult, type HeadlessApi } from "../src/agent/subagents/headless.ts";
|
||||||
|
import { demoModelOverride, runDemoJob, runVisualJob, spawnPendingJobs, visualModelOverride } from "../src/agent/subagents/runner.ts";
|
||||||
|
import { runVisualMaker } from "../src/agent/subagents/visual-maker.ts";
|
||||||
|
import { createChat, getChat, saveMessages } from "../src/store/chats.ts";
|
||||||
|
import { createJob, findJobByToolCallId, getJob, listJobs } from "../src/store/jobs.ts";
|
||||||
|
import { handleChatRequest } from "../src/server/chat-handler.ts";
|
||||||
|
import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
|
||||||
|
import { artifactRenderers } from "../components/artifacts/index.ts";
|
||||||
|
|
||||||
|
const JOBS_DIR = path.join(PATHS.data, "jobs");
|
||||||
|
const createdChats: string[] = [];
|
||||||
|
const createdJobs: string[] = [];
|
||||||
|
|
||||||
|
after(() => {
|
||||||
|
for (const id of createdChats) {
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(path.join(PATHS.chats, `${id}.json`));
|
||||||
|
} catch {
|
||||||
|
/* already gone */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const id of createdJobs) {
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(path.join(JOBS_DIR, `${id}.json`));
|
||||||
|
} catch {
|
||||||
|
/* already gone */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
headlessOverride.current = undefined;
|
||||||
|
demoModelOverride.current = undefined;
|
||||||
|
visualModelOverride.current = undefined;
|
||||||
|
});
|
||||||
|
|
||||||
|
function usage() {
|
||||||
|
return {
|
||||||
|
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
|
||||||
|
outputTokens: { total: 5, text: 5, reasoning: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A model that answers a scripted sequence of doGenerate turns (tool calls or final text). */
|
||||||
|
function scriptedModel(turns: Array<{ toolCalls?: Array<{ toolCallId: string; toolName: string; input: unknown }>; text?: string }>): MockLanguageModelV4 {
|
||||||
|
let turn = -1;
|
||||||
|
return new MockLanguageModelV4({
|
||||||
|
doGenerate: async () => {
|
||||||
|
turn++;
|
||||||
|
const step = turns[turn];
|
||||||
|
if (!step) throw new Error(`scripted model exhausted after ${turns.length} turns`);
|
||||||
|
if (step.toolCalls) {
|
||||||
|
return {
|
||||||
|
content: step.toolCalls.map((c) => ({ type: "tool-call" as const, toolCallId: c.toolCallId, toolName: c.toolName, input: JSON.stringify(c.input) })),
|
||||||
|
finishReason: { unified: "tool-calls" as const, raw: "tool-calls" },
|
||||||
|
usage: usage(),
|
||||||
|
warnings: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
content: [{ type: "text" as const, text: step.text ?? "" }],
|
||||||
|
finishReason: { unified: "stop" as const, raw: "stop" },
|
||||||
|
usage: usage(),
|
||||||
|
warnings: [],
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── fake headless ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const CLEAN_RUN: DemoRunResult = { consoleErrors: [], pageErrors: [], failedRequests: [], ok: true };
|
||||||
|
const BROKEN_RUN: DemoRunResult = {
|
||||||
|
consoleErrors: ["Uncaught ReferenceError: foo is not defined"],
|
||||||
|
pageErrors: [],
|
||||||
|
failedRequests: [],
|
||||||
|
ok: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
function fakeHeadless(opts: {
|
||||||
|
/** Return BROKEN_RUN for html matching this substring, else CLEAN_RUN. */
|
||||||
|
brokenWhenIncludes?: string;
|
||||||
|
/** Throw for the first N renderSvg calls (then succeed). */
|
||||||
|
renderFailures?: number;
|
||||||
|
} = {}): HeadlessApi & { demoRuns: string[]; svgRenders: string[] } {
|
||||||
|
const api = {
|
||||||
|
demoRuns: [] as string[],
|
||||||
|
svgRenders: [] as string[],
|
||||||
|
available() {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
unavailableReason() {
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
async runDemo(html: string): Promise<DemoRunResult> {
|
||||||
|
api.demoRuns.push(html);
|
||||||
|
return opts.brokenWhenIncludes && html.includes(opts.brokenWhenIncludes) ? BROKEN_RUN : CLEAN_RUN;
|
||||||
|
},
|
||||||
|
async renderSvg(svg: string): Promise<Buffer> {
|
||||||
|
api.svgRenders.push(svg);
|
||||||
|
if (opts.renderFailures && api.svgRenders.length <= opts.renderFailures) {
|
||||||
|
throw new Error("XML parse error");
|
||||||
|
}
|
||||||
|
return Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); // PNG magic
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return api as HeadlessApi & { demoRuns: string[]; svgRenders: string[] };
|
||||||
|
}
|
||||||
|
|
||||||
|
function noChromeHeadless(): HeadlessApi {
|
||||||
|
return {
|
||||||
|
available: () => false,
|
||||||
|
unavailableReason: () => "No Chrome/Chromium found for headless testing. Install Google Chrome or set CHROME_PATH.",
|
||||||
|
runDemo: async () => {
|
||||||
|
throw new Error("should not run");
|
||||||
|
},
|
||||||
|
renderSvg: async () => {
|
||||||
|
throw new Error("should not render");
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── 1. bridge injection + reducer ───────────────────────────────────────────
|
||||||
|
|
||||||
|
describe("demo bridge", () => {
|
||||||
|
it("injects the resize + error hooks before </body>", () => {
|
||||||
|
const srcDoc = buildDemoSrcDoc("<!doctype html><html><body><canvas id=\"c\"></canvas></body></html>");
|
||||||
|
assert.match(srcDoc, /demo-resize/);
|
||||||
|
assert.match(srcDoc, /demo-error/);
|
||||||
|
assert.match(srcDoc, /ResizeObserver/);
|
||||||
|
assert.match(srcDoc, /window\.addEventListener\("error"/);
|
||||||
|
// the bridge lands inside the document, and the original content survives
|
||||||
|
assert.ok(srcDoc.indexOf("ResizeObserver") < srcDoc.indexOf("</body>"));
|
||||||
|
assert.match(srcDoc, /<canvas id="c">/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("appends the bridge when the document has no </body>", () => {
|
||||||
|
const srcDoc = buildDemoSrcDoc("<div>fragment demo</div>");
|
||||||
|
assert.match(srcDoc, /demo-resize/);
|
||||||
|
assert.ok(srcDoc.endsWith("</script>"));
|
||||||
|
assert.match(srcDoc, /fragment demo/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reduceDemoBridgeState handles resize, error, and garbage", () => {
|
||||||
|
let state = INITIAL_DEMO_BRIDGE_STATE;
|
||||||
|
state = reduceDemoBridgeState(state, { type: "demo-resize", height: 421.7 });
|
||||||
|
assert.equal(state.height, 422);
|
||||||
|
assert.equal(state.error, null);
|
||||||
|
// clamped
|
||||||
|
state = reduceDemoBridgeState(state, { type: "demo-resize", height: 99999 });
|
||||||
|
assert.equal(state.height, 2400);
|
||||||
|
state = reduceDemoBridgeState(state, { type: "demo-resize", height: 5 });
|
||||||
|
assert.equal(state.height, 80);
|
||||||
|
// invalid payloads ignored
|
||||||
|
assert.equal(reduceDemoBridgeState(state, { type: "demo-resize", height: "tall" }).height, 80);
|
||||||
|
assert.equal(reduceDemoBridgeState(state, { type: "demo-resize" }).height, 80);
|
||||||
|
// errors recorded, with a fallback message
|
||||||
|
state = reduceDemoBridgeState(state, { type: "demo-error", message: "x is not defined" });
|
||||||
|
assert.equal(state.error, "x is not defined");
|
||||||
|
assert.equal(state.height, 80, "error does not clobber height");
|
||||||
|
state = reduceDemoBridgeState(state, { type: "demo-error" });
|
||||||
|
assert.equal(state.error, "unknown error");
|
||||||
|
// unknown types and non-objects ignored
|
||||||
|
assert.deepEqual(reduceDemoBridgeState(state, { type: "something-else" }), state);
|
||||||
|
assert.deepEqual(reduceDemoBridgeState(state, "demo-resize"), state);
|
||||||
|
assert.deepEqual(reduceDemoBridgeState(state, null), state);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── 2. card rendering ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const DEMO_HTML = "<!doctype html><html><body><h1>oscillator</h1><script>let t=0;</script></body></html>";
|
||||||
|
|
||||||
|
function part(state: string, output?: unknown) {
|
||||||
|
return { type: "tool-demo", state, toolCallId: "call-1", input: { title: "Oscillator", brief: "b" }, output };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("demo card rendering", () => {
|
||||||
|
it("renders a sandboxed iframe (allow-scripts, NO allow-same-origin) with the bridge injected", () => {
|
||||||
|
const html = renderToStaticMarkup(<DemoCard part={part("output-available", { title: "Damped oscillator", html: DEMO_HTML, tested: true }) as never} />);
|
||||||
|
assert.match(html, /sandbox="allow-scripts"/);
|
||||||
|
assert.ok(!html.includes("allow-same-origin"), "never allow-same-origin");
|
||||||
|
assert.match(html, /demo-resize/);
|
||||||
|
assert.match(html, /demo-error/);
|
||||||
|
assert.match(html, /Damped oscillator/);
|
||||||
|
assert.match(html, /interactive demo/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders the failure note with no iframe when the build FAILED", () => {
|
||||||
|
const html = renderToStaticMarkup(
|
||||||
|
<DemoCard part={part("output-available", { title: "T", html: "", tested: true, notes: "FAILED: never passed verification" }) as never} />,
|
||||||
|
);
|
||||||
|
assert.match(html, /FAILED: never passed verification/);
|
||||||
|
assert.ok(!html.includes("<iframe"), "no iframe for a failed build");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders notes in muted small text", () => {
|
||||||
|
const html = renderToStaticMarkup(
|
||||||
|
<DemoCard part={part("output-available", { title: "T", html: DEMO_HTML, tested: true, notes: "Drag the damping slider." }) as never} />,
|
||||||
|
);
|
||||||
|
assert.match(html, /Drag the damping slider\./);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders the activity card while the job runs", () => {
|
||||||
|
const html = renderToStaticMarkup(<DemoCard part={part("input-available") as never} />);
|
||||||
|
assert.match(html, /Building demo…/);
|
||||||
|
assert.ok(!html.includes("<iframe"));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders the error chip with a Send to teacher button", () => {
|
||||||
|
const html = renderToStaticMarkup(<DemoErrorChip title="T" message="boom is not defined" onSend={() => {}} />);
|
||||||
|
assert.match(html, /This demo threw an error/);
|
||||||
|
assert.match(html, /boom is not defined/);
|
||||||
|
assert.match(html, /Send to teacher/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("visual card rendering", () => {
|
||||||
|
it("renders the verified svg inline (sanitized) with notes", () => {
|
||||||
|
const svg = '<svg xmlns="http://www.w3.org/2000/svg" width="200" height="100"><circle cx="50" cy="50" r="20"/></svg>';
|
||||||
|
const html = renderToStaticMarkup(
|
||||||
|
<VisualCard part={{ type: "tool-visual", state: "output-available", toolCallId: "c", output: { svg, notes: "The unit circle." } } as never} />,
|
||||||
|
);
|
||||||
|
assert.match(html, /The unit circle\./);
|
||||||
|
assert.ok(html.includes("circle"), "svg content rendered inline");
|
||||||
|
assert.match(html, /Figure/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders the failure note with no svg when the build FAILED", () => {
|
||||||
|
const html = renderToStaticMarkup(
|
||||||
|
<VisualCard part={{ type: "tool-visual", state: "output-available", toolCallId: "c", output: { svg: "", notes: "FAILED: iteration limit" } } as never} />,
|
||||||
|
);
|
||||||
|
assert.match(html, /FAILED: iteration limit/);
|
||||||
|
assert.ok(!html.includes("circle") && !html.includes("<svg"), "no svg for a failed build");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders the activity card while the job runs", () => {
|
||||||
|
const html = renderToStaticMarkup(<VisualCard part={{ type: "tool-visual", state: "input-available", toolCallId: "c" } as never} />);
|
||||||
|
assert.match(html, /Drawing figure…/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("sanitizeSvg", () => {
|
||||||
|
it("strips script blocks, event handlers, and javascript: urls, keeps geometry", () => {
|
||||||
|
const dirty = '<svg width="10" height="10" onload="alert(1)"><script>alert(2)</script><circle r="1" onclick="pwn()"/><a href="javascript:alert(3)"><rect width="2" height="2"/></a></svg>';
|
||||||
|
const clean = sanitizeSvg(dirty);
|
||||||
|
assert.ok(!/<script/i.test(clean), "script tags removed");
|
||||||
|
assert.ok(!/\sonload/i.test(clean), "onload removed");
|
||||||
|
assert.ok(!/\sonclick/i.test(clean), "onclick removed");
|
||||||
|
assert.ok(!/javascript:/i.test(clean), "javascript: urls removed");
|
||||||
|
assert.ok(clean.includes("circle") && clean.includes("rect"), "geometry preserved");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── 3. the demo-builder loop ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const BROKEN_HTML = "<!doctype html><html><body>BROKEN<script>foo();</script></body></html>";
|
||||||
|
const GOOD_HTML = "<!doctype html><html><body><canvas id=\"c\"></canvas><script>requestAnimationFrame(function step(){});</script></body></html>";
|
||||||
|
|
||||||
|
describe("demo builder (fake model + fake headless)", () => {
|
||||||
|
it("iterates write → run → fix → run → clean → delivers with notes", async () => {
|
||||||
|
const headless = fakeHeadless({ brokenWhenIncludes: "BROKEN" });
|
||||||
|
const activities: string[] = [];
|
||||||
|
const model = scriptedModel([
|
||||||
|
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: BROKEN_HTML } }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "r1", toolName: "run_demo", input: {} }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "w2", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "r2", toolName: "run_demo", input: { probe: "typeof step === 'function'" } }] },
|
||||||
|
{ text: "Drag the damping slider to see the oscillation decay." },
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = await runDemoBuilder(
|
||||||
|
{ title: "Damped oscillator", brief: "A mass on a spring with a damping slider." },
|
||||||
|
{ model, headless, onActivity: (line) => activities.push(line) },
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(result.title, "Damped oscillator");
|
||||||
|
assert.equal(result.html, GOOD_HTML, "delivers the FIXED html");
|
||||||
|
assert.equal(result.tested, true);
|
||||||
|
assert.equal(result.notes, "Drag the damping slider to see the oscillation decay.");
|
||||||
|
assert.equal(headless.demoRuns.length, 2, "ran the broken demo, then the fixed one — no extra verification needed");
|
||||||
|
assert.ok(activities.some((a) => a.startsWith("run_demo: 1 console error")), `activity recorded per iteration: ${activities.join(" | ")}`);
|
||||||
|
assert.ok(activities.includes("run_demo: clean"), "clean run recorded in activity");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("verifies headlessly even when the model forgets run_demo before finishing", async () => {
|
||||||
|
const headless = fakeHeadless();
|
||||||
|
const model = scriptedModel([
|
||||||
|
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||||
|
{ text: "Done." },
|
||||||
|
]);
|
||||||
|
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless });
|
||||||
|
assert.equal(result.html, GOOD_HTML);
|
||||||
|
assert.equal(result.tested, true, "tested by the final verification gate");
|
||||||
|
assert.equal(headless.demoRuns.length, 1, "the gate ran the demo once itself");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails (empty html + FAILED notes) when the demo never passes verification", async () => {
|
||||||
|
const headless = fakeHeadless({ brokenWhenIncludes: "BROKEN" });
|
||||||
|
const model = scriptedModel([
|
||||||
|
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: BROKEN_HTML } }] },
|
||||||
|
{ text: "It works, trust me." },
|
||||||
|
]);
|
||||||
|
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless });
|
||||||
|
assert.equal(result.html, "");
|
||||||
|
assert.equal(result.tested, true, "the loop ran");
|
||||||
|
assert.match(result.notes ?? "", /^FAILED:/);
|
||||||
|
assert.match(result.notes ?? "", /ReferenceError/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails fast with a clear message when no Chrome is available (never skips testing)", async () => {
|
||||||
|
const model = scriptedModel([{ text: "should never be reached" }]);
|
||||||
|
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless: noChromeHeadless() });
|
||||||
|
assert.equal(result.html, "");
|
||||||
|
assert.match(result.notes ?? "", /^FAILED: .*Chrome/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails when the model never writes a demo", async () => {
|
||||||
|
const model = scriptedModel([{ text: "I cannot build this." }]);
|
||||||
|
const result = await runDemoBuilder({ title: "T", brief: "b" }, { model, headless: fakeHeadless() });
|
||||||
|
assert.equal(result.html, "");
|
||||||
|
assert.match(result.notes ?? "", /^FAILED: the builder never wrote a demo/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── 4. the visual-maker loop ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const SVG_1 = '<svg xmlns="http://www.w3.org/2000/svg" width="300" height="100"><line x1="10" y1="50" x2="290" y2="50"/></svg>';
|
||||||
|
|
||||||
|
describe("visual maker (fake model + fake headless)", () => {
|
||||||
|
it("write → render → look → finish delivers the svg with notes", async () => {
|
||||||
|
const headless = fakeHeadless();
|
||||||
|
const model = scriptedModel([
|
||||||
|
{ toolCalls: [{ toolCallId: "w1", toolName: "write_svg", input: { svg: SVG_1 } }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "r1", toolName: "render_svg", input: {} }] },
|
||||||
|
{ text: "A number line from -3 to 3 with the root marked at 0." },
|
||||||
|
]);
|
||||||
|
const activities: string[] = [];
|
||||||
|
const result = await runVisualMaker({ brief: "Number line, root at 0." }, { model, headless, onActivity: (l) => activities.push(l) });
|
||||||
|
|
||||||
|
assert.equal(result.svg, SVG_1);
|
||||||
|
assert.equal(result.notes, "A number line from -3 to 3 with the root marked at 0.");
|
||||||
|
assert.equal(headless.svgRenders.length, 1);
|
||||||
|
assert.ok(activities.some((a) => a.startsWith("render_svg")), "render activity recorded");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forces a look when the model finishes without rendering the final svg", async () => {
|
||||||
|
const headless = fakeHeadless();
|
||||||
|
const model = scriptedModel([
|
||||||
|
{ toolCalls: [{ toolCallId: "w1", toolName: "write_svg", input: { svg: SVG_1 } }] },
|
||||||
|
{ text: "Done." }, // premature finish — no render yet
|
||||||
|
{ text: "Looked at it — correct." }, // after the forced look
|
||||||
|
]);
|
||||||
|
const result = await runVisualMaker({ brief: "b" }, { model, headless });
|
||||||
|
assert.equal(result.svg, SVG_1, "delivered after the forced look");
|
||||||
|
assert.equal(result.notes, "Looked at it — correct.");
|
||||||
|
assert.equal(headless.svgRenders.length, 1, "the gate rendered the svg once itself");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("feeds render errors back to the model instead of crashing", async () => {
|
||||||
|
const headless = fakeHeadless({ renderFailures: 1 });
|
||||||
|
const model = scriptedModel([
|
||||||
|
{ toolCalls: [{ toolCallId: "w1", toolName: "write_svg", input: { svg: "<svg>broken" } }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "r1", toolName: "render_svg", input: {} }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "w2", toolName: "write_svg", input: { svg: SVG_1 } }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "r2", toolName: "render_svg", input: {} }] },
|
||||||
|
{ text: "Fixed and verified." },
|
||||||
|
]);
|
||||||
|
const result = await runVisualMaker({ brief: "b" }, { model, headless });
|
||||||
|
assert.equal(result.svg, SVG_1);
|
||||||
|
assert.equal(result.notes, "Fixed and verified.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails when the model never writes an svg", async () => {
|
||||||
|
const model = scriptedModel([{ text: "This idea is purely relational — belongs in mermaid." }]);
|
||||||
|
const result = await runVisualMaker({ brief: "b" }, { model, headless: fakeHeadless() });
|
||||||
|
assert.equal(result.svg, "");
|
||||||
|
assert.match(result.notes ?? "", /^FAILED: the maker never wrote an SVG/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails with a clear message when no Chrome is available", async () => {
|
||||||
|
const result = await runVisualMaker({ brief: "b" }, { model: scriptedModel([{ text: "x" }]), headless: noChromeHeadless() });
|
||||||
|
assert.equal(result.svg, "");
|
||||||
|
assert.match(result.notes ?? "", /^FAILED: .*Chrome/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── 5. job wrappers + spawn dispatch ────────────────────────────────────────
|
||||||
|
|
||||||
|
const DEMO_INPUT = { title: "Compound interest", brief: "Slider for rate, curve of growth over 30 years." };
|
||||||
|
|
||||||
|
function waitFor(cond: () => boolean, timeoutMs = 5000, what = "condition"): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const started = Date.now();
|
||||||
|
const tick = () => {
|
||||||
|
if (cond()) return resolve();
|
||||||
|
if (Date.now() - started > timeoutMs) return reject(new Error(`timeout waiting for ${what}`));
|
||||||
|
setTimeout(tick, 25);
|
||||||
|
};
|
||||||
|
tick();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function drain(res: Response): Promise<string> {
|
||||||
|
if (res.status !== 200) assert.fail(`expected 200, got ${res.status}: ${await res.text()}`);
|
||||||
|
return res.text();
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("demo/visual job runners", () => {
|
||||||
|
it("runDemoJob fulfills the part with a tested DemoResult and marks the job done", async () => {
|
||||||
|
const chat = createChat("Test: demo runner");
|
||||||
|
createdChats.push(chat.id);
|
||||||
|
saveMessages(chat.id, [
|
||||||
|
{ id: "u1", role: "user", parts: [{ type: "text", text: "show me compounding" }] },
|
||||||
|
{
|
||||||
|
id: "a1",
|
||||||
|
role: "assistant",
|
||||||
|
parts: [{ type: "tool-demo", toolCallId: "call-d1", state: "input-available", input: DEMO_INPUT }],
|
||||||
|
},
|
||||||
|
] as never);
|
||||||
|
|
||||||
|
const job = createJob({ kind: "demo", chatId: chat.id, toolCallId: "call-d1" });
|
||||||
|
createdJobs.push(job.id);
|
||||||
|
|
||||||
|
// runDemoJob has no headless injection point — the module override is the seam.
|
||||||
|
headlessOverride.current = fakeHeadless();
|
||||||
|
try {
|
||||||
|
await runDemoJob(job.id, {
|
||||||
|
model: scriptedModel([
|
||||||
|
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "r1", toolName: "run_demo", input: {} }] },
|
||||||
|
{ text: "Sweep the rate slider." },
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
headlessOverride.current = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const done = getJob(job.id)!;
|
||||||
|
assert.equal(done.status, "done");
|
||||||
|
assert.equal(done.activity, "run_demo: clean", "final activity line recorded");
|
||||||
|
const result = done.result as { title: string; html: string; tested: boolean };
|
||||||
|
assert.equal(result.title, "Compound interest");
|
||||||
|
assert.equal(result.tested, true);
|
||||||
|
|
||||||
|
const saved = getChat(chat.id)!;
|
||||||
|
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-demo");
|
||||||
|
assert.ok(part && part.type === "tool-demo", "demo tool part persisted");
|
||||||
|
assert.equal(part.state, "output-available", "part fulfilled");
|
||||||
|
assert.equal((part as { output?: { html?: string } }).output?.html, GOOD_HTML);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("runVisualJob fulfills the part even on failure and marks the job error", async () => {
|
||||||
|
const chat = createChat("Test: visual runner failure");
|
||||||
|
createdChats.push(chat.id);
|
||||||
|
saveMessages(chat.id, [
|
||||||
|
{ id: "u1", role: "user", parts: [{ type: "text", text: "draw it" }] },
|
||||||
|
{
|
||||||
|
id: "a1",
|
||||||
|
role: "assistant",
|
||||||
|
parts: [{ type: "tool-visual", toolCallId: "call-v1", state: "input-available", input: { brief: "b" } }],
|
||||||
|
},
|
||||||
|
] as never);
|
||||||
|
|
||||||
|
const job = createJob({ kind: "visual", chatId: chat.id, toolCallId: "call-v1" });
|
||||||
|
createdJobs.push(job.id);
|
||||||
|
|
||||||
|
await runVisualJob(job.id, { model: scriptedModel([{ text: "cannot" }]) });
|
||||||
|
|
||||||
|
const errored = getJob(job.id)!;
|
||||||
|
assert.equal(errored.status, "error");
|
||||||
|
assert.match(errored.error ?? "", /never wrote an SVG/);
|
||||||
|
|
||||||
|
const saved = getChat(chat.id)!;
|
||||||
|
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-visual");
|
||||||
|
assert.ok(part && part.type === "tool-visual", "visual tool part persisted");
|
||||||
|
assert.equal(part.state, "output-available", "failure still fulfills the part");
|
||||||
|
const output = (part as { output?: { svg?: string; notes?: string } }).output!;
|
||||||
|
assert.equal(output.svg, "");
|
||||||
|
assert.match(output.notes ?? "", /^FAILED:/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("the spawn hook dispatches a demo part to the demo runner (kind→runner map)", async () => {
|
||||||
|
const chat = createChat("Test: demo spawn dispatch");
|
||||||
|
createdChats.push(chat.id);
|
||||||
|
|
||||||
|
// A teacher that calls demo on turn 1 (scripted stream, like jobs.test.ts).
|
||||||
|
let turn = 0;
|
||||||
|
const teacher = new MockLanguageModelV4({
|
||||||
|
doStream: async () => {
|
||||||
|
turn++;
|
||||||
|
const chunks: LanguageModelV4StreamPart[] =
|
||||||
|
turn === 1
|
||||||
|
? [
|
||||||
|
{ type: "stream-start", warnings: [] },
|
||||||
|
{ type: "tool-input-start", id: "call-d2", toolName: "demo" },
|
||||||
|
{ type: "tool-input-delta", id: "call-d2", delta: JSON.stringify(DEMO_INPUT) },
|
||||||
|
{ type: "tool-input-end", id: "call-d2" },
|
||||||
|
{ type: "tool-call", toolCallId: "call-d2", toolName: "demo", input: JSON.stringify(DEMO_INPUT) },
|
||||||
|
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||||
|
]
|
||||||
|
: [
|
||||||
|
{ type: "stream-start", warnings: [] },
|
||||||
|
{ type: "text-start", id: "t2" },
|
||||||
|
{ type: "text-delta", id: "t2", delta: "Here it is." },
|
||||||
|
{ type: "text-end", id: "t2" },
|
||||||
|
{ type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: usage() },
|
||||||
|
];
|
||||||
|
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
demoModelOverride.current = scriptedModel([
|
||||||
|
{ toolCalls: [{ toolCallId: "w1", toolName: "write_demo", input: { html: GOOD_HTML } }] },
|
||||||
|
{ toolCalls: [{ toolCallId: "r1", toolName: "run_demo", input: {} }] },
|
||||||
|
{ text: "Sweep it." },
|
||||||
|
]);
|
||||||
|
headlessOverride.current = fakeHeadless();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await drain(
|
||||||
|
await handleChatRequest(
|
||||||
|
new Request("http://test/api/chat", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
chatId: chat.id,
|
||||||
|
messages: [{ id: "u1", role: "user", parts: [{ type: "text", text: "teach me compounding" }] }],
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
createTeacherAgent(teacher),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const job = findJobByToolCallId(chat.id, "call-d2");
|
||||||
|
assert.ok(job, "spawn hook created a job for the demo part");
|
||||||
|
assert.equal(job!.kind, "demo");
|
||||||
|
createdJobs.push(job!.id);
|
||||||
|
|
||||||
|
// The background runner (fire-and-forget) completes with the overrides.
|
||||||
|
await waitFor(() => getJob(job!.id)!.status !== "running", 5000, "demo job completion");
|
||||||
|
const done = getJob(job!.id)!;
|
||||||
|
assert.equal(done.status, "done");
|
||||||
|
assert.equal(done.activity, "run_demo: clean");
|
||||||
|
|
||||||
|
const saved = getChat(chat.id)!;
|
||||||
|
const part = saved.messages[1]!.parts.find((p) => p.type === "tool-demo");
|
||||||
|
assert.ok(part && part.type === "tool-demo", "demo tool part persisted");
|
||||||
|
assert.equal(part.state, "output-available", "demo part fulfilled by the background runner");
|
||||||
|
|
||||||
|
// Idempotency: re-running the spawn hook creates no second job.
|
||||||
|
spawnPendingJobs(chat.id, saved.messages);
|
||||||
|
assert.equal(listJobs(chat.id).length, 1, "no duplicate job for the same tool part");
|
||||||
|
} finally {
|
||||||
|
demoModelOverride.current = undefined;
|
||||||
|
headlessOverride.current = undefined;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── 6. registry wiring ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe("tool + renderer registry wiring", () => {
|
||||||
|
it("demo and visual are CLIENT-SIDE tools with the contract schemas", () => {
|
||||||
|
const tools = jobTools({ chatId: "c" });
|
||||||
|
assert.deepEqual(Object.keys(tools).sort(), ["demo", "research", "visual"]);
|
||||||
|
for (const name of ["demo", "visual"] as const) {
|
||||||
|
assert.equal(typeof (tools[name] as { execute?: unknown }).execute, "undefined", `${name} has no execute — client-side like quiz/ask`);
|
||||||
|
}
|
||||||
|
assert.equal(demoInputSchema.safeParse({ title: "T", brief: "b" }).success, true);
|
||||||
|
assert.equal(demoInputSchema.safeParse({ brief: "b" }).success, false, "demo requires a title");
|
||||||
|
assert.equal(visualInputSchema.safeParse({ brief: "b" }).success, true);
|
||||||
|
assert.equal(visualInputSchema.safeParse({}).success, false, "visual requires a brief");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("the artifact renderer registry maps demo/visual to the W6 cards", () => {
|
||||||
|
assert.equal(artifactRenderers.demo, DemoCard);
|
||||||
|
assert.equal(artifactRenderers.visual, VisualCard);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user