v1 web interface: dispatch-mode quiz/ask, Fastify+WS server, React frontend, deterministic test suite
- extensions/core/question-core.ts: pure quiz/ask logic (normalize/shuffle/resolve/grade/format) shared by both interfaces - extensions/core/bridge.ts: in-process server<->extension bridge (answer/dismiss/re-arm) - extensions/quiz-web.ts + ask-web.ts: web dispatch mode — tool returns immediately, graded answer arrives as next user message; sessions resumable mid-question via lh_question/lh_question_answered custom entries (never in LLM context) - extensions/quiz.ts: refactored onto shared core (terminal blocking mode unchanged) - src/server/index.ts: Fastify + WebSocket; REST (list/create/resume), entry sync (diff-based, all message kinds), streaming deltas, pending-question push - web/: Vite + React — home view (graph placeholder, session list, new-session input), session view (markdown+KaTeX+mermaid transcript, quiz cards, streaming) - src/testing/scripted-model.ts: deterministic fake LLM (scripted model runtime injected via factory overrides) - tests/: 48 deterministic tests (unit: question-core + bridge; integration: full pi loop with scripted model incl. resume re-arm) - scripts/e2e.ts: live smoke test (manual, real LLM, never a gate) - teacher/system-prompt.md: end-your-turn rule for question tools
This commit is contained in:
Amos
+5
-2
@@ -3,7 +3,9 @@
|
||||
* learning-harness CLI
|
||||
*
|
||||
* teach [message] launch an interactive teaching session (full pi TUI)
|
||||
* doctor non-interactive smoke test of the teacher wiring
|
||||
* doctor [--web] non-interactive smoke test of the teacher wiring
|
||||
* (--web checks the web-dispatch extension set)
|
||||
* serve start the web server (teacher agent + REST + WebSocket)
|
||||
* ingest <file> normalize a session JSONL and print the canonical document
|
||||
* (Stage 1 only — extractor/merger not yet implemented)
|
||||
* graph print graph/ledger stats
|
||||
@@ -26,7 +28,8 @@ async function main() {
|
||||
break;
|
||||
}
|
||||
case "doctor": {
|
||||
await doctor();
|
||||
const mode = rest.includes("--web") ? "web" : "tui";
|
||||
await doctor(mode);
|
||||
break;
|
||||
}
|
||||
case "ingest": {
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
/**
|
||||
* web server — Fastify + WebSocket server embedding the teacher agent via the
|
||||
* pi SDK (web dispatch mode: quiz/ask return immediately, answers arrive as
|
||||
* user messages — nothing blocks, so sessions can be exited/resumed mid-question).
|
||||
*
|
||||
* v1 scope (deliberately minimal):
|
||||
* - one active session at a time
|
||||
* - REST: list/create/resume sessions
|
||||
* - WS: entry stream (backfill + live), streaming deltas, pending question,
|
||||
* prompt / answer / dismiss / abort from the client
|
||||
*
|
||||
* NOT wired yet: extraction pipeline (post-session), graph queries.
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import Fastify from "fastify";
|
||||
import fastifyWebsocket from "@fastify/websocket";
|
||||
import fastifyStatic from "@fastify/static";
|
||||
import {
|
||||
AgentSessionRuntime,
|
||||
createAgentSessionRuntime,
|
||||
SessionManager,
|
||||
type AgentSessionEvent,
|
||||
} from "@earendil-works/pi-coding-agent";
|
||||
import { PATHS } from "../../config.ts";
|
||||
import { buildTeacherRuntimeFactory } from "../teacher/create-session.ts";
|
||||
import {
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
findPendingQuestion,
|
||||
type QuestionAnswerData,
|
||||
type QuestionData,
|
||||
toClientQuestion,
|
||||
} from "../../extensions/core/question-core.ts";
|
||||
import { getBridge } from "../../extensions/core/bridge.ts";
|
||||
import type { ClientEntry, ServerMessage, SessionSummary } from "../shared/protocol.ts";
|
||||
|
||||
const PORT = Number(process.env.LH_PORT ?? 8787);
|
||||
|
||||
// ─── active session state ───────────────────────────────────────────────────
|
||||
|
||||
interface ActiveSession {
|
||||
runtime: AgentSessionRuntime;
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
let active: ActiveSession | null = null;
|
||||
const sockets = new Set<WebSocket>();
|
||||
|
||||
function broadcast(msg: ServerMessage): void {
|
||||
const raw = JSON.stringify(msg);
|
||||
for (const ws of sockets) {
|
||||
if (ws.readyState === ws.OPEN) ws.send(raw);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── entry serialization ────────────────────────────────────────────────────
|
||||
|
||||
function extractText(content: unknown[]): string {
|
||||
return content
|
||||
.filter((c): c is { type: "text"; text: string } => (c as { type?: string }).type === "text")
|
||||
.map((c) => c.text)
|
||||
.join("");
|
||||
}
|
||||
|
||||
function serializeEntry(entry: {
|
||||
type: string;
|
||||
id: string;
|
||||
customType?: string;
|
||||
data?: unknown;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
message?: any;
|
||||
}): ClientEntry | null {
|
||||
if (entry.type === "message" && entry.message) {
|
||||
const m = entry.message;
|
||||
if (m.role === "user") {
|
||||
return { type: "user", id: entry.id, text: extractText(m.content ?? []) };
|
||||
}
|
||||
if (m.role === "assistant") {
|
||||
const toolCalls = ((m.content ?? []) as unknown[])
|
||||
.filter(
|
||||
(c): c is { type: "toolCall"; name: string; arguments?: unknown } =>
|
||||
(c as { type?: string }).type === "toolCall",
|
||||
)
|
||||
.map((c) => ({ name: c.name, args: (c.arguments ?? {}) as Record<string, unknown> }));
|
||||
return { type: "assistant", id: entry.id, text: extractText(m.content ?? []), toolCalls };
|
||||
}
|
||||
if (m.role === "toolResult") {
|
||||
return { type: "toolResult", id: entry.id, toolName: m.toolName ?? "", text: extractText(m.content ?? []) };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (entry.type === "custom" && entry.customType === QUESTION_ENTRY) {
|
||||
const data = entry.data as QuestionData | undefined;
|
||||
if (!data) return null;
|
||||
return { type: "question", id: entry.id, data: toClientQuestion(data) };
|
||||
}
|
||||
if (entry.type === "custom" && entry.customType === QUESTION_ANSWERED_ENTRY) {
|
||||
const data = entry.data as QuestionAnswerData | undefined;
|
||||
if (!data) return null;
|
||||
return { type: "questionAnswered", id: entry.id, data };
|
||||
}
|
||||
return { type: "meta", id: entry.id, kind: entry.type };
|
||||
}
|
||||
|
||||
function currentEntries(): Array<{ type: string; customType?: string; data?: unknown }> {
|
||||
if (!active) return [];
|
||||
return active.runtime.session.sessionManager.getEntries();
|
||||
}
|
||||
|
||||
function broadcastPending(): void {
|
||||
broadcast({ type: "pending", question: findPendingQuestion(currentEntries()) });
|
||||
}
|
||||
|
||||
// ─── session lifecycle ──────────────────────────────────────────────────────
|
||||
|
||||
// ─── entry synchronization ────────────────────────────────────────────────────────────
|
||||
// `entry_appended` in the session event stream only fires for EXTENSION-appended
|
||||
// custom entries — NOT for messages pi appends itself (user prompts, assistant
|
||||
// messages, tool results). So instead of relying on it, we diff the session
|
||||
// manager's entries against what we have already broadcast, on every session
|
||||
// event. Sessions are small; this is cheap and can not miss anything.
|
||||
|
||||
const broadcastEntryIds = new Set<string>();
|
||||
|
||||
function syncEntries(): void {
|
||||
if (!active) return;
|
||||
const entries = active.runtime.session.sessionManager.getEntries();
|
||||
for (const entry of entries) {
|
||||
if (broadcastEntryIds.has(entry.id)) continue;
|
||||
broadcastEntryIds.add(entry.id);
|
||||
const client = serializeEntry(entry as never);
|
||||
if (client) broadcast({ type: "entry", entry: client });
|
||||
if (
|
||||
entry.type === "custom" &&
|
||||
(entry.customType === QUESTION_ENTRY || entry.customType === QUESTION_ANSWERED_ENTRY)
|
||||
) {
|
||||
broadcastPending();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function handleSessionEvent(evt: AgentSessionEvent): void {
|
||||
if (!active) return;
|
||||
switch (evt.type) {
|
||||
case "message_update": {
|
||||
if (evt.assistantMessageEvent?.type === "text_delta") {
|
||||
broadcast({ type: "delta", delta: evt.assistantMessageEvent.delta });
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "agent_start":
|
||||
broadcast({ type: "streaming", value: true });
|
||||
break;
|
||||
case "agent_settled":
|
||||
syncEntries();
|
||||
broadcast({ type: "streaming", value: false });
|
||||
break;
|
||||
}
|
||||
// Catch every appended entry (messages + custom) regardless of event type.
|
||||
syncEntries();
|
||||
}
|
||||
|
||||
async function activate(sessionManager: SessionManager): Promise<ActiveSession> {
|
||||
if (active) {
|
||||
active.runtime.session.dispose();
|
||||
active = null;
|
||||
}
|
||||
const createRuntime = buildTeacherRuntimeFactory("web");
|
||||
const runtime = await createAgentSessionRuntime(createRuntime, {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager,
|
||||
});
|
||||
for (const d of runtime.diagnostics) {
|
||||
if (d.type === "error") console.error(`[teacher] ${d.message}`);
|
||||
}
|
||||
runtime.session.subscribe(handleSessionEvent);
|
||||
active = { runtime, sessionId: sessionManager.getSessionId() };
|
||||
broadcastEntryIds.clear();
|
||||
// Notify every client of the switch + backfill.
|
||||
const rawEntries = currentEntries();
|
||||
const entries = rawEntries
|
||||
.map((e) => serializeEntry(e as never))
|
||||
.filter((e): e is ClientEntry => e !== null);
|
||||
// Everything backfilled here is marked as already-broadcast so a later
|
||||
// syncEntries() does not re-send the session history as live entries.
|
||||
for (const e of rawEntries) broadcastEntryIds.add((e as unknown as { id: string }).id);
|
||||
broadcast({ type: "hello", sessionId: active.sessionId });
|
||||
broadcast({ type: "backfill", entries });
|
||||
broadcastPending();
|
||||
broadcast({ type: "streaming", value: runtime.session.isStreaming });
|
||||
return active;
|
||||
}
|
||||
|
||||
async function createSession(topic: string): Promise<ActiveSession> {
|
||||
const sessionManager = SessionManager.create(PATHS.root, PATHS.sessions);
|
||||
const result = await activate(sessionManager);
|
||||
// Name the session after the topic (shown in the session list).
|
||||
const name = topic.trim().slice(0, 60) || "New session";
|
||||
result.runtime.session.setSessionName(name);
|
||||
// Fire the first prompt (don't await — the client follows along via WS).
|
||||
void result.runtime.session.prompt(topic).catch((err) => {
|
||||
console.error("[teacher] prompt failed:", err);
|
||||
broadcast({ type: "error", message: String(err) });
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
async function resumeSession(file: string): Promise<ActiveSession> {
|
||||
const sessionManager = SessionManager.open(path.join(PATHS.sessions, file), PATHS.sessions, PATHS.root);
|
||||
return activate(sessionManager);
|
||||
}
|
||||
|
||||
// ─── session listing (reads data/sessions/*.jsonl) ──────────────────────────
|
||||
|
||||
function listSessions(): SessionSummary[] {
|
||||
const dir = PATHS.sessions;
|
||||
const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith(".jsonl")) : [];
|
||||
const summaries: SessionSummary[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const lines = fs.readFileSync(path.join(dir, file), "utf8").split("\n").filter(Boolean);
|
||||
if (lines.length === 0) continue;
|
||||
const header = JSON.parse(lines[0] ?? "") as { id?: string; timestamp?: string };
|
||||
const id = header.id;
|
||||
if (!id) continue;
|
||||
let name: string | undefined;
|
||||
let preview: string | undefined;
|
||||
let updatedAt = header.timestamp ?? "";
|
||||
let messageCount = 0;
|
||||
for (const line of lines.slice(1)) {
|
||||
let entry: { type?: string; message?: { role?: string; content?: unknown[] }; timestamp?: string; name?: string };
|
||||
try {
|
||||
entry = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (entry.timestamp) updatedAt = entry.timestamp;
|
||||
if (entry.type === "session_info" && entry.name) name = entry.name;
|
||||
if (entry.type === "message" && entry.message?.role === "user") {
|
||||
messageCount++;
|
||||
if (!preview) preview = extractText(entry.message.content ?? []).slice(0, 120);
|
||||
}
|
||||
}
|
||||
summaries.push({
|
||||
id,
|
||||
file,
|
||||
name,
|
||||
createdAt: header.timestamp ?? "",
|
||||
updatedAt,
|
||||
messageCount,
|
||||
preview,
|
||||
});
|
||||
} catch {
|
||||
// unreadable session file — skip
|
||||
}
|
||||
}
|
||||
summaries.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
||||
return summaries;
|
||||
}
|
||||
|
||||
// ─── server ─────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const app = Fastify({ logger: false });
|
||||
await app.register(fastifyWebsocket);
|
||||
|
||||
// Serve the built frontend (web/dist) when it exists.
|
||||
const distDir = path.join(PATHS.root, "web", "dist");
|
||||
if (fs.existsSync(distDir)) {
|
||||
await app.register(fastifyStatic, { root: distDir });
|
||||
app.setNotFoundHandler((req, reply) => {
|
||||
if (req.url.startsWith("/api") || req.url.startsWith("/ws")) {
|
||||
reply.code(404).send({ error: "not found" });
|
||||
} else {
|
||||
reply.sendFile("index.html");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
app.get("/api/sessions", async () => ({
|
||||
sessions: listSessions(),
|
||||
activeId: active?.sessionId ?? null,
|
||||
}));
|
||||
|
||||
app.post<{ Body: { topic?: string } }>("/api/sessions", async (req, reply) => {
|
||||
const topic = (req.body?.topic ?? "").trim();
|
||||
if (!topic) return reply.code(400).send({ error: "topic is required" });
|
||||
const result = await createSession(topic);
|
||||
return { id: result.sessionId };
|
||||
});
|
||||
|
||||
app.post<{ Params: { file: string } }>("/api/sessions/:file/resume", async (req, reply) => {
|
||||
const file = req.params.file;
|
||||
const full = path.join(PATHS.sessions, file);
|
||||
if (!file.endsWith(".jsonl") || !fs.existsSync(full)) {
|
||||
return reply.code(404).send({ error: "session not found" });
|
||||
}
|
||||
try {
|
||||
const result = await resumeSession(file);
|
||||
return { id: result.sessionId };
|
||||
} catch (err) {
|
||||
return reply.code(500).send({ error: String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/ws", { websocket: true }, (socket) => {
|
||||
sockets.add(socket);
|
||||
socket.on("close", () => sockets.delete(socket));
|
||||
|
||||
// Initial state snapshot.
|
||||
socket.send(
|
||||
JSON.stringify({ type: "hello", sessionId: active?.sessionId ?? null } satisfies ServerMessage),
|
||||
);
|
||||
if (active) {
|
||||
const entries = currentEntries()
|
||||
.map((e) => serializeEntry(e as never))
|
||||
.filter((e): e is ClientEntry => e !== null);
|
||||
socket.send(JSON.stringify({ type: "backfill", entries } satisfies ServerMessage));
|
||||
socket.send(
|
||||
JSON.stringify({ type: "pending", question: findPendingQuestion(currentEntries()) } satisfies ServerMessage),
|
||||
);
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: "streaming",
|
||||
value: active.runtime.session.isStreaming,
|
||||
} satisfies ServerMessage),
|
||||
);
|
||||
}
|
||||
|
||||
socket.on("message", (raw: Buffer) => {
|
||||
let msg: { type?: string } & Record<string, unknown>;
|
||||
try {
|
||||
msg = JSON.parse(raw.toString());
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!active) {
|
||||
socket.send(JSON.stringify({ type: "error", message: "no active session" } satisfies ServerMessage));
|
||||
return;
|
||||
}
|
||||
const session = active.runtime.session;
|
||||
switch (msg.type) {
|
||||
case "prompt": {
|
||||
const text = String(msg.text ?? "").trim();
|
||||
if (!text) return;
|
||||
// One question at a time: don't accept prompts while a question
|
||||
// is waiting to be answered.
|
||||
if (findPendingQuestion(currentEntries())) {
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
message: "Answer the pending question first (or dismiss it).",
|
||||
} satisfies ServerMessage),
|
||||
);
|
||||
return;
|
||||
}
|
||||
void session.prompt(text).catch((err) => {
|
||||
console.error("[teacher] prompt failed:", err);
|
||||
broadcast({ type: "error", message: String(err) });
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "answer": {
|
||||
const outcome = getBridge().answer(String(msg.questionId ?? ""), {
|
||||
dontKnow: Boolean(msg.dontKnow),
|
||||
values: Array.isArray(msg.values) ? msg.values.map(String) : undefined,
|
||||
other: msg.other ? String(msg.other) : undefined,
|
||||
text: msg.text ? String(msg.text) : undefined,
|
||||
note: msg.note ? String(msg.note) : undefined,
|
||||
});
|
||||
if (!outcome.ok) {
|
||||
socket.send(
|
||||
JSON.stringify({ type: "error", message: outcome.error ?? "answer failed" } satisfies ServerMessage),
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "dismiss": {
|
||||
const outcome = getBridge().dismiss(String(msg.questionId ?? ""));
|
||||
if (!outcome.ok) {
|
||||
socket.send(
|
||||
JSON.stringify({ type: "error", message: outcome.error ?? "dismiss failed" } satisfies ServerMessage),
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "abort": {
|
||||
void session.abort();
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
await app.listen({ port: PORT, host: "127.0.0.1" });
|
||||
console.log(`learning-harness web server: http://127.0.0.1:${PORT}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* protocol — shared types between the web server and the browser client.
|
||||
* One source of truth for the WS protocol and REST DTOs.
|
||||
*/
|
||||
|
||||
import type { ClientQuestion, QuestionAnswerData } from "../../extensions/core/question-core.ts";
|
||||
|
||||
export type { ClientQuestion, QuestionAnswerData } from "../../extensions/core/question-core.ts";
|
||||
|
||||
// ─── entries (session → client) ─────────────────────────────────────────────
|
||||
|
||||
export interface ClientToolCall {
|
||||
name: string;
|
||||
args: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type ClientEntry =
|
||||
| { type: "user"; id: string; text: string }
|
||||
| { type: "assistant"; id: string; text: string; toolCalls: ClientToolCall[] }
|
||||
| { type: "toolResult"; id: string; toolName: string; text: string }
|
||||
| { type: "question"; id: string; data: ClientQuestion }
|
||||
| { type: "questionAnswered"; id: string; data: QuestionAnswerData }
|
||||
| { type: "meta"; id: string; kind: string };
|
||||
|
||||
// ─── REST DTOs ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SessionSummary {
|
||||
id: string;
|
||||
file: string;
|
||||
name?: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
messageCount: number;
|
||||
preview?: string;
|
||||
}
|
||||
|
||||
export interface NewSessionResponse {
|
||||
id: string;
|
||||
}
|
||||
|
||||
// ─── WS protocol ────────────────────────────────────────────────────────────
|
||||
|
||||
export type ServerMessage =
|
||||
| { type: "hello"; sessionId: string | null }
|
||||
| { type: "backfill"; entries: ClientEntry[] }
|
||||
| { type: "entry"; entry: ClientEntry }
|
||||
| { type: "pending"; question: ClientQuestion | null }
|
||||
| { type: "delta"; delta: string }
|
||||
| { type: "streaming"; value: boolean }
|
||||
| { type: "error"; message: string };
|
||||
|
||||
export type ClientMessage =
|
||||
| { type: "prompt"; text: string }
|
||||
| {
|
||||
type: "answer";
|
||||
questionId: string;
|
||||
dontKnow?: boolean;
|
||||
values?: string[];
|
||||
other?: string;
|
||||
text?: string;
|
||||
note?: string;
|
||||
}
|
||||
| { type: "dismiss"; questionId: string }
|
||||
| { type: "abort" };
|
||||
@@ -23,7 +23,25 @@ import {
|
||||
} from "@earendil-works/pi-coding-agent";
|
||||
import { PATHS } from "../../config.ts";
|
||||
|
||||
export function buildTeacherRuntimeFactory(): CreateAgentSessionRuntimeFactory {
|
||||
export type TeacherMode = "tui" | "web";
|
||||
|
||||
/** Test-only injection: replace the model + model runtime with a scripted fake. */
|
||||
export interface TeacherRuntimeOverrides {
|
||||
model?: unknown;
|
||||
modelRuntime?: unknown;
|
||||
}
|
||||
|
||||
export function buildTeacherRuntimeFactory(
|
||||
mode: TeacherMode = "tui",
|
||||
overrides?: TeacherRuntimeOverrides,
|
||||
): CreateAgentSessionRuntimeFactory {
|
||||
// Terminal mode: quiz/ask block on interactive pi-tui components.
|
||||
// Web mode: quiz/ask use the dispatch shape (return immediately; the graded
|
||||
// answer arrives as the next user message — see extensions/quiz-web.ts).
|
||||
const extensionFiles =
|
||||
mode === "web"
|
||||
? [path.join(PATHS.extensions, "quiz-web.ts"), path.join(PATHS.extensions, "ask-web.ts")]
|
||||
: [path.join(PATHS.extensions, "quiz.ts"), path.join(PATHS.extensions, "ask-user-question.ts")];
|
||||
return async ({ cwd, sessionManager, sessionStartEvent }) => {
|
||||
const services = await createAgentSessionServices({
|
||||
cwd,
|
||||
@@ -31,10 +49,7 @@ export function buildTeacherRuntimeFactory(): CreateAgentSessionRuntimeFactory {
|
||||
resourceLoaderOptions: {
|
||||
// Only the extensions we explicitly list. No discovery of ~/.pi/agent
|
||||
// or .pi/extensions beyond what agentDir/cwd contain (they contain none).
|
||||
additionalExtensionPaths: [
|
||||
path.join(PATHS.extensions, "quiz.ts"),
|
||||
path.join(PATHS.extensions, "ask-user-question.ts"),
|
||||
],
|
||||
additionalExtensionPaths: extensionFiles,
|
||||
// The teacher system prompt — replaces pi's coding prompt entirely.
|
||||
systemPrompt: PATHS.teacherPrompt,
|
||||
// Hermetic: no skills, no prompt templates, no AGENTS.md context files.
|
||||
@@ -44,6 +59,11 @@ export function buildTeacherRuntimeFactory(): CreateAgentSessionRuntimeFactory {
|
||||
noContextFiles: true,
|
||||
},
|
||||
});
|
||||
// Test injection: swap the real model runtime for a scripted fake BEFORE
|
||||
// the session is constructed from services. Production never passes this.
|
||||
if (overrides?.modelRuntime) {
|
||||
(services as { modelRuntime: unknown }).modelRuntime = overrides.modelRuntime;
|
||||
}
|
||||
return {
|
||||
...(await createAgentSessionFromServices({
|
||||
services,
|
||||
@@ -52,6 +72,7 @@ export function buildTeacherRuntimeFactory(): CreateAgentSessionRuntimeFactory {
|
||||
// No read/bash/edit/write. quiz + ask_user_question (extension tools)
|
||||
// remain enabled.
|
||||
noTools: "builtin",
|
||||
model: overrides?.model as never,
|
||||
})),
|
||||
services,
|
||||
diagnostics: services.diagnostics,
|
||||
@@ -91,9 +112,9 @@ export async function runTeachingSession(initialMessage?: string): Promise<void>
|
||||
}
|
||||
|
||||
/** Non-interactive smoke test: verify the whole wiring boots correctly. */
|
||||
export async function doctor(): Promise<void> {
|
||||
export async function doctor(mode: TeacherMode = "tui"): Promise<void> {
|
||||
const sessionManager = SessionManager.inMemory(PATHS.root);
|
||||
const runtime = await buildTeacherRuntimeFactory()({
|
||||
const runtime = await buildTeacherRuntimeFactory(mode)({
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager,
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* scripted-model — a deterministic fake LLM for integration tests.
|
||||
*
|
||||
* Replaces the real model runtime inside the teacher session: the FULL pi
|
||||
* agent loop runs (extensions, tool execution, session persistence, event
|
||||
* stream), but every assistant response comes from a fixed script. No network,
|
||||
* no tokens, no variance — the same script always produces the same session.
|
||||
*/
|
||||
|
||||
// AgentMessage comes from a transitive dep that is not directly importable;
|
||||
// a structural local type is enough for the helpers below.
|
||||
export interface AnyMessage {
|
||||
role?: string;
|
||||
content?: Array<{ type?: string; text?: string }>;
|
||||
}
|
||||
|
||||
// ─── types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ScriptedToolCall {
|
||||
name: string;
|
||||
arguments: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** One scripted assistant turn: tool calls and/or final text. */
|
||||
export interface ScriptedTurn {
|
||||
text?: string;
|
||||
toolCalls?: ScriptedToolCall[];
|
||||
}
|
||||
|
||||
interface FakeAssistantMessage {
|
||||
role: "assistant";
|
||||
content: Array<Record<string, unknown>>;
|
||||
api: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
usage: Record<string, unknown>;
|
||||
stopReason: string;
|
||||
}
|
||||
|
||||
// ─── fake event stream (what the agent loop consumes) ──────────────────────
|
||||
|
||||
class FakeEventStream {
|
||||
private queue: Array<Record<string, unknown>> = [];
|
||||
private done = false;
|
||||
private resolvers: Array<(r: IteratorResult<Record<string, unknown>>) => void> = [];
|
||||
private finalMessage: FakeAssistantMessage | null = null;
|
||||
private resultResolve!: (m: FakeAssistantMessage) => void;
|
||||
readonly resultPromise: Promise<FakeAssistantMessage>;
|
||||
|
||||
constructor() {
|
||||
this.resultPromise = new Promise((resolve) => {
|
||||
this.resultResolve = resolve;
|
||||
});
|
||||
}
|
||||
|
||||
push(event: Record<string, unknown>): void {
|
||||
if (this.done) return;
|
||||
const r = this.resolvers.shift();
|
||||
if (r) r({ value: event, done: false });
|
||||
else this.queue.push(event);
|
||||
}
|
||||
|
||||
end(message: FakeAssistantMessage): void {
|
||||
this.done = true;
|
||||
this.finalMessage = message;
|
||||
this.resultResolve(message);
|
||||
while (this.resolvers.length > 0) {
|
||||
this.resolvers.shift()!({ value: undefined as never, done: true });
|
||||
}
|
||||
}
|
||||
|
||||
async result(): Promise<FakeAssistantMessage> {
|
||||
return this.resultPromise;
|
||||
}
|
||||
|
||||
[Symbol.asyncIterator](): AsyncIterator<Record<string, unknown>> {
|
||||
return {
|
||||
next: (): Promise<IteratorResult<Record<string, unknown>>> => {
|
||||
const queued = this.queue.shift();
|
||||
if (queued !== undefined) return Promise.resolve({ value: queued, done: false });
|
||||
if (this.done) return Promise.resolve({ value: undefined as never, done: true });
|
||||
return new Promise((resolve) => this.resolvers.push(resolve));
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ─── fake model + runtime ───────────────────────────────────────────────────
|
||||
|
||||
function baseMessage(turnIndex: number): FakeAssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: "openai-completions",
|
||||
provider: "scripted",
|
||||
model: `scripted-turn-${turnIndex}`,
|
||||
usage: { input: 100, output: 10, cacheRead: 0, cacheWrite: 0, totalTokens: 110, cost: { input: 0, output: 0, total: 0 } },
|
||||
stopReason: "stop",
|
||||
};
|
||||
}
|
||||
|
||||
export interface ScriptedRuntime {
|
||||
/** The model object to pass as `model` when creating the session. */
|
||||
model: Record<string, unknown>;
|
||||
/** The ModelRuntime replacement to inject into services. */
|
||||
modelRuntime: {
|
||||
getModel: () => unknown;
|
||||
hasConfiguredAuth: () => boolean;
|
||||
streamSimple: (model: unknown, context: unknown, options?: unknown) => FakeEventStream;
|
||||
};
|
||||
/** All contexts the "model" was called with (for assertions). */
|
||||
contextsSeen: Array<{ systemPrompt?: string; messages: unknown[] }>;
|
||||
}
|
||||
|
||||
export function createScriptedRuntime(script: ScriptedTurn[]): ScriptedRuntime {
|
||||
let turnIndex = 0;
|
||||
const contextsSeen: Array<{ systemPrompt?: string; messages: unknown[] }> = [];
|
||||
|
||||
const model = {
|
||||
id: "scripted-model",
|
||||
name: "Scripted Test Model",
|
||||
api: "openai-completions",
|
||||
provider: "scripted",
|
||||
baseUrl: "http://localhost:0",
|
||||
reasoning: false,
|
||||
input: ["text"] as const,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 8_192,
|
||||
};
|
||||
|
||||
function nextStream(): FakeEventStream {
|
||||
const stream = new FakeEventStream();
|
||||
const turn = script[turnIndex] ?? { text: "(script exhausted)" };
|
||||
turnIndex++;
|
||||
|
||||
const msg = baseMessage(turnIndex);
|
||||
const content: Array<Record<string, unknown>> = [];
|
||||
|
||||
// "start"
|
||||
stream.push({ type: "start", partial: { ...msg, content: [] } });
|
||||
|
||||
// tool calls
|
||||
for (const tc of turn.toolCalls ?? []) {
|
||||
const toolCall = {
|
||||
type: "toolCall",
|
||||
id: `call_${turnIndex}_${content.length}`,
|
||||
name: tc.name,
|
||||
arguments: tc.arguments,
|
||||
};
|
||||
content.push(toolCall);
|
||||
stream.push({
|
||||
type: "toolcall_end",
|
||||
contentIndex: content.length - 1,
|
||||
toolCall,
|
||||
partial: { ...msg, content: [...content] },
|
||||
});
|
||||
}
|
||||
|
||||
// text
|
||||
if (turn.text !== undefined && turn.text.length > 0) {
|
||||
content.push({ type: "text", text: turn.text });
|
||||
stream.push({
|
||||
type: "text_start",
|
||||
contentIndex: content.length - 1,
|
||||
partial: { ...msg, content: [...content] },
|
||||
});
|
||||
stream.push({
|
||||
type: "text_delta",
|
||||
contentIndex: content.length - 1,
|
||||
delta: turn.text,
|
||||
partial: { ...msg, content: [...content] },
|
||||
});
|
||||
stream.push({
|
||||
type: "text_end",
|
||||
contentIndex: content.length - 1,
|
||||
content: turn.text,
|
||||
partial: { ...msg, content: [...content] },
|
||||
});
|
||||
}
|
||||
|
||||
// "done" + result
|
||||
const final = { ...msg, content, stopReason: turn.toolCalls?.length ? "toolUse" : "stop" };
|
||||
stream.push({
|
||||
type: "done",
|
||||
reason: turn.toolCalls?.length ? "toolUse" : "stop",
|
||||
message: final,
|
||||
});
|
||||
stream.end(final);
|
||||
return stream;
|
||||
}
|
||||
|
||||
return {
|
||||
model,
|
||||
contextsSeen,
|
||||
modelRuntime: {
|
||||
getModel: () => model,
|
||||
hasConfiguredAuth: () => true,
|
||||
streamSimple: (_model: unknown, context: unknown) => {
|
||||
const ctx = context as { systemPrompt?: string; messages: unknown[] };
|
||||
contextsSeen.push(ctx);
|
||||
return nextStream();
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ─── helpers for assertions on session messages ─────────────────────────────
|
||||
|
||||
export function messageText(m: AnyMessage): string {
|
||||
const content = (m as { content?: Array<{ type?: string; text?: string }> }).content ?? [];
|
||||
return content
|
||||
.filter((c) => c.type === "text" && typeof c.text === "string")
|
||||
.map((c) => c.text)
|
||||
.join("");
|
||||
}
|
||||
Reference in New Issue
Block a user