sorokin.oleg.edu
d4a0ad6ac1
- e2e/fixtures.ts - e2e/application-flows.spec.ts GSD context: - Milestone: M001 - Merge upstream v0.4.0 into the Gitea fork - Slice: S02 - Verify combined application behavior - Task: T03 - Added six offline real-browser application and quiz cases with synthetic native records, acknowledged SSE replies, and real artifact persistence. GSD-Task: S02/T03
263 lines
15 KiB
TypeScript
263 lines
15 KiB
TypeScript
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { test as base, expect, type Route } from "@playwright/test";
|
|
import type { ModelCatalogEntry, ModelCatalogResponse } from "../src/core/contracts/models";
|
|
import type { UserSettings } from "../src/core/settings";
|
|
import type { ChatGPTStatus } from "../src/core/contracts/chatgpt";
|
|
import type { NativeSubscriptionStatus } from "../src/core/contracts/subscriptions";
|
|
import type { ProviderKeyInfo } from "../components/settings/keys-section";
|
|
import type { ChatRecord } from "../src/core/contracts/chat";
|
|
import type { GraphSnapshot } from "../src/core/contracts/graph";
|
|
import type { JobRecord } from "../src/core/contracts/jobs";
|
|
import type { LhMessage } from "../src/core/native-messages";
|
|
import { compactAnswerForModel } from "../src/core/question-core";
|
|
import type { ChatInput } from "../src/core/input-events";
|
|
|
|
/** Setup/teardown owns only the UUID run directory beneath this worktree. */
|
|
export default function setup() {
|
|
const root = process.env.LH_E2E_RUN_DIR;
|
|
if (!root || path.dirname(root) !== path.resolve(".playwright-data") || !/^run-[a-f0-9-]{36}$/.test(path.basename(root))) {
|
|
throw new Error("Unsafe/missing Playwright data directory; refusing to run or clean up.");
|
|
}
|
|
return async () => { await rm(root, { recursive: true, force: true }); };
|
|
}
|
|
|
|
export const syntheticModel: ModelCatalogEntry = {
|
|
id: "synthetic-smoke-model", name: "Synthetic smoke model", provider: "openai",
|
|
reasoning: false, input: ["text", "image"], contextWindow: 8192, maxTokens: 1024,
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, supportedLevels: ["off"],
|
|
};
|
|
const provider: ProviderKeyInfo = {
|
|
id: "openai", envVar: "OPENAI_API_KEY", label: "OpenAI",
|
|
};
|
|
export interface ApplicationFixture {
|
|
chats: ChatRecord[];
|
|
jobs: JobRecord[];
|
|
graph: GraphSnapshot;
|
|
inputs: Array<{ chatId: string; input: ChatInput }>;
|
|
reads: string[];
|
|
reply: LhMessage | null;
|
|
failures: Array<{ status: number; body: unknown }>;
|
|
}
|
|
export interface HarnessState {
|
|
keyPresent: boolean;
|
|
settings: UserSettings | null;
|
|
writes: { path: string; body: unknown }[];
|
|
blocked: string[];
|
|
generation: string[];
|
|
catalogFailure: boolean;
|
|
models?: ModelCatalogEntry[];
|
|
modelsWrite: { status: number; wait?: Promise<void> };
|
|
transcription: { configured: boolean; status: number; body: { text?: string; message?: string } };
|
|
seedGeneral: (general: NonNullable<UserSettings["general"]>) => Promise<void>;
|
|
application: ApplicationFixture | null;
|
|
seedChat: (chat: ChatRecord) => Promise<void>;
|
|
}
|
|
const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, json: body });
|
|
// Native shapes from native-messages.ts; avoid its transitive locale JSON imports
|
|
// in Playwright's Node loader. These messages never leave the offline boundary.
|
|
export function syntheticUserMessage(text: string): LhMessage {
|
|
return { id: `msg_${crypto.randomUUID()}`, role: "user", content: text, timestamp: Date.now() };
|
|
}
|
|
function hasPendingQuiz(chat: ChatRecord): boolean {
|
|
const newest = chat.messages.findLast((message) => message.role === "assistant");
|
|
return newest?.role === "assistant" && newest.content.some((block) => block.type === "toolCall" && block.name === "quiz"
|
|
&& !chat.messages.some((message) => message.role === "toolResult" && message.toolCallId === block.id));
|
|
}
|
|
|
|
export const test = base.extend<{ harness: HarnessState }>({
|
|
harness: async ({ context }, use) => {
|
|
// Layout reads general settings on the server, before browser routes exist.
|
|
// Bridge ONLY synthetic general settings to the webServer's UUID-owned data.
|
|
setup(); // validate run directory before any filesystem access
|
|
const settingsFile = path.join(process.env.LH_E2E_RUN_DIR!, "data", "settings.json");
|
|
await mkdir(path.dirname(settingsFile), { recursive: true });
|
|
const ownedChats = new Set<string>();
|
|
const streams = new Map<string, LhMessage>();
|
|
const persistChat = async (chat: ChatRecord) => {
|
|
if (!/^chat_[a-z0-9_]+$/.test(chat.id)) throw new Error("Invalid synthetic chat id");
|
|
const file = path.join(path.dirname(settingsFile), "chats", `${chat.id}.json`);
|
|
await mkdir(path.dirname(file), { recursive: true });
|
|
ownedChats.add(file);
|
|
await writeFile(file, JSON.stringify(chat), "utf8");
|
|
};
|
|
const state: HarnessState = {
|
|
keyPresent: false, settings: null, writes: [], blocked: [], generation: [], catalogFailure: false,
|
|
application: null,
|
|
seedChat: async (chat) => {
|
|
if (!state.application) throw new Error("Application fixture must be explicitly enabled");
|
|
state.application.chats = [...state.application.chats.filter((c) => c.id !== chat.id), chat];
|
|
await persistChat(chat);
|
|
// RSC pages read these synthetic records/settings through production stores.
|
|
await writeFile(settingsFile, JSON.stringify(state.settings), "utf8");
|
|
},
|
|
modelsWrite: { status: 200 },
|
|
transcription: { configured: true, status: 200, body: { text: "Synthetic transcript" } },
|
|
seedGeneral: async (general) => {
|
|
state.settings = { ...state.settings, version: 1, general };
|
|
await writeFile(settingsFile, JSON.stringify({ version: 1, general }), "utf8");
|
|
},
|
|
};
|
|
await state.seedGeneral({});
|
|
await context.routeWebSocket("**/*", (socket) => {
|
|
const url = new URL(socket.url());
|
|
if (url.origin === "ws://127.0.0.1:3005" && url.pathname.startsWith("/_next/")) {
|
|
socket.connectToServer(); // Next dev HMR only; no provider sockets.
|
|
return;
|
|
}
|
|
state.blocked.push(`WS ${url.origin}${url.pathname}`);
|
|
socket.close({ code: 1008, reason: "Offline test boundary" });
|
|
});
|
|
// Installed on the CONTEXT before a page navigates, including popups.
|
|
// Never let a missed provider fixture fall through to a live endpoint.
|
|
await context.route("**/*", async (route) => {
|
|
const request = route.request();
|
|
const url = new URL(request.url());
|
|
const method = request.method();
|
|
const p = url.pathname;
|
|
const block = async () => {
|
|
state.blocked.push(`${method} ${url.origin}${p}`); // no query strings/payloads in diagnostics
|
|
await route.abort("blockedbyclient");
|
|
};
|
|
if (url.origin !== "http://127.0.0.1:3005") return block();
|
|
if (!p.startsWith("/api/")) {
|
|
if (method === "GET" || method === "HEAD") return route.continue();
|
|
return block();
|
|
}
|
|
const app = state.application;
|
|
if (app) {
|
|
if (method === "GET") app.reads.push(p);
|
|
if (p === "/api/graph" && method === "GET") return json(route, app.graph);
|
|
if (p === "/api/chats" && method === "GET") return json(route, { chats: app.chats.map((chat) => ({
|
|
id: chat.id, title: chat.title, createdAt: chat.createdAt, updatedAt: chat.updatedAt,
|
|
kind: chat.kind ?? "teach", messageCount: chat.messages.length, phase: "probe",
|
|
pendingQuestion: hasPendingQuiz(chat), pendingProblem: false, busy: false, compacting: false,
|
|
})) });
|
|
if (p === "/api/jobs" && method === "GET") return json(route, { jobs: app.jobs.filter((job) => !url.searchParams.has("chatId") || job.chatId === url.searchParams.get("chatId")), turnInFlight: false, compacting: false });
|
|
const chatPath = p.match(/^\/api\/chats\/(chat_[a-z0-9_]+)(?:\/(stream|artifact-ui))?$/);
|
|
if (chatPath) {
|
|
const chat = app.chats.find((c) => c.id === chatPath[1]);
|
|
if (!chat) return json(route, { error: "Not found" }, 404);
|
|
if (!chatPath[2] && method === "GET") return json(route, { ...chat, turnInFlight: false, compacting: false });
|
|
if (chatPath[2] === "artifact-ui" && method === "POST") {
|
|
const body = request.postDataJSON() as { input: { messageId: string; inverted: boolean } };
|
|
state.writes.push({ path: p, body });
|
|
// Exercise the real silent persistence route against our owned chat.
|
|
const response = await route.fetch();
|
|
if (response.ok()) {
|
|
const saved = JSON.parse(await readFile(path.join(path.dirname(settingsFile), "chats", `${chat.id}.json`), "utf8")) as ChatRecord;
|
|
chat.messages = saved.messages;
|
|
}
|
|
return route.fulfill({ response });
|
|
}
|
|
if (chatPath[2] === "stream" && method === "GET") {
|
|
const reply = streams.get(chat.id);
|
|
streams.delete(chat.id);
|
|
if (reply) { chat.messages.push(reply); await persistChat(chat); }
|
|
// Pi SSE carries no app record id; SessionModel assigns local ids.
|
|
const wire = reply && Object.fromEntries(Object.entries(reply).filter(([key]) => key !== "id"));
|
|
const events = reply ? [{ type: "agent_start" }, { type: "message_start", message: wire }, { type: "message_update", message: wire }, { type: "message_end", message: wire }, { type: "agent_end", messages: [wire] }] : [{ type: "turn-settled" }];
|
|
return route.fulfill({ contentType: "text/event-stream", body: events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("") + "data: [DONE]\n\n" });
|
|
}
|
|
}
|
|
if (p === "/api/chat" && method === "POST" && app.reply) {
|
|
const body = request.postDataJSON() as { chatId: string; input: ChatInput };
|
|
app.inputs.push(body);
|
|
const failure = app.failures.shift();
|
|
if (failure) return json(route, failure.body, failure.status);
|
|
const chat = app.chats.find((c) => c.id === body.chatId);
|
|
if (!chat) throw new Error("Generation targeted an unseeded chat");
|
|
if (body.input.type === "text") chat.messages.push(syntheticUserMessage(body.input.text));
|
|
else if (body.input.type === "answer") chat.messages.push({
|
|
id: `msg_${crypto.randomUUID()}`, role: "toolResult", toolCallId: body.input.toolCallId, toolName: "quiz",
|
|
content: [{ type: "text", text: JSON.stringify(compactAnswerForModel(body.input.answer)) }],
|
|
details: body.input.answer as unknown as Extract<LhMessage, { role: "toolResult" }>["details"], isError: false, timestamp: Date.now(),
|
|
});
|
|
else if (body.input.type !== "retry") throw new Error(`Unscripted input: ${body.input.type}`);
|
|
await persistChat(chat);
|
|
streams.set(chat.id, app.reply);
|
|
return json(route, { ok: true });
|
|
}
|
|
}
|
|
if ((/\/(turn|generate|transcribe)(\/|$)/.test(p) && p !== "/api/stt/transcribe") || (p.startsWith("/api/chats") && method !== "GET")) {
|
|
state.generation.push(`${method} ${p}`);
|
|
return block();
|
|
}
|
|
if (p === "/api/keys" && method === "GET") return json(route, {
|
|
keys: state.keyPresent ? [{ id: provider.envVar, info: null, present: true, source: "file", masked: "synthetic…only (41 chars)", shadowedByEnv: false }] : [],
|
|
providers: [provider],
|
|
});
|
|
if (p === "/api/keys/OPENAI_API_KEY" && method === "PUT") {
|
|
const body: unknown = request.postDataJSON();
|
|
state.writes.push({ path: p, body });
|
|
if (JSON.stringify(body) !== JSON.stringify({ value: "synthetic-smoke-key-not-a-credential" })) return json(route, { error: "Only the synthetic fixture key is accepted" }, 400);
|
|
state.keyPresent = true;
|
|
return json(route, { status: { id: provider.envVar, info: null, present: true, source: "file", masked: "synthetic…only (41 chars)", shadowedByEnv: false } });
|
|
}
|
|
if (p === "/api/models" && method === "GET") {
|
|
if (state.catalogFailure) return json(route, { error: "Synthetic catalog unavailable" }, 503);
|
|
const slots = {
|
|
strong: state.settings?.models?.strong?.model ? { ...state.settings.models.strong, model: state.settings.models.strong.model } : null,
|
|
cheap: state.settings?.models?.cheap?.model ? { ...state.settings.models.cheap, model: state.settings.models.cheap.model } : null,
|
|
vision: state.settings?.models?.vision?.model ? { ...state.settings.models.vision, model: state.settings.models.vision.model } : null,
|
|
};
|
|
const catalog: ModelCatalogResponse = {
|
|
models: state.models ?? (state.keyPresent ? [syntheticModel] : []), defaults: slots.strong, slots,
|
|
rolePins: { researcher: slots.cheap, demoBuilder: slots.cheap, visualMaker: slots.vision, extractor: slots.cheap, auditor: slots.cheap },
|
|
providers: [{ id: "openai", envVar: provider.envVar, keyPresent: state.keyPresent }],
|
|
};
|
|
return json(route, catalog);
|
|
}
|
|
if (p === "/api/settings" && method === "GET") return json(route, { settings: state.settings });
|
|
if (p === "/api/settings/models" && method === "PUT") {
|
|
const body = request.postDataJSON() as NonNullable<UserSettings["models"]>;
|
|
state.writes.push({ path: p, body });
|
|
await state.modelsWrite.wait;
|
|
if (state.modelsWrite.status !== 200) return json(route, { error: "Synthetic settings rejection" }, state.modelsWrite.status);
|
|
state.settings = { ...state.settings, version: 1, models: body };
|
|
return json(route, { settings: state.settings });
|
|
}
|
|
if (p === "/api/settings/general" && method === "PUT") {
|
|
const body = request.postDataJSON() as UserSettings["general"];
|
|
state.writes.push({ path: p, body });
|
|
await state.seedGeneral(body ?? {});
|
|
return json(route, { settings: state.settings });
|
|
}
|
|
if (p === "/api/search" && method === "GET") return json(route, { available: false, engine: null, reason: "No synthetic search key" });
|
|
if (p === "/api/settings/prompts" && method === "GET") return json(route, { prober: [], teacher: [] });
|
|
if (p === "/api/stt/transcribe") {
|
|
if (method === "GET") return json(route, { configured: state.transcription.configured });
|
|
if (method === "POST") return json(route, state.transcription.body, state.transcription.status);
|
|
}
|
|
if (p === "/api/subscriptions" && method === "GET") {
|
|
const chatgpt: ChatGPTStatus = { available: false, pending: false, allowBackgroundUse: false, activeId: null, accounts: [], error: null };
|
|
const native: NativeSubscriptionStatus[] = ["github-copilot", "xai"].map((providerId) => ({
|
|
providerId: providerId as NativeSubscriptionStatus["providerId"], available: false, connected: false,
|
|
pending: false, allowBackgroundUse: false, connectionId: null, epoch: 0, authorityRevision: 0, revision: 0,
|
|
}));
|
|
return json(route, { chatgpt, native });
|
|
}
|
|
if (p === "/api/custom-provider" && method === "GET") return json(route, {
|
|
baseUrl: "", authMode: "keyless", models: [], overrides: {}, manualModels: [],
|
|
keyPresent: false, keySource: "none", keyReuseRequired: false,
|
|
});
|
|
if (p === "/api/stt/token" && method === "GET") return json(route, { message: "No DEEPGRAM_API_KEY — synthetic offline fixture." }, 503);
|
|
// Real, read-only application contracts against empty fixture-owned data.
|
|
if (method === "GET" && ["/api/activity", "/api/chats", "/api/events", "/api/graph", "/api/graph/status", "/api/graph/costs", "/api/graph/render-settings"].includes(p)) return route.continue();
|
|
return block();
|
|
});
|
|
try { await use(state); }
|
|
finally {
|
|
await rm(settingsFile, { force: true });
|
|
for (const file of ownedChats) await rm(file, { force: true });
|
|
}
|
|
expect(state.blocked, "Unexpected network request (blocked before dispatch)").toEqual([]);
|
|
expect(state.generation, "Setup/settings must not generate or send").toEqual([]);
|
|
},
|
|
page: async ({ page, harness }, use) => {
|
|
void harness; // force interception before the test receives its page
|
|
await use(page);
|
|
},
|
|
});
|
|
export { expect };
|