test: Added real-browser language persistence, synthetic OpenRouter d...
- e2e/fixtures.ts - e2e/fork-preservation.spec.ts GSD context: - Milestone: M001 - Merge upstream v0.4.0 into the Gitea fork - Slice: S02 - Verify combined application behavior - Task: T02 - Added real-browser language persistence, synthetic OpenRouter dictation, and ChatGPT Fast-mode failure/retry coverage without product changes. GSD-Task: S02/T02
This commit is contained in:
+35
-4
@@ -1,4 +1,4 @@
|
||||
import { rm } from "node:fs/promises";
|
||||
import { mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { test as base, expect, type Route } from "@playwright/test";
|
||||
import type { ModelCatalogEntry, ModelCatalogResponse } from "../src/core/contracts/models";
|
||||
@@ -31,14 +31,30 @@ export interface HarnessState {
|
||||
blocked: string[];
|
||||
generation: string[];
|
||||
catalogFailure: boolean;
|
||||
models?: ModelCatalogEntry[];
|
||||
modelsWrite: { status: number; wait?: Promise<void> };
|
||||
transcription: { configured: boolean; status: number; body: { text?: string; message?: string } };
|
||||
seedGeneral: (general: NonNullable<UserSettings["general"]>) => Promise<void>;
|
||||
}
|
||||
const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, json: body });
|
||||
|
||||
export const test = base.extend<{ harness: HarnessState }>({
|
||||
harness: async ({ context }, use) => {
|
||||
// Layout reads general settings on the server, before browser routes exist.
|
||||
// Bridge ONLY synthetic general settings to the webServer's UUID-owned data.
|
||||
setup(); // validate run directory before any filesystem access
|
||||
const settingsFile = path.join(process.env.LH_E2E_RUN_DIR!, "data", "settings.json");
|
||||
await mkdir(path.dirname(settingsFile), { recursive: true });
|
||||
const state: HarnessState = {
|
||||
keyPresent: false, settings: null, writes: [], blocked: [], generation: [], catalogFailure: false,
|
||||
modelsWrite: { status: 200 },
|
||||
transcription: { configured: true, status: 200, body: { text: "Synthetic transcript" } },
|
||||
seedGeneral: async (general) => {
|
||||
state.settings = { ...state.settings, version: 1, general };
|
||||
await writeFile(settingsFile, JSON.stringify({ version: 1, general }), "utf8");
|
||||
},
|
||||
};
|
||||
await state.seedGeneral({});
|
||||
await context.routeWebSocket("**/*", (socket) => {
|
||||
const url = new URL(socket.url());
|
||||
if (url.origin === "ws://127.0.0.1:3005" && url.pathname.startsWith("/_next/")) {
|
||||
@@ -64,7 +80,7 @@ export const test = base.extend<{ harness: HarnessState }>({
|
||||
if (method === "GET" || method === "HEAD") return route.continue();
|
||||
return block();
|
||||
}
|
||||
if (/\/(turn|generate|transcribe)(\/|$)/.test(p) || (p.startsWith("/api/chats") && method !== "GET")) {
|
||||
if ((/\/(turn|generate|transcribe)(\/|$)/.test(p) && p !== "/api/stt/transcribe") || (p.startsWith("/api/chats") && method !== "GET")) {
|
||||
state.generation.push(`${method} ${p}`);
|
||||
return block();
|
||||
}
|
||||
@@ -87,7 +103,7 @@ export const test = base.extend<{ harness: HarnessState }>({
|
||||
vision: state.settings?.models?.vision?.model ? { ...state.settings.models.vision, model: state.settings.models.vision.model } : null,
|
||||
};
|
||||
const catalog: ModelCatalogResponse = {
|
||||
models: state.keyPresent ? [syntheticModel] : [], defaults: slots.strong, slots,
|
||||
models: state.models ?? (state.keyPresent ? [syntheticModel] : []), defaults: slots.strong, slots,
|
||||
rolePins: { researcher: slots.cheap, demoBuilder: slots.cheap, visualMaker: slots.vision, extractor: slots.cheap, auditor: slots.cheap },
|
||||
providers: [{ id: "openai", envVar: provider.envVar, keyPresent: state.keyPresent }],
|
||||
};
|
||||
@@ -97,9 +113,23 @@ export const test = base.extend<{ harness: HarnessState }>({
|
||||
if (p === "/api/settings/models" && method === "PUT") {
|
||||
const body = request.postDataJSON() as NonNullable<UserSettings["models"]>;
|
||||
state.writes.push({ path: p, body });
|
||||
await state.modelsWrite.wait;
|
||||
if (state.modelsWrite.status !== 200) return json(route, { error: "Synthetic settings rejection" }, state.modelsWrite.status);
|
||||
state.settings = { ...state.settings, version: 1, models: body };
|
||||
return json(route, { settings: state.settings });
|
||||
}
|
||||
if (p === "/api/settings/general" && method === "PUT") {
|
||||
const body = request.postDataJSON() as UserSettings["general"];
|
||||
state.writes.push({ path: p, body });
|
||||
await state.seedGeneral(body ?? {});
|
||||
return json(route, { settings: state.settings });
|
||||
}
|
||||
if (p === "/api/search" && method === "GET") return json(route, { available: false, engine: null, reason: "No synthetic search key" });
|
||||
if (p === "/api/settings/prompts" && method === "GET") return json(route, { prober: [], teacher: [] });
|
||||
if (p === "/api/stt/transcribe") {
|
||||
if (method === "GET") return json(route, { configured: state.transcription.configured });
|
||||
if (method === "POST") return json(route, state.transcription.body, state.transcription.status);
|
||||
}
|
||||
if (p === "/api/subscriptions" && method === "GET") {
|
||||
const chatgpt: ChatGPTStatus = { available: false, pending: false, allowBackgroundUse: false, activeId: null, accounts: [], error: null };
|
||||
const native: NativeSubscriptionStatus[] = ["github-copilot", "xai"].map((providerId) => ({
|
||||
@@ -117,7 +147,8 @@ export const test = base.extend<{ harness: HarnessState }>({
|
||||
if (method === "GET" && ["/api/activity", "/api/chats", "/api/events", "/api/graph", "/api/graph/status", "/api/graph/costs", "/api/graph/render-settings"].includes(p)) return route.continue();
|
||||
return block();
|
||||
});
|
||||
await use(state);
|
||||
try { await use(state); }
|
||||
finally { await rm(settingsFile, { force: true }); }
|
||||
expect(state.blocked, "Unexpected network request (blocked before dispatch)").toEqual([]);
|
||||
expect(state.generation, "Setup/settings must not generate or send").toEqual([]);
|
||||
},
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
import type { Page } from "@playwright/test";
|
||||
import { test, expect, syntheticModel, type HarnessState } from "./fixtures";
|
||||
import type { GeneralSettings } from "../src/core/settings";
|
||||
import type { ModelCatalogEntry } from "../src/core/contracts/models";
|
||||
|
||||
const openrouter = "microsoft/mai-transcribe-2";
|
||||
const chatgpt: ModelCatalogEntry = { ...syntheticModel, provider: "chatgpt", id: "synthetic-chatgpt-model", name: "Synthetic ChatGPT", reasoning: true, supportedLevels: ["low", "high"] };
|
||||
const compactLanguage = "language-select-sttLanguage-compact";
|
||||
const sttLanguage = "language-select-sttLanguage";
|
||||
const uiLanguage = "language-select-language";
|
||||
|
||||
function configure(harness: HarnessState) {
|
||||
harness.keyPresent = true;
|
||||
harness.settings = { version: 1, models: {
|
||||
strong: { model: { provider: "openai", id: syntheticModel.id } },
|
||||
cheap: { model: { provider: "openai", id: syntheticModel.id } },
|
||||
vision: { model: { provider: "openai", id: syntheticModel.id } },
|
||||
} };
|
||||
}
|
||||
|
||||
async function settings(page: Page, section: string) {
|
||||
await page.getByRole("button", { name: /^(Settings|Настройки)$/ }).click();
|
||||
await page.getByTestId(`settings-nav-${section}`).click();
|
||||
await expect(page.getByTestId(`section-${section}`)).toBeVisible();
|
||||
}
|
||||
|
||||
async function choose(page: Page, id: string, label: string) {
|
||||
const button = page.getByTestId(id);
|
||||
await expect(button).toBeEnabled();
|
||||
await button.click();
|
||||
await page.getByRole("menuitemradio", { name: label, exact: true }).click();
|
||||
await expect(button).toBeEnabled();
|
||||
}
|
||||
|
||||
async function modelTooltip(page: Page, text: string) {
|
||||
const trigger = page.getByTestId("stats-model");
|
||||
if (text.includes(":")) await expect(trigger).toContainText(text.slice(text.indexOf(":") + 1));
|
||||
await page.mouse.move(0, 0);
|
||||
await trigger.hover();
|
||||
await expect(page.getByRole("tooltip")).toContainText(text);
|
||||
await page.mouse.move(0, 0);
|
||||
await expect(page.getByRole("tooltip")).toHaveCount(0);
|
||||
}
|
||||
|
||||
function generalWrites(harness: HarnessState) {
|
||||
return harness.writes.filter((write) => write.path === "/api/settings/general").map((write) => write.body);
|
||||
}
|
||||
|
||||
// No host microphone permission or physical device: the production AudioWorklet
|
||||
// consumes a synthetic oscillator MediaStream through the normal getUserMedia seam.
|
||||
async function syntheticMicrophone(page: Page) {
|
||||
await page.addInitScript(() => {
|
||||
const resources = globalThis as typeof globalThis & { __syntheticTracks: MediaStreamTrack[] };
|
||||
resources.__syntheticTracks = [];
|
||||
let audio: AudioContext;
|
||||
Object.defineProperty(navigator.mediaDevices, "getUserMedia", { value: async () => {
|
||||
audio ??= new AudioContext();
|
||||
await audio.resume();
|
||||
const tone = audio.createOscillator();
|
||||
const destination = audio.createMediaStreamDestination();
|
||||
tone.connect(destination);
|
||||
tone.start();
|
||||
resources.__syntheticTracks.push(...destination.stream.getTracks());
|
||||
return destination.stream;
|
||||
} });
|
||||
});
|
||||
}
|
||||
|
||||
async function expectMicrophoneStopped(page: Page) {
|
||||
await expect.poll(() => page.evaluate(() => {
|
||||
const tracks = (globalThis as typeof globalThis & { __syntheticTracks: MediaStreamTrack[] }).__syntheticTracks;
|
||||
return tracks.length > 0 && tracks.every((track) => track.readyState === "ended");
|
||||
})).toBe(true);
|
||||
}
|
||||
|
||||
test.describe("language preference authority", () => {
|
||||
test.use({ locale: "ru-RU" });
|
||||
|
||||
test("unset STT follows system, not interface language, and localized model fallback is accessible", async ({ page, harness }) => {
|
||||
await harness.seedGeneral({ language: "en" });
|
||||
await page.goto("/");
|
||||
await expect(page.getByTestId(compactLanguage)).toContainText("Русский");
|
||||
await expect(page.getByRole("textbox", { name: "What do you want to learn?", exact: true })).toBeVisible();
|
||||
await modelTooltip(page, "Model selector");
|
||||
await settings(page, "general");
|
||||
await choose(page, uiLanguage, "Русский");
|
||||
await expect(page.getByTestId("settings-dialog")).toHaveAttribute("aria-label", "Настройки");
|
||||
await expect.poll(() => generalWrites(harness)).toEqual([{ language: "ru" }]);
|
||||
expect(harness.settings?.general).not.toHaveProperty("sttLanguage");
|
||||
await page.getByTestId("settings-close").click();
|
||||
await page.reload();
|
||||
await modelTooltip(page, "Выбор модели");
|
||||
await expect(page.getByTestId(compactLanguage)).toContainText("Русский");
|
||||
await settings(page, "stt");
|
||||
await page.getByTestId(sttLanguage).click();
|
||||
await expect(page.getByRole("menuitemradio", { name: "Язык системы", exact: true })).toHaveAttribute("aria-checked", "true");
|
||||
await page.keyboard.press("Escape");
|
||||
expect(harness.settings?.general).toEqual({ language: "ru" });
|
||||
});
|
||||
|
||||
test("English/Russian/multilingual STT values persist independently of interface and both STT models", async ({ page, harness }) => {
|
||||
configure(harness);
|
||||
await harness.seedGeneral({ language: "en", sttModel: openrouter, autoUpdate: false, sttPreferredLanguages: ["en", "ru"] });
|
||||
await page.goto("/");
|
||||
await modelTooltip(page, `openai:${syntheticModel.id}`);
|
||||
const expected: GeneralSettings = { ...harness.settings!.general };
|
||||
const payloads: GeneralSettings[] = [];
|
||||
await settings(page, "stt");
|
||||
for (const model of ["deepgram/nova-3", openrouter] as const) {
|
||||
const modelLabel = model === openrouter ? "MAI-Transcribe-2 · OpenRouter" : "Nova-3 · Deepgram";
|
||||
await choose(page, "stt-model-select", modelLabel);
|
||||
expected.sttModel = model;
|
||||
payloads.push({ ...expected });
|
||||
for (const [code, label] of [["en", "English"], ["ru", "Русский"], ["multi", "Multilingual"]] as const) {
|
||||
await choose(page, sttLanguage, label);
|
||||
expected.sttLanguage = code;
|
||||
payloads.push({ ...expected });
|
||||
await expect.poll(() => generalWrites(harness)).toEqual(payloads);
|
||||
expect(harness.settings?.general).toEqual(expected);
|
||||
await page.getByTestId("settings-close").click();
|
||||
await page.reload();
|
||||
await expect(page.getByTestId(compactLanguage)).toContainText(code === "multi" ? "Multi" : label);
|
||||
await settings(page, "stt");
|
||||
await expect(page.getByTestId("stt-model-select")).toContainText(modelLabel);
|
||||
await expect(page.getByTestId(sttLanguage)).toContainText(label);
|
||||
}
|
||||
}
|
||||
await page.getByTestId("settings-nav-general").click();
|
||||
await choose(page, uiLanguage, "Русский");
|
||||
expected.language = "ru";
|
||||
payloads.push({ ...expected });
|
||||
await expect.poll(() => generalWrites(harness)).toEqual(payloads);
|
||||
await page.getByTestId("settings-close").click();
|
||||
await page.reload();
|
||||
await expect(page.locator("html")).toHaveAttribute("lang", "ru");
|
||||
await settings(page, "stt");
|
||||
await expect(page.getByTestId("stt-model-select")).toContainText("MAI-Transcribe-2 · OpenRouter");
|
||||
await expect(page.getByTestId(sttLanguage)).toContainText("Несколько языков");
|
||||
await page.getByTestId("settings-nav-general").click();
|
||||
await expect(page.getByTestId(uiLanguage)).toContainText("Русский");
|
||||
await choose(page, uiLanguage, "English");
|
||||
expected.language = "en";
|
||||
payloads.push({ ...expected });
|
||||
await expect.poll(() => generalWrites(harness)).toEqual(payloads);
|
||||
await page.getByTestId("settings-close").click();
|
||||
await page.reload();
|
||||
await expect(page.getByRole("textbox", { name: "What do you want to learn?", exact: true })).toBeVisible();
|
||||
await expect(page.getByTestId(compactLanguage)).toContainText("Multi");
|
||||
});
|
||||
});
|
||||
|
||||
test("OpenRouter dictation inserts synthetic audio transcript; failure retains draft and retry recovers", async ({ page, harness }) => {
|
||||
configure(harness);
|
||||
await harness.seedGeneral({ language: "en", sttLanguage: "ru", sttModel: openrouter });
|
||||
await syntheticMicrophone(page);
|
||||
const requests: { model: string | null; language: string | null; size: number; type: string | undefined; riff: string }[] = [];
|
||||
// Production sends raw WAV with model/language in the URL, not multipart.
|
||||
page.on("request", (request) => {
|
||||
const url = new URL(request.url());
|
||||
if (url.pathname !== "/api/stt/transcribe" || request.method() !== "POST") return;
|
||||
const audio = request.postDataBuffer()!;
|
||||
requests.push({ model: url.searchParams.get("model"), language: url.searchParams.get("language"), size: audio.length, type: request.headers()["content-type"], riff: audio.subarray(0, 4).toString() });
|
||||
});
|
||||
await page.goto("/");
|
||||
const draft = page.getByRole("textbox", { name: "What do you want to learn?", exact: true });
|
||||
await draft.fill("Keep this draft.");
|
||||
async function record() {
|
||||
await page.getByTestId("dictation-toggle").click();
|
||||
await expect(page.getByTestId("dictation-bar")).toBeVisible();
|
||||
await expect(page.getByTestId(compactLanguage)).toBeDisabled();
|
||||
// Bounded sample window lets the real AudioWorklet emit PCM frames.
|
||||
await page.waitForTimeout(400);
|
||||
await page.getByTestId("dictation-toggle").click();
|
||||
}
|
||||
await record();
|
||||
await expect(draft).toHaveText("Keep this draft. Synthetic transcript");
|
||||
await expectMicrophoneStopped(page);
|
||||
harness.transcription.status = 503;
|
||||
harness.transcription.body = { message: "Synthetic transcription unavailable" };
|
||||
const retained = await draft.innerText();
|
||||
await record();
|
||||
await expect(page.getByTestId("dictation-toggle")).toContainText("Synthetic transcription unavailable");
|
||||
await expect(draft).toHaveText(retained);
|
||||
await expectMicrophoneStopped(page);
|
||||
harness.transcription.status = 200;
|
||||
harness.transcription.body = { text: "Recovered transcript" };
|
||||
await record();
|
||||
await expect(draft).toHaveText(`${retained} Recovered transcript`);
|
||||
await expectMicrophoneStopped(page);
|
||||
expect(requests).toHaveLength(3);
|
||||
for (const request of requests) {
|
||||
expect(request).toMatchObject({ model: openrouter, language: "ru", type: "audio/wav", riff: "RIFF" });
|
||||
expect(request.size).toBeGreaterThan(44);
|
||||
expect(request.size).toBeLessThan(25 * 1024 * 1024);
|
||||
}
|
||||
expect(harness.generation).toEqual([]);
|
||||
});
|
||||
|
||||
test("unavailable OpenRouter prevents upload, stops capture and deep-links to the missing key", async ({ page, harness }) => {
|
||||
configure(harness);
|
||||
await harness.seedGeneral({ language: "en", sttModel: openrouter });
|
||||
harness.transcription.configured = false;
|
||||
// The existing latency contract acquires mic and provider status concurrently.
|
||||
await syntheticMicrophone(page);
|
||||
const uploads: string[] = [];
|
||||
page.on("request", (request) => {
|
||||
if (new URL(request.url()).pathname === "/api/stt/transcribe" && request.method() === "POST") uploads.push(request.url());
|
||||
});
|
||||
await page.goto("/");
|
||||
const draft = page.getByRole("textbox", { name: "What do you want to learn?", exact: true });
|
||||
await draft.fill("Keep unavailable draft");
|
||||
await page.getByTestId("dictation-toggle").click();
|
||||
await expect(page.getByTestId("dictation-toggle")).toContainText("Dictation needs an OpenRouter key");
|
||||
await expect(page.getByTestId("dictation-bar")).toHaveCount(0);
|
||||
await expectMicrophoneStopped(page);
|
||||
await expect(draft).toHaveText("Keep unavailable draft");
|
||||
expect(uploads).toEqual([]);
|
||||
expect(harness.generation).toEqual([]);
|
||||
await page.getByTestId("dictation-toggle").click();
|
||||
await expect(page.getByTestId("section-keys")).toBeVisible();
|
||||
});
|
||||
|
||||
test("ChatGPT Fast-mode pending, rejected save and retry preserve all other preferences without sending", async ({ page, harness }) => {
|
||||
configure(harness);
|
||||
harness.models = [chatgpt, syntheticModel];
|
||||
harness.settings = { ...harness.settings!, models: {
|
||||
strong: { model: { provider: "chatgpt", id: chatgpt.id }, thinkingLevel: "high" },
|
||||
cheap: { model: { provider: "openai", id: syntheticModel.id } },
|
||||
vision: { model: { provider: "chatgpt", id: chatgpt.id }, thinkingLevel: "low", fastMode: false },
|
||||
} };
|
||||
await harness.seedGeneral({ language: "en", sttLanguage: "ru", sttModel: openrouter, autoUpdate: false });
|
||||
const before = structuredClone(harness.settings);
|
||||
await page.goto("/");
|
||||
const draft = page.getByRole("textbox", { name: "What do you want to learn?", exact: true });
|
||||
await draft.fill("Never send this Fast-mode draft");
|
||||
await settings(page, "agent");
|
||||
const toggle = page.getByTestId("slot-fast-strong");
|
||||
await expect(toggle).toHaveAttribute("aria-pressed", "false");
|
||||
await expect(page.getByTestId("slot-fast-cheap")).toBeDisabled();
|
||||
let release!: () => void;
|
||||
harness.modelsWrite.wait = new Promise<void>((resolve) => { release = resolve; });
|
||||
harness.modelsWrite.status = 503;
|
||||
const payload = { ...before!.models, strong: { ...before!.models!.strong, fastMode: true } };
|
||||
await toggle.click();
|
||||
try {
|
||||
for (const slot of ["strong", "vision"]) for (const kind of ["fast", "model", "thinking"]) {
|
||||
await expect(page.getByTestId(`slot-${kind}-${slot}`)).toBeDisabled();
|
||||
}
|
||||
await expect(toggle).toHaveAttribute("aria-pressed", "false");
|
||||
await expect.poll(() => harness.writes.filter((write) => write.path === "/api/settings/models")).toEqual([{ path: "/api/settings/models", body: payload }]);
|
||||
expect(harness.settings).toEqual(before);
|
||||
} finally { release(); }
|
||||
await expect(page.getByTestId("settings-error")).toHaveText("models: Synthetic settings rejection");
|
||||
await expect(toggle).toBeEnabled();
|
||||
await expect(toggle).toHaveAttribute("aria-pressed", "false");
|
||||
expect(harness.settings).toEqual(before);
|
||||
harness.modelsWrite = { status: 200 };
|
||||
await toggle.click();
|
||||
await expect(toggle).toHaveAttribute("aria-pressed", "true");
|
||||
await expect(page.getByTestId("settings-error")).toHaveCount(0);
|
||||
expect(harness.settings).toEqual({ ...before, models: payload });
|
||||
expect(harness.writes.filter((write) => write.path === "/api/settings/models")).toEqual([
|
||||
{ path: "/api/settings/models", body: payload }, { path: "/api/settings/models", body: payload },
|
||||
]);
|
||||
await page.getByTestId("settings-close").click();
|
||||
await expect(draft).toHaveText("Never send this Fast-mode draft");
|
||||
await settings(page, "agent");
|
||||
await expect(page.getByTestId("slot-fast-strong")).toHaveAttribute("aria-pressed", "true");
|
||||
expect(harness.generation).toEqual([]);
|
||||
});
|
||||
Reference in New Issue
Block a user