feat: add OpenRouter dictation and speech recognition settings
This commit is contained in:
+113
-15
@@ -3,14 +3,17 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Loader2Icon, MicIcon } from "lucide-react";
|
||||
import { useSettingsDialog } from "@/components/settings/settings-dialog";
|
||||
import { dictationUrl } from "@/src/core";
|
||||
import { DEFAULT_STT_MODEL, OPENROUTER_STT_MODEL, STT_MAX_AUDIO_BYTES, WAV_HEADER_BYTES, dictationUrl, dictationWav, type SttModel, type SttLanguage } from "@/src/core";
|
||||
import { LanguageSelect, useI18n } from "@/components/i18n";
|
||||
import { Hint } from "@/components/ui/hint";
|
||||
|
||||
/**
|
||||
* Voice dictation — the cloud engine (STT wave 1, 2026-09-17).
|
||||
* Voice dictation — direct Deepgram streaming or whole-recording OpenRouter STT.
|
||||
* MAI-Transcribe-2 records locally, drains the final PCM frame on Stop, then
|
||||
* sends one WAV through /api/stt/transcribe. That route holds the API key.
|
||||
* Both engines buffer text and append once when dictation finishes.
|
||||
*
|
||||
* Architecture (docs/research/stt.md): the renderer captures mic audio in
|
||||
* Deepgram architecture: the renderer captures mic audio in
|
||||
* an AudioWorklet (public/worklets/pcm-capture.js → linear16 16kHz mono),
|
||||
* opens the Deepgram streaming WebSocket DIRECTLY (audio never loops
|
||||
* through our server) with a short-lived token minted by /api/stt/token
|
||||
@@ -107,6 +110,7 @@ export const DEEPGRAM_KEY_ID = "DEEPGRAM_API_KEY";
|
||||
* instead. Exported for pins; the errored idle button's click recognizes
|
||||
* it and deep-links Settings → Keys → the Deepgram row. */
|
||||
export const DICTATION_KEYLESS = "Dictation needs a Deepgram key";
|
||||
export const OPENROUTER_DICTATION_KEYLESS = "Dictation needs an OpenRouter key";
|
||||
|
||||
/** Mint a fresh short-lived token (the route holds the real key
|
||||
* server-side). Module-level: the start path's warm cache and the
|
||||
@@ -239,7 +243,7 @@ export function useDictationTarget(append: (text: string) => void, claimed: bool
|
||||
}, [append, claimed]);
|
||||
}
|
||||
|
||||
export type DictationPhase = "idle" | "connecting" | "recording" | "stopping";
|
||||
export type DictationPhase = "idle" | "connecting" | "recording" | "stopping" | "transcribing";
|
||||
|
||||
export type Dictation = {
|
||||
phase: DictationPhase;
|
||||
@@ -283,6 +287,14 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
const selectedUrl = useRef("");
|
||||
selectedUrl.current = dictationUrl(sttLanguage, general.sttPreferredLanguages);
|
||||
const recordingUrl = useRef(selectedUrl.current);
|
||||
const selected = useRef({ model: general.sttModel ?? DEFAULT_STT_MODEL, language: sttLanguage });
|
||||
selected.current = { model: general.sttModel ?? DEFAULT_STT_MODEL, language: sttLanguage };
|
||||
const recordingConfig = useRef<{ model: SttModel; language: SttLanguage }>(selected.current);
|
||||
const audioFrames = useRef<Uint8Array[]>([]);
|
||||
const audioBytes = useRef(0);
|
||||
const upload = useRef<AbortController | null>(null);
|
||||
const drain = useRef<(() => void) | null>(null);
|
||||
const stopping = useRef(false);
|
||||
const [phase, setPhase] = useState<DictationPhase>("idle");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
/** Latest RMS from the worklet — the meter samples this (dictate pattern). */
|
||||
@@ -303,7 +315,7 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
/** Frames captured before the socket is OPEN (mitigation A): flushed in
|
||||
* order on open; capped at 256 frames (≈32s) with the oldest dropped;
|
||||
* cleared at teardown so a dead run's audio can't leak into the next. */
|
||||
const pendingRef = useRef<ArrayBuffer[]>([]);
|
||||
const pendingRef = useRef<Array<ArrayBuffer | Int16Array>>([]);
|
||||
const genRef = useRef(0); // stale-handler guard (dictate pattern)
|
||||
const safetyRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const reconnectsRef = useRef(0);
|
||||
@@ -327,6 +339,13 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
/** Full teardown of one run's machinery. Idempotent; generation-guarded by callers. */
|
||||
const cleanup = useCallback(() => {
|
||||
genRef.current++;
|
||||
upload.current?.abort();
|
||||
upload.current = null;
|
||||
drain.current?.();
|
||||
drain.current = null;
|
||||
audioFrames.current = [];
|
||||
audioBytes.current = 0;
|
||||
stopping.current = false;
|
||||
dictationActiveFlag = false;
|
||||
setDictating(false);
|
||||
flush(); // the deferred transcript appears when the run ends (owner round 2)
|
||||
@@ -356,7 +375,7 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
for (const w of waitersRef.current.splice(0)) w();
|
||||
// Pre-mint for the next run (mitigation D) — best-effort; a keyless
|
||||
// page just doesn't warm.
|
||||
warmTokens.refresh();
|
||||
if (selected.current.model === DEFAULT_STT_MODEL) warmTokens.refresh();
|
||||
}, []);
|
||||
|
||||
/** Open the Deepgram socket and wire its messages. `gen` guards staleness. */
|
||||
@@ -427,6 +446,11 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
const start = useCallback(async () => {
|
||||
// Pin the complete configuration once; reconnects must retain both language and hints.
|
||||
recordingUrl.current = selectedUrl.current;
|
||||
recordingConfig.current = { ...selected.current };
|
||||
const batch = recordingConfig.current.model === OPENROUTER_STT_MODEL;
|
||||
audioFrames.current = [];
|
||||
audioBytes.current = 0;
|
||||
stopping.current = false;
|
||||
setDictating(true);
|
||||
dictationActiveFlag = true;
|
||||
setError(null);
|
||||
@@ -442,7 +466,17 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
// the token runs concurrently with it. 127.0.0.1 is a secure
|
||||
// context, so getUserMedia works without TLS; macOS TCC prompts on
|
||||
// first use.
|
||||
const tokenPromise = warmTokens.mint();
|
||||
const controller = new AbortController();
|
||||
upload.current = controller;
|
||||
const tokenPromise = batch ? fetch("/api/stt/transcribe", { signal: controller.signal }).then(async (response) => {
|
||||
if (!response.ok) throw new Error("OpenRouter transcription is unavailable");
|
||||
const status = await response.json() as { configured?: boolean };
|
||||
if (!status.configured) throw new Error(OPENROUTER_DICTATION_KEYLESS);
|
||||
return "";
|
||||
}).catch((error: unknown) => {
|
||||
if (error instanceof Error && error.message === OPENROUTER_DICTATION_KEYLESS) throw error;
|
||||
throw new Error("OpenRouter transcription is unavailable");
|
||||
}) : warmTokens.mint();
|
||||
tokenPromise.catch(() => {}); // surfaced below; no unhandled rejection
|
||||
const micPromise = navigator.mediaDevices.getUserMedia({
|
||||
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
@@ -466,10 +500,26 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
const worklet = new AudioWorkletNode(ctx, "pcm-capture");
|
||||
workletRef.current = worklet;
|
||||
worklet.port.onmessage = (e) => {
|
||||
const msg = e.data as { type: string; rms?: number; buffer?: ArrayBuffer };
|
||||
if (gen !== genRef.current) return;
|
||||
const msg = e.data as { type: string; rms?: number; buffer?: ArrayBuffer | Int16Array };
|
||||
if (msg.type === "level") {
|
||||
levelRef.current = msg.rms ?? 0; // never reset to zero (dictate pattern)
|
||||
} else if (msg.type === "flushed") {
|
||||
drain.current?.();
|
||||
drain.current = null;
|
||||
} else if (msg.type === "audio") {
|
||||
if (batch) {
|
||||
const raw = msg.buffer!;
|
||||
const bytes = raw instanceof ArrayBuffer ? new Uint8Array(raw) : new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength);
|
||||
if (audioBytes.current + bytes.byteLength + WAV_HEADER_BYTES > STT_MAX_AUDIO_BYTES) {
|
||||
setError("Recording exceeds OpenRouter's 25 MB audio limit");
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
audioFrames.current.push(bytes);
|
||||
audioBytes.current += bytes.byteLength;
|
||||
return;
|
||||
}
|
||||
const ws = wsRef.current;
|
||||
if (ws && ws.readyState === WebSocket.OPEN) ws.send(msg.buffer!);
|
||||
// Mitigation A: not OPEN yet (handshake or reconnect gap) —
|
||||
@@ -511,12 +561,15 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
if (gen !== genRef.current) return;
|
||||
// Mitigation B: the handshake fires the moment the graph is live,
|
||||
// overlapping any remaining token wait — mic-gated, never before it.
|
||||
openSocket(gen, token);
|
||||
if (batch) { if (!stopping.current) setPhase("recording"); }
|
||||
else openSocket(gen, token);
|
||||
}, [openSocket, cleanup]);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
const gen = genRef.current;
|
||||
if (phase !== "recording" && phase !== "connecting") return;
|
||||
if (stopping.current) return;
|
||||
stopping.current = true;
|
||||
setPhase("stopping");
|
||||
|
||||
// Mic dies FIRST — no more audio enqueues (dictate pattern). The
|
||||
@@ -525,6 +578,43 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
streamRef.current = null;
|
||||
sourceRef.current?.disconnect();
|
||||
sourceRef.current = null;
|
||||
if (recordingConfig.current.model === OPENROUTER_STT_MODEL) {
|
||||
const worklet = workletRef.current;
|
||||
if (!worklet) { cleanup(); return; }
|
||||
void (async () => {
|
||||
try {
|
||||
// Source is disconnected; drain its last partial PCM frame before wrapping the WAV.
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => { drain.current = null; reject(new Error("Could not finish microphone capture")); }, 2000);
|
||||
drain.current = () => { clearTimeout(timer); resolve(); };
|
||||
worklet.port.postMessage({ type: "flush" });
|
||||
});
|
||||
if (gen !== genRef.current) return;
|
||||
worklet.disconnect();
|
||||
workletRef.current = null;
|
||||
if (audioBytes.current === 0) { cleanup(); return; }
|
||||
setPhase("transcribing");
|
||||
const wav = dictationWav(audioFrames.current);
|
||||
audioFrames.current = [];
|
||||
const params = new URLSearchParams(recordingConfig.current);
|
||||
const response = await fetch(`/api/stt/transcribe?${params}`, {
|
||||
method: "POST", headers: { "Content-Type": "audio/wav" }, body: wav,
|
||||
signal: AbortSignal.any([upload.current!.signal, AbortSignal.timeout(75_000)]),
|
||||
});
|
||||
const body = await response.json() as { text?: string; message?: string };
|
||||
if (gen !== genRef.current) return;
|
||||
if (!response.ok) throw new Error(body.message ?? "OpenRouter transcription is unavailable");
|
||||
if (typeof body.text !== "string") throw new Error("OpenRouter returned an invalid transcript");
|
||||
if (body.text.trim()) bufferRef.current.push(body.text.trim());
|
||||
cleanup();
|
||||
} catch (error) {
|
||||
if (gen !== genRef.current) return;
|
||||
setError(error instanceof Error && error.name === "TimeoutError" ? "OpenRouter transcription timed out" : error instanceof Error && error.name === "Error" ? error.message : "OpenRouter transcription is unavailable");
|
||||
cleanup();
|
||||
}
|
||||
})();
|
||||
return;
|
||||
}
|
||||
workletRef.current?.disconnect();
|
||||
workletRef.current = null;
|
||||
|
||||
@@ -587,8 +677,13 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
|
||||
// page owns ONE hook instance, and only the mounted page's listener is
|
||||
// attached.
|
||||
useEffect(() => {
|
||||
warmTokens.refresh(); // pre-mint at mount (mitigation D) — silent if keyless
|
||||
}, []);
|
||||
if ((general.sttModel ?? DEFAULT_STT_MODEL) === DEFAULT_STT_MODEL) warmTokens.refresh();
|
||||
}, [general.sttModel]);
|
||||
useEffect(() => () => {
|
||||
// Navigation cancels a pending upload and releases the microphone without inserting text.
|
||||
bufferRef.current = [];
|
||||
cleanup();
|
||||
}, [cleanup]);
|
||||
useEffect(() => {
|
||||
window.addEventListener("lh:kbd-toggle-dictation", toggle);
|
||||
return () => window.removeEventListener("lh:kbd-toggle-dictation", toggle);
|
||||
@@ -631,9 +726,8 @@ export function DictationBar({ dictation }: { dictation: Dictation }) {
|
||||
* idle button deep-links Settings → Keys → the Deepgram row and clears
|
||||
* the error — the surface is clean when the user returns. Any other
|
||||
* error keeps the old behavior (click = retry, which clears). */
|
||||
const keyless = error === DICTATION_KEYLESS;
|
||||
const keyless = error === DICTATION_KEYLESS || error === OPENROUTER_DICTATION_KEYLESS;
|
||||
const [levels, setLevels] = useState<number[]>(() => new Array(METER_CELLS).fill(0));
|
||||
const recording = phase === "recording" || phase === "stopping";
|
||||
/** Adaptive cell count (owner round 2): derived from the meter's
|
||||
* measured width — narrow rows show fewer bars instead of
|
||||
* overflowing under neighbors. */
|
||||
@@ -691,12 +785,12 @@ export function DictationBar({ dictation }: { dictation: Dictation }) {
|
||||
onClick={() => {
|
||||
if (keyless) {
|
||||
clearError();
|
||||
openSettings("keys", DEEPGRAM_KEY_ID);
|
||||
openSettings("keys", error === OPENROUTER_DICTATION_KEYLESS ? "OPENROUTER_API_KEY" : DEEPGRAM_KEY_ID);
|
||||
} else {
|
||||
toggle();
|
||||
}
|
||||
}}
|
||||
aria-label={error ? t("Dictation failed: {v0}", { v0: error }) : t("Start dictation")}
|
||||
aria-label={error ? t("Dictation failed: {v0}", { v0: t(error) }) : t("Start dictation")}
|
||||
data-testid="dictation-toggle"
|
||||
// Same blurring as the floating chrome (owner, fix wave round 3),
|
||||
// but lower opacity — the mic stays quieter than the model pills.
|
||||
@@ -710,6 +804,10 @@ export function DictationBar({ dictation }: { dictation: Dictation }) {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (phase === "transcribing") return <div className="flex h-7 w-full min-w-0 items-center gap-2 text-xs text-muted-foreground" role="status" data-testid="dictation-transcribing">
|
||||
<Loader2Icon className="size-3.5 shrink-0 animate-spin" /><span className="min-w-0 flex-1 truncate">{t("Transcribing…")}</span>
|
||||
<button type="button" onClick={dictation.cancel} className="shrink-0 rounded-sm border border-border px-1.5 py-1 hover:bg-muted">{t("Cancel")}</button>
|
||||
</div>;
|
||||
|
||||
return (
|
||||
<div className="flex w-full min-w-0 items-center gap-2" data-testid="dictation-bar">
|
||||
|
||||
Reference in New Issue
Block a user