feat: voice dictation (STT wave 1 — Deepgram cloud)
Two-band architecture from docs/research/stt.md: renderer captures mic audio in an AudioWorklet (linear16 16kHz mono, ~128ms frames, per-quantum RMS for the meter) and streams it directly to Deepgram over WebSocket — audio never loops through our server. The real key stays server-side; /api/stt/token mints a short-lived grant token (TTL 300s) per run. Patterns ported from the owner's pi dictate extension (tuned in real use): generation counter guarding every async handler, the latest-RMS-never-resets meter (-50..-10dB window, 60ms tick), mic-first stop + CloseStream + 3s safety net, endpointing=300 with the utterance_end_ms=1000 backstop. UI (owner placement): mic button in the stats bar's middle slot on all three surfaces (home graph-view, chat, draft); while recording the meter unfurls to fill the row's free width with the interim ghost at its flank. Esc ends dictation from the composer. Token mint comes before getUserMedia so a keyless app fails fast without a TCC prompt; one automatic reconnect on abnormal socket close, errors surface inline. Live-smoked the full error path in the dev shell (bar renders on all three pages, 403 grant failure shows inline, mic untouched). The full audio path needs a Deepgram key with Member permission to mint tokens.
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* POST-less BYOK mint: the renderer never holds the real Deepgram key.
|
||||
*
|
||||
* The key lives server-side only (auth.json → process.env in the desktop
|
||||
* app; .env.local in dev — same seam as OPENROUTER_API_KEY). A dictation
|
||||
* run starts here: mint a short-lived access token (TTL 300s — a run can
|
||||
* outlast the 30s default), the renderer opens its WebSocket to Deepgram
|
||||
* with it, and audio flows renderer → Deepgram directly, never through
|
||||
* this server. The token is only checked at the WebSocket handshake, so
|
||||
* long dictation sessions survive the TTL fine (Deepgram docs).
|
||||
*
|
||||
* NOTE: the key needs ≥ Member permission in the Deepgram console to call
|
||||
* /auth/grant (their docs, token-based auth FAQ) — plain read/transcribe
|
||||
* keys return FORBIDDEN here.
|
||||
*/
|
||||
export async function GET() {
|
||||
const key = process.env.DEEPGRAM_API_KEY;
|
||||
if (!key) {
|
||||
return Response.json(
|
||||
{ message: "No DEEPGRAM_API_KEY — add it to auth.json (desktop) or .env.local (dev)." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await fetch("https://api.deepgram.com/v1/auth/grant", {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Token ${key}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ ttl_seconds: 300 }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const detail = await res.text();
|
||||
return Response.json(
|
||||
{ message: `Deepgram token grant failed (${res.status}): ${detail.slice(0, 200)}` },
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
const data = (await res.json()) as { token?: string };
|
||||
if (!data.token) {
|
||||
return Response.json({ message: "Deepgram grant returned no token" }, { status: 502 });
|
||||
}
|
||||
return Response.json({ token: data.token });
|
||||
} catch {
|
||||
return Response.json({ message: "Deepgram unreachable" }, { status: 502 });
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import { AnsweredCard } from "@/components/answered-card";
|
||||
import { artifactRenderers } from "@/components/artifacts";
|
||||
import { SubagentSteerCard } from "@/components/artifacts/steer-card";
|
||||
import { Composer } from "@/components/composer";
|
||||
import { DictationBar, useDictation } from "@/components/dictation";
|
||||
import { JobsContext } from "@/components/jobs-context";
|
||||
import { StatsBar } from "@/components/stats-bar";
|
||||
import { GraphContextMarker } from "@/components/graph-context-marker";
|
||||
@@ -246,6 +247,12 @@ export function Chat({
|
||||
(messages.at(-1)?.parts.some((p) => isToolUIPart(p) && p.state === "input-streaming") ?? false);
|
||||
|
||||
const [text, setText] = useState("");
|
||||
// Voice dictation (STT wave, 2026-09-17): finals append at the end of
|
||||
// the current text (the page owns the separator, dictate-flush style);
|
||||
// interims ghost in the stats bar's middle slot next to the meter.
|
||||
const dictation = useDictation({
|
||||
append: (t) => setText((v) => (v ? v.replace(/\s+$/, "") + " " + t : t)),
|
||||
});
|
||||
// Jobs never gate the composer (contracts.ts "THE ASYNC PROTOCOL" §4): job
|
||||
// tools resolve with an ack mid-turn, so the learner can interject while
|
||||
// sub-agents run — a feature, not a race: the wake pipeline queues steers
|
||||
@@ -864,6 +871,7 @@ export function Chat({
|
||||
phase={phases.at(-1) ?? "probe"}
|
||||
turns={turns}
|
||||
onReserve={setDockReserve}
|
||||
dictationSlot={<DictationBar dictation={dictation} />}
|
||||
composer={
|
||||
<Composer
|
||||
// Stop applies only to a LOCAL stream: a server-side turn has no
|
||||
@@ -874,6 +882,7 @@ export function Chat({
|
||||
onSend={onSend}
|
||||
onStop={() => stop()}
|
||||
canSend={canSend}
|
||||
onEscape={dictation.active ? dictation.stop : undefined}
|
||||
verdict={
|
||||
planVerdict
|
||||
? {
|
||||
@@ -940,6 +949,7 @@ function DockOverlay({
|
||||
onReserve,
|
||||
plan,
|
||||
onOpenPlan,
|
||||
dictationSlot,
|
||||
}: {
|
||||
pending: ReturnType<typeof findPendingToolCall>;
|
||||
/** Teach-phase pending question: arrives collapsed behind the reveal button (W5). */
|
||||
@@ -962,6 +972,9 @@ function DockOverlay({
|
||||
plan?: React.ReactNode;
|
||||
/** Opens the full-DAG popup (the marginalia + chip share it). */
|
||||
onOpenPlan: () => void;
|
||||
/** The dictation strip (STT wave): rendered in the stats bar's middle
|
||||
* slot — the mic button + live meter (components/dictation.tsx). */
|
||||
dictationSlot?: React.ReactNode;
|
||||
}) {
|
||||
const { isAtBottom, scrollToBottom, scrollRef } = useStickToBottomContext();
|
||||
// Teach-phase mount (session reload with a pending check): the dock starts
|
||||
@@ -1226,6 +1239,7 @@ function DockOverlay({
|
||||
chatId={chatId}
|
||||
settings={settings}
|
||||
onSettingsSaved={onSettingsSaved}
|
||||
dictationSlot={dictationSlot}
|
||||
/>
|
||||
</div>
|
||||
{/* The plan marginalia (owner, plan-placement round): the bare
|
||||
|
||||
@@ -20,6 +20,7 @@ export function Composer({
|
||||
canSend,
|
||||
autoFocus,
|
||||
verdict,
|
||||
onEscape,
|
||||
}: {
|
||||
busy: boolean;
|
||||
value: string;
|
||||
@@ -32,6 +33,9 @@ export function Composer({
|
||||
* word, the send square becomes two stacked buttons — green accept,
|
||||
* red decline. Typed text rides the verdict as his note. */
|
||||
verdict?: { onAccept: () => void; onDecline: () => void };
|
||||
/** Escape while a dictation runs stops it (STT wave, 2026-09-17). No-op
|
||||
* when absent — Esc otherwise means nothing in the composer. */
|
||||
onEscape?: () => void;
|
||||
}) {
|
||||
return (
|
||||
<form
|
||||
@@ -47,6 +51,13 @@ export function Composer({
|
||||
value={value}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
// Esc ends dictation before anything else Esc might someday mean
|
||||
// here (the dictate decisions: click-again or Esc to stop).
|
||||
if (e.key === "Escape" && onEscape) {
|
||||
e.preventDefault();
|
||||
onEscape();
|
||||
return;
|
||||
}
|
||||
// Chat convention: Enter sends, Shift+Enter is a newline.
|
||||
// ⌘/Ctrl+Enter also sends — the universal submit shortcut
|
||||
// across every input in the app (thinking field, answers).
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Loader2Icon, MicIcon } from "lucide-react";
|
||||
|
||||
/**
|
||||
* Voice dictation — the cloud engine (STT wave 1, 2026-09-17).
|
||||
*
|
||||
* Architecture (docs/research/stt.md): the renderer captures mic audio in
|
||||
* an AudioWorklet (public/worklets/pcm-capture.js → linear16 16kHz mono),
|
||||
* opens the Deepgram streaming WebSocket DIRECTLY (audio never loops
|
||||
* through our server) with a short-lived token minted by /api/stt/token
|
||||
* (the real key stays server-side), and streams frames while it listens.
|
||||
* Finals append into the composer; interims ghost beside the meter.
|
||||
*
|
||||
* Ported patterns from the owner's pi `dictate` extension (his words:
|
||||
* "very well tuned") — see ~/.pi/agent/extensions/dictate:
|
||||
* - generation counter: every start/stop increments it; all async
|
||||
* handlers no-op when stale, so a dying session's socket can never
|
||||
* tear down the current one (a real bug he hit in the terminal).
|
||||
* - meter: latest-RMS-never-resets — the level ref keeps the last
|
||||
* observed value, so a tick with no fresh audio re-renders the last
|
||||
* level instead of dropping to silence (60ms tick samples the ref).
|
||||
* - dB mapping floor −50 / ceiling −10: the sweet spot for showing
|
||||
* conversational speech as mid-height bars without pegging.
|
||||
* - stop: mic dies FIRST (no more audio enqueues), then CloseStream,
|
||||
* then a 3s safety net in case Deepgram never closes the socket.
|
||||
*
|
||||
* Meter rendering lives in <DictationBar> (own 60ms timer over the level
|
||||
* ref) so the 16-renders/sec never touch the page component — only the
|
||||
* tiny bar strip repaints.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Deepgram streaming params (tuned in the owner's real-use dictate
|
||||
* extension + the research pass): nova-3, his endpointing=300 (faster
|
||||
* finals after a pause), utterance_end_ms=1000 as the noise-robust
|
||||
* backstop, interim_results for the live ghost. Silence streams too
|
||||
* (the worklet never pauses), so the 10s idle timeout can't bite — no
|
||||
* KeepAlive needed.
|
||||
*/
|
||||
const DG_URL =
|
||||
"wss://api.deepgram.com/v1/listen" +
|
||||
"?model=nova-3" +
|
||||
"&encoding=linear16" +
|
||||
"&sample_rate=16000" +
|
||||
"&channels=1" +
|
||||
"&interim_results=true" +
|
||||
"&smart_format=true" +
|
||||
"&punctuate=true" +
|
||||
"&endpointing=300" +
|
||||
"&utterance_end_ms=1000";
|
||||
|
||||
/** Meter tuning — the owner's tested values, see the file header. */
|
||||
const METER_FLOOR_DB = -50;
|
||||
const METER_CEILING_DB = -10;
|
||||
export const METER_TICK_MS = 60;
|
||||
/** Rolling history: 48 cells × 60ms ≈ 2.9s of recent loudness. */
|
||||
export const METER_CELLS = 48;
|
||||
|
||||
/** RMS (0..1) → normalized 0..1 across the audible dB window. */
|
||||
export function rmsToMeter(rms: number): number {
|
||||
if (rms <= 0) return 0;
|
||||
const db = 20 * Math.log10(rms);
|
||||
const t = (db - METER_FLOOR_DB) / (METER_CEILING_DB - METER_FLOOR_DB);
|
||||
return Math.max(0, Math.min(1, t));
|
||||
}
|
||||
|
||||
export type DictationPhase = "idle" | "connecting" | "recording" | "stopping";
|
||||
|
||||
export type Dictation = {
|
||||
phase: DictationPhase;
|
||||
/** Terminal-state explanation surfaced next to the mic (cleared on the next start). */
|
||||
error: string | null;
|
||||
/** Live in-flight partial (the ghost); cleared when the final commits. */
|
||||
interim: string;
|
||||
/** Latest observed RMS, 0..1 — sampled by the meter's own tick, never reset. */
|
||||
levelRef: { current: number };
|
||||
/** Recording or finalizing — true from start to stop completion. */
|
||||
active: boolean;
|
||||
toggle: () => void;
|
||||
stop: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* One dictation instance per composer-bearing page. `append` receives each
|
||||
* committed utterance (final) — the page joins it into its text state
|
||||
* (separator handling is the page's: it owns the current text).
|
||||
*/
|
||||
export function useDictation({ append }: { append: (text: string) => void }): Dictation {
|
||||
const [phase, setPhase] = useState<DictationPhase>("idle");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [interim, setInterim] = useState("");
|
||||
/** Latest RMS from the worklet — the meter samples this (dictate pattern). */
|
||||
const levelRef = useRef(0);
|
||||
|
||||
// Live machinery (non-render values). AudioContext/MediaStream are
|
||||
// closed on stop; the socket is recreated per run.
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const ctxRef = useRef<AudioContext | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const genRef = useRef(0); // stale-handler guard (dictate pattern)
|
||||
const safetyRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const reconnectsRef = useRef(0);
|
||||
|
||||
/** Full teardown of one run's machinery. Idempotent; generation-guarded by callers. */
|
||||
const cleanup = useCallback(() => {
|
||||
genRef.current++;
|
||||
if (safetyRef.current) {
|
||||
clearTimeout(safetyRef.current);
|
||||
safetyRef.current = null;
|
||||
}
|
||||
for (const track of streamRef.current?.getTracks() ?? []) track.stop();
|
||||
streamRef.current = null;
|
||||
void ctxRef.current?.close().catch(() => {});
|
||||
ctxRef.current = null;
|
||||
const ws = wsRef.current;
|
||||
wsRef.current = null;
|
||||
if (ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) {
|
||||
try {
|
||||
ws.close();
|
||||
} catch {}
|
||||
}
|
||||
setInterim("");
|
||||
setPhase("idle");
|
||||
}, []);
|
||||
|
||||
/** Mint a short-lived token — the run's first step, BEFORE the mic: a
|
||||
* keyless app fails fast without ever triggering the macOS TCC prompt. */
|
||||
const mintToken = useCallback(async () => {
|
||||
const res = await fetch("/api/stt/token");
|
||||
const data = (await res.json()) as { token?: string; message?: string };
|
||||
if (!res.ok || !data.token) throw new Error(data.message ?? `token route ${res.status}`);
|
||||
return data.token;
|
||||
}, []);
|
||||
|
||||
/** Open the Deepgram socket and wire its messages. `gen` guards staleness. */
|
||||
const openSocket = useCallback(
|
||||
(gen: number, token: string) => {
|
||||
const ws = new WebSocket(DG_URL, ["token", token]);
|
||||
wsRef.current = ws;
|
||||
|
||||
ws.addEventListener("open", () => {
|
||||
if (gen !== genRef.current) return;
|
||||
setPhase("recording");
|
||||
});
|
||||
ws.addEventListener("message", (ev) => {
|
||||
if (gen !== genRef.current) return;
|
||||
let msg: any;
|
||||
try {
|
||||
msg = JSON.parse(ev.data as string);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (msg.type === "Results") {
|
||||
const text: string | undefined = msg.channel?.alternatives?.[0]?.transcript;
|
||||
if (msg.is_final) {
|
||||
// The committed utterance: hand to the page, clear the ghost.
|
||||
if (text) append(text);
|
||||
setInterim("");
|
||||
} else if (text) {
|
||||
setInterim(text);
|
||||
}
|
||||
} else if (msg.type === "UtteranceEnd") {
|
||||
// Backstop: the endpointing VAD missed, the word-gap one didn't.
|
||||
setInterim("");
|
||||
}
|
||||
});
|
||||
ws.addEventListener("close", (ev) => {
|
||||
if (gen !== genRef.current) return;
|
||||
if (ev.code === 1000) {
|
||||
// Our own CloseStream landing: finalize now — the stop path's
|
||||
// safety net stays as the no-close backstop only.
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
// Abnormal close mid-run: ONE reconnect with a fresh token (the
|
||||
// research pass: resume is our job, no SDK to lean on).
|
||||
if (reconnectsRef.current < 1) {
|
||||
reconnectsRef.current++;
|
||||
mintToken()
|
||||
.then((t) => {
|
||||
if (gen === genRef.current) openSocket(gen, t);
|
||||
})
|
||||
.catch(() => {
|
||||
if (gen === genRef.current) {
|
||||
setError("Deepgram reconnect failed");
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
setError(`Deepgram connection lost (${ev.code})`);
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
},
|
||||
[append, cleanup, mintToken],
|
||||
);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
setError(null);
|
||||
setPhase("connecting");
|
||||
reconnectsRef.current = 0;
|
||||
const gen = ++genRef.current;
|
||||
|
||||
// 1. Token first — fail fast, no TCC prompt for a keyless app.
|
||||
try {
|
||||
const token = await mintToken();
|
||||
if (gen !== genRef.current) return;
|
||||
openSocket(gen, token);
|
||||
} catch (e) {
|
||||
if (gen !== genRef.current) return;
|
||||
setError(e instanceof Error ? e.message : "token fetch failed");
|
||||
setPhase("idle");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// 2. Mic + capture graph. 127.0.0.1 is a secure context —
|
||||
// getUserMedia works without TLS (research); macOS TCC prompts
|
||||
// on first use.
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
});
|
||||
if (gen !== genRef.current) {
|
||||
for (const track of stream.getTracks()) track.stop();
|
||||
return;
|
||||
}
|
||||
streamRef.current = stream;
|
||||
|
||||
const ctx = new AudioContext({ sampleRate: 16000 });
|
||||
ctxRef.current = ctx;
|
||||
await ctx.audioWorklet.addModule("/worklets/pcm-capture.js");
|
||||
|
||||
const worklet = new AudioWorkletNode(ctx, "pcm-capture");
|
||||
worklet.port.onmessage = (e) => {
|
||||
const msg = e.data as { type: string; rms?: number; buffer?: ArrayBuffer };
|
||||
if (msg.type === "level") {
|
||||
levelRef.current = msg.rms ?? 0; // never reset to zero (dictate pattern)
|
||||
} else if (msg.type === "audio") {
|
||||
const ws = wsRef.current;
|
||||
if (ws && ws.readyState === WebSocket.OPEN) ws.send(msg.buffer!);
|
||||
}
|
||||
};
|
||||
ctx.createMediaStreamSource(stream).connect(worklet);
|
||||
// Pull the graph with silence out (the worklet writes no output) —
|
||||
// an unconnected worklet node is never clocked.
|
||||
worklet.connect(ctx.destination);
|
||||
} catch (e) {
|
||||
if (gen !== genRef.current) return;
|
||||
const msg =
|
||||
e instanceof DOMException && (e.name === "NotAllowedError" || e.name === "SecurityError")
|
||||
? "Microphone permission denied"
|
||||
: e instanceof Error
|
||||
? e.message
|
||||
: "mic unavailable";
|
||||
setError(msg);
|
||||
cleanup();
|
||||
}
|
||||
}, [openSocket, cleanup, mintToken]);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
const gen = genRef.current;
|
||||
if (phase !== "recording" && phase !== "connecting") return;
|
||||
setPhase("stopping");
|
||||
|
||||
// Mic dies FIRST — no more audio enqueues (dictate pattern).
|
||||
for (const track of streamRef.current?.getTracks() ?? []) track.stop();
|
||||
streamRef.current = null;
|
||||
void ctxRef.current?.close().catch(() => {});
|
||||
ctxRef.current = null;
|
||||
|
||||
const ws = wsRef.current;
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
ws.send(JSON.stringify({ type: "CloseStream" }));
|
||||
} catch {
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
// Deepgram flushes remaining finals then closes; if it never
|
||||
// does, the safety net ends the run anyway (dictate pattern).
|
||||
safetyRef.current = setTimeout(() => {
|
||||
if (gen === genRef.current) cleanup();
|
||||
}, 3000);
|
||||
} else {
|
||||
cleanup();
|
||||
}
|
||||
}, [phase, cleanup]);
|
||||
|
||||
const toggle = useCallback(() => {
|
||||
if (phase === "idle") void start();
|
||||
else if (phase === "recording" || phase === "connecting") stop();
|
||||
// "stopping" presses are ignored — the flush is in flight.
|
||||
}, [phase, start, stop]);
|
||||
|
||||
return { phase, error, interim, levelRef, active: phase !== "idle", toggle, stop };
|
||||
}
|
||||
|
||||
/**
|
||||
* The stats-bar dictation slot (owner placement, 2026-09-17): the mic
|
||||
* button rides the middle of the stats row — idle it is just the quiet mic
|
||||
* affordance; recording, the meter unfurls to fill the free middle space
|
||||
* (width follows what the row has), with the live interim ghost at its
|
||||
* flank. Red dot + red mic while live; spinner while Deepgram flushes.
|
||||
*/
|
||||
export function DictationBar({ dictation }: { dictation: Dictation }) {
|
||||
const { phase, error, interim, levelRef, toggle, active } = dictation;
|
||||
const [levels, setLevels] = useState<number[]>(() => new Array(METER_CELLS).fill(0));
|
||||
const recording = phase === "recording" || phase === "stopping";
|
||||
|
||||
// The meter's own heartbeat — 60ms samples of the level ref into a
|
||||
// rolling ring. Only THIS strip re-renders; the page never does.
|
||||
useEffect(() => {
|
||||
if (!recording) return;
|
||||
setLevels(new Array(METER_CELLS).fill(0));
|
||||
const id = setInterval(() => {
|
||||
setLevels((ring) => {
|
||||
const next = ring.slice(1);
|
||||
next.push(rmsToMeter(levelRef.current));
|
||||
return next;
|
||||
});
|
||||
}, METER_TICK_MS);
|
||||
return () => clearInterval(id);
|
||||
}, [recording, levelRef]);
|
||||
|
||||
return (
|
||||
<div className="flex min-w-0 items-center gap-2" data-testid="dictation-bar">
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggle}
|
||||
disabled={phase === "stopping"}
|
||||
title={active ? "Stop dictation (Esc)" : "Dictate"}
|
||||
aria-label={active ? "Stop dictation" : "Start dictation"}
|
||||
data-testid="dictation-toggle"
|
||||
className="flex size-6 shrink-0 items-center justify-center rounded-md transition-colors hover:bg-accent hover:text-accent-foreground dark:hover:bg-accent/50"
|
||||
>
|
||||
{phase === "stopping" ? (
|
||||
<Loader2Icon className="size-3.5 animate-spin text-muted-foreground" />
|
||||
) : (
|
||||
<MicIcon className={active ? "size-3.5 text-destructive" : "size-3.5 text-muted-foreground"} />
|
||||
)}
|
||||
</button>
|
||||
{error && (
|
||||
<span className="max-w-64 truncate text-destructive" title={error}>
|
||||
{error}
|
||||
</span>
|
||||
)}
|
||||
{!error && recording && (
|
||||
<>
|
||||
<span className="size-1.5 shrink-0 animate-pulse rounded-full bg-destructive" aria-hidden />
|
||||
{/* Bars flex to the row's free width — the owner's "fill up
|
||||
the whole middle section" ask: pitch is whatever space
|
||||
exists between session data and the model control. */}
|
||||
<div className="flex h-4 min-w-0 flex-1 items-center gap-[2px]" data-testid="dictation-meter">
|
||||
{levels.map((v, i) => (
|
||||
<span
|
||||
key={i}
|
||||
className="min-w-[2px] flex-1 rounded-full bg-foreground/40"
|
||||
style={{ height: `${2 + Math.round(v * 14)}px` }}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
{interim && (
|
||||
<span className="max-w-[45%] truncate text-muted-foreground/80" data-testid="dictation-interim">
|
||||
{interim}
|
||||
</span>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { useRouter } from "next/navigation";
|
||||
import { CornerDownLeftIcon } from "lucide-react";
|
||||
import { GraphView } from "@/components/graph-view";
|
||||
import { GraphContextControl, effectiveContextTokens } from "@/components/graph-context-control";
|
||||
import { DictationBar, useDictation } from "@/components/dictation";
|
||||
import { SidebarToggle } from "@/components/sidebar";
|
||||
import { StatsBar } from "@/components/stats-bar";
|
||||
import type { ChatSettings, TurnUsage } from "@/src/core";
|
||||
@@ -18,6 +19,11 @@ import type { ChatSettings, TurnUsage } from "@/src/core";
|
||||
export function Home() {
|
||||
const router = useRouter();
|
||||
const [topic, setTopic] = useState("");
|
||||
// Voice dictation (STT wave, 2026-09-17): finals append at the end of the
|
||||
// topic draft; the strip lives in the stats bar's middle slot.
|
||||
const dictation = useDictation({
|
||||
append: (t) => setTopic((v) => (v ? v.replace(/\s+$/, "") + " " + t : t)),
|
||||
});
|
||||
const [busy, setBusy] = useState(false);
|
||||
// The model+thinking selection made pre-chat — lifted from the StatsBar's
|
||||
// ModelControl, carried to the chat POST then PUT-on-create (the model
|
||||
@@ -97,6 +103,12 @@ export function Home() {
|
||||
value={topic}
|
||||
onChange={(e) => setTopic(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
// Esc ends dictation (the dictate decisions: click-again or Esc).
|
||||
if (e.key === "Escape" && dictation.active) {
|
||||
e.preventDefault();
|
||||
dictation.stop();
|
||||
return;
|
||||
}
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
void start();
|
||||
@@ -121,6 +133,7 @@ export function Home() {
|
||||
budget (session-data slot), while the RIGHT keeps the model +
|
||||
thinking control (deferred save). */}
|
||||
<StatsBar
|
||||
dictationSlot={<DictationBar dictation={dictation} />}
|
||||
turns={[] as TurnUsage[]}
|
||||
chatId=""
|
||||
settings={settings}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { useCallback, useState } from "react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import { Conversation, ConversationContent, ConversationEmptyState } from "@/components/ai-elements/conversation";
|
||||
import { Composer } from "@/components/composer";
|
||||
import { DictationBar, useDictation } from "@/components/dictation";
|
||||
import { GraphContextControl, effectiveContextTokens } from "@/components/graph-context-control";
|
||||
import { PhaseChip } from "@/components/phase-divider";
|
||||
import { SidebarToggle } from "@/components/sidebar";
|
||||
@@ -74,6 +75,11 @@ export function DraftShell({
|
||||
onSettingsSaved: (s: ChatSettings) => void;
|
||||
}) {
|
||||
const canSend = text.trim().length > 0;
|
||||
// Voice dictation (STT wave, 2026-09-17): finals append at the end of the
|
||||
// draft text; the strip lives in the stats bar's middle slot.
|
||||
const dictation = useDictation({
|
||||
append: (t) => onChange(text ? text.replace(/\s+$/, "") + " " + t : t),
|
||||
});
|
||||
return (
|
||||
<div className="flex h-dvh">
|
||||
<div className="flex min-w-0 flex-1 flex-col">
|
||||
@@ -110,9 +116,11 @@ export function DraftShell({
|
||||
onStop={() => {}}
|
||||
canSend={canSend}
|
||||
autoFocus
|
||||
onEscape={dictation.active ? dictation.stop : undefined}
|
||||
/>
|
||||
</div>
|
||||
<StatsBar
|
||||
dictationSlot={<DictationBar dictation={dictation} />}
|
||||
turns={[]}
|
||||
chatId=""
|
||||
settings={settings}
|
||||
|
||||
@@ -33,6 +33,7 @@ export function StatsBar({
|
||||
settings,
|
||||
onSettingsSaved,
|
||||
leftSlot,
|
||||
dictationSlot,
|
||||
}: {
|
||||
turns: TurnUsage[];
|
||||
chatId: string;
|
||||
@@ -41,6 +42,11 @@ export function StatsBar({
|
||||
/** Pre-chat only: replaces the (empty) session-data summary — home and
|
||||
* the /chat/new draft mount the knowledge-context slider here. */
|
||||
leftSlot?: ReactNode;
|
||||
/** The dictation strip (STT wave, 2026-09-17): the row's middle is dead
|
||||
* space between the session data and the model control — the mic
|
||||
* button lives there, and while recording the meter unfurls to fill
|
||||
* whatever width the row has (owner placement). */
|
||||
dictationSlot?: ReactNode;
|
||||
}) {
|
||||
const [open, setOpen] = useState(false);
|
||||
// Same dropdown discipline as the model control (owner, df4433f treatment,
|
||||
@@ -155,6 +161,12 @@ export function StatsBar({
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{/* The middle slot: dictation (owner, 2026-09-17). flex-1 lets the
|
||||
meter expand into the row's free width; justify-center keeps the
|
||||
lone idle mic button visually centered. */}
|
||||
{dictationSlot && (
|
||||
<div className="flex min-w-0 flex-1 items-center justify-center">{dictationSlot}</div>
|
||||
)}
|
||||
{/* The model + thinking control — one controller for the conversation
|
||||
agents (prober/teacher); the record's `turns` above still folds
|
||||
the ACTUAL models billed, regardless of what the picker says. */}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* pcm-capture — the dictation audio path (STT wave, 2026-09-17).
|
||||
*
|
||||
* Runs OFF the main thread (AudioWorklet): converts the mic's Float32
|
||||
* samples to 16-bit little-endian PCM (linear16) and hands buffers to the
|
||||
* main thread in two streams:
|
||||
*
|
||||
* { type: "level", rms } — every render quantum (128 samples ≈ 8ms at
|
||||
* 16kHz): the quantum's RMS level, 0..1. This
|
||||
* is the meter's heartbeat; posting per quantum
|
||||
* is what makes the meter feel instant (the
|
||||
* dominant latency in the owner's terminal
|
||||
* version was a 256ms IO buffer — see
|
||||
* ~/.pi/agent/extensions/dictate).
|
||||
* { type: "audio", buffer } — Int16Array frames of 2048 samples (128ms),
|
||||
* zero-copy (transferred). Deepgram recommends
|
||||
* ~100–200ms chunks; silence is streamed too,
|
||||
* so the socket never idles (no KeepAlive
|
||||
* needed — the 10s NET-0001 timeout only bites
|
||||
* a stream that stops SENDING).
|
||||
*
|
||||
* Ships as a static file under public/ so it rides the standalone bundle.
|
||||
* Plain JS by necessity (worklets run outside the module graph).
|
||||
*/
|
||||
class PcmCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.frame = new Int16Array(2048);
|
||||
this.count = 0;
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const channel = inputs[0] && inputs[0][0];
|
||||
if (!channel) return true; // no input yet — stay alive
|
||||
|
||||
// RMS for the meter (float domain, before conversion).
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < channel.length; i++) sumSquares += channel[i] * channel[i];
|
||||
this.port.postMessage({ type: "level", rms: Math.sqrt(sumSquares / channel.length) });
|
||||
|
||||
// Float32 → Int16 (clamp first: a hot mic can exceed ±1.0).
|
||||
for (let i = 0; i < channel.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, channel[i]));
|
||||
this.frame[this.count++] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
if (this.count === this.frame.length) {
|
||||
this.port.postMessage({ type: "audio", buffer: this.frame }, [this.frame.buffer]);
|
||||
this.frame = new Int16Array(2048);
|
||||
this.count = 0;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("pcm-capture", PcmCaptureProcessor);
|
||||
Reference in New Issue
Block a user