From 4dbbfa6fe765ccb7fa4ad3400109f1d7912a4659 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Thu, 17 Sep 2026 19:59:14 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20dictation=20round=201=20=E2=80=94=20pri?= =?UTF-8?q?mary-input=20routing,=20no=20interims,=20optimistic=20meter,=20?= =?UTF-8?q?wide=20button?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner's first live mic test (2026-09-17) drove four changes: - ONE primary transcription input at all times: while a quiz/ask question waits, finals land in its field (text mode → 'Your answer', select modes → 'Your thinking') via a module-level claim registry (useDictationTarget in dictation.tsx); the composer is the fallback. Esc in a question field also stops dictation (threaded through DockOverlay → QuestionDock). - No interim results: live words were distracting and shifted the meter. Params now match the pi dictate extension exactly (no interim_results, endpointing=300, no utterance_end_ms); the meter's width stays fixed. - Optimistic meter: the click enters the meter view immediately (Stop pill + meter within one frame, verified at 150ms in the dev shell); token mint and mic acquisition run concurrently behind it. - Wide idle button: the stats-bar middle is one full-width affordance — a quiet bordered 'Dictate' button when idle (failure text replaces the label), Stop pill + fixed meter while live. Live-verified in the Electron dev shell: wide button renders on home (167px) and chat (358px), meter + Stop appear at 150ms after click, flat 2px bars in silence, clean return to 'Dictate' on stop. --- components/chat.tsx | 6 + components/dictation.tsx | 221 ++++++++++++++++++++--------------- components/question-dock.tsx | 37 +++++- docs/research/stt.md | 10 +- 4 files changed, 180 insertions(+), 94 deletions(-) diff --git a/components/chat.tsx b/components/chat.tsx index d3e0a7b..921b45f 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -872,6 +872,7 @@ export function Chat({ turns={turns} onReserve={setDockReserve} dictationSlot={} + dictationStop={dictation.stop} composer={ ; /** Teach-phase pending question: arrives collapsed behind the reveal button (W5). */ @@ -975,6 +977,9 @@ function DockOverlay({ /** The dictation strip (STT wave): rendered in the stats bar's middle * slot — the mic button + live meter (components/dictation.tsx). */ dictationSlot?: React.ReactNode; + /** Esc in a pending question's field stops dictation (the strip is the + * stats bar's; the stop belongs to the page-owned hook, threaded down). */ + dictationStop?: () => void; }) { const { isAtBottom, scrollToBottom, scrollRef } = useStickToBottomContext(); // Teach-phase mount (session reload with a pending check): the dock starts @@ -1225,6 +1230,7 @@ function DockOverlay({ phase={phase} onExpand={expandAndReveal} onSubmit={onSubmitAnswer} + onEscape={dictationStop} /> ) : ( composer diff --git a/components/dictation.tsx b/components/dictation.tsx index 5f17e25..bc02cd9 100644 --- a/components/dictation.tsx +++ b/components/dictation.tsx @@ -11,13 +11,13 @@ import { Loader2Icon, MicIcon } from "lucide-react"; * opens the Deepgram streaming WebSocket DIRECTLY (audio never loops * through our server) with a short-lived token minted by /api/stt/token * (the real key stays server-side), and streams frames while it listens. - * Finals append into the composer; interims ghost beside the meter. + * Finals append into the PRIMARY INPUT (see below). * * Ported patterns from the owner's pi `dictate` extension (his words: * "very well tuned") — see ~/.pi/agent/extensions/dictate: * - generation counter: every start/stop increments it; all async * handlers no-op when stale, so a dying session's socket can never - * tear down the current one (a real bug he hit in the terminal). + * tear down the current one. * - meter: latest-RMS-never-resets — the level ref keeps the last * observed value, so a tick with no fresh audio re-renders the last * level instead of dropping to silence (60ms tick samples the ref). @@ -25,6 +25,21 @@ import { Loader2Icon, MicIcon } from "lucide-react"; * conversational speech as mid-height bars without pegging. * - stop: mic dies FIRST (no more audio enqueues), then CloseStream, * then a 3s safety net in case Deepgram never closes the socket. + * - interim_results=false, endpointing=300: his live-tuned params — no + * partial hypotheses, finals commit ~300ms after a pause (owner + * round 1: live words are distracting and shift the meter; the + * meter's width is FIXED while recording). + * + * Owner round 1 (2026-09-17, first live mic test): + * - ONE primary transcription input at all times: the pending + * quiz/ask question's field while one waits (text mode → the answer + * field, select modes → "Your thinking"), else the page's composer. + * Implemented as a module-level registry the question form claims on + * mount — see useDictationTarget below. + * - Click → meter IMMEDIATELY (optimistic): the meter strip renders + * from the click, connection + TCC happen behind it. The token mint + * and the mic acquisition run CONCURRENTLY (Promise.all) to shrink + * the real gap. * * Meter rendering lives in (own 60ms timer over the level * ref) so the 16-renders/sec never touch the page component — only the @@ -32,12 +47,9 @@ import { Loader2Icon, MicIcon } from "lucide-react"; */ /* - * Deepgram streaming params (tuned in the owner's real-use dictate - * extension + the research pass): nova-3, his endpointing=300 (faster - * finals after a pause), utterance_end_ms=1000 as the noise-robust - * backstop, interim_results for the live ghost. Silence streams too - * (the worklet never pauses), so the 10s idle timeout can't bite — no - * KeepAlive needed. + * Deepgram streaming params (the owner's live-tuned dictate config): nova-3, + * endpointing=300, no interims. Silence streams too (the worklet never + * pauses), so the socket never idles — no KeepAlive needed. */ const DG_URL = "wss://api.deepgram.com/v1/listen" + @@ -45,11 +57,9 @@ const DG_URL = "&encoding=linear16" + "&sample_rate=16000" + "&channels=1" + - "&interim_results=true" + "&smart_format=true" + "&punctuate=true" + - "&endpointing=300" + - "&utterance_end_ms=1000"; + "&endpointing=300"; /** Meter tuning — the owner's tested values, see the file header. */ const METER_FLOOR_DB = -50; @@ -66,33 +76,64 @@ export function rmsToMeter(rms: number): number { return Math.max(0, Math.min(1, t)); } +/** + * The primary-input registry: at most ONE non-composer input can claim the + * dictation target while mounted — the pending question's answer/thinking + * field (question-dock.tsx). The page's `append` (the composer) is the + * fallback when nobody claims. Module-level by design: the claimant is a + * sibling subtree far from the hook's owner, and there is exactly one + * window, one pending question, one target at a time. + */ +let questionTarget: ((text: string) => void) | null = null; + +/** Join a dictated final onto the current text (shared separator rule). */ +export function joinDictation(current: string, final: string): string { + return current ? current.replace(/\s+$/, "") + " " + final : final; +} + +/** + * Claim the dictation target while mounted (and `claimed` is true). The + * unmount guard is identity-based so dev-mode double-invoked effects + * can't clear another claimant. + */ +export function useDictationTarget(append: (text: string) => void, claimed: boolean) { + useEffect(() => { + if (!claimed) return; + questionTarget = append; + return () => { + if (questionTarget === append) questionTarget = null; + }; + }, [append, claimed]); +} + export type DictationPhase = "idle" | "connecting" | "recording" | "stopping"; export type Dictation = { phase: DictationPhase; - /** Terminal-state explanation surfaced next to the mic (cleared on the next start). */ + /** Terminal-state explanation, shown inside the wide button (cleared on the next start). */ error: string | null; - /** Live in-flight partial (the ghost); cleared when the final commits. */ - interim: string; /** Latest observed RMS, 0..1 — sampled by the meter's own tick, never reset. */ levelRef: { current: number }; - /** Recording or finalizing — true from start to stop completion. */ + /** Not idle — from the click until the stop completes (the optimistic window). */ active: boolean; toggle: () => void; stop: () => void; }; /** - * One dictation instance per composer-bearing page. `append` receives each - * committed utterance (final) — the page joins it into its text state - * (separator handling is the page's: it owns the current text). + * One dictation instance per composer-bearing page. `append` is the + * composer fallback; the mounted question form's claim wins while a + * question waits. */ export function useDictation({ append }: { append: (text: string) => void }): Dictation { const [phase, setPhase] = useState("idle"); const [error, setError] = useState(null); - const [interim, setInterim] = useState(""); /** Latest RMS from the worklet — the meter samples this (dictate pattern). */ const levelRef = useRef(0); + /** The page's fallback append — kept fresh per render so a mid-recording + * retarget (composer↔question) is honored on the very next final. */ + const appendRef = useRef(append); + appendRef.current = append; // Live machinery (non-render values). AudioContext/MediaStream are // closed on stop; the socket is recreated per run. @@ -121,12 +162,10 @@ export function useDictation({ append }: { append: (text: string) => void }): Di ws.close(); } catch {} } - setInterim(""); setPhase("idle"); }, []); - /** Mint a short-lived token — the run's first step, BEFORE the mic: a - * keyless app fails fast without ever triggering the macOS TCC prompt. */ + /** Mint a short-lived token (the route holds the real key server-side). */ const mintToken = useCallback(async () => { const res = await fetch("/api/stt/token"); const data = (await res.json()) as { token?: string; message?: string }; @@ -157,16 +196,12 @@ export function useDictation({ append }: { append: (text: string) => void }): Di } if (msg.type === "Results") { const text: string | undefined = msg.channel?.alternatives?.[0]?.transcript; - if (msg.is_final) { - // The committed utterance: hand to the page, clear the ghost. - if (text) append(text); - setInterim(""); - } else if (text) { - setInterim(text); + // Only finals stream (interim_results=false): each Results + // message is a committed utterance — route it to the + // primary input, question claim first, composer fallback. + if (msg.is_final && text) { + (questionTarget ?? appendRef.current)(text); } - } else if (msg.type === "UtteranceEnd") { - // Backstop: the endpointing VAD missed, the word-gap one didn't. - setInterim(""); } }); ws.addEventListener("close", (ev) => { @@ -197,7 +232,7 @@ export function useDictation({ append }: { append: (text: string) => void }): Di } }); }, - [append, cleanup, mintToken], + [cleanup, mintToken], ); const start = useCallback(async () => { @@ -206,25 +241,17 @@ export function useDictation({ append }: { append: (text: string) => void }): Di reconnectsRef.current = 0; const gen = ++genRef.current; - // 1. Token first — fail fast, no TCC prompt for a keyless app. + // Token + mic CONCURRENTLY (owner round 1: the sequential wait was the + // felt delay) — the optimistic meter already covers the gap. 127.0.0.1 + // is a secure context, so getUserMedia works without TLS; macOS TCC + // prompts on first use. + const mic = navigator.mediaDevices.getUserMedia({ + audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); + let token: string; try { - const token = await mintToken(); - if (gen !== genRef.current) return; - openSocket(gen, token); - } catch (e) { - if (gen !== genRef.current) return; - setError(e instanceof Error ? e.message : "token fetch failed"); - setPhase("idle"); - return; - } - - try { - // 2. Mic + capture graph. 127.0.0.1 is a secure context — - // getUserMedia works without TLS (research); macOS TCC prompts - // on first use. - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true }, - }); + const [t, stream] = await Promise.all([mintToken(), mic]); + token = t; if (gen !== genRef.current) { for (const track of stream.getTracks()) track.stop(); return; @@ -259,7 +286,10 @@ export function useDictation({ append }: { append: (text: string) => void }): Di : "mic unavailable"; setError(msg); cleanup(); + return; } + if (gen !== genRef.current) return; + openSocket(gen, token); }, [openSocket, cleanup, mintToken]); const stop = useCallback(() => { @@ -297,25 +327,30 @@ export function useDictation({ append }: { append: (text: string) => void }): Di // "stopping" presses are ignored — the flush is in flight. }, [phase, start, stop]); - return { phase, error, interim, levelRef, active: phase !== "idle", toggle, stop }; + return { phase, error, levelRef, active: phase !== "idle", toggle, stop }; } /** - * The stats-bar dictation slot (owner placement, 2026-09-17): the mic - * button rides the middle of the stats row — idle it is just the quiet mic - * affordance; recording, the meter unfurls to fill the free middle space - * (width follows what the row has), with the live interim ghost at its - * flank. Red dot + red mic while live; spinner while Deepgram flushes. + * The stats-bar dictation slot (owner placement, round 1): the row's middle + * is one strip that is always the full free width — + * + * idle a wide, quiet "Dictate" button (mic + label, grayed out — the + * whole middle is the affordance, per the owner). + * active optimistic from the CLICK: red stop pill + the fixed-width + * meter (no interim text ever shifts it). Spinner while + * Deepgram flushes the last final. + * error the wide button again, label swapped for the failure reason + * (full text in the tooltip), cleared on the next click. */ export function DictationBar({ dictation }: { dictation: Dictation }) { - const { phase, error, interim, levelRef, toggle, active } = dictation; + const { phase, error, levelRef, toggle, active } = dictation; const [levels, setLevels] = useState(() => new Array(METER_CELLS).fill(0)); const recording = phase === "recording" || phase === "stopping"; // The meter's own heartbeat — 60ms samples of the level ref into a // rolling ring. Only THIS strip re-renders; the page never does. useEffect(() => { - if (!recording) return; + if (!active) return; setLevels(new Array(METER_CELLS).fill(0)); const id = setInterval(() => { setLevels((ring) => { @@ -325,52 +360,54 @@ export function DictationBar({ dictation }: { dictation: Dictation }) { }); }, METER_TICK_MS); return () => clearInterval(id); - }, [recording, levelRef]); + }, [active, levelRef]); + + if (!active) { + return ( + + ); + } return ( -
+
- {error && ( - - {error} - - )} - {!error && recording && ( - <> - - {/* Bars flex to the row's free width — the owner's "fill up - the whole middle section" ask: pitch is whatever space - exists between session data and the model control. */} -
- {levels.map((v, i) => ( - - ))} -
- {interim && ( - - {interim} - - )} - - )} + + {/* Fixed width while recording — nothing ever appends text beside it + (no interims, owner round 1). Bars flex to the row's free space. */} +
+ {levels.map((v, i) => ( + + ))} +
); } diff --git a/components/question-dock.tsx b/components/question-dock.tsx index bd4803d..e179470 100644 --- a/components/question-dock.tsx +++ b/components/question-dock.tsx @@ -1,8 +1,9 @@ "use client"; -import { useMemo, useState } from "react"; +import { useCallback, useMemo, useState } from "react"; import { ArrowDownIcon, ChevronRightIcon, PlayIcon } from "lucide-react"; import { Button } from "@/components/ui/button"; +import { joinDictation, useDictationTarget } from "@/components/dictation"; import { MessageResponse } from "@/components/ai-elements/message"; import { cn } from "@/lib/utils"; import { @@ -63,6 +64,9 @@ export interface QuestionDockProps { reveal?: boolean; onExpand: () => void; onSubmit: (output: QuestionAnswerData) => void; + /** Escape in a question field stops dictation (STT wave) — threaded + * from the page that owns the dictation hook. */ + onEscape?: () => void; /** Session phase at the question — scales the thinking field: probing * stays quick (one row), teaching invites dumping (three rows). */ phase?: Phase3; @@ -76,6 +80,7 @@ export function QuestionDock({ reveal = false, onExpand, onSubmit, + onEscape, phase = "probe", }: QuestionDockProps) { // Parse, don't cast (src/core/tool-parts.ts): the model authored this @@ -109,6 +114,7 @@ export function QuestionDock({ input={parsed} phase={phase} onSubmit={onSubmit} + onEscape={onEscape} />
@@ -189,6 +195,7 @@ function QuestionDockForm({ input, phase = "probe", onSubmit, + onEscape, }: Omit) { const kind = toolName === "quiz" ? ("quiz" as const) : ("ask" as const); @@ -243,6 +250,22 @@ function QuestionDockForm({ const [text, setText] = useState(""); const selectMode: SelectMode = qd.mode === "multi-select" ? "multi-select" : "single-select"; + // The PRIMARY dictation input (owner, STT round 1): while a question + // waits, dictated finals land HERE, never in the hidden composer — text + // mode owns "Your answer", select modes own "Your thinking". The claim + // is effect-scoped, so the question unmounting releases it and + // dictation falls back to the composer automatically. + useDictationTarget( + useCallback( + (t: string) => { + if (qd.mode === "text") setText((v) => joinDictation(v, t)); + else setThinking((v) => joinDictation(v, t)); + }, + [qd.mode], + ), + true, + ); + const toggle = (idx: number) => setSel(toggleOption(selectMode, sel, idx)); const canSubmit = @@ -420,6 +443,12 @@ function QuestionDockForm({ value={text} onChange={(e) => setText(e.target.value)} onKeyDown={(e) => { + // Esc ends dictation (the dictate decisions: click-again or Esc). + if (e.key === "Escape" && onEscape) { + e.preventDefault(); + onEscape(); + return; + } // Chat convention: Enter submits, Shift+Enter is a newline; // ⌘/Ctrl+Enter also submits (universal across the app). if (e.key === "Enter" && (e.metaKey || e.ctrlKey || !e.shiftKey)) { @@ -436,6 +465,12 @@ function QuestionDockForm({ value={thinking} onChange={(e) => setThinking(e.target.value)} onKeyDown={(e) => { + // Esc ends dictation (the dictate decisions: click-again or Esc). + if (e.key === "Escape" && onEscape) { + e.preventDefault(); + onEscape(); + return; + } // ⌘↵ on macOS, Ctrl↵ elsewhere — the Gmail convention; plain // Enter stays a newline (this field is for dumping). if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) { diff --git a/docs/research/stt.md b/docs/research/stt.md index bf9f9ec..0ffdddb 100644 --- a/docs/research/stt.md +++ b/docs/research/stt.md @@ -57,5 +57,13 @@ against our codebase. Decisions at the bottom. - **Activation: toggle mic** — click to start a dictation run, click again (or Esc) to end; no push-to-talk before the shortcuts milestone. - **Wave scope: cloud first, local second** — capture component + Deepgram path ship and get live-smoked as one wave; whisper.cpp follows as a second wave. - Renderer never holds the real key (server mints short-lived Deepgram tokens); audio never loops through our server (renderer → wss direct). -- Interim text ghosts in the composer; finals commit. Engine picker = small popover off the mic button (localStorage for now; settings milestone absorbs it). +- Engine picker = small popover off the mic button (localStorage for now; settings milestone absorbs it). - Bands: 1) capture + mic states (no engine), 2) cloud end-to-end, 3) local end-to-end, 4) smoke + docs. + +## Round-1 refinements (owner, 2026-09-17 — first live mic test) + +- **No interims.** Live words beside the meter are distracting AND shift the meter around — the meter's width stays fixed while recording. Params now match the pi dictate extension exactly: `interim_results` omitted (false), `endpointing=300`, no `utterance_end_ms`. +- **One primary transcription input at all times**: while a quiz/ask question waits, finals land in ITS field (text mode → "Your answer", select modes → "Your thinking") — never the hidden composer. Implemented as a module-level claim registry (`useDictationTarget`); the composer is the fallback when nothing claims. +- **Optimistic meter**: the click enters the meter view immediately (Stop pill + meter within one frame); the token mint and mic acquisition run concurrently (Promise.all) behind it. Esc or click-again stops. +- **Wide idle button**: the stats-bar middle is one full-width affordance — a quiet bordered "Dictate" button when idle (error text replaces the label on failure), the Stop pill + fixed meter while live. +- Final-appending keeps the dictate-flush separator rule (`joinDictation`).