From 93c6513df6651d57f1e306b40b45f40f38c2649c Mon Sep 17 00:00:00 2001 From: Amin Mousavi Date: Thu, 20 Aug 2026 18:43:04 +0330 Subject: [PATCH] feat(frontend): voice capture hook, API client and types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MediaRecorder handling and the API call live in lib/, not in ui/, so TreatmentDetailsEditor can stay presentational and take only a `voice` prop. Container choice is made at record time and needs no transcode: Chrome and Android give webm/opus, Safari and iPad give mp4/aac, and the transcription endpoint accepts both. Safari's `audio/mp4` is sent as `m4a`, the name the vendor's container list actually uses, so iPad recordings do not fail while Chrome works. Older Safari shipped MediaRecorder without isTypeSupported, so that path lets the browser choose rather than refusing outright. From review of this commit: - The auto-stop at maxMs guaranteed a 413. The client measures the final length after the recorder has stopped, so a recording that runs to the cap always reports slightly over it, and the server rejected exactly the recording the auto-stop existed to save. The server now allows a documented 2s tolerance and the client keeps reporting the true length, so telemetry stays honest. - getUserMedia is async, so a permission granted after unmount installed a live stream the cleanup effect had already run past — leaving the browser's recording indicator lit with nothing listening. Guarded with a mounted ref. - Client-side failures are now ApiError-shaped ({code, statusCode}) rather than bare Errors, because getUserFacingError only resolves that shape; without it errors.VOICE_MIC_DENIED was dead in all three locales. Cancelling aborts the request, which closes the connection and aborts the metered vendor call server-side rather than letting it settle unseen. The level meter is best-effort: a blocked AudioContext costs the meter, not the recording. Co-Authored-By: Claude Opus 5 (1M context) --- backend/src/modules/voice/voice.service.ts | 19 +- frontend/src/lib/api/voice.ts | 28 +++ frontend/src/lib/voice/audioFormat.ts | 62 +++++ frontend/src/lib/voice/useVoiceCapture.ts | 253 +++++++++++++++++++++ frontend/src/types/voice.ts | 58 +++++ 5 files changed, 416 insertions(+), 4 deletions(-) create mode 100644 frontend/src/lib/api/voice.ts create mode 100644 frontend/src/lib/voice/audioFormat.ts create mode 100644 frontend/src/lib/voice/useVoiceCapture.ts create mode 100644 frontend/src/types/voice.ts diff --git a/backend/src/modules/voice/voice.service.ts b/backend/src/modules/voice/voice.service.ts index 209cee0..a5f9ee3 100644 --- a/backend/src/modules/voice/voice.service.ts +++ b/backend/src/modules/voice/voice.service.ts @@ -138,7 +138,7 @@ export class VoiceService { this.logTelemetry({ locale: catalogLocale, - durationMs: dto.durationMs ?? null, + durationMs: dto.durationMs, elapsedMs: Date.now() - startedAt, asrCost, llmCost, @@ -196,9 +196,20 @@ export class VoiceService { return profile; } - private assertWithinCap(durationMs?: number) { + /** + * Grace above the configured cap. + * + * The client auto-stops when elapsed >= maxMs, then measures the final length after the + * recorder has actually stopped — so a recording that runs to the cap always reports + * slightly over it. Without this tolerance the auto-stop would guarantee a rejection, + * discarding exactly the recording it was meant to save. The client still reports the + * true length, so telemetry stays honest. + */ + private static readonly CAP_TOLERANCE_MS = 2_000; + + private assertWithinCap(durationMs: number) { const max = this.voiceConfig.maxRecordingMs; - if (max != null && durationMs != null && durationMs > max) { + if (max != null && durationMs > max + VoiceService.CAP_TOLERANCE_MS) { throw new AppException( ErrorCode.VOICE_CLIP_TOO_LONG, HttpStatus.PAYLOAD_TOO_LARGE, @@ -300,7 +311,7 @@ export class VoiceService { */ private logTelemetry(input: { locale: string; - durationMs: number | null; + durationMs: number; elapsedMs: number; asrCost: number | null; llmCost: number | null; diff --git a/frontend/src/lib/api/voice.ts b/frontend/src/lib/api/voice.ts new file mode 100644 index 0000000..a9563c3 --- /dev/null +++ b/frontend/src/lib/api/voice.ts @@ -0,0 +1,28 @@ +import { apiClient } from './client'; +import type { VoiceAvailability, VoiceExtractionResult } from '@/types/voice'; + +export interface ExtractVoicePayload { + /** Base64 audio, no data: prefix. */ + audio: string; + format: string; + /** IANA zone — the server derives "today" from it for relative deadlines. */ + timeZone: string; + durationMs: number; +} + +export const voiceApi = { + availability: async (): Promise<{ success: boolean; data: VoiceAvailability }> => { + const response = await apiClient.get('/voice/availability'); + return response.data; + }, + + extract: async ( + payload: ExtractVoicePayload, + signal?: AbortSignal, + ): Promise<{ success: boolean; data: VoiceExtractionResult }> => { + // The signal is forwarded so cancelling closes the connection, which aborts the + // metered vendor call server-side rather than letting it settle unseen. + const response = await apiClient.post('/voice/extract', payload, { signal }); + return response.data; + }, +}; diff --git a/frontend/src/lib/voice/audioFormat.ts b/frontend/src/lib/voice/audioFormat.ts new file mode 100644 index 0000000..1199dc2 --- /dev/null +++ b/frontend/src/lib/voice/audioFormat.ts @@ -0,0 +1,62 @@ +/** Containers the backend accepts, in the order we prefer to record them. */ +const PREFERRED_MIME_TYPES = [ + 'audio/webm;codecs=opus', + 'audio/webm', + 'audio/mp4', + 'audio/aac', + 'audio/ogg;codecs=opus', + 'audio/ogg', +] as const; + +/** + * Pick a container this browser can record AND the backend accepts. + * + * Chrome and Android produce webm/opus; Safari and iPad produce mp4/aac. Both go to the + * vendor unmodified, so there is no transcode step — but the choice still has to be made + * at record time, and `isTypeSupported` is missing entirely on older Safari. + */ +export function pickRecordingMimeType(): string | null { + if (typeof MediaRecorder === 'undefined') return null; + if (typeof MediaRecorder.isTypeSupported !== 'function') { + // Safari <14.1 shipped MediaRecorder without the feature check; let it choose. + return ''; + } + for (const type of PREFERRED_MIME_TYPES) { + if (MediaRecorder.isTypeSupported(type)) return type; + } + return null; +} + +/** `audio/webm;codecs=opus` → `webm`, which is what the API's `format` field wants. */ +export function mimeTypeToFormat(mimeType: string): string { + const base = mimeType.split(';')[0]?.trim().toLowerCase() ?? ''; + const subtype = base.startsWith('audio/') ? base.slice('audio/'.length) : base; + // Safari/iOS records `audio/mp4`, but the transcription endpoint's documented container + // list names m4a, not mp4. Same container; send the name the vendor documents, so iPad + // recordings do not fail while Chrome's webm works. + if (subtype === 'x-m4a' || subtype === 'm4a' || subtype === 'mp4') return 'm4a'; + if (subtype === 'mpeg') return 'mp3'; + return subtype || 'webm'; +} + +/** Blob → base64 without the `data:` prefix, which the API does not want. */ +export async function blobToBase64(blob: Blob): Promise { + const buffer = await blob.arrayBuffer(); + let binary = ''; + const bytes = new Uint8Array(buffer); + // Chunked to avoid blowing the argument limit on a two-minute recording. + const chunkSize = 0x8000; + for (let i = 0; i < bytes.length; i += chunkSize) { + binary += String.fromCharCode(...bytes.subarray(i, i + chunkSize)); + } + return btoa(binary); +} + +export function isMediaRecorderSupported(): boolean { + return ( + typeof window !== 'undefined' && + typeof MediaRecorder !== 'undefined' && + typeof navigator !== 'undefined' && + Boolean(navigator.mediaDevices?.getUserMedia) + ); +} diff --git a/frontend/src/lib/voice/useVoiceCapture.ts b/frontend/src/lib/voice/useVoiceCapture.ts new file mode 100644 index 0000000..37574d7 --- /dev/null +++ b/frontend/src/lib/voice/useVoiceCapture.ts @@ -0,0 +1,253 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { voiceApi } from '@/lib/api/voice'; +import type { ApiError } from '@/types/api'; +import type { VoiceExtractionResult, VoicePhase } from '@/types/voice'; +import { + blobToBase64, + isMediaRecorderSupported, + mimeTypeToFormat, + pickRecordingMimeType, +} from './audioFormat'; + +export interface UseVoiceCaptureOptions { + /** null means uncapped; otherwise the recorder auto-stops here. */ + maxMs: number | null; + onExtracted: (result: VoiceExtractionResult) => void; + onError: (error: unknown) => void; +} + +export interface VoiceCaptureState { + phase: VoicePhase; + elapsedMs: number; + /** 0..1, for the level meter — proves the microphone is actually hearing something. */ + level: number; + maxMs: number | null; + onStart: () => void; + onStop: () => void; + onCancel: () => void; +} + +const LEVEL_POLL_MS = 100; + +/** + * Client-side failures must be ApiError-shaped or getUserFacingError cannot resolve them + * and every one renders the generic fallback, leaving errors.VOICE_MIC_DENIED dead. + */ +function clientError(code: string): ApiError { + return { statusCode: 0, code }; +} + +/** + * Microphone capture for treatment voice entry. + * + * Lives in lib/ rather than in the editor: TreatmentDetailsEditor stays presentational + * and receives only a `voice` prop, so MediaRecorder and the API call never enter ui/. + */ +export function useVoiceCapture({ + maxMs, + onExtracted, + onError, +}: UseVoiceCaptureOptions): VoiceCaptureState { + const [phase, setPhase] = useState('idle'); + const [elapsedMs, setElapsedMs] = useState(0); + const [level, setLevel] = useState(0); + + const recorderRef = useRef(null); + const streamRef = useRef(null); + const chunksRef = useRef([]); + const startedAtRef = useRef(0); + const timerRef = useRef | null>(null); + const audioContextRef = useRef(null); + const abortRef = useRef(null); + /** Set when the user cancels, so the recorder's stop handler discards instead of sending. */ + const cancelledRef = useRef(false); + /** getUserMedia is async; without this a permission granted after unmount leaks the mic. */ + const mountedRef = useRef(true); + + const teardown = useCallback(() => { + if (timerRef.current) { + clearInterval(timerRef.current); + timerRef.current = null; + } + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + void audioContextRef.current?.close().catch(() => undefined); + audioContextRef.current = null; + recorderRef.current = null; + setLevel(0); + }, []); + + // Releasing the microphone on unmount matters: the browser shows a recording indicator + // for as long as the track is live, and an orphaned one looks like the app is listening. + useEffect(() => () => { + mountedRef.current = false; + cancelledRef.current = true; + abortRef.current?.abort(); + try { + recorderRef.current?.stop(); + } catch { + // already stopped + } + teardown(); + }, [teardown]); + + const send = useCallback( + async (blob: Blob, mimeType: string, durationMs: number) => { + setPhase('processing'); + const controller = new AbortController(); + abortRef.current = controller; + try { + const audio = await blobToBase64(blob); + const response = await voiceApi.extract( + { + audio, + format: mimeTypeToFormat(mimeType), + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + durationMs, + }, + controller.signal, + ); + if (cancelledRef.current) return; + onExtracted(response.data); + } catch (error) { + if (cancelledRef.current || controller.signal.aborted) return; + onError(error); + } finally { + abortRef.current = null; + setPhase('idle'); + setElapsedMs(0); + } + }, + [onExtracted, onError], + ); + + const stop = useCallback(() => { + try { + recorderRef.current?.stop(); + } catch { + teardown(); + setPhase('idle'); + } + }, [teardown]); + + const onStart = useCallback(() => { + if (phase !== 'idle') return; + if (!isMediaRecorderSupported()) { + onError(clientError('VOICE_MIC_DENIED')); + return; + } + + cancelledRef.current = false; + void (async () => { + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch { + // Permission refused, or no input device. Never a server round-trip. + onError(clientError('VOICE_MIC_DENIED')); + return; + } + + if (!mountedRef.current) { + // Permission resolved after the component went away — release it immediately + // rather than leaving the browser's recording indicator lit. + stream.getTracks().forEach((track) => track.stop()); + return; + } + + const mimeType = pickRecordingMimeType(); + if (mimeType === null) { + stream.getTracks().forEach((track) => track.stop()); + onError(clientError('VOICE_MIC_DENIED')); + return; + } + + streamRef.current = stream; + chunksRef.current = []; + const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + recorderRef.current = recorder; + + recorder.ondataavailable = (event) => { + if (event.data.size > 0) chunksRef.current.push(event.data); + }; + recorder.onstop = () => { + const durationMs = Date.now() - startedAtRef.current; + const blob = new Blob(chunksRef.current, { type: recorder.mimeType || mimeType }); + teardown(); + if (cancelledRef.current || blob.size === 0) { + setPhase('idle'); + setElapsedMs(0); + return; + } + void send(blob, recorder.mimeType || mimeType || 'audio/webm', durationMs); + }; + + attachLevelMeter(stream, audioContextRef, setLevel); + + startedAtRef.current = Date.now(); + recorder.start(); + setPhase('recording'); + setElapsedMs(0); + + timerRef.current = setInterval(() => { + const elapsed = Date.now() - startedAtRef.current; + setElapsedMs(elapsed); + // Auto-stop proceeds to processing with what was captured; discarding two minutes + // of dictation because a timer expired would be the worst possible failure. + if (maxMs != null && elapsed >= maxMs) stop(); + }, LEVEL_POLL_MS); + })(); + }, [maxMs, onError, phase, send, stop, teardown]); + + const onCancel = useCallback(() => { + cancelledRef.current = true; + // Aborting closes the connection, which aborts the vendor call server-side. It is + // metered per minute, so letting it settle costs money for a result nobody sees. + abortRef.current?.abort(); + try { + recorderRef.current?.stop(); + } catch { + // already stopped + } + teardown(); + setPhase('idle'); + setElapsedMs(0); + }, [teardown]); + + return { phase, elapsedMs, level, maxMs, onStart, onStop: stop, onCancel }; +} + +/** Drives the level meter from the live stream; failure here must not stop recording. */ +function attachLevelMeter( + stream: MediaStream, + contextRef: React.MutableRefObject, + setLevel: (value: number) => void, +) { + try { + const AudioContextCtor = + window.AudioContext ?? (window as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; + if (!AudioContextCtor) return; + + const context = new AudioContextCtor(); + contextRef.current = context; + const source = context.createMediaStreamSource(stream); + const analyser = context.createAnalyser(); + analyser.fftSize = 512; + source.connect(analyser); + + const data = new Uint8Array(analyser.frequencyBinCount); + const tick = () => { + if (contextRef.current !== context || context.state === 'closed') return; + analyser.getByteTimeDomainData(data); + let peak = 0; + for (const sample of data) peak = Math.max(peak, Math.abs(sample - 128)); + setLevel(Math.min(1, peak / 128)); + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + } catch { + // A missing or blocked AudioContext costs the meter, not the recording. + } +} diff --git a/frontend/src/types/voice.ts b/frontend/src/types/voice.ts new file mode 100644 index 0000000..3360839 --- /dev/null +++ b/frontend/src/types/voice.ts @@ -0,0 +1,58 @@ +import type { FdiToothId, ToothSelectionGroup } from '@/types/treatment'; + +/** Mirrors the backend's ResolvedExtraction — values already resolved, plus what was not. */ + +export type VoiceUnresolvedReason = + | 'not_permanent_tooth' + | 'position_out_of_range' + | 'malformed' + | 'span_not_same_arch' + | 'unknown_catalog_code' + | 'tooth_not_selected' + | 'invalid_date'; + +export interface VoiceUnresolvedItem { + /** The transcript span that could not be resolved, so the clinician sees what was heard. */ + spoken: string; + reason: VoiceUnresolvedReason; +} + +export interface VoiceProsthesisResult { + byTooth: Record; + /** False means the case cannot ship — every tooth needs a prosthesis type. */ + complete: boolean; + missingTeeth: FdiToothId[]; +} + +export interface VoiceExtractionResult { + transcript: string; + treatmentType: string | null; + teeth: FdiToothId[]; + toothSelectionGroups: ToothSelectionGroup[]; + comment: string | null; + prosthesis: VoiceProsthesisResult | null; + labId: string | null; + /** When false, the lab row must not tick itself — the name only approximately matched. */ + labMatchExact: boolean; + dueDate: string | null; + unresolved: VoiceUnresolvedItem[]; +} + +export interface VoiceAvailability { + enabled: boolean; + locales: string[]; + /** null means uncapped. */ + maxRecordingMs: number | null; +} + +/** Which review rows the clinician ticked. */ +export interface VoiceApplySelection { + treatmentType: boolean; + teeth: boolean; + comment: boolean; + prosthesis: boolean; + lab: boolean; + dueDate: boolean; +} + +export type VoicePhase = 'idle' | 'recording' | 'processing';