diff --git a/backend/src/modules/voice/voice.service.ts b/backend/src/modules/voice/voice.service.ts index 209cee0..a5f9ee3 100644 --- a/backend/src/modules/voice/voice.service.ts +++ b/backend/src/modules/voice/voice.service.ts @@ -138,7 +138,7 @@ export class VoiceService { this.logTelemetry({ locale: catalogLocale, - durationMs: dto.durationMs ?? null, + durationMs: dto.durationMs, elapsedMs: Date.now() - startedAt, asrCost, llmCost, @@ -196,9 +196,20 @@ export class VoiceService { return profile; } - private assertWithinCap(durationMs?: number) { + /** + * Grace above the configured cap. + * + * The client auto-stops when elapsed >= maxMs, then measures the final length after the + * recorder has actually stopped — so a recording that runs to the cap always reports + * slightly over it. Without this tolerance the auto-stop would guarantee a rejection, + * discarding exactly the recording it was meant to save. The client still reports the + * true length, so telemetry stays honest. + */ + private static readonly CAP_TOLERANCE_MS = 2_000; + + private assertWithinCap(durationMs: number) { const max = this.voiceConfig.maxRecordingMs; - if (max != null && durationMs != null && durationMs > max) { + if (max != null && durationMs > max + VoiceService.CAP_TOLERANCE_MS) { throw new AppException( ErrorCode.VOICE_CLIP_TOO_LONG, HttpStatus.PAYLOAD_TOO_LARGE, @@ -300,7 +311,7 @@ export class VoiceService { */ private logTelemetry(input: { locale: string; - durationMs: number | null; + durationMs: number; elapsedMs: number; asrCost: number | null; llmCost: number | null; diff --git a/frontend/src/lib/api/voice.ts b/frontend/src/lib/api/voice.ts new file mode 100644 index 0000000..a9563c3 --- /dev/null +++ b/frontend/src/lib/api/voice.ts @@ -0,0 +1,28 @@ +import { apiClient } from './client'; +import type { VoiceAvailability, VoiceExtractionResult } from '@/types/voice'; + +export interface ExtractVoicePayload { + /** Base64 audio, no data: prefix. */ + audio: string; + format: string; + /** IANA zone — the server derives "today" from it for relative deadlines. */ + timeZone: string; + durationMs: number; +} + +export const voiceApi = { + availability: async (): Promise<{ success: boolean; data: VoiceAvailability }> => { + const response = await apiClient.get('/voice/availability'); + return response.data; + }, + + extract: async ( + payload: ExtractVoicePayload, + signal?: AbortSignal, + ): Promise<{ success: boolean; data: VoiceExtractionResult }> => { + // The signal is forwarded so cancelling closes the connection, which aborts the + // metered vendor call server-side rather than letting it settle unseen. + const response = await apiClient.post('/voice/extract', payload, { signal }); + return response.data; + }, +}; diff --git a/frontend/src/lib/voice/audioFormat.ts b/frontend/src/lib/voice/audioFormat.ts new file mode 100644 index 0000000..1199dc2 --- /dev/null +++ b/frontend/src/lib/voice/audioFormat.ts @@ -0,0 +1,62 @@ +/** Containers the backend accepts, in the order we prefer to record them. */ +const PREFERRED_MIME_TYPES = [ + 'audio/webm;codecs=opus', + 'audio/webm', + 'audio/mp4', + 'audio/aac', + 'audio/ogg;codecs=opus', + 'audio/ogg', +] as const; + +/** + * Pick a container this browser can record AND the backend accepts. + * + * Chrome and Android produce webm/opus; Safari and iPad produce mp4/aac. Both go to the + * vendor unmodified, so there is no transcode step — but the choice still has to be made + * at record time, and `isTypeSupported` is missing entirely on older Safari. + */ +export function pickRecordingMimeType(): string | null { + if (typeof MediaRecorder === 'undefined') return null; + if (typeof MediaRecorder.isTypeSupported !== 'function') { + // Safari <14.1 shipped MediaRecorder without the feature check; let it choose. + return ''; + } + for (const type of PREFERRED_MIME_TYPES) { + if (MediaRecorder.isTypeSupported(type)) return type; + } + return null; +} + +/** `audio/webm;codecs=opus` → `webm`, which is what the API's `format` field wants. */ +export function mimeTypeToFormat(mimeType: string): string { + const base = mimeType.split(';')[0]?.trim().toLowerCase() ?? ''; + const subtype = base.startsWith('audio/') ? base.slice('audio/'.length) : base; + // Safari/iOS records `audio/mp4`, but the transcription endpoint's documented container + // list names m4a, not mp4. Same container; send the name the vendor documents, so iPad + // recordings do not fail while Chrome's webm works. + if (subtype === 'x-m4a' || subtype === 'm4a' || subtype === 'mp4') return 'm4a'; + if (subtype === 'mpeg') return 'mp3'; + return subtype || 'webm'; +} + +/** Blob → base64 without the `data:` prefix, which the API does not want. */ +export async function blobToBase64(blob: Blob): Promise { + const buffer = await blob.arrayBuffer(); + let binary = ''; + const bytes = new Uint8Array(buffer); + // Chunked to avoid blowing the argument limit on a two-minute recording. + const chunkSize = 0x8000; + for (let i = 0; i < bytes.length; i += chunkSize) { + binary += String.fromCharCode(...bytes.subarray(i, i + chunkSize)); + } + return btoa(binary); +} + +export function isMediaRecorderSupported(): boolean { + return ( + typeof window !== 'undefined' && + typeof MediaRecorder !== 'undefined' && + typeof navigator !== 'undefined' && + Boolean(navigator.mediaDevices?.getUserMedia) + ); +} diff --git a/frontend/src/lib/voice/useVoiceCapture.ts b/frontend/src/lib/voice/useVoiceCapture.ts new file mode 100644 index 0000000..37574d7 --- /dev/null +++ b/frontend/src/lib/voice/useVoiceCapture.ts @@ -0,0 +1,253 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { voiceApi } from '@/lib/api/voice'; +import type { ApiError } from '@/types/api'; +import type { VoiceExtractionResult, VoicePhase } from '@/types/voice'; +import { + blobToBase64, + isMediaRecorderSupported, + mimeTypeToFormat, + pickRecordingMimeType, +} from './audioFormat'; + +export interface UseVoiceCaptureOptions { + /** null means uncapped; otherwise the recorder auto-stops here. */ + maxMs: number | null; + onExtracted: (result: VoiceExtractionResult) => void; + onError: (error: unknown) => void; +} + +export interface VoiceCaptureState { + phase: VoicePhase; + elapsedMs: number; + /** 0..1, for the level meter — proves the microphone is actually hearing something. */ + level: number; + maxMs: number | null; + onStart: () => void; + onStop: () => void; + onCancel: () => void; +} + +const LEVEL_POLL_MS = 100; + +/** + * Client-side failures must be ApiError-shaped or getUserFacingError cannot resolve them + * and every one renders the generic fallback, leaving errors.VOICE_MIC_DENIED dead. + */ +function clientError(code: string): ApiError { + return { statusCode: 0, code }; +} + +/** + * Microphone capture for treatment voice entry. + * + * Lives in lib/ rather than in the editor: TreatmentDetailsEditor stays presentational + * and receives only a `voice` prop, so MediaRecorder and the API call never enter ui/. + */ +export function useVoiceCapture({ + maxMs, + onExtracted, + onError, +}: UseVoiceCaptureOptions): VoiceCaptureState { + const [phase, setPhase] = useState('idle'); + const [elapsedMs, setElapsedMs] = useState(0); + const [level, setLevel] = useState(0); + + const recorderRef = useRef(null); + const streamRef = useRef(null); + const chunksRef = useRef([]); + const startedAtRef = useRef(0); + const timerRef = useRef | null>(null); + const audioContextRef = useRef(null); + const abortRef = useRef(null); + /** Set when the user cancels, so the recorder's stop handler discards instead of sending. */ + const cancelledRef = useRef(false); + /** getUserMedia is async; without this a permission granted after unmount leaks the mic. */ + const mountedRef = useRef(true); + + const teardown = useCallback(() => { + if (timerRef.current) { + clearInterval(timerRef.current); + timerRef.current = null; + } + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + void audioContextRef.current?.close().catch(() => undefined); + audioContextRef.current = null; + recorderRef.current = null; + setLevel(0); + }, []); + + // Releasing the microphone on unmount matters: the browser shows a recording indicator + // for as long as the track is live, and an orphaned one looks like the app is listening. + useEffect(() => () => { + mountedRef.current = false; + cancelledRef.current = true; + abortRef.current?.abort(); + try { + recorderRef.current?.stop(); + } catch { + // already stopped + } + teardown(); + }, [teardown]); + + const send = useCallback( + async (blob: Blob, mimeType: string, durationMs: number) => { + setPhase('processing'); + const controller = new AbortController(); + abortRef.current = controller; + try { + const audio = await blobToBase64(blob); + const response = await voiceApi.extract( + { + audio, + format: mimeTypeToFormat(mimeType), + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + durationMs, + }, + controller.signal, + ); + if (cancelledRef.current) return; + onExtracted(response.data); + } catch (error) { + if (cancelledRef.current || controller.signal.aborted) return; + onError(error); + } finally { + abortRef.current = null; + setPhase('idle'); + setElapsedMs(0); + } + }, + [onExtracted, onError], + ); + + const stop = useCallback(() => { + try { + recorderRef.current?.stop(); + } catch { + teardown(); + setPhase('idle'); + } + }, [teardown]); + + const onStart = useCallback(() => { + if (phase !== 'idle') return; + if (!isMediaRecorderSupported()) { + onError(clientError('VOICE_MIC_DENIED')); + return; + } + + cancelledRef.current = false; + void (async () => { + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch { + // Permission refused, or no input device. Never a server round-trip. + onError(clientError('VOICE_MIC_DENIED')); + return; + } + + if (!mountedRef.current) { + // Permission resolved after the component went away — release it immediately + // rather than leaving the browser's recording indicator lit. + stream.getTracks().forEach((track) => track.stop()); + return; + } + + const mimeType = pickRecordingMimeType(); + if (mimeType === null) { + stream.getTracks().forEach((track) => track.stop()); + onError(clientError('VOICE_MIC_DENIED')); + return; + } + + streamRef.current = stream; + chunksRef.current = []; + const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + recorderRef.current = recorder; + + recorder.ondataavailable = (event) => { + if (event.data.size > 0) chunksRef.current.push(event.data); + }; + recorder.onstop = () => { + const durationMs = Date.now() - startedAtRef.current; + const blob = new Blob(chunksRef.current, { type: recorder.mimeType || mimeType }); + teardown(); + if (cancelledRef.current || blob.size === 0) { + setPhase('idle'); + setElapsedMs(0); + return; + } + void send(blob, recorder.mimeType || mimeType || 'audio/webm', durationMs); + }; + + attachLevelMeter(stream, audioContextRef, setLevel); + + startedAtRef.current = Date.now(); + recorder.start(); + setPhase('recording'); + setElapsedMs(0); + + timerRef.current = setInterval(() => { + const elapsed = Date.now() - startedAtRef.current; + setElapsedMs(elapsed); + // Auto-stop proceeds to processing with what was captured; discarding two minutes + // of dictation because a timer expired would be the worst possible failure. + if (maxMs != null && elapsed >= maxMs) stop(); + }, LEVEL_POLL_MS); + })(); + }, [maxMs, onError, phase, send, stop, teardown]); + + const onCancel = useCallback(() => { + cancelledRef.current = true; + // Aborting closes the connection, which aborts the vendor call server-side. It is + // metered per minute, so letting it settle costs money for a result nobody sees. + abortRef.current?.abort(); + try { + recorderRef.current?.stop(); + } catch { + // already stopped + } + teardown(); + setPhase('idle'); + setElapsedMs(0); + }, [teardown]); + + return { phase, elapsedMs, level, maxMs, onStart, onStop: stop, onCancel }; +} + +/** Drives the level meter from the live stream; failure here must not stop recording. */ +function attachLevelMeter( + stream: MediaStream, + contextRef: React.MutableRefObject, + setLevel: (value: number) => void, +) { + try { + const AudioContextCtor = + window.AudioContext ?? (window as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; + if (!AudioContextCtor) return; + + const context = new AudioContextCtor(); + contextRef.current = context; + const source = context.createMediaStreamSource(stream); + const analyser = context.createAnalyser(); + analyser.fftSize = 512; + source.connect(analyser); + + const data = new Uint8Array(analyser.frequencyBinCount); + const tick = () => { + if (contextRef.current !== context || context.state === 'closed') return; + analyser.getByteTimeDomainData(data); + let peak = 0; + for (const sample of data) peak = Math.max(peak, Math.abs(sample - 128)); + setLevel(Math.min(1, peak / 128)); + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + } catch { + // A missing or blocked AudioContext costs the meter, not the recording. + } +} diff --git a/frontend/src/types/voice.ts b/frontend/src/types/voice.ts new file mode 100644 index 0000000..3360839 --- /dev/null +++ b/frontend/src/types/voice.ts @@ -0,0 +1,58 @@ +import type { FdiToothId, ToothSelectionGroup } from '@/types/treatment'; + +/** Mirrors the backend's ResolvedExtraction — values already resolved, plus what was not. */ + +export type VoiceUnresolvedReason = + | 'not_permanent_tooth' + | 'position_out_of_range' + | 'malformed' + | 'span_not_same_arch' + | 'unknown_catalog_code' + | 'tooth_not_selected' + | 'invalid_date'; + +export interface VoiceUnresolvedItem { + /** The transcript span that could not be resolved, so the clinician sees what was heard. */ + spoken: string; + reason: VoiceUnresolvedReason; +} + +export interface VoiceProsthesisResult { + byTooth: Record; + /** False means the case cannot ship — every tooth needs a prosthesis type. */ + complete: boolean; + missingTeeth: FdiToothId[]; +} + +export interface VoiceExtractionResult { + transcript: string; + treatmentType: string | null; + teeth: FdiToothId[]; + toothSelectionGroups: ToothSelectionGroup[]; + comment: string | null; + prosthesis: VoiceProsthesisResult | null; + labId: string | null; + /** When false, the lab row must not tick itself — the name only approximately matched. */ + labMatchExact: boolean; + dueDate: string | null; + unresolved: VoiceUnresolvedItem[]; +} + +export interface VoiceAvailability { + enabled: boolean; + locales: string[]; + /** null means uncapped. */ + maxRecordingMs: number | null; +} + +/** Which review rows the clinician ticked. */ +export interface VoiceApplySelection { + treatmentType: boolean; + teeth: boolean; + comment: boolean; + prosthesis: boolean; + lab: boolean; + dueDate: boolean; +} + +export type VoicePhase = 'idle' | 'recording' | 'processing';