diff --git a/backend/.env.example b/backend/.env.example index 3d2145e..d387d78 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -47,3 +47,28 @@ SMTP_PASSWORD=your_app_password # SMS_IR_API_KEY=4QKMiSU4Kh7tWPLCdRMV0QpDh8WgF33YkWRS18BcG3vf4QHi SMS_IR_API_KEY=lwbK7hxmjimNjFS4g5DWahh75EKCgJUfcUIinUQzfQXwXkSp SMS_IR_TEMPLATE_ID=123456 + +# ── Voice treatment entry ────────────────────────────────────────────────────── +# Without OPENROUTER_API_KEY the microphone button does not render at all. +OPENROUTER_API_KEY= +# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 + +# Locales the microphone is offered in. An unknown locale here fails at boot. +# VOICE_ENABLED_LOCALES=fa,en,nl + +# Models, overridable per locale (VOICE_ASR_MODEL_FA, VOICE_LLM_MODEL_NL, ...). +# All locales share these today; the per-locale override exists so Persian can be +# repointed at a specialist ASR vendor without a code change. +# VOICE_ASR_MODEL=openai/whisper-1 +# VOICE_LLM_MODEL=google/gemini-3.7-flash +# VOICE_ASR_PROVIDER_FA=openrouter +# VOICE_LLM_PROVIDER_FA=openrouter + +# Recording cap in ms (0 = uncapped). 2 minutes bounds worst-case vendor spend at +# about 1.3 cents per recording. +# VOICE_MAX_RECORDING_MS=120000 + +# Per-user rate limit on the extract endpoint. Unreachable by a human — a recording +# plus processing takes ten seconds at minimum — so it is purely an abuse guard. +# VOICE_THROTTLE_TTL=60 +# VOICE_THROTTLE_LIMIT=6 diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 2dfad1d..66e6583 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; +import { ConfigModule, ConfigService } from '@nestjs/config'; +import { ThrottlerModule } from '@nestjs/throttler'; import configurations from './configs/configurations'; import { AuthModule } from './modules/auth/auth.module'; import { AppController } from './app.controller'; @@ -20,6 +21,7 @@ import { LabCaseCommentsModule } from './modules/lab-case-comments/lab-case-comm import { TodayModule } from './modules/today/today.module'; import { NotificationsModule } from './modules/notifications/notifications.module'; import { RealtimeModule } from './realtime/realtime.module'; +import { VoiceModule } from './modules/voice/voice.module'; @Module({ imports: [ @@ -27,6 +29,22 @@ import { RealtimeModule } from './realtime/realtime.module'; isGlobal: true, load: [configurations], }), + // First use of @nestjs/throttler in this app. Deliberately NOT bound as a global + // APP_GUARD: a globally-bound ThrottlerGuard rate-limits every route against every + // named throttler, which would cap the whole API at the voice limit. ThrottlerGuard + // is applied to the one expensive route instead, so nothing else changes behaviour. + ThrottlerModule.forRootAsync({ + inject: [ConfigService], + useFactory: (config: ConfigService) => ({ + throttlers: [ + { + name: 'voice', + ttl: (config.get('voice.throttle.ttl') ?? 60) * 1000, + limit: config.get('voice.throttle.limit') ?? 6, + }, + ], + }), + }), PrismaModule, // ✅ ADD THIS CatalogModule, TreatmentCatalogModule, @@ -43,9 +61,10 @@ import { RealtimeModule } from './realtime/realtime.module'; TodayModule, NotificationsModule, RealtimeModule, + VoiceModule, AdminModule.forRoot(), ], controllers: [AppController], - providers: [AppService], + providers: [AppService], }) -export class AppModule {} \ No newline at end of file +export class AppModule {} diff --git a/backend/src/common/errors/error-codes.ts b/backend/src/common/errors/error-codes.ts index 5c17305..73d7385 100644 --- a/backend/src/common/errors/error-codes.ts +++ b/backend/src/common/errors/error-codes.ts @@ -29,8 +29,10 @@ export const ErrorCode = { PERMISSION_LAB_ONLY: 'PERMISSION_LAB_ONLY', PERMISSION_NOT_MEMBER: 'PERMISSION_NOT_MEMBER', PERMISSION_OWNER_ONLY: 'PERMISSION_OWNER_ONLY', - PERMISSION_PARTICIPATION_SUBSCRIPTION: 'PERMISSION_PARTICIPATION_SUBSCRIPTION', - PERMISSION_ENABLE_PARTICIPATION_FIRST: 'PERMISSION_ENABLE_PARTICIPATION_FIRST', + PERMISSION_PARTICIPATION_SUBSCRIPTION: + 'PERMISSION_PARTICIPATION_SUBSCRIPTION', + PERMISSION_ENABLE_PARTICIPATION_FIRST: + 'PERMISSION_ENABLE_PARTICIPATION_FIRST', PERMISSION_CLINIC_WORKING_HOURS: 'PERMISSION_CLINIC_WORKING_HOURS', PERMISSION_ACCESS_APPOINTMENTS: 'PERMISSION_ACCESS_APPOINTMENTS', PERMISSION_EDIT_APPOINTMENTS: 'PERMISSION_EDIT_APPOINTMENTS', @@ -51,7 +53,8 @@ export const ErrorCode = { VALIDATION_PASSWORD_REQUIRED: 'VALIDATION_PASSWORD_REQUIRED', VALIDATION_MOBILE_INVALID: 'VALIDATION_MOBILE_INVALID', VALIDATION_NAME_TOO_SHORT: 'VALIDATION_NAME_TOO_SHORT', - VALIDATION_ORGANIZATION_NAME_REQUIRED: 'VALIDATION_ORGANIZATION_NAME_REQUIRED', + VALIDATION_ORGANIZATION_NAME_REQUIRED: + 'VALIDATION_ORGANIZATION_NAME_REQUIRED', VALIDATION_ORGANIZATION_TYPE_INVALID: 'VALIDATION_ORGANIZATION_TYPE_INVALID', VALIDATION_TOKEN_REQUIRED: 'VALIDATION_TOKEN_REQUIRED', VALIDATION_FIELD_REQUIRED: 'VALIDATION_FIELD_REQUIRED', @@ -69,8 +72,10 @@ export const ErrorCode = { APPOINTMENT_INVALID_DATE: 'APPOINTMENT_INVALID_DATE', APPOINTMENT_PROVIDER_NOT_MEMBER: 'APPOINTMENT_PROVIDER_NOT_MEMBER', APPOINTMENT_PROVIDER_INACTIVE: 'APPOINTMENT_PROVIDER_INACTIVE', - APPOINTMENT_PROVIDER_NO_TREATMENT_EDIT: 'APPOINTMENT_PROVIDER_NO_TREATMENT_EDIT', - APPOINTMENT_PROVIDER_NO_WORKING_HOURS: 'APPOINTMENT_PROVIDER_NO_WORKING_HOURS', + APPOINTMENT_PROVIDER_NO_TREATMENT_EDIT: + 'APPOINTMENT_PROVIDER_NO_TREATMENT_EDIT', + APPOINTMENT_PROVIDER_NO_WORKING_HOURS: + 'APPOINTMENT_PROVIDER_NO_WORKING_HOURS', APPOINTMENT_PROVIDER_NOT_WORKING_DAY: 'APPOINTMENT_PROVIDER_NOT_WORKING_DAY', APPOINTMENT_OUTSIDE_WORKING_HOURS: 'APPOINTMENT_OUTSIDE_WORKING_HOURS', APPOINTMENT_NOT_FOUND: 'APPOINTMENT_NOT_FOUND', @@ -82,7 +87,8 @@ export const ErrorCode = { WORKING_HOURS_INVALID: 'WORKING_HOURS_INVALID', WORKING_HOURS_OWNER_NOT_ALLOWED: 'WORKING_HOURS_OWNER_NOT_ALLOWED', - WORKING_HOURS_CONFLICTS_WITH_APPOINTMENTS: 'WORKING_HOURS_CONFLICTS_WITH_APPOINTMENTS', + WORKING_HOURS_CONFLICTS_WITH_APPOINTMENTS: + 'WORKING_HOURS_CONFLICTS_WITH_APPOINTMENTS', STAFF_UNKNOWN_PERMISSIONS: 'STAFF_UNKNOWN_PERMISSIONS', STAFF_NO_SUBSCRIPTION: 'STAFF_NO_SUBSCRIPTION', @@ -147,7 +153,8 @@ export const ErrorCode = { TREATMENT_ATTACHMENT_NOT_FOUND: 'TREATMENT_ATTACHMENT_NOT_FOUND', TREATMENT_FILE_UNAVAILABLE: 'TREATMENT_FILE_UNAVAILABLE', TREATMENT_CASE_INVALID_ATTACHMENTS: 'TREATMENT_CASE_INVALID_ATTACHMENTS', - TREATMENT_TOOTH_PROSTHESIS_INCOMPLETE: 'TREATMENT_TOOTH_PROSTHESIS_INCOMPLETE', + TREATMENT_TOOTH_PROSTHESIS_INCOMPLETE: + 'TREATMENT_TOOTH_PROSTHESIS_INCOMPLETE', TREATMENT_DETAIL_SENT: 'TREATMENT_DETAIL_SENT', TREATMENT_NOT_FOUND: 'TREATMENT_NOT_FOUND', TREATMENT_PATIENT_OR_WALK_IN: 'TREATMENT_PATIENT_OR_WALK_IN', @@ -177,6 +184,14 @@ export const ErrorCode = { TODAY_INVALID_RANGE: 'TODAY_INVALID_RANGE', TODAY_INVALID_RANGE_ORDER: 'TODAY_INVALID_RANGE_ORDER', + // Voice treatment entry + VOICE_NOT_AVAILABLE: 'VOICE_NOT_AVAILABLE', + VOICE_CLIP_TOO_LONG: 'VOICE_CLIP_TOO_LONG', + VOICE_ASR_FAILED: 'VOICE_ASR_FAILED', + VOICE_EXTRACT_FAILED: 'VOICE_EXTRACT_FAILED', + VOICE_NOTHING_RECOGNIZED: 'VOICE_NOTHING_RECOGNIZED', + VOICE_RATE_LIMITED: 'VOICE_RATE_LIMITED', + // Generic HTTP NOT_FOUND: 'NOT_FOUND', CONFLICT: 'CONFLICT', diff --git a/backend/src/main.ts b/backend/src/main.ts index 4af42c2..7509b3a 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -1,5 +1,6 @@ // backend/src/main.ts import { NestFactory } from '@nestjs/core'; +import { json, urlencoded } from 'express'; import { AppModule } from './app.module'; import { ValidationPipe } from '@nestjs/common'; import cookieParser from 'cookie-parser'; // 👈 Change this line! @@ -23,7 +24,18 @@ console.log = (...args) => { }; async function bootstrap() { - const app = await NestFactory.create(AppModule); + // bodyParser is disabled here so the JSON parsers can be registered in an explicit + // order below; Nest's built-in one is installed during create() and would otherwise + // reject a voice recording at its 100 kb default before any later middleware ran. + const app = await NestFactory.create(AppModule, { bodyParser: false }); + + // Voice recordings are base64 JSON and pass 100 kb at roughly 20 seconds of audio. + // Registered first and scoped to the one route: body-parser marks the request handled, + // so the default-limit parser below skips it and every other endpoint keeps the + // standard limit. + app.use('/api/voice/extract', json({ limit: '10mb' })); + app.use(json()); + app.use(urlencoded({ extended: true })); app.useGlobalFilters(new HttpExceptionFilter()); diff --git a/backend/src/modules/voice/dto/voice.dto.ts b/backend/src/modules/voice/dto/voice.dto.ts new file mode 100644 index 0000000..2a5a32d --- /dev/null +++ b/backend/src/modules/voice/dto/voice.dto.ts @@ -0,0 +1,55 @@ +import { + IsBase64, + IsIn, + IsInt, + IsString, + MaxLength, + Min, +} from 'class-validator'; + +/** Containers OpenRouter's transcription endpoint accepts, and MediaRecorder can produce. */ +export const VOICE_AUDIO_FORMATS = [ + 'webm', + 'mp4', + 'm4a', + 'aac', + 'ogg', + 'wav', + 'mp3', + 'flac', +] as const; + +export type VoiceAudioFormat = (typeof VOICE_AUDIO_FORMATS)[number]; + +export class ExtractVoiceDto { + /** + * Base64 audio, no data: prefix. Capped well above a 2-minute opus clip (~400 KB) but + * far below OpenRouter's 25 MB ceiling, so an oversized upload is rejected before it + * costs a vendor call. + */ + @IsString() + @IsBase64() + @MaxLength(8_000_000) + audio: string; + + @IsIn(VOICE_AUDIO_FORMATS) + format: VoiceAudioFormat; + + /** + * The clinician's IANA zone. The server derives "today" from it rather than trusting a + * client-supplied date, which is what relative deadlines resolve against. + */ + @IsString() + @MaxLength(64) + timeZone: string; + + /** + * Recording length as measured by the client. + * + * Required, not optional: an optional value means omitting it bypasses + * VOICE_MAX_RECORDING_MS entirely, which would make the cap advisory. + */ + @IsInt() + @Min(0) + durationMs: number; +} diff --git a/backend/src/modules/voice/voice-throttler.guard.ts b/backend/src/modules/voice/voice-throttler.guard.ts new file mode 100644 index 0000000..0a42949 --- /dev/null +++ b/backend/src/modules/voice/voice-throttler.guard.ts @@ -0,0 +1,35 @@ +import { HttpStatus, Injectable } from '@nestjs/common'; +import { ThrottlerGuard } from '@nestjs/throttler'; +import { AppException, ErrorCode } from '../../common/errors'; + +/** + * Rate limits voice extraction per user rather than per IP. + * + * The default tracker keys on `req.ip`, which behind nginx means the whole deployment + * shares one bucket unless `trust proxy` is set — and an abuser rotating IPs would bypass + * it entirely. Since v1 ships with no plan gate, this is the only control on metered + * vendor spend, so it has to key on something the client cannot change. + * + * Guard order matters: the controller's JwtAuthGuard runs before this method-level guard, + * so `req.user` is populated by the time `getTracker` is called. + */ +@Injectable() +export class VoiceThrottlerGuard extends ThrottlerGuard { + protected getTracker(req: Record): Promise { + const user = req?.user as { id?: unknown } | undefined; + if (typeof user?.id === 'string' && user.id) { + return Promise.resolve(`voice:user:${user.id}`); + } + // Unauthenticated requests never reach here, but fall back rather than share a bucket. + const ip = typeof req?.ip === 'string' ? req.ip : 'unknown'; + return Promise.resolve(`voice:ip:${ip}`); + } + + /** Without this, ThrottlerException surfaces as INTERNAL_ERROR — there is no 429 fallback. */ + protected throwThrottlingException(): Promise { + throw new AppException( + ErrorCode.VOICE_RATE_LIMITED, + HttpStatus.TOO_MANY_REQUESTS, + ); + } +} diff --git a/backend/src/modules/voice/voice.controller.ts b/backend/src/modules/voice/voice.controller.ts new file mode 100644 index 0000000..9c63b5f --- /dev/null +++ b/backend/src/modules/voice/voice.controller.ts @@ -0,0 +1,71 @@ +import { + Body, + Controller, + Get, + Post, + Req, + Res, + UseGuards, +} from '@nestjs/common'; +import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger'; +import type { Request, Response } from 'express'; +import { ClinicOrgGuard } from '../../common/guards/clinic-org.guard'; +import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard'; +import { ExtractVoiceDto } from './dto/voice.dto'; +import { VoiceThrottlerGuard } from './voice-throttler.guard'; +import { VoiceService } from './voice.service'; + +type VoiceRequestUser = { + id: string; + organizationId?: string; + language?: string | null; +}; + +@ApiTags('voice') +@ApiBearerAuth('JWT-auth') +@UseGuards(JwtAuthGuard, ClinicOrgGuard) +@Controller('voice') +export class VoiceController { + constructor(private readonly voiceService: VoiceService) {} + + @Get('availability') + @ApiOperation({ + summary: 'Whether voice entry is configured, and for which locales', + }) + getAvailability() { + // The frontend cannot learn this from NEXT_PUBLIC_* — those are baked in at build + // time, so enabling a locale would otherwise require rebuilding the image. + return { success: true, data: this.voiceService.getAvailability() }; + } + + @Post('extract') + // Guarded here rather than globally, and configured by VOICE_THROTTLE_* rather than + // hardcoded. Availability is deliberately left unthrottled — it is cheap and the + // frontend calls it on load. + @UseGuards(VoiceThrottlerGuard) + @ApiOperation({ + summary: + 'Transcribe a recording and extract treatment detail intents (TAB_TREATMENT_EDIT)', + }) + async extract( + @Req() req: Request & { user: VoiceRequestUser }, + @Res({ passthrough: true }) res: Response, + @Body() dto: ExtractVoiceDto, + ) { + // Cancelling in the browser closes the connection; propagate that as an abort so the + // in-flight vendor call stops rather than settling and being discarded. It is metered + // per minute, so letting it run costs real money for a result nobody will see. + const aborter = new AbortController(); + res.on('close', () => { + if (!res.writableFinished) aborter.abort(); + }); + + const data = await this.voiceService.extract( + req.user, + dto, + req.user?.language ?? 'en', + aborter.signal, + ); + return { success: true, data }; + } +} diff --git a/backend/src/modules/voice/voice.module.ts b/backend/src/modules/voice/voice.module.ts new file mode 100644 index 0000000..7c5c366 --- /dev/null +++ b/backend/src/modules/voice/voice.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { PrismaService } from '../../../prisma/prisma.service'; +import { ProsthesisCatalogModule } from '../prosthesis-catalog/prosthesis-catalog.module'; +import { VoiceController } from './voice.controller'; +import { VoiceService } from './voice.service'; + +@Module({ + imports: [ProsthesisCatalogModule], + controllers: [VoiceController], + providers: [VoiceService, PrismaService], +}) +export class VoiceModule {} diff --git a/backend/src/modules/voice/voice.service.ts b/backend/src/modules/voice/voice.service.ts new file mode 100644 index 0000000..209cee0 --- /dev/null +++ b/backend/src/modules/voice/voice.service.ts @@ -0,0 +1,329 @@ +import { HttpStatus, Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { LinkStatus } from '@prisma/client'; +import { PrismaService } from '../../../prisma/prisma.service'; +import { AppException, ErrorCode } from '../../common/errors'; +import { + civilDateInZone, + isValidIanaTimeZone, +} from '../../common/zoned-civil-time'; +import type { VoiceConfig, VoiceProfile } from '../../configs/configurations'; +import { hasEffectivePermission } from '../../common/membership-permissions'; +import { normalizeCatalogLocale } from '../catalog/catalog-label.service'; +import { ProsthesisCatalogService } from '../prosthesis-catalog/prosthesis-catalog.service'; +import { TreatmentCatalogService } from '../treatment-catalog/treatment-catalog.service'; +import { + resolveVoiceIntent, + type ResolvedExtraction, +} from './extraction.resolver'; +import { + OpenRouterAsrProvider, + OpenRouterExtractionProvider, +} from './openrouter.provider'; +import { + VoiceProviderError, + type AsrProvider, + type ExtractionCatalog, + type ExtractionProvider, +} from './voice.providers'; +import type { ExtractVoiceDto } from './dto/voice.dto'; + +export type VoiceAvailability = { + enabled: boolean; + locales: string[]; + maxRecordingMs: number | null; +}; + +export type VoiceExtractionResponse = ResolvedExtraction & { + transcript: string; +}; + +@Injectable() +export class VoiceService { + private readonly logger = new Logger(VoiceService.name); + + constructor( + private readonly prisma: PrismaService, + private readonly config: ConfigService, + private readonly treatmentCatalog: TreatmentCatalogService, + private readonly prosthesisCatalog: ProsthesisCatalogService, + ) {} + + private get voiceConfig(): VoiceConfig { + return this.config.get('voice')!; + } + + /** + * What the frontend needs to decide whether to render the microphone at all. + * + * v1 ships ungated beyond a configured locale profile — no plan check. The + * Plan.features design is deferred, not dropped. + */ + getAvailability(): VoiceAvailability { + const voice = this.voiceConfig; + const hasKey = Boolean(voice.openRouter.apiKey); + const locales = hasKey ? Object.keys(voice.profiles) : []; + return { + enabled: locales.length > 0, + locales, + maxRecordingMs: voice.maxRecordingMs, + }; + } + + async extract( + user: { id: string; organizationId?: string }, + dto: ExtractVoiceDto, + locale: string, + signal?: AbortSignal, + ): Promise { + const startedAt = Date.now(); + const organizationId = this.assertOrganization(user); + await this.assertCanEditTreatment(user.id, organizationId); + + const catalogLocale = normalizeCatalogLocale(locale); + const profile = this.resolveProfile(catalogLocale); + this.assertWithinCap(dto.durationMs); + + const timeZone = isValidIanaTimeZone(dto.timeZone) ? dto.timeZone : 'UTC'; + const todayIso = civilDateInZone(new Date(), timeZone); + + const { asr, extraction } = this.buildProviders(profile); + + // Stage 1 — audio never touches disk and is not retained beyond this call. + let transcript: string; + let asrCost: number | null = null; + try { + const result = await asr.transcribe( + { data: dto.audio, format: dto.format }, + catalogLocale, + signal, + ); + transcript = result.text; + asrCost = result.usage.costUsd; + } catch (error) { + throw this.toAppException(error, 'asr'); + } + + if (!transcript.trim()) { + throw new AppException( + ErrorCode.VOICE_NOTHING_RECOGNIZED, + HttpStatus.UNPROCESSABLE_ENTITY, + ); + } + + // Stage 2 — structure it. On failure the transcript still goes back to the client so + // the words the clinician already paid for are not lost (transcript salvage). + const catalog = await this.buildCatalog(organizationId, catalogLocale); + let resolved: ResolvedExtraction; + let llmCost: number | null = null; + try { + const result = await extraction.extract( + transcript, + catalog, + catalogLocale, + signal, + ); + llmCost = result.costUsd; + resolved = resolveVoiceIntent(result.intent, { + todayIso, + treatmentTypeCodes: new Set(catalog.treatmentTypes.map((t) => t.code)), + prosthesisTypeCodes: new Set( + catalog.prosthesisTypes.map((t) => t.code), + ), + linkedLabIds: new Set(catalog.labs.map((l) => l.id)), + }); + } catch (error) { + throw this.toAppException(error, 'extraction', transcript); + } + + this.logTelemetry({ + locale: catalogLocale, + durationMs: dto.durationMs ?? null, + elapsedMs: Date.now() - startedAt, + asrCost, + llmCost, + resolved, + }); + + return { ...resolved, transcript }; + } + + private assertOrganization(user: { organizationId?: string }): string { + if (!user?.organizationId) { + throw new AppException( + ErrorCode.AUTH_ORG_NOT_SELECTED, + HttpStatus.BAD_REQUEST, + ); + } + return user.organizationId; + } + + private async assertCanEditTreatment(userId: string, organizationId: string) { + const membership = await this.prisma.membership.findFirst({ + where: { + userId, + organizationId, + OR: [{ isOwner: true }, { isActive: true }], + }, + include: { + permissions: { include: { permission: true } }, + organization: { include: { type: true, plan: true } }, + }, + }); + if (!membership) { + throw new AppException( + ErrorCode.PERMISSION_NOT_MEMBER, + HttpStatus.FORBIDDEN, + ); + } + if (!hasEffectivePermission(membership, 'TAB_TREATMENT_EDIT')) { + throw new AppException( + ErrorCode.PERMISSION_EDIT_TREATMENTS, + HttpStatus.FORBIDDEN, + ); + } + } + + private resolveProfile(locale: string): VoiceProfile { + const voice = this.voiceConfig; + const profile = voice.profiles[locale]; + if (!profile || !voice.openRouter.apiKey) { + throw new AppException( + ErrorCode.VOICE_NOT_AVAILABLE, + HttpStatus.FORBIDDEN, + ); + } + return profile; + } + + private assertWithinCap(durationMs?: number) { + const max = this.voiceConfig.maxRecordingMs; + if (max != null && durationMs != null && durationMs > max) { + throw new AppException( + ErrorCode.VOICE_CLIP_TOO_LONG, + HttpStatus.PAYLOAD_TOO_LARGE, + ); + } + } + + private buildProviders(profile: VoiceProfile): { + asr: AsrProvider; + extraction: ExtractionProvider; + } { + const { apiKey, baseUrl } = this.voiceConfig.openRouter; + const base = { apiKey: apiKey!, baseUrl }; + return { + asr: new OpenRouterAsrProvider({ ...base, model: profile.asr.model }), + extraction: new OpenRouterExtractionProvider({ + ...base, + model: profile.llm.model, + }), + }; + } + + /** Codes with labels in the actor's locale, plus the clinic's linked labs. */ + private async buildCatalog( + organizationId: string, + locale: string, + ): Promise { + const [treatmentTypes, prosthesisTypes, labs] = await Promise.all([ + this.treatmentCatalog.list(locale, null), + this.prosthesisCatalog.list(locale), + this.listLinkedLabs(organizationId), + ]); + + return { + treatmentTypes: treatmentTypes + .filter((entry) => entry.availableInTreatment) + .map((entry) => ({ code: entry.code, label: entry.label })), + prosthesisTypes: prosthesisTypes.map((entry) => ({ + code: entry.code, + label: entry.label, + })), + labs, + }; + } + + private async listLinkedLabs( + organizationId: string, + ): Promise<{ id: string; name: string }[]> { + const [linksA, linksB] = await Promise.all([ + this.prisma.organizationLink.findMany({ + where: { organizationAId: organizationId, status: LinkStatus.ACTIVE }, + include: { organizationB: { select: { id: true, name: true } } }, + }), + this.prisma.organizationLink.findMany({ + where: { organizationBId: organizationId, status: LinkStatus.ACTIVE }, + include: { organizationA: { select: { id: true, name: true } } }, + }), + ]); + return [ + ...linksA.map((l) => ({ + id: l.organizationB.id, + name: l.organizationB.name, + })), + ...linksB.map((l) => ({ + id: l.organizationA.id, + name: l.organizationA.name, + })), + ]; + } + + private toAppException( + error: unknown, + stage: 'asr' | 'extraction', + transcript?: string, + ): AppException { + if (error instanceof Error && error.name === 'AbortError') { + // The clinician cancelled; not a failure worth a translated message. + return new AppException(ErrorCode.BAD_REQUEST, HttpStatus.BAD_REQUEST); + } + if (error instanceof VoiceProviderError) { + this.logger.warn(`voice ${stage} failed: ${error.message}`); + } else { + this.logger.error(`voice ${stage} failed unexpectedly`, error as Error); + } + const code = + stage === 'asr' + ? ErrorCode.VOICE_ASR_FAILED + : ErrorCode.VOICE_EXTRACT_FAILED; + return new AppException( + code, + HttpStatus.BAD_GATEWAY, + transcript ? { transcript } : undefined, + ); + } + + /** + * Structured, patient-free. Never the transcript, never audio, never a patient id. + * Log lines are the interim sink until this repo has metrics infrastructure. + */ + private logTelemetry(input: { + locale: string; + durationMs: number | null; + elapsedMs: number; + asrCost: number | null; + llmCost: number | null; + resolved: ResolvedExtraction; + }) { + const { resolved } = input; + this.logger.log( + JSON.stringify({ + event: 'voice.extract', + locale: input.locale, + clipMs: input.durationMs, + elapsedMs: input.elapsedMs, + costUsd: (input.asrCost ?? 0) + (input.llmCost ?? 0), + resolvedFields: { + treatmentType: resolved.treatmentType != null, + teeth: resolved.teeth.length, + comment: resolved.comment != null, + prosthesisComplete: resolved.prosthesis?.complete ?? null, + lab: resolved.labId != null, + dueDate: resolved.dueDate != null, + }, + unresolvedCount: resolved.unresolved.length, + }), + ); + } +} diff --git a/frontend/messages/en.json b/frontend/messages/en.json index 65b9760..05cee8c 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -1240,6 +1240,13 @@ "LAB_CASE_START_INCOMPLETE": "Complete teeth and prosthesis types before starting this case.", "LAB_CASE_CLIENT_REQUIRED": "Enter a clinic name or a patient name for this case.", "BAD_REQUEST": "The request could not be processed.", - "INTERNAL_ERROR": "Something went wrong on our end. Please try again later." + "INTERNAL_ERROR": "Something went wrong on our end. Please try again later.", + "VOICE_MIC_DENIED": "Microphone access was blocked. Allow it in your browser settings and try again.", + "VOICE_NOT_AVAILABLE": "Voice entry is not available for this language yet.", + "VOICE_CLIP_TOO_LONG": "That recording is too long. Please keep it under two minutes.", + "VOICE_ASR_FAILED": "Could not turn the recording into text. Please try again.", + "VOICE_EXTRACT_FAILED": "Could not read the treatment details from the recording.", + "VOICE_NOTHING_RECOGNIZED": "No speech was recognised. Check the microphone and try again.", + "VOICE_RATE_LIMITED": "Too many recordings in a short time. Please wait a moment." } } diff --git a/frontend/messages/fa.json b/frontend/messages/fa.json index 6909975..e301954 100644 --- a/frontend/messages/fa.json +++ b/frontend/messages/fa.json @@ -1241,6 +1241,13 @@ "LAB_CASE_START_INCOMPLETE": "قبل از شروع پرونده، دندان‌ها و نوع پروتز را کامل کنید.", "LAB_CASE_CLIENT_REQUIRED": "نام کلینیک یا نام بیمار را وارد کنید.", "BAD_REQUEST": "درخواست قابل پردازش نبود.", - "INTERNAL_ERROR": "مشکلی در سرور رخ داد. لطفاً بعداً تلاش کنید." + "INTERNAL_ERROR": "مشکلی در سرور رخ داد. لطفاً بعداً تلاش کنید.", + "VOICE_MIC_DENIED": "دسترسی به میکروفون مسدود شده است. در تنظیمات مرورگر اجازه دهید و دوباره تلاش کنید.", + "VOICE_NOT_AVAILABLE": "ثبت گفتاری هنوز برای این زبان در دسترس نیست.", + "VOICE_CLIP_TOO_LONG": "مدت ضبط بیش از حد است. لطفاً کمتر از دو دقیقه صحبت کنید.", + "VOICE_ASR_FAILED": "تبدیل گفتار به متن انجام نشد. لطفاً دوباره تلاش کنید.", + "VOICE_EXTRACT_FAILED": "اطلاعات درمان از روی گفتار استخراج نشد.", + "VOICE_NOTHING_RECOGNIZED": "گفتاری شناسایی نشد. میکروفون را بررسی کنید و دوباره تلاش کنید.", + "VOICE_RATE_LIMITED": "تعداد ضبط‌ها در بازه کوتاه زیاد بود. کمی صبر کنید." } -} \ No newline at end of file +} diff --git a/frontend/messages/nl.json b/frontend/messages/nl.json index a8f2ffb..c4ef4f4 100644 --- a/frontend/messages/nl.json +++ b/frontend/messages/nl.json @@ -1240,6 +1240,13 @@ "LAB_CASE_START_INCOMPLETE": "Vul tanden en prothesetypes in voordat u deze case start.", "LAB_CASE_CLIENT_REQUIRED": "Voer een klinieknaam of een patiëntnaam in voor deze case.", "BAD_REQUEST": "Het verzoek kon niet worden verwerkt.", - "INTERNAL_ERROR": "Er is iets misgegaan aan onze kant. Probeer het later opnieuw." + "INTERNAL_ERROR": "Er is iets misgegaan aan onze kant. Probeer het later opnieuw.", + "VOICE_MIC_DENIED": "Microfoontoegang is geblokkeerd. Sta dit toe in uw browserinstellingen en probeer opnieuw.", + "VOICE_NOT_AVAILABLE": "Spraakinvoer is nog niet beschikbaar voor deze taal.", + "VOICE_CLIP_TOO_LONG": "Die opname is te lang. Houd het onder twee minuten.", + "VOICE_ASR_FAILED": "De opname kon niet naar tekst worden omgezet. Probeer het opnieuw.", + "VOICE_EXTRACT_FAILED": "De behandelgegevens konden niet uit de opname worden gelezen.", + "VOICE_NOTHING_RECOGNIZED": "Er is geen spraak herkend. Controleer de microfoon en probeer opnieuw.", + "VOICE_RATE_LIMITED": "Te veel opnames in korte tijd. Wacht even." } -} \ No newline at end of file +}