Files
dyolink/backend/src/modules/voice/voice.service.ts
Amin Mousavi 56d413944a feat: wire voice entry into the treatment workspace
Makes the feature reachable end to end: availability is fetched alongside the
catalogs, the capture hook drives the segmented control, and confirming the
review sheet appends a new detail.

Confirm always appends — it never edits an existing detail and never calls
onAddDetail. Ticked rows land on top of the seeded defaults, so unticking the
type row leaves the appointment-purpose default rather than a blank. Lab-side
rows ride on a lab case draft keyed by the detail's *client* id, so a brand-new
unsaved detail can carry a lab, due date and per-tooth prosthesis map.

Availability comes from the API rather than a NEXT_PUBLIC_* var, since those are
baked in at build time; a failure fetching it degrades to no microphone rather
than taking the treatment tab down.

From review of this commit:

- Unticking "teeth" while leaving "prosthesis" ticked attached prosthesis rows
  for teeth the detail does not contain. Nothing downstream filters them —
  assertCompleteToothProsthesisMap only checks detail-teeth ⊆ map, never the
  reverse — so they would have reached task generation as lab work for teeth
  nobody is treating. The map is now filtered to the detail's own teeth.
- The microphone was gated on the URL locale while the server resolved
  everything from req.user.language. Those diverge (a bookmarked /fa/ URL, a
  language toggle whose save failed), which would transcribe Persian with an
  English hint and anchor "next Thursday" to a Monday week instead of a Saturday
  one — or 403 from a visibly-enabled button. The client now sends the locale the
  microphone was offered in, so the gate and the request agree by construction.

Also fixed from the previous review: a civil YYYY-MM-DD date rendered a day
early west of Greenwich (parsed as UTC midnight); the missing-teeth list
hardcoded the Arabic comma for all locales; and voiceApply had no ICU plural, so
the common single-field case read "Apply 1 fields".

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-20 20:22:42 +03:30

355 lines
11 KiB
TypeScript

import { HttpStatus, Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { LinkStatus } from '@prisma/client';
import { PrismaService } from '../../../prisma/prisma.service';
import { AppException, ErrorCode } from '../../common/errors';
import {
civilDateInZone,
isValidIanaTimeZone,
} from '../../common/zoned-civil-time';
import type { VoiceConfig, VoiceProfile } from '../../configs/configurations';
import { hasEffectivePermission } from '../../common/membership-permissions';
import { normalizeCatalogLocale } from '../catalog/catalog-label.service';
import { ProsthesisCatalogService } from '../prosthesis-catalog/prosthesis-catalog.service';
import { TreatmentCatalogService } from '../treatment-catalog/treatment-catalog.service';
import { weekStartForLocale } from './due-date.resolver';
import {
resolveVoiceIntent,
type ResolvedExtraction,
} from './extraction.resolver';
import {
OpenRouterAsrProvider,
OpenRouterExtractionProvider,
} from './openrouter.provider';
import {
VoiceProviderError,
type AsrProvider,
type ExtractionCatalog,
type ExtractionProvider,
} from './voice.providers';
import type { ExtractVoiceDto } from './dto/voice.dto';
export type VoiceAvailability = {
enabled: boolean;
locales: string[];
maxRecordingMs: number | null;
};
export type VoiceExtractionResponse = ResolvedExtraction & {
transcript: string;
};
@Injectable()
export class VoiceService {
private readonly logger = new Logger(VoiceService.name);
constructor(
private readonly prisma: PrismaService,
private readonly config: ConfigService,
private readonly treatmentCatalog: TreatmentCatalogService,
private readonly prosthesisCatalog: ProsthesisCatalogService,
) {}
private get voiceConfig(): VoiceConfig {
return this.config.get<VoiceConfig>('voice')!;
}
/**
* What the frontend needs to decide whether to render the microphone at all.
*
* v1 ships ungated beyond a configured locale profile — no plan check. The
* Plan.features design is deferred, not dropped.
*/
getAvailability(): VoiceAvailability {
const voice = this.voiceConfig;
const hasKey = Boolean(voice.openRouter.apiKey);
const locales = hasKey ? Object.keys(voice.profiles) : [];
return {
enabled: locales.length > 0,
locales,
maxRecordingMs: voice.maxRecordingMs,
};
}
async extract(
user: { id: string; organizationId?: string },
dto: ExtractVoiceDto,
locale: string,
signal?: AbortSignal,
): Promise<VoiceExtractionResponse> {
const startedAt = Date.now();
const organizationId = this.assertOrganization(user);
await this.assertCanEditTreatment(user.id, organizationId);
const catalogLocale = normalizeCatalogLocale(locale);
const profile = this.resolveProfile(catalogLocale);
this.assertWithinCap(dto.durationMs);
const timeZone = isValidIanaTimeZone(dto.timeZone) ? dto.timeZone : 'UTC';
const todayIso = civilDateInZone(new Date(), timeZone);
const { asr, extraction } = this.buildProviders(profile);
// Stage 1 — audio never touches disk and is not retained beyond this call.
let transcript: string;
let asrCost: number | null = null;
let asrSeconds: number | null = null;
try {
const result = await asr.transcribe(
{ data: dto.audio, format: dto.format },
catalogLocale,
signal,
);
transcript = result.text;
asrCost = result.usage.costUsd;
asrSeconds = result.usage.seconds;
} catch (error) {
throw this.toAppException(error, 'asr');
}
// durationMs is client-reported and therefore not enforcement. usage.seconds is the
// vendor's own measurement of the audio it decoded, so a client under-reporting length
// to slip past the cap is caught here — after the ASR spend, but before the extraction
// call, and visibly in telemetry.
if (asrSeconds != null) {
this.assertWithinCap(asrSeconds * 1000);
}
if (!transcript.trim()) {
throw new AppException(
ErrorCode.VOICE_NOTHING_RECOGNIZED,
HttpStatus.UNPROCESSABLE_ENTITY,
);
}
// Stage 2 — structure it. On failure the transcript still goes back to the client so
// the words the clinician already paid for are not lost (transcript salvage).
let resolved: ResolvedExtraction;
let llmCost: number | null = null;
try {
// Inside the try: the transcript is already paid for, so a catalog/DB failure here
// must still salvage it rather than becoming a generic 500 that throws it away.
const catalog = await this.buildCatalog(organizationId, catalogLocale);
const result = await extraction.extract(
transcript,
catalog,
catalogLocale,
signal,
);
llmCost = result.costUsd;
resolved = resolveVoiceIntent(result.intent, {
todayIso,
weekStartJs: weekStartForLocale(catalogLocale),
treatmentTypeCodes: new Set(catalog.treatmentTypes.map((t) => t.code)),
prosthesisTypeCodes: new Set(
catalog.prosthesisTypes.map((t) => t.code),
),
linkedLabIds: new Set(catalog.labs.map((l) => l.id)),
});
} catch (error) {
throw this.toAppException(error, 'extraction', transcript);
}
this.logTelemetry({
locale: catalogLocale,
durationMs: dto.durationMs,
elapsedMs: Date.now() - startedAt,
asrCost,
llmCost,
resolved,
});
return { ...resolved, transcript };
}
private assertOrganization(user: { organizationId?: string }): string {
if (!user?.organizationId) {
throw new AppException(
ErrorCode.AUTH_ORG_NOT_SELECTED,
HttpStatus.BAD_REQUEST,
);
}
return user.organizationId;
}
private async assertCanEditTreatment(userId: string, organizationId: string) {
const membership = await this.prisma.membership.findFirst({
where: {
userId,
organizationId,
OR: [{ isOwner: true }, { isActive: true }],
},
include: {
permissions: { include: { permission: true } },
organization: { include: { type: true, plan: true } },
},
});
if (!membership) {
throw new AppException(
ErrorCode.PERMISSION_NOT_MEMBER,
HttpStatus.FORBIDDEN,
);
}
if (!hasEffectivePermission(membership, 'TAB_TREATMENT_EDIT')) {
throw new AppException(
ErrorCode.PERMISSION_EDIT_TREATMENTS,
HttpStatus.FORBIDDEN,
);
}
}
private resolveProfile(locale: string): VoiceProfile {
const voice = this.voiceConfig;
const profile = voice.profiles[locale];
if (!profile || !voice.openRouter.apiKey) {
throw new AppException(
ErrorCode.VOICE_NOT_AVAILABLE,
HttpStatus.FORBIDDEN,
);
}
return profile;
}
/**
* Grace above the configured cap.
*
* The client auto-stops when elapsed >= maxMs, then measures the final length after the
* recorder has actually stopped — so a recording that runs to the cap always reports
* slightly over it. Without this tolerance the auto-stop would guarantee a rejection,
* discarding exactly the recording it was meant to save. The client still reports the
* true length, so telemetry stays honest.
*/
private static readonly CAP_TOLERANCE_MS = 2_000;
private assertWithinCap(durationMs: number) {
const max = this.voiceConfig.maxRecordingMs;
if (max != null && durationMs > max + VoiceService.CAP_TOLERANCE_MS) {
throw new AppException(
ErrorCode.VOICE_CLIP_TOO_LONG,
HttpStatus.PAYLOAD_TOO_LARGE,
);
}
}
private buildProviders(profile: VoiceProfile): {
asr: AsrProvider;
extraction: ExtractionProvider;
} {
const { apiKey, baseUrl } = this.voiceConfig.openRouter;
const base = { apiKey: apiKey!, baseUrl };
return {
asr: new OpenRouterAsrProvider({ ...base, model: profile.asr.model }),
extraction: new OpenRouterExtractionProvider({
...base,
model: profile.llm.model,
}),
};
}
/** Codes with labels in the actor's locale, plus the clinic's linked labs. */
private async buildCatalog(
organizationId: string,
locale: string,
): Promise<ExtractionCatalog> {
const [treatmentTypes, prosthesisTypes, labs] = await Promise.all([
this.treatmentCatalog.list(locale, null),
this.prosthesisCatalog.list(locale),
this.listLinkedLabs(organizationId),
]);
return {
treatmentTypes: treatmentTypes
.filter((entry) => entry.availableInTreatment)
.map((entry) => ({ code: entry.code, label: entry.label })),
prosthesisTypes: prosthesisTypes.map((entry) => ({
code: entry.code,
label: entry.label,
})),
labs,
};
}
private async listLinkedLabs(
organizationId: string,
): Promise<{ id: string; name: string }[]> {
const [linksA, linksB] = await Promise.all([
this.prisma.organizationLink.findMany({
where: { organizationAId: organizationId, status: LinkStatus.ACTIVE },
include: { organizationB: { select: { id: true, name: true } } },
}),
this.prisma.organizationLink.findMany({
where: { organizationBId: organizationId, status: LinkStatus.ACTIVE },
include: { organizationA: { select: { id: true, name: true } } },
}),
]);
return [
...linksA.map((l) => ({
id: l.organizationB.id,
name: l.organizationB.name,
})),
...linksB.map((l) => ({
id: l.organizationA.id,
name: l.organizationA.name,
})),
];
}
private toAppException(
error: unknown,
stage: 'asr' | 'extraction',
transcript?: string,
): AppException {
if (error instanceof Error && error.name === 'AbortError') {
// The clinician cancelled; not a failure worth a translated message.
return new AppException(ErrorCode.BAD_REQUEST, HttpStatus.BAD_REQUEST);
}
if (error instanceof VoiceProviderError) {
this.logger.warn(`voice ${stage} failed: ${error.message}`);
} else {
this.logger.error(`voice ${stage} failed unexpectedly`, error as Error);
}
const code =
stage === 'asr'
? ErrorCode.VOICE_ASR_FAILED
: ErrorCode.VOICE_EXTRACT_FAILED;
return new AppException(
code,
HttpStatus.BAD_GATEWAY,
transcript ? { transcript } : undefined,
);
}
/**
* Structured, patient-free. Never the transcript, never audio, never a patient id.
* Log lines are the interim sink until this repo has metrics infrastructure.
*/
private logTelemetry(input: {
locale: string;
durationMs: number;
elapsedMs: number;
asrCost: number | null;
llmCost: number | null;
resolved: ResolvedExtraction;
}) {
const { resolved } = input;
this.logger.log(
JSON.stringify({
event: 'voice.extract',
locale: input.locale,
clipMs: input.durationMs,
elapsedMs: input.elapsedMs,
costUsd: (input.asrCost ?? 0) + (input.llmCost ?? 0),
resolvedFields: {
treatmentType: resolved.treatmentType != null,
teeth: resolved.teeth.length,
comment: resolved.comment != null,
prosthesisComplete: resolved.prosthesis?.complete ?? null,
lab: resolved.labId != null,
dueDate: resolved.dueDate != null,
},
unresolvedCount: resolved.unresolved.length,
}),
);
}
}