feat(treatment): add voice detail entry #65
@@ -1,13 +1,14 @@
|
||||
// backend/src/config/configuration.ts
|
||||
|
||||
/** Matches values accepted by jsonwebtoken `expiresIn` (via ms), e.g. 7d, 15m, or plain seconds. */
|
||||
const JWT_TIMESPAN_PATTERN =
|
||||
/^\d+(\.\d+)?(ms|s|m|h|d|w|y)?$/i;
|
||||
const JWT_TIMESPAN_PATTERN = /^\d+(\.\d+)?(ms|s|m|h|d|w|y)?$/i;
|
||||
|
||||
function assertJwtSecret(value: string, envKey: string): string {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error(`❌ Environment variable ${envKey} is required but not set`);
|
||||
throw new Error(
|
||||
`❌ Environment variable ${envKey} is required but not set`,
|
||||
);
|
||||
}
|
||||
if (trimmed.length < 16) {
|
||||
throw new Error(`❌ ${envKey} must be at least 16 characters`);
|
||||
@@ -21,7 +22,9 @@ function assertJwtSecret(value: string, envKey: string): string {
|
||||
function assertJwtTimespan(value: string, envKey: string): string {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error(`❌ Environment variable ${envKey} is required but not set`);
|
||||
throw new Error(
|
||||
`❌ Environment variable ${envKey} is required but not set`,
|
||||
);
|
||||
}
|
||||
if (!/^\d+$/.test(trimmed) && !JWT_TIMESPAN_PATTERN.test(trimmed)) {
|
||||
throw new Error(
|
||||
@@ -31,7 +34,10 @@ function assertJwtTimespan(value: string, envKey: string): string {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function parseEnvBoolean(value: string | undefined, defaultValue: boolean): boolean {
|
||||
function parseEnvBoolean(
|
||||
value: string | undefined,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
if (value === undefined || value.trim() === '') {
|
||||
return defaultValue;
|
||||
}
|
||||
@@ -67,8 +73,37 @@ export interface Config {
|
||||
apiKey: string | null;
|
||||
templateId: number;
|
||||
};
|
||||
voice: VoiceConfig;
|
||||
}
|
||||
|
||||
/** Only OpenRouter today. The indirection exists so a locale can diverge without code. */
|
||||
export type VoiceProviderId = 'openrouter';
|
||||
|
||||
export type VoiceProfile = {
|
||||
asr: { provider: VoiceProviderId; model: string };
|
||||
llm: { provider: VoiceProviderId; model: string };
|
||||
};
|
||||
|
||||
export interface VoiceConfig {
|
||||
openRouter: {
|
||||
apiKey: string | null;
|
||||
baseUrl: string;
|
||||
};
|
||||
/**
|
||||
* Locale → provider profile. A locale absent here has no microphone button at all —
|
||||
* clean absence rather than a dead control.
|
||||
*/
|
||||
profiles: Record<string, VoiceProfile>;
|
||||
/** Recording cap in ms; null means uncapped. */
|
||||
maxRecordingMs: number | null;
|
||||
throttle: { ttl: number; limit: number };
|
||||
}
|
||||
|
||||
export const VOICE_LOCALES = ['fa', 'en', 'nl'] as const;
|
||||
|
||||
const DEFAULT_ASR_MODEL = 'openai/whisper-1';
|
||||
const DEFAULT_LLM_MODEL = 'google/gemini-3.7-flash';
|
||||
|
||||
export default (): Config => {
|
||||
// Helper function to get required env var with type safety
|
||||
const getEnvVar = (key: string): string => {
|
||||
@@ -129,5 +164,96 @@ export default (): Config => {
|
||||
apiKey: process.env.SMS_IR_API_KEY?.trim() || null,
|
||||
templateId: getEnvVarAsNumber('SMS_IR_TEMPLATE_ID', 123456),
|
||||
},
|
||||
voice: buildVoiceConfig(getEnvVarWithDefault, getEnvVarAsNumber),
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const VOICE_PROVIDER_IDS: readonly VoiceProviderId[] = ['openrouter'];
|
||||
|
||||
/** Unknown provider ids fail at boot; silently coercing a typo would ship the wrong vendor. */
|
||||
function parseProviderId(
|
||||
value: string | undefined,
|
||||
envKey: string,
|
||||
): VoiceProviderId {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed) return 'openrouter';
|
||||
if ((VOICE_PROVIDER_IDS as readonly string[]).includes(trimmed)) {
|
||||
return trimmed as VoiceProviderId;
|
||||
}
|
||||
throw new Error(
|
||||
`❌ ${envKey}="${trimmed}" is not a known voice provider (${VOICE_PROVIDER_IDS.join(', ')})`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every enabled locale gets its own ASR and LLM provider+model, each overridable
|
||||
* independently. They all point at the same OpenRouter models today; the per-locale
|
||||
* indirection is kept because Persian ASR is the weakest link and swapping only `fa` must
|
||||
* not be a code change.
|
||||
*/
|
||||
function buildVoiceConfig(
|
||||
getEnvVarWithDefault: (key: string, defaultValue: string) => string,
|
||||
getEnvVarAsNumber: (key: string, defaultValue: number) => number,
|
||||
): VoiceConfig {
|
||||
const enabled = getEnvVarWithDefault(
|
||||
'VOICE_ENABLED_LOCALES',
|
||||
VOICE_LOCALES.join(','),
|
||||
)
|
||||
.split(',')
|
||||
.map((locale) => locale.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
for (const locale of enabled) {
|
||||
// Fail loudly, like parseProviderId. Silently filtering a typo would disable the
|
||||
// microphone everywhere with nothing to explain why.
|
||||
if (!(VOICE_LOCALES as readonly string[]).includes(locale)) {
|
||||
throw new Error(
|
||||
`❌ VOICE_ENABLED_LOCALES contains unknown locale "${locale}" (known: ${VOICE_LOCALES.join(', ')})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const profiles: Record<string, VoiceProfile> = {};
|
||||
for (const locale of enabled) {
|
||||
const suffix = locale.toUpperCase();
|
||||
profiles[locale] = {
|
||||
asr: {
|
||||
provider: parseProviderId(
|
||||
process.env[`VOICE_ASR_PROVIDER_${suffix}`],
|
||||
`VOICE_ASR_PROVIDER_${suffix}`,
|
||||
),
|
||||
model: getEnvVarWithDefault(
|
||||
`VOICE_ASR_MODEL_${suffix}`,
|
||||
getEnvVarWithDefault('VOICE_ASR_MODEL', DEFAULT_ASR_MODEL),
|
||||
),
|
||||
},
|
||||
llm: {
|
||||
provider: parseProviderId(
|
||||
process.env[`VOICE_LLM_PROVIDER_${suffix}`],
|
||||
`VOICE_LLM_PROVIDER_${suffix}`,
|
||||
),
|
||||
model: getEnvVarWithDefault(
|
||||
`VOICE_LLM_MODEL_${suffix}`,
|
||||
getEnvVarWithDefault('VOICE_LLM_MODEL', DEFAULT_LLM_MODEL),
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const maxRecordingMs = getEnvVarAsNumber('VOICE_MAX_RECORDING_MS', 120_000);
|
||||
|
||||
return {
|
||||
openRouter: {
|
||||
apiKey: process.env.OPENROUTER_API_KEY?.trim() || null,
|
||||
baseUrl: getEnvVarWithDefault(
|
||||
'OPENROUTER_BASE_URL',
|
||||
'https://openrouter.ai/api/v1',
|
||||
),
|
||||
},
|
||||
profiles,
|
||||
maxRecordingMs: maxRecordingMs > 0 ? maxRecordingMs : null,
|
||||
throttle: {
|
||||
ttl: getEnvVarAsNumber('VOICE_THROTTLE_TTL', 60),
|
||||
limit: getEnvVarAsNumber('VOICE_THROTTLE_LIMIT', 6),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
78
backend/src/modules/voice/extraction.prompt.ts
Normal file
78
backend/src/modules/voice/extraction.prompt.ts
Normal file
@@ -0,0 +1,78 @@
|
||||
import type { ExtractionCatalog } from './voice.providers';
|
||||
|
||||
/** Locale-specific guidance. Only the tooth vocabulary and numbering habits differ. */
|
||||
const LOCALE_NOTES: Record<string, string> = {
|
||||
fa: [
|
||||
'The clinician is speaking Persian. Tooth references are usually quadrant-relative:',
|
||||
'"شش بالا راست" = upper right six -> arch "upper", side "patient_right", position 6.',
|
||||
'Digits may appear in Persian or Latin script. Two-digit FDI notation ("یک چهار") does',
|
||||
'occur — use the "fdi" field only for that.',
|
||||
].join(' '),
|
||||
nl: [
|
||||
'The clinician is speaking Dutch and uses FDI notation, which is standard in the',
|
||||
'Netherlands. "rechtsboven zes" = upper right six. A bare two-digit number is FDI.',
|
||||
].join(' '),
|
||||
en: [
|
||||
'The clinician is speaking English. IMPORTANT: a bare two-digit number is ambiguous,',
|
||||
'because Universal numbering and FDI disagree ("tooth 14" is a different tooth in each).',
|
||||
'Set "fdi" ONLY when the speaker made the notation explicit (e.g. "FDI one four").',
|
||||
'Otherwise describe the tooth with arch/side/position, or leave it unresolved.',
|
||||
].join(' '),
|
||||
};
|
||||
|
||||
function codeList(entries: { code: string; label: string }[]): string {
|
||||
if (entries.length === 0) return '(none available)';
|
||||
return entries.map((e) => `- ${e.code} = ${e.label}`).join('\n');
|
||||
}
|
||||
|
||||
export function buildExtractionPrompt(
|
||||
transcript: string,
|
||||
catalog: ExtractionCatalog,
|
||||
localeHint: string,
|
||||
) {
|
||||
const localeNote = LOCALE_NOTES[localeHint] ?? LOCALE_NOTES.en;
|
||||
|
||||
const system = [
|
||||
'You extract structured dental treatment data from a transcript of a clinician speaking.',
|
||||
'You are a parser, not an assistant: report only what was said.',
|
||||
'',
|
||||
'HARD RULES',
|
||||
'1. Never invent a code. treatmentType, prosthesisDefaultType and prosthesisOverrides[].type',
|
||||
' must be codes from the lists below. labId must be an id from the lab list. If what you',
|
||||
' heard is not in a list, use null.',
|
||||
'2. Never output an FDI tooth code unless the speaker used FDI notation. Prefer',
|
||||
' arch + side + position.',
|
||||
'3. "side" is always the PATIENT\'s side. The patient\'s upper right is quadrant 1. Never',
|
||||
" flip to the viewer's point of view.",
|
||||
'4. Never do calendar arithmetic. Report the deadline as it was said, using due.kind.',
|
||||
' If no deadline was mentioned, use due.kind = "none".',
|
||||
'5. Copy the exact spoken words for each tooth into "spoken", so the clinician can see',
|
||||
' what was heard.',
|
||||
'6. If you are unsure about a value, use null. A missing field is recoverable; a wrong',
|
||||
' one is not.',
|
||||
'',
|
||||
localeNote,
|
||||
'',
|
||||
'TREATMENT TYPE CODES',
|
||||
codeList(catalog.treatmentTypes),
|
||||
'',
|
||||
'PROSTHESIS TYPE CODES',
|
||||
codeList(catalog.prosthesisTypes),
|
||||
'',
|
||||
'LABS THIS CLINIC CAN SEND TO',
|
||||
catalog.labs.length > 0
|
||||
? catalog.labs.map((l) => `- ${l.id} = ${l.name}`).join('\n')
|
||||
: '(none linked — labId must be null)',
|
||||
'',
|
||||
'OTHER FIELDS',
|
||||
'- connectedSpans: only for bridges or splinted units. Endpoints inclusive.',
|
||||
'- comment: clinical notes, in the language spoken. Omit the parts already captured as',
|
||||
' treatment type, teeth or deadline.',
|
||||
'- labMatchExact: true only when the spoken name matched a lab name exactly.',
|
||||
].join('\n');
|
||||
|
||||
return [
|
||||
{ role: 'system' as const, content: system },
|
||||
{ role: 'user' as const, content: transcript },
|
||||
];
|
||||
}
|
||||
209
backend/src/modules/voice/extraction.wire.spec.ts
Normal file
209
backend/src/modules/voice/extraction.wire.spec.ts
Normal file
@@ -0,0 +1,209 @@
|
||||
import {
|
||||
toVoiceIntent,
|
||||
VOICE_INTENT_JSON_SCHEMA,
|
||||
type WireVoiceIntent,
|
||||
} from './extraction.wire';
|
||||
|
||||
const emptyDue = {
|
||||
kind: 'none' as const,
|
||||
weekday: null,
|
||||
which: null,
|
||||
unit: null,
|
||||
amount: null,
|
||||
jy: null,
|
||||
jm: null,
|
||||
jd: null,
|
||||
y: null,
|
||||
m: null,
|
||||
d: null,
|
||||
};
|
||||
|
||||
const wire = (overrides: Partial<WireVoiceIntent> = {}): WireVoiceIntent => ({
|
||||
treatmentType: null,
|
||||
teeth: [],
|
||||
connectedSpans: [],
|
||||
comment: null,
|
||||
prosthesisDefaultType: null,
|
||||
prosthesisOverrides: [],
|
||||
labId: null,
|
||||
labMatchExact: false,
|
||||
due: emptyDue,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const positionalTooth = {
|
||||
spoken: 'شش بالا راست',
|
||||
fdi: null,
|
||||
arch: 'upper' as const,
|
||||
side: 'patient_right' as const,
|
||||
position: 6,
|
||||
};
|
||||
|
||||
describe('VOICE_INTENT_JSON_SCHEMA', () => {
|
||||
it('is strict — every property required, no extras', () => {
|
||||
expect(VOICE_INTENT_JSON_SCHEMA.additionalProperties).toBe(false);
|
||||
expect([...VOICE_INTENT_JSON_SCHEMA.required].sort()).toEqual(
|
||||
Object.keys(VOICE_INTENT_JSON_SCHEMA.properties).sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it('requires every field of the due object, since strict mode allows no optionals', () => {
|
||||
const due = VOICE_INTENT_JSON_SCHEMA.properties.due;
|
||||
expect([...due.required].sort()).toEqual(
|
||||
Object.keys(due.properties).sort(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('toVoiceIntent', () => {
|
||||
it('narrows a positional tooth', () => {
|
||||
const result = toVoiceIntent(wire({ teeth: [positionalTooth] }));
|
||||
expect(result.teeth[0]).toEqual({
|
||||
kind: 'positional',
|
||||
arch: 'upper',
|
||||
side: 'patient_right',
|
||||
position: 6,
|
||||
spoken: 'شش بالا راست',
|
||||
});
|
||||
});
|
||||
|
||||
it('narrows an explicit FDI tooth, which wins over positional fields', () => {
|
||||
const result = toVoiceIntent(
|
||||
wire({ teeth: [{ ...positionalTooth, fdi: '14', spoken: 'یک چهار' }] }),
|
||||
);
|
||||
expect(result.teeth[0]).toEqual({
|
||||
kind: 'explicit',
|
||||
fdi: '14',
|
||||
spoken: 'یک چهار',
|
||||
});
|
||||
});
|
||||
|
||||
it('maps due.kind "none" to no deadline', () => {
|
||||
expect(toVoiceIntent(wire()).due).toBeNull();
|
||||
});
|
||||
|
||||
it('narrows each due kind', () => {
|
||||
expect(
|
||||
toVoiceIntent(
|
||||
wire({
|
||||
due: {
|
||||
...emptyDue,
|
||||
kind: 'weekday',
|
||||
weekday: 'thursday',
|
||||
which: 'next',
|
||||
},
|
||||
}),
|
||||
).due,
|
||||
).toEqual({ kind: 'weekday', weekday: 'thursday', which: 'next' });
|
||||
|
||||
expect(
|
||||
toVoiceIntent(
|
||||
wire({ due: { ...emptyDue, kind: 'offset', unit: 'week', amount: 2 } }),
|
||||
).due,
|
||||
).toEqual({ kind: 'offset', unit: 'week', amount: 2 });
|
||||
|
||||
expect(
|
||||
toVoiceIntent(
|
||||
wire({ due: { ...emptyDue, kind: 'jalali', jy: 1404, jm: 7, jd: 25 } }),
|
||||
).due,
|
||||
).toEqual({ kind: 'jalali', jy: 1404, jm: 7, jd: 25 });
|
||||
|
||||
expect(
|
||||
toVoiceIntent(
|
||||
wire({
|
||||
due: { ...emptyDue, kind: 'gregorian', y: 2025, m: 10, d: 17 },
|
||||
}),
|
||||
).due,
|
||||
).toEqual({ kind: 'gregorian', y: 2025, m: 10, d: 17 });
|
||||
});
|
||||
|
||||
it('ignores a non-FDI-shaped fdi and keeps the positional fields', () => {
|
||||
// A model emitting fdi:"6" alongside correct arch/side/position must still yield 16,
|
||||
// not lose the tooth to the explicit branch.
|
||||
const result = toVoiceIntent(
|
||||
wire({ teeth: [{ ...positionalTooth, fdi: '6' }] }),
|
||||
);
|
||||
expect(result.teeth[0]).toEqual({
|
||||
kind: 'positional',
|
||||
arch: 'upper',
|
||||
side: 'patient_right',
|
||||
position: 6,
|
||||
spoken: 'شش بالا راست',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects impossible FDI shapes from the explicit branch', () => {
|
||||
for (const fdi of ['99', '0', '140', '9', 'ab']) {
|
||||
expect(
|
||||
toVoiceIntent(wire({ teeth: [{ ...positionalTooth, fdi }] })).teeth[0]
|
||||
.kind,
|
||||
).toBe('positional');
|
||||
}
|
||||
});
|
||||
|
||||
it('passes an unrecognised due kind through so it can be flagged', () => {
|
||||
// Collapsing it to null would make a misunderstood deadline indistinguishable from
|
||||
// no deadline at all, and the resolver's flagging path unreachable.
|
||||
const result = toVoiceIntent(
|
||||
wire({ due: { ...emptyDue, kind: 'lunar_month' as never } }),
|
||||
);
|
||||
expect(result.due).toEqual({ kind: 'lunar_month' });
|
||||
});
|
||||
|
||||
it('reports no prosthesis when neither a default nor an override was given', () => {
|
||||
expect(toVoiceIntent(wire()).prosthesis).toBeNull();
|
||||
});
|
||||
|
||||
it('builds a prosthesis intent from a default alone', () => {
|
||||
const result = toVoiceIntent(wire({ prosthesisDefaultType: 'pfm_crown' }));
|
||||
expect(result.prosthesis).toEqual({
|
||||
defaultType: 'pfm_crown',
|
||||
overrides: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('builds a prosthesis intent from overrides alone', () => {
|
||||
const result = toVoiceIntent(
|
||||
wire({
|
||||
prosthesisOverrides: [{ tooth: positionalTooth, type: 'pfm_crown' }],
|
||||
}),
|
||||
);
|
||||
expect(result.prosthesis?.defaultType).toBeNull();
|
||||
expect(result.prosthesis?.overrides).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('narrows connected spans', () => {
|
||||
const result = toVoiceIntent(
|
||||
wire({
|
||||
connectedSpans: [
|
||||
{
|
||||
from: { ...positionalTooth, fdi: '14' },
|
||||
to: { ...positionalTooth, fdi: '16' },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(result.connectedSpans[0].from).toEqual({
|
||||
kind: 'explicit',
|
||||
fdi: '14',
|
||||
spoken: 'شش بالا راست',
|
||||
});
|
||||
});
|
||||
|
||||
it('is total — a missing or malformed payload yields a resolvable shape, not a throw', () => {
|
||||
// Whatever survives here is reported as unresolved downstream rather than crashing.
|
||||
for (const bad of [undefined, null, {}, { teeth: 'nope', due: 5 }]) {
|
||||
expect(() => toVoiceIntent(bad as never)).not.toThrow();
|
||||
const result = toVoiceIntent(bad as never);
|
||||
expect(result.teeth).toEqual([]);
|
||||
expect(result.due).toBeNull();
|
||||
expect(result.labMatchExact).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('coerces a non-boolean labMatchExact to false', () => {
|
||||
expect(
|
||||
toVoiceIntent(wire({ labMatchExact: 'yes' as never })).labMatchExact,
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
271
backend/src/modules/voice/extraction.wire.ts
Normal file
271
backend/src/modules/voice/extraction.wire.ts
Normal file
@@ -0,0 +1,271 @@
|
||||
import type {
|
||||
ConnectedSpanIntent,
|
||||
DueIntent,
|
||||
ProsthesisIntent,
|
||||
ToothIntent,
|
||||
VoiceIntent,
|
||||
Weekday,
|
||||
} from './voice.types';
|
||||
import { WEEKDAYS } from './voice.types';
|
||||
|
||||
/**
|
||||
* The shape the model actually emits, and its JSON schema.
|
||||
*
|
||||
* Deliberately flat: strict `json_schema` mode has poor support for discriminated unions,
|
||||
* so every variant field is present and nullable on the wire. `toVoiceIntent` narrows the
|
||||
* flat shape into the internal union the resolvers consume, and is total — anything it
|
||||
* cannot classify becomes a shape the resolvers will report as unresolved rather than
|
||||
* something that throws here.
|
||||
*/
|
||||
|
||||
export type WireToothIntent = {
|
||||
spoken: string;
|
||||
/** Two-digit FDI code, only when the speaker genuinely used FDI notation. */
|
||||
fdi: string | null;
|
||||
arch: 'upper' | 'lower' | null;
|
||||
side: 'patient_right' | 'patient_left' | null;
|
||||
position: number | null;
|
||||
};
|
||||
|
||||
export type WireDue = {
|
||||
kind: 'weekday' | 'offset' | 'jalali' | 'gregorian' | 'none';
|
||||
weekday: Weekday | null;
|
||||
which: 'this' | 'next' | null;
|
||||
unit: 'day' | 'week' | 'month' | null;
|
||||
amount: number | null;
|
||||
jy: number | null;
|
||||
jm: number | null;
|
||||
jd: number | null;
|
||||
y: number | null;
|
||||
m: number | null;
|
||||
d: number | null;
|
||||
};
|
||||
|
||||
export type WireVoiceIntent = {
|
||||
treatmentType: string | null;
|
||||
teeth: WireToothIntent[];
|
||||
connectedSpans: { from: WireToothIntent; to: WireToothIntent }[];
|
||||
comment: string | null;
|
||||
prosthesisDefaultType: string | null;
|
||||
prosthesisOverrides: { tooth: WireToothIntent; type: string }[];
|
||||
labId: string | null;
|
||||
labMatchExact: boolean;
|
||||
due: WireDue;
|
||||
};
|
||||
|
||||
const TOOTH_SCHEMA = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['spoken', 'fdi', 'arch', 'side', 'position'],
|
||||
properties: {
|
||||
spoken: {
|
||||
type: 'string',
|
||||
description: 'The exact transcript words for this tooth.',
|
||||
},
|
||||
fdi: {
|
||||
type: ['string', 'null'],
|
||||
description:
|
||||
'Two-digit FDI code ONLY if the speaker used FDI notation. Otherwise null.',
|
||||
},
|
||||
arch: { type: ['string', 'null'], enum: ['upper', 'lower', null] },
|
||||
side: {
|
||||
type: ['string', 'null'],
|
||||
enum: ['patient_right', 'patient_left', null],
|
||||
description: "The PATIENT's side, never the viewer's.",
|
||||
},
|
||||
position: {
|
||||
type: ['integer', 'null'],
|
||||
description: '1 = central incisor … 8 = third molar.',
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
export const VOICE_INTENT_JSON_SCHEMA = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: [
|
||||
'treatmentType',
|
||||
'teeth',
|
||||
'connectedSpans',
|
||||
'comment',
|
||||
'prosthesisDefaultType',
|
||||
'prosthesisOverrides',
|
||||
'labId',
|
||||
'labMatchExact',
|
||||
'due',
|
||||
],
|
||||
properties: {
|
||||
treatmentType: {
|
||||
type: ['string', 'null'],
|
||||
description: 'A treatment type CODE from the supplied list, or null.',
|
||||
},
|
||||
teeth: { type: 'array', items: TOOTH_SCHEMA },
|
||||
connectedSpans: {
|
||||
type: 'array',
|
||||
description: 'Bridges / splinted units. Endpoints inclusive.',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['from', 'to'],
|
||||
properties: { from: TOOTH_SCHEMA, to: TOOTH_SCHEMA },
|
||||
},
|
||||
},
|
||||
comment: {
|
||||
type: ['string', 'null'],
|
||||
description: 'Clinical notes, in the spoken language.',
|
||||
},
|
||||
prosthesisDefaultType: {
|
||||
type: ['string', 'null'],
|
||||
description:
|
||||
'A prosthesis type CODE applied to every tooth unless overridden.',
|
||||
},
|
||||
prosthesisOverrides: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['tooth', 'type'],
|
||||
properties: { tooth: TOOTH_SCHEMA, type: { type: 'string' } },
|
||||
},
|
||||
},
|
||||
labId: {
|
||||
type: ['string', 'null'],
|
||||
description: 'An id from the supplied lab list. Never invent one.',
|
||||
},
|
||||
labMatchExact: {
|
||||
type: 'boolean',
|
||||
description: 'True only when the spoken name matched a lab name exactly.',
|
||||
},
|
||||
due: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: [
|
||||
'kind',
|
||||
'weekday',
|
||||
'which',
|
||||
'unit',
|
||||
'amount',
|
||||
'jy',
|
||||
'jm',
|
||||
'jd',
|
||||
'y',
|
||||
'm',
|
||||
'd',
|
||||
],
|
||||
properties: {
|
||||
kind: {
|
||||
type: 'string',
|
||||
enum: ['weekday', 'offset', 'jalali', 'gregorian', 'none'],
|
||||
},
|
||||
weekday: { type: ['string', 'null'], enum: [...WEEKDAYS, null] },
|
||||
which: { type: ['string', 'null'], enum: ['this', 'next', null] },
|
||||
unit: {
|
||||
type: ['string', 'null'],
|
||||
enum: ['day', 'week', 'month', null],
|
||||
},
|
||||
amount: { type: ['integer', 'null'] },
|
||||
jy: { type: ['integer', 'null'] },
|
||||
jm: { type: ['integer', 'null'] },
|
||||
jd: { type: ['integer', 'null'] },
|
||||
y: { type: ['integer', 'null'] },
|
||||
m: { type: ['integer', 'null'] },
|
||||
d: { type: ['integer', 'null'] },
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
/** Two digits, quadrant 1-8, position 1-8 — the only thing that can be an FDI code. */
|
||||
const FDI_SHAPE = /^[1-8][1-8]$/;
|
||||
|
||||
function toToothIntent(wire: WireToothIntent | undefined | null): ToothIntent {
|
||||
const spoken = typeof wire?.spoken === 'string' ? wire.spoken : '';
|
||||
const fdi = typeof wire?.fdi === 'string' ? wire.fdi.trim() : '';
|
||||
// Only take the explicit branch for something actually FDI-shaped. A model that emits
|
||||
// fdi:"6" alongside correct arch/side/position would otherwise lose the tooth entirely.
|
||||
if (FDI_SHAPE.test(fdi)) {
|
||||
return { kind: 'explicit', fdi, spoken };
|
||||
}
|
||||
return {
|
||||
kind: 'positional',
|
||||
arch: wire?.arch as 'upper' | 'lower',
|
||||
side: wire?.side as 'patient_right' | 'patient_left',
|
||||
position: typeof wire?.position === 'number' ? wire.position : Number.NaN,
|
||||
spoken,
|
||||
};
|
||||
}
|
||||
|
||||
function toDueIntent(wire: WireDue | undefined | null): DueIntent | null {
|
||||
switch (wire?.kind) {
|
||||
case 'weekday':
|
||||
return {
|
||||
kind: 'weekday',
|
||||
weekday: wire.weekday as Weekday,
|
||||
which: wire.which as 'this',
|
||||
};
|
||||
case 'offset':
|
||||
return {
|
||||
kind: 'offset',
|
||||
unit: wire.unit as 'day',
|
||||
amount: typeof wire.amount === 'number' ? wire.amount : Number.NaN,
|
||||
};
|
||||
case 'jalali':
|
||||
return {
|
||||
kind: 'jalali',
|
||||
jy: wire.jy as number,
|
||||
jm: wire.jm as number,
|
||||
jd: wire.jd as number,
|
||||
};
|
||||
case 'gregorian':
|
||||
return {
|
||||
kind: 'gregorian',
|
||||
y: wire.y as number,
|
||||
m: wire.m as number,
|
||||
d: wire.d as number,
|
||||
};
|
||||
case 'none':
|
||||
case undefined:
|
||||
return null;
|
||||
default:
|
||||
// An unrecognised kind means a deadline WAS spoken and we failed to classify it.
|
||||
// Passing it through lets the resolver flag it; collapsing it to null would make a
|
||||
// misunderstood deadline indistinguishable from no deadline at all.
|
||||
return { kind: wire?.kind } as unknown as DueIntent;
|
||||
}
|
||||
}
|
||||
|
||||
export function toVoiceIntent(wire: WireVoiceIntent): VoiceIntent {
|
||||
const teeth = Array.isArray(wire?.teeth) ? wire.teeth : [];
|
||||
const spans = Array.isArray(wire?.connectedSpans) ? wire.connectedSpans : [];
|
||||
const overrides = Array.isArray(wire?.prosthesisOverrides)
|
||||
? wire.prosthesisOverrides
|
||||
: [];
|
||||
|
||||
const connectedSpans: ConnectedSpanIntent[] = spans.map((span) => ({
|
||||
from: toToothIntent(span?.from),
|
||||
to: toToothIntent(span?.to),
|
||||
}));
|
||||
|
||||
const hasProsthesis =
|
||||
wire?.prosthesisDefaultType != null || overrides.length > 0;
|
||||
const prosthesis: ProsthesisIntent | null = hasProsthesis
|
||||
? {
|
||||
defaultType: wire?.prosthesisDefaultType ?? null,
|
||||
overrides: overrides.map((o) => ({
|
||||
tooth: toToothIntent(o?.tooth),
|
||||
type: o?.type,
|
||||
})),
|
||||
}
|
||||
: null;
|
||||
|
||||
return {
|
||||
treatmentType: wire?.treatmentType ?? null,
|
||||
teeth: teeth.map(toToothIntent),
|
||||
connectedSpans,
|
||||
comment: wire?.comment ?? null,
|
||||
prosthesis,
|
||||
labId: wire?.labId ?? null,
|
||||
labMatchExact: wire?.labMatchExact === true,
|
||||
due: toDueIntent(wire?.due),
|
||||
};
|
||||
}
|
||||
171
backend/src/modules/voice/openrouter.provider.ts
Normal file
171
backend/src/modules/voice/openrouter.provider.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import {
|
||||
type AsrProvider,
|
||||
type AsrResult,
|
||||
type AudioInput,
|
||||
type ExtractionCatalog,
|
||||
type ExtractionProvider,
|
||||
type ExtractionResult,
|
||||
VoiceProviderError,
|
||||
} from './voice.providers';
|
||||
import { buildExtractionPrompt } from './extraction.prompt';
|
||||
import {
|
||||
VOICE_INTENT_JSON_SCHEMA,
|
||||
toVoiceIntent,
|
||||
type WireVoiceIntent,
|
||||
} from './extraction.wire';
|
||||
|
||||
type OpenRouterConfig = {
|
||||
apiKey: string;
|
||||
baseUrl: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
type TranscriptionResponse = {
|
||||
text?: unknown;
|
||||
usage?: { seconds?: unknown; cost?: unknown };
|
||||
};
|
||||
|
||||
type ChatResponse = {
|
||||
choices?: { message?: { content?: unknown } }[];
|
||||
usage?: { cost?: unknown };
|
||||
};
|
||||
|
||||
function numberOrNull(value: unknown): number | null {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
const errorLogger = new Logger('OpenRouterVoice');
|
||||
|
||||
async function readError(
|
||||
response: Response,
|
||||
stage: 'asr' | 'extraction',
|
||||
): Promise<never> {
|
||||
// A vendor 4xx can echo the request back, transcript included. Keep the body out of the
|
||||
// thrown message — which callers log and could forward — and out of the default log
|
||||
// level; it stays available at debug when someone is actively diagnosing.
|
||||
try {
|
||||
errorLogger.debug(
|
||||
`${stage} ${response.status} body: ${(await response.text()).slice(0, 500)}`,
|
||||
);
|
||||
} catch {
|
||||
errorLogger.debug(`${stage} ${response.status} body unreadable`);
|
||||
}
|
||||
throw new VoiceProviderError(
|
||||
stage,
|
||||
`OpenRouter ${stage} failed with status ${response.status}`,
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
|
||||
/** Speech → text via OpenRouter's transcription endpoint (whisper-1 and friends). */
|
||||
export class OpenRouterAsrProvider implements AsrProvider {
|
||||
private readonly logger = new Logger(OpenRouterAsrProvider.name);
|
||||
|
||||
constructor(private readonly config: OpenRouterConfig) {}
|
||||
|
||||
async transcribe(
|
||||
audio: AudioInput,
|
||||
localeHint: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<AsrResult> {
|
||||
const response = await fetch(
|
||||
`${this.config.baseUrl}/audio/transcriptions`,
|
||||
{
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.config.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.config.model,
|
||||
input_audio: { data: audio.data, format: audio.format },
|
||||
language: localeHint,
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) await readError(response, 'asr');
|
||||
|
||||
const body = (await response.json()) as TranscriptionResponse;
|
||||
const text = typeof body.text === 'string' ? body.text.trim() : '';
|
||||
this.logger.debug(
|
||||
`transcribed ${numberOrNull(body.usage?.seconds) ?? '?'}s`,
|
||||
);
|
||||
|
||||
return {
|
||||
text,
|
||||
usage: {
|
||||
seconds: numberOrNull(body.usage?.seconds),
|
||||
costUsd: numberOrNull(body.usage?.cost),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Transcript → VoiceIntent via OpenRouter chat completions with a JSON schema. */
|
||||
export class OpenRouterExtractionProvider implements ExtractionProvider {
|
||||
constructor(private readonly config: OpenRouterConfig) {}
|
||||
|
||||
async extract(
|
||||
transcript: string,
|
||||
catalog: ExtractionCatalog,
|
||||
localeHint: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ExtractionResult> {
|
||||
const response = await fetch(`${this.config.baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.config.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.config.model,
|
||||
temperature: 0,
|
||||
// Only route to endpoints that actually honour the JSON schema. Without this,
|
||||
// OpenRouter may pick a provider that treats it as a hint and returns prose,
|
||||
// which fails parsing intermittently and unreproducibly.
|
||||
provider: { require_parameters: true },
|
||||
messages: buildExtractionPrompt(transcript, catalog, localeHint),
|
||||
response_format: {
|
||||
type: 'json_schema',
|
||||
json_schema: {
|
||||
name: 'voice_intent',
|
||||
strict: true,
|
||||
schema: VOICE_INTENT_JSON_SCHEMA,
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) await readError(response, 'extraction');
|
||||
|
||||
const body = (await response.json()) as ChatResponse;
|
||||
const content = body.choices?.[0]?.message?.content;
|
||||
if (typeof content !== 'string' || !content.trim()) {
|
||||
throw new VoiceProviderError(
|
||||
'extraction',
|
||||
'OpenRouter returned no content',
|
||||
);
|
||||
}
|
||||
|
||||
let wire: WireVoiceIntent;
|
||||
try {
|
||||
wire = JSON.parse(content) as WireVoiceIntent;
|
||||
} catch {
|
||||
// Schema-constrained output should be valid JSON; if it is not, the resolvers can do
|
||||
// nothing with it, so fail here rather than pass rubbish downstream.
|
||||
throw new VoiceProviderError(
|
||||
'extraction',
|
||||
'OpenRouter returned unparseable JSON',
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
intent: toVoiceIntent(wire),
|
||||
costUsd: numberOrNull(body.usage?.cost),
|
||||
};
|
||||
}
|
||||
}
|
||||
68
backend/src/modules/voice/voice.providers.ts
Normal file
68
backend/src/modules/voice/voice.providers.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import type { VoiceIntent } from './voice.types';
|
||||
|
||||
/**
|
||||
* ASR and extraction are separate, independently swappable roles — they will not come
|
||||
* from the same vendor for every locale. Both are resolved per locale from
|
||||
* `config.voice.profiles`, so pointing `fa` at a Persian-specialist vendor while `en`
|
||||
* and `nl` keep OpenRouter is configuration, not code.
|
||||
*/
|
||||
|
||||
export type AudioInput = {
|
||||
/** Raw base64, no data: prefix. */
|
||||
data: string;
|
||||
/** Container as the recorder produced it: webm, mp4, m4a, wav, … */
|
||||
format: string;
|
||||
};
|
||||
|
||||
export type AsrUsage = {
|
||||
seconds: number | null;
|
||||
costUsd: number | null;
|
||||
};
|
||||
|
||||
export type AsrResult = {
|
||||
text: string;
|
||||
usage: AsrUsage;
|
||||
};
|
||||
|
||||
export interface AsrProvider {
|
||||
/** `localeHint` is ISO-639-1 and materially improves accuracy — always pass it. */
|
||||
transcribe(
|
||||
audio: AudioInput,
|
||||
localeHint: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<AsrResult>;
|
||||
}
|
||||
|
||||
export type ExtractionCatalog = {
|
||||
/** Catalog codes with their labels in the actor's locale, so the model matches spoken words. */
|
||||
treatmentTypes: { code: string; label: string }[];
|
||||
prosthesisTypes: { code: string; label: string }[];
|
||||
/** The clinic's linked labs — a closed choice list. */
|
||||
labs: { id: string; name: string }[];
|
||||
};
|
||||
|
||||
export type ExtractionResult = {
|
||||
intent: VoiceIntent;
|
||||
costUsd: number | null;
|
||||
};
|
||||
|
||||
export interface ExtractionProvider {
|
||||
extract(
|
||||
transcript: string,
|
||||
catalog: ExtractionCatalog,
|
||||
localeHint: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ExtractionResult>;
|
||||
}
|
||||
|
||||
/** Raised when a vendor call fails; the service maps this onto the staged error codes. */
|
||||
export class VoiceProviderError extends Error {
|
||||
constructor(
|
||||
readonly stage: 'asr' | 'extraction',
|
||||
message: string,
|
||||
readonly status?: number,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'VoiceProviderError';
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user