feat(backend): OpenRouter voice providers and per-locale registry
ASR and extraction are separate, independently swappable roles resolved per locale from config. All three locales point at the same OpenRouter models today (whisper-1, gemini-3.7-flash); the indirection stays because Persian ASR is the weakest link and repointing only `fa` must not be a code change. The model emits a deliberately flat wire shape rather than the internal discriminated unions — strict json_schema mode has poor union support — and toVoiceIntent narrows it. That normalizer is total: a missing or malformed payload yields a shape the resolvers report as unresolved rather than one that throws. The prompt supplies catalog codes with labels in the actor's locale, so the model matches spoken words rather than translating, and carries per-locale tooth vocabulary. English gets an explicit warning that a bare two-digit number is ambiguous under Universal numbering, and must not be treated as FDI unless the speaker said so. From review of this commit: - only an actually FDI-shaped code takes the explicit branch; fdi:"6" alongside valid arch/side/position used to lose the tooth entirely - an unrecognised due kind passes through to be flagged, instead of collapsing to null and looking like no deadline was ever spoken - vendor error bodies stay out of the thrown message and the default log level; a 4xx can echo the request back, transcript included - the chat call sets provider.require_parameters so OpenRouter only routes to endpoints that honour the JSON schema, rather than ones treating it as a hint - an unknown locale in VOICE_ENABLED_LOCALES now fails at boot like an unknown provider id, instead of silently disabling the microphone everywhere Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
171
backend/src/modules/voice/openrouter.provider.ts
Normal file
171
backend/src/modules/voice/openrouter.provider.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import {
|
||||
type AsrProvider,
|
||||
type AsrResult,
|
||||
type AudioInput,
|
||||
type ExtractionCatalog,
|
||||
type ExtractionProvider,
|
||||
type ExtractionResult,
|
||||
VoiceProviderError,
|
||||
} from './voice.providers';
|
||||
import { buildExtractionPrompt } from './extraction.prompt';
|
||||
import {
|
||||
VOICE_INTENT_JSON_SCHEMA,
|
||||
toVoiceIntent,
|
||||
type WireVoiceIntent,
|
||||
} from './extraction.wire';
|
||||
|
||||
type OpenRouterConfig = {
|
||||
apiKey: string;
|
||||
baseUrl: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
type TranscriptionResponse = {
|
||||
text?: unknown;
|
||||
usage?: { seconds?: unknown; cost?: unknown };
|
||||
};
|
||||
|
||||
type ChatResponse = {
|
||||
choices?: { message?: { content?: unknown } }[];
|
||||
usage?: { cost?: unknown };
|
||||
};
|
||||
|
||||
function numberOrNull(value: unknown): number | null {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
const errorLogger = new Logger('OpenRouterVoice');
|
||||
|
||||
async function readError(
|
||||
response: Response,
|
||||
stage: 'asr' | 'extraction',
|
||||
): Promise<never> {
|
||||
// A vendor 4xx can echo the request back, transcript included. Keep the body out of the
|
||||
// thrown message — which callers log and could forward — and out of the default log
|
||||
// level; it stays available at debug when someone is actively diagnosing.
|
||||
try {
|
||||
errorLogger.debug(
|
||||
`${stage} ${response.status} body: ${(await response.text()).slice(0, 500)}`,
|
||||
);
|
||||
} catch {
|
||||
errorLogger.debug(`${stage} ${response.status} body unreadable`);
|
||||
}
|
||||
throw new VoiceProviderError(
|
||||
stage,
|
||||
`OpenRouter ${stage} failed with status ${response.status}`,
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
|
||||
/** Speech → text via OpenRouter's transcription endpoint (whisper-1 and friends). */
|
||||
export class OpenRouterAsrProvider implements AsrProvider {
|
||||
private readonly logger = new Logger(OpenRouterAsrProvider.name);
|
||||
|
||||
constructor(private readonly config: OpenRouterConfig) {}
|
||||
|
||||
async transcribe(
|
||||
audio: AudioInput,
|
||||
localeHint: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<AsrResult> {
|
||||
const response = await fetch(
|
||||
`${this.config.baseUrl}/audio/transcriptions`,
|
||||
{
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.config.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.config.model,
|
||||
input_audio: { data: audio.data, format: audio.format },
|
||||
language: localeHint,
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) await readError(response, 'asr');
|
||||
|
||||
const body = (await response.json()) as TranscriptionResponse;
|
||||
const text = typeof body.text === 'string' ? body.text.trim() : '';
|
||||
this.logger.debug(
|
||||
`transcribed ${numberOrNull(body.usage?.seconds) ?? '?'}s`,
|
||||
);
|
||||
|
||||
return {
|
||||
text,
|
||||
usage: {
|
||||
seconds: numberOrNull(body.usage?.seconds),
|
||||
costUsd: numberOrNull(body.usage?.cost),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Transcript → VoiceIntent via OpenRouter chat completions with a JSON schema. */
|
||||
export class OpenRouterExtractionProvider implements ExtractionProvider {
|
||||
constructor(private readonly config: OpenRouterConfig) {}
|
||||
|
||||
async extract(
|
||||
transcript: string,
|
||||
catalog: ExtractionCatalog,
|
||||
localeHint: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ExtractionResult> {
|
||||
const response = await fetch(`${this.config.baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
signal,
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.config.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.config.model,
|
||||
temperature: 0,
|
||||
// Only route to endpoints that actually honour the JSON schema. Without this,
|
||||
// OpenRouter may pick a provider that treats it as a hint and returns prose,
|
||||
// which fails parsing intermittently and unreproducibly.
|
||||
provider: { require_parameters: true },
|
||||
messages: buildExtractionPrompt(transcript, catalog, localeHint),
|
||||
response_format: {
|
||||
type: 'json_schema',
|
||||
json_schema: {
|
||||
name: 'voice_intent',
|
||||
strict: true,
|
||||
schema: VOICE_INTENT_JSON_SCHEMA,
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) await readError(response, 'extraction');
|
||||
|
||||
const body = (await response.json()) as ChatResponse;
|
||||
const content = body.choices?.[0]?.message?.content;
|
||||
if (typeof content !== 'string' || !content.trim()) {
|
||||
throw new VoiceProviderError(
|
||||
'extraction',
|
||||
'OpenRouter returned no content',
|
||||
);
|
||||
}
|
||||
|
||||
let wire: WireVoiceIntent;
|
||||
try {
|
||||
wire = JSON.parse(content) as WireVoiceIntent;
|
||||
} catch {
|
||||
// Schema-constrained output should be valid JSON; if it is not, the resolvers can do
|
||||
// nothing with it, so fail here rather than pass rubbish downstream.
|
||||
throw new VoiceProviderError(
|
||||
'extraction',
|
||||
'OpenRouter returned unparseable JSON',
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
intent: toVoiceIntent(wire),
|
||||
costUsd: numberOrNull(body.usage?.cost),
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user