110 lines
3.4 KiB
TypeScript
110 lines
3.4 KiB
TypeScript
import { QwenAsrRelatime } from "@kevisual/video-tools/src/asr/index.ts";
|
|
|
|
import { Listener, WebSocketListenerFun, WebSocketReq } from "@kevisual/router";
|
|
import { callText } from "@/routes/ha-api/ha.ts";
|
|
import { assistantConfig } from "@/app.ts";
|
|
|
|
const func: WebSocketListenerFun = async (req: WebSocketReq<{ asr: QwenAsrRelatime, msgId: string, startTime?: number, loading?: boolean }>, res) => {
|
|
const { ws, emitter, id, data } = req;
|
|
let asr = ws.data.asr;
|
|
|
|
if (!id) {
|
|
ws.send(JSON.stringify({ type: 'error', message: 'not found id' }));
|
|
ws.close();
|
|
return;
|
|
}
|
|
if (!asr) {
|
|
if (ws.data.loading) return;
|
|
ws.data.loading = true;
|
|
|
|
const confg = assistantConfig.getConfig();
|
|
const asrConfig = confg?.asr;
|
|
if (!asrConfig?.enabled) {
|
|
ws.send(JSON.stringify({ type: 'error', message: 'asr服务未启用' }));
|
|
ws.close();
|
|
return;
|
|
}
|
|
const token = asrConfig?.token;
|
|
if (!token) {
|
|
ws.send(JSON.stringify({ type: 'error', message: 'asr服务未配置token' }));
|
|
ws.close();
|
|
return;
|
|
}
|
|
const onConnect = () => {
|
|
const asr = ws.data.asr as QwenAsrRelatime;
|
|
ws.send(JSON.stringify({ type: 'asr', code: 200, message: 'asr服务已连接', time: Date.now() }));
|
|
if (!asr) return;
|
|
asr.emitter.on('message', (message) => {
|
|
// console.log('ASR message', message);
|
|
});
|
|
asr.emitter.on('partial', (message) => {
|
|
// console.log('ASR message', message);
|
|
let msgId = ws.data.msgId || Math.random().toString(36).substring(2, 10);
|
|
ws.data.msgId = msgId;
|
|
ws.send(JSON.stringify({
|
|
type: 'partial',
|
|
msgId: msgId,
|
|
time: Date.now(),
|
|
text: message?.text,
|
|
raw: message?.raw,
|
|
}));
|
|
});
|
|
asr.emitter.on('result', async ({ text, raw }) => {
|
|
let msgId = ws.data.msgId;
|
|
ws.data.msgId = Math.random().toString(36).substring(2, 10);
|
|
const endTime = Date.now();
|
|
console.log('cost time', ws.data.startTime ? (endTime - ws.data.startTime) : 0);
|
|
ws.send(JSON.stringify({
|
|
type: 'result',
|
|
msgId: msgId,
|
|
time: Date.now(),
|
|
text,
|
|
}));
|
|
if (!text) return;
|
|
await callText(text);
|
|
console.log('toogle light time', Date.now() - endTime);
|
|
});
|
|
asr.start();
|
|
}
|
|
// 第一次请求
|
|
asr = new QwenAsrRelatime({
|
|
token,
|
|
emitter,
|
|
onConnect,
|
|
})
|
|
ws.data.asr = asr;
|
|
ws.data.loading = false;
|
|
emitter.on('close--' + id, () => {
|
|
console.log('ASR websocket 关闭, 释放资源');
|
|
asr?.close?.();
|
|
});
|
|
}
|
|
const isConnected = await asr.checkConnected();
|
|
if (!isConnected) return;
|
|
if (data?.type === 'blankVoice') {
|
|
asr.sendBlank();
|
|
console.log('ASR receive data', 'blank voice');
|
|
} else if (data?.voice) {
|
|
if (!data?.isRelatime) {
|
|
console.log('ASR receive data', 'has voice', !!data?.voice, data?.isRelatime);
|
|
const time = data?.time || 0;
|
|
console.log('receiveDelay', Date.now() - time);
|
|
}
|
|
const isBrowserFormat = data.format === 'float32';
|
|
|
|
let voice: Buffer;
|
|
if (isBrowserFormat) {
|
|
voice = await asr.fixBrowerBuffer(data.voice);
|
|
} else {
|
|
voice = Buffer.from(data.voice, 'base64');
|
|
}
|
|
ws.data.startTime = Date.now();
|
|
asr.sendBuffer(voice);
|
|
}
|
|
}
|
|
export const qwenAsr: Listener = {
|
|
id: 'qwen-asr',
|
|
path: '/ws/asr',
|
|
io: true,
|
|
func
|
|
} |