import { QwenAsrRelatime } from "@kevisual/video-tools/src/asr/index.ts"; import { Listener, WebSocketListenerFun, WebSocketReq } from "@kevisual/router"; import { lightHA } from "@/routes/ha-api/ha.ts"; import { assistantConfig } from "@/app.ts"; const func: WebSocketListenerFun = async (req: WebSocketReq<{ asr: QwenAsrRelatime, msgId: string, startTime?: number, loading?: boolean }>, res) => { const { ws, emitter, id, data } = req; let asr = ws.data.asr; if (!id) { ws.send(JSON.stringify({ type: 'error', message: 'not found id' })); ws.close(); return; } if (!asr) { if (ws.data.loading) return; ws.data.loading = true; const confg = assistantConfig.getConfig(); const asrConfig = confg?.asr; if (!asrConfig?.enabled) { ws.send(JSON.stringify({ type: 'error', message: 'asr服务未启用' })); ws.close(); return; } const token = asrConfig?.token; if (!token) { ws.send(JSON.stringify({ type: 'error', message: 'asr服务未配置token' })); ws.close(); return; } const onConnect = () => { const asr = ws.data.asr as QwenAsrRelatime; ws.send(JSON.stringify({ type: 'asr', code: 200, message: 'asr服务已连接', time: Date.now() })); if (!asr) return; asr.emitter.on('message', (message) => { // console.log('ASR message', message); }); asr.emitter.on('partial', (message) => { // console.log('ASR message', message); let msgId = ws.data.msgId || Math.random().toString(36).substring(2, 10); ws.data.msgId = msgId; ws.send(JSON.stringify({ type: 'partial', msgId: msgId, time: Date.now(), text: message?.text, raw: message?.raw, })); }); asr.emitter.on('result', async ({ text, raw }) => { let msgId = ws.data.msgId; ws.data.msgId = Math.random().toString(36).substring(2, 10); const endTime = Date.now(); console.log('cost time', ws.data.startTime ? (endTime - ws.data.startTime) : 0); ws.send(JSON.stringify({ type: 'result', msgId: msgId, time: Date.now(), text, })); if (!text) return; const command = text?.trim().slice(0, 20); type ParseCommand = { type?: '打开' | '关闭', appName?: string, command?: string, } let obj: ParseCommand = {}; if (command.startsWith('打开')) { obj.appName = command.replace('打开', '').trim(); obj.type = '打开'; } else if (command.startsWith('关闭')) { obj.appName = command.replace('关闭', '').trim(); obj.type = '关闭'; } if (obj.type) { try { const search = await lightHA.searchLight(obj.appName || ''); console.log('searchTime', Date.now() - endTime); if (search.id) { await lightHA.runService({ entity_id: search.id, service: obj.type === '打开' ? 'turn_on' : 'turn_off' }); } else if (search.hasMore) { const [first] = search.result; await lightHA.runService({ entity_id: first.entity_id, service: obj.type === '打开' ? 'turn_on' : 'turn_off' }); } else { console.log('未找到对应设备:', obj.appName); } console.log('解析到控制指令', obj); } catch (e) { console.error('控制失败', e); } } console.log('toogle light time', Date.now() - endTime); }); asr.start(); } // 第一次请求 asr = new QwenAsrRelatime({ token, emitter, onConnect, }) ws.data.asr = asr; ws.data.loading = false; emitter.on('close--' + id, () => { console.log('ASR websocket 关闭, 释放资源'); asr?.close?.(); }); } const isConnected = await asr.checkConnected(); if (!isConnected) return; if (data?.type === 'blankVoice') { asr.sendBlank(); console.log('ASR receive data', 'blank voice'); } else if (data?.voice) { if (!data?.isRelatime) { console.log('ASR receive data', 'has voice', !!data?.voice, data?.isRelatime); const time = data?.time || 0; console.log('receiveDelay', Date.now() - time); } const isBrowserFormat = data.format === 'float32'; let voice: Buffer; if (isBrowserFormat) { voice = await asr.fixBrowerBuffer(data.voice); } else { voice = Buffer.from(data.voice, 'base64'); } ws.data.startTime = Date.now(); asr.sendBuffer(voice); } } export const qwenAsr: Listener = { id: 'qwen-asr', path: '/ws/asr', io: true, func }