cli/assistant/src/services/asr/qwen-asr.ts

import { QwenAsrRelatime } from "@kevisual/video-tools/src/asr/index.ts";

import { Listener, WebSocketListenerFun, WebSocketReq } from "@kevisual/router";
import { lightHA } from "@/routes/ha-api/ha.ts";
import { assistantConfig } from "@/app.ts";

const func: WebSocketListenerFun = async (req: WebSocketReq<{ asr: QwenAsrRelatime, msgId: string, startTime?: number, loading?: boolean }>, res) => {
  const { ws, emitter, id, data } = req;
  let asr = ws.data.asr;

  if (!id) {
    ws.send(JSON.stringify({ type: 'error', message: 'not found id' }));
    ws.close();
    return;
  }
  if (!asr) {
    if (ws.data.loading) return;
    ws.data.loading = true;

    const confg = assistantConfig.getConfig();
    const asrConfig = confg?.asr;
    if (!asrConfig?.enabled) {
      ws.send(JSON.stringify({ type: 'error', message: 'asr服务未启用' }));
      ws.close();
      return;
    }
    const token = asrConfig?.token;
    if (!token) {
      ws.send(JSON.stringify({ type: 'error', message: 'asr服务未配置token' }));
      ws.close();
      return;
    }
    const onConnect = () => {
      const asr = ws.data.asr as QwenAsrRelatime;
      ws.send(JSON.stringify({ type: 'asr', code: 200, message: 'asr服务已连接', time: Date.now() }));
      if (!asr) return;
      asr.emitter.on('message', (message) => {
        // console.log('ASR message', message);
      });
      asr.emitter.on('partial', (message) => {
        // console.log('ASR message', message);
        let msgId = ws.data.msgId || Math.random().toString(36).substring(2, 10);
        ws.data.msgId = msgId;
        ws.send(JSON.stringify({
          type: 'partial',
          msgId: msgId,
          time: Date.now(),
          text: message?.text,
          raw: message?.raw,
        }));
      });
      asr.emitter.on('result', async ({ text, raw }) => {
        let msgId = ws.data.msgId;
        ws.data.msgId = Math.random().toString(36).substring(2, 10);
        const endTime = Date.now();
        console.log('cost time', ws.data.startTime ? (endTime - ws.data.startTime) : 0);
        ws.send(JSON.stringify({
          type: 'result',
          msgId: msgId,
          time: Date.now(),
          text,
        }));
        if (!text) return;
        const command = text?.trim().slice(0, 20);
        type ParseCommand = {
          type?: '打开' | '关闭',
          appName?: string,
          command?: string,
        }
        let obj: ParseCommand = {};
        if (command.startsWith('打开')) {
          obj.appName = command.replace('打开', '').trim();
          obj.type = '打开';
        } else if (command.startsWith('关闭')) {
          obj.appName = command.replace('关闭', '').trim();
          obj.type = '关闭';
        }
        if (obj.type) {
          try {
            const search = await lightHA.searchLight(obj.appName || '');
            console.log('searchTime', Date.now() - endTime);
            if (search.id) {
              await lightHA.runService({ entity_id: search.id, service: obj.type === '打开' ? 'turn_on' : 'turn_off' });
            } else if (search.hasMore) {
              const [first] = search.result;
              await lightHA.runService({ entity_id: first.entity_id, service: obj.type === '打开' ? 'turn_on' : 'turn_off' });
            } else {
              console.log('未找到对应设备:', obj.appName);
            }
            console.log('解析到控制指令', obj);
          } catch (e) {
            console.error('控制失败', e);
          }
        }
        console.log('toogle light time', Date.now() - endTime);
      });
      asr.start();
    }
    // 第一次请求
    asr = new QwenAsrRelatime({
      token,
      emitter,
      onConnect,
    })
    ws.data.asr = asr;
    ws.data.loading = false;
    emitter.on('close--' + id, () => {
      console.log('ASR websocket 关闭, 释放资源');
      asr?.close?.();
    });
  }
  const isConnected = await asr.checkConnected();
  if (!isConnected) return;
  if (data?.type === 'blankVoice') {
    asr.sendBlank();
    console.log('ASR receive data', 'blank voice');
  } else if (data?.voice) {
    if (!data?.isRelatime) {
      console.log('ASR receive data', 'has voice', !!data?.voice, data?.isRelatime);
      const time = data?.time || 0;
      console.log('receiveDelay', Date.now() - time);
    }
    const isBrowserFormat = data.format === 'float32';

    let voice: Buffer;
    if (isBrowserFormat) {
      voice = await asr.fixBrowerBuffer(data.voice);
    } else {
      voice = Buffer.from(data.voice, 'base64');
    }
    ws.data.startTime = Date.now();
    asr.sendBuffer(voice);
  }
}
export const qwenAsr: Listener = {
  id: 'qwen-asr',
  path: '/ws/asr',
  io: true,
  func
}