From 4bdebd66d4f3a74c649963752171951261e30f95 Mon Sep 17 00:00:00 2001 From: xiongxiao Date: Tue, 23 Dec 2025 00:40:09 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E9=A1=B9=E7=89=88=E6=9C=AC=EF=BC=8C=E6=B7=BB=E5=8A=A0=E6=96=B0?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=B9=B6=E9=87=8D=E6=9E=84=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=BC=BA=20ASR=20=E6=9C=8D=E5=8A=A1=E5=92=8C?= =?UTF-8?q?=E7=81=AF=E5=85=89=E6=8E=A7=E5=88=B6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- assistant/package.json | 4 +- assistant/src/routes/ha-api/ha.ts | 2 + assistant/src/routes/index.ts | 2 + assistant/src/server.ts | 4 +- assistant/src/services/asr/qwen-asr.ts | 107 +++++++++++++++++++++++++ pnpm-lock.yaml | 73 ++++++++++++++++- 6 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 assistant/src/routes/ha-api/ha.ts create mode 100644 assistant/src/services/asr/qwen-asr.ts diff --git a/assistant/package.json b/assistant/package.json index d5a8152..cb1fb6c 100644 --- a/assistant/package.json +++ b/assistant/package.json @@ -48,7 +48,7 @@ "@kevisual/logger": "^0.0.4", "@kevisual/query": "0.0.33", "@kevisual/query-login": "0.0.7", - "@kevisual/router": "^0.0.48", + "@kevisual/router": "^0.0.49", "@kevisual/types": "^0.0.10", "@kevisual/use-config": "^1.0.21", "@types/bun": "^1.3.5", @@ -76,6 +76,8 @@ "access": "public" }, "dependencies": { + "@kevisual/ha-api": "^0.0.1", + "@kevisual/video-tools": "^0.0.12", "eventemitter3": "^5.0.1", "lowdb": "^7.0.1", "lru-cache": "^11.2.4", diff --git a/assistant/src/routes/ha-api/ha.ts b/assistant/src/routes/ha-api/ha.ts new file mode 100644 index 0000000..8c7ac7c --- /dev/null +++ b/assistant/src/routes/ha-api/ha.ts @@ -0,0 +1,2 @@ +import { LightHA } from "@kevisual/ha-api"; +export const lightHA = new LightHA({ token: process.env.HAAS_TOKEN || '', homeassistantURL: process.env.HAAS_URL }); diff --git a/assistant/src/routes/index.ts b/assistant/src/routes/index.ts index 0df6125..83d8053 100644 --- a/assistant/src/routes/index.ts +++ b/assistant/src/routes/index.ts @@ -5,6 +5,8 @@ import './ai/index.ts'; // TODO: // import './light-code/index.ts'; import './user/index.ts'; + +// TODO: 移除 import './hot-api/key-sender/index.ts'; import os from 'node:os'; diff --git a/assistant/src/server.ts b/assistant/src/server.ts index c197320..52338f7 100644 --- a/assistant/src/server.ts +++ b/assistant/src/server.ts @@ -9,6 +9,7 @@ import path from 'node:path' import chalk from 'chalk'; import { AssistantApp } from './lib.ts'; import { getBunPath } from './module/get-bun-path.ts'; +import { qwenAsr } from './services/asr/qwen-asr.ts'; export const runServer = async (port: number = 51015, listenPath = '127.0.0.1') => { let _port: number | undefined; if (port) { @@ -42,7 +43,8 @@ export const runServer = async (port: number = 51015, listenPath = '127.0.0.1') id: 'handle-all', func: proxyRoute as any, }, - ...proxyWs() + ...proxyWs(), + qwenAsr, ]); const manager = new AssistantApp(assistantConfig, app); setTimeout(() => { diff --git a/assistant/src/services/asr/qwen-asr.ts b/assistant/src/services/asr/qwen-asr.ts new file mode 100644 index 0000000..e1d51fb --- /dev/null +++ b/assistant/src/services/asr/qwen-asr.ts @@ -0,0 +1,107 @@ +import { QwenAsrRelatime } from "@kevisual/video-tools/src/asr/index.ts"; + +import { Listener, WebSocketListenerFun, WebSocketReq } from "@kevisual/router"; +import { lightHA } from "@/routes/ha-api/ha.ts"; + +const func: WebSocketListenerFun = async (req: WebSocketReq<{ asr: QwenAsrRelatime, msgId: string }>, res) => { + const { ws, emitter, id, data } = req; + let asr = ws.data.asr; + + if (!id) { + ws.send(JSON.stringify({ type: 'error', message: 'not found id' })); + ws.close(); + return; + } + if (!asr) { + const token = process.env.BAILIAN_API_KEY || '' + // 第一次请求 + asr = new QwenAsrRelatime({ + token, + onConnect: () => { + const asr = ws.data.asr as QwenAsrRelatime; + ws.send(JSON.stringify({ type: 'asr', code: 200, message: 'asr服务已连接', time: Date.now() })); + if (!asr) return; + asr.emitter.on('message', (message) => { + console.log('ASR message', message); + }); + asr.emitter.on('partial', (message) => { + // console.log('ASR message', message); + let msgId = ws.data.msgId || Math.random().toString(36).substring(2, 10); + ws.data.msgId = msgId; + ws.send(JSON.stringify({ + type: 'partial', + msgId: msgId, + time: Date.now(), + text: message?.text, + raw: message?.raw, + })); + }); + asr.emitter.on('result', async ({ text, raw }) => { + let msgId = ws.data.msgId; + ws.data.msgId = Math.random().toString(36).substring(2, 10); + ws.send(JSON.stringify({ + type: 'result', + msgId: msgId, + time: Date.now(), + text, + })); + if (!text) return; + const command = text?.trim().slice(0, 20); + type ParseCommand = { + type?: '打开' | '关闭', + appName?: string, + command?: string, + } + let obj: ParseCommand = {}; + if (command.startsWith('打开')) { + obj.appName = command.replace('打开', '').trim(); + obj.type = '打开'; + } else if (command.startsWith('关闭')) { + obj.appName = command.replace('关闭', '').trim(); + obj.type = '关闭'; + } + if (obj.type) { + try { + const search = await lightHA.searchLight(obj.appName || ''); + if (search.id) { + lightHA.toggleLight({ entity_id: search.id, service: obj.type === '打开' ? 'turn_on' : 'turn_off' }); + } else if (search.hasMore) { + const [first] = search.result; + lightHA.toggleLight({ entity_id: first.entity_id, service: obj.type === '打开' ? 'turn_on' : 'turn_off' }); + } else { + console.log('未找到对应设备:', obj.appName); + } + console.log('解析到控制指令', obj); + } catch (e) { + console.error('控制失败', e); + } + } + }); + asr.start(); + } + }) + ws.data.asr = asr; + } + const isConnected = await asr.checkConnected(); + if (!isConnected) return; + console.log('ASR receive data', 'has voice', !!data?.voice); + if (data?.type === 'blankVoice') { + asr.sendBlank(); + } else if (data?.voice) { + console.log('ASR receive voice', data.voice.slice(0, 30)); + const isBrowserFormat = data.format === 'float32'; + let voice: Buffer; + if (isBrowserFormat) { + voice = await asr.fixBrowerBuffer(data.voice); + } else { + voice = Buffer.from(data.voice, 'base64'); + } + asr.sendBuffer(voice); + } +} +export const qwenAsr: Listener = { + id: 'qwen-asr', + path: '/ws/asr', + io: true, + func +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 952f022..55ae963 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,6 +111,12 @@ importers: assistant: dependencies: + '@kevisual/ha-api': + specifier: ^0.0.1 + version: 0.0.1 + '@kevisual/video-tools': + specifier: ^0.0.12 + version: 0.0.12(dotenv@17.2.3)(supports-color@10.2.2) eventemitter3: specifier: ^5.0.1 version: 5.0.1 @@ -146,8 +152,8 @@ importers: specifier: 0.0.7 version: 0.0.7(@kevisual/query@0.0.33) '@kevisual/router': - specifier: ^0.0.48 - version: 0.0.48(supports-color@10.2.2) + specifier: ^0.0.49 + version: 0.0.49(supports-color@10.2.2) '@kevisual/types': specifier: ^0.0.10 version: 0.0.10 @@ -775,6 +781,10 @@ packages: cpu: [x64] os: [win32] + '@gradio/client@2.0.1': + resolution: {integrity: sha512-NLaQNj5fn+Klgtf9ESL2NhlfBo9GHYjxBCbLMXamRev36nQ/fVmhKV2V2DLV91IVTbL/gAMzeTsCmZ1Cl2CLlQ==} + engines: {node: '>=18.0.0'} + '@img/colour@1.0.0': resolution: {integrity: sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==} engines: {node: '>=18'} @@ -1263,6 +1273,9 @@ packages: resolution: {integrity: sha512-4T/m2LqhtwWEW+lWmg7jLxKFW7VtIAftsWFDDZvh10bZunqFf8iXxChHcVSQWikghJb4cq1IkWzPkvc2l+Asdw==} hasBin: true + '@kevisual/ha-api@0.0.1': + resolution: {integrity: sha512-6wFXbHIVvQhK08XMdsCm5A5P/CrepDiqd4k5x+kwOlhHHhFlu+WbGYh+wxdi4+62+W7WxZK7jjNt2x8ioQEFgg==} + '@kevisual/hot-api@0.0.3': resolution: {integrity: sha512-qZ4CNK08StZP4+DR1vWwJhKVDoSXXC+PBFG4ZxtkXF5vO2rybE055zp1n3dg5jo8GwW5wxpqMIG3KBp3pYSTkg==} @@ -1313,6 +1326,9 @@ packages: '@kevisual/router@0.0.48': resolution: {integrity: sha512-WsSvT+NpfC/bZbaAzE3WSKD2DRZP0JuPQJGr4YucSdO/lOLB4cEpOZRbPlV3l7G064ow8QJRAN2DUW+bRjrp1A==} + '@kevisual/router@0.0.49': + resolution: {integrity: sha512-2HXuOnnWdRfkO0LyqolWU9cvWHGXi8FV3OqEvWgfO+f7wx8GT8T6Bb8dCzdldDaAxve1dgLBavtdmnHyCkp+1Q==} + '@kevisual/types@0.0.10': resolution: {integrity: sha512-Q73uzzjk9UidumnmCvOpgzqDDvQxsblz22bIFuoiioUFJWwaparx8bpd8ArRyFojicYL1YJoFDzDZ9j9NN8grA==} @@ -1321,6 +1337,12 @@ packages: peerDependencies: dotenv: ^17 + '@kevisual/video-tools@0.0.12': + resolution: {integrity: sha512-yjLbijFzbSvThwKWlDpjF2zZPLtc4ar2LJHjHopmtukzPv/F0bXUEtrNXlkr40PnlE76nzBljmzdUd+b2ww2Cg==} + + '@kevisual/video@0.0.2': + resolution: {integrity: sha512-v2k9CC6Nq2UDzGwR9V7BMFf4jUsyCRKes1+3V7odPqOrbu+DskirWZVnMQFCkndB2Mmhkz1BugFVFrYak8bBew==} + '@kevisual/ws@8.0.0': resolution: {integrity: sha512-jlFxSlXUEz93cFW+UYT5BXv/rFVgiMQnIfqRYZ0gj1hSP8PMGRqMqUoHSLfKvfRRS4jseLSvTTeEKSQpZJtURg==} engines: {node: '>=10.0.0'} @@ -3083,6 +3105,9 @@ packages: picomatch: optional: true + fetch-event-stream@0.1.6: + resolution: {integrity: sha512-GREtJ5HNikdU2AXtZ6E/5bk+aslMU6ie5mPG6H9nvsdDkkHQ6m5lHwmmmDTOBexok9hApQ7EprsXCdmz9ZC68w==} + figures@6.1.0: resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} engines: {node: '>=18'} @@ -3145,6 +3170,10 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + fuse.js@7.1.0: + resolution: {integrity: sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ==} + engines: {node: '>=10'} + gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -5814,6 +5843,10 @@ snapshots: '@esbuild/win32-x64@0.25.12': optional: true + '@gradio/client@2.0.1': + dependencies: + fetch-event-stream: 0.1.6 + '@img/colour@1.0.0': optional: true @@ -6339,6 +6372,11 @@ snapshots: transitivePeerDependencies: - typescript + '@kevisual/ha-api@0.0.1': + dependencies: + dotenv: 17.2.3 + fuse.js: 7.1.0 + '@kevisual/hot-api@0.0.3(dotenv@17.2.3)': dependencies: '@kevisual/ai': 0.0.16 @@ -6467,6 +6505,14 @@ snapshots: transitivePeerDependencies: - supports-color + '@kevisual/router@0.0.49(supports-color@10.2.2)': + dependencies: + path-to-regexp: 8.3.0 + selfsigned: 5.2.0 + send: 1.2.1(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + '@kevisual/types@0.0.10': {} '@kevisual/use-config@1.0.21(dotenv@17.2.3)': @@ -6474,6 +6520,23 @@ snapshots: '@kevisual/load': 0.0.6 dotenv: 17.2.3 + '@kevisual/video-tools@0.0.12(dotenv@17.2.3)(supports-color@10.2.2)': + dependencies: + '@gradio/client': 2.0.1 + '@kevisual/ai': 0.0.19 + '@kevisual/router': 0.0.48(supports-color@10.2.2) + '@kevisual/use-config': 1.0.21(dotenv@17.2.3) + '@kevisual/video': 0.0.2 + crypto-js: 4.2.0 + dayjs: 1.11.19 + eventemitter3: 5.0.1 + nanoid: 5.1.6 + transitivePeerDependencies: + - dotenv + - supports-color + + '@kevisual/video@0.0.2': {} + '@kevisual/ws@8.0.0': {} '@lezer/common@1.4.0': {} @@ -8051,7 +8114,7 @@ snapshots: centra@2.7.0: dependencies: - follow-redirects: 1.15.9(debug@4.3.7(supports-color@10.2.2)) + follow-redirects: 1.15.9(debug@4.3.7) transitivePeerDependencies: - debug @@ -8610,6 +8673,8 @@ snapshots: optionalDependencies: picomatch: 4.0.3 + fetch-event-stream@0.1.6: {} + figures@6.1.0: dependencies: is-unicode-supported: 2.1.0 @@ -8681,6 +8746,8 @@ snapshots: function-bind@1.1.2: {} + fuse.js@7.1.0: {} + gensync@1.0.0-beta.2: {} get-east-asian-width@1.4.0: {}