From da249ad779781596a968ac73f7daff0b414f24eb Mon Sep 17 00:00:00 2001 From: abearxiong Date: Wed, 24 Dec 2025 11:21:07 +0800 Subject: [PATCH] update --- pnpm-lock.yaml | 148 ++++++++++++++++++ web/package.json | 1 + .../apps/muse/voice/modules/AudioRecorder.ts | 16 +- web/src/apps/muse/voice/modules/ShowText.tsx | 40 +++++ web/src/apps/muse/voice/modules/VadVoice.tsx | 95 +++++------ web/src/apps/muse/voice/store/relatime.ts | 13 +- web/src/components/ui/tooltip.tsx | 59 +++++++ 7 files changed, 308 insertions(+), 64 deletions(-) create mode 100644 web/src/apps/muse/voice/modules/ShowText.tsx create mode 100644 web/src/components/ui/tooltip.tsx diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5859799..77ad182 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -90,6 +90,9 @@ importers: '@radix-ui/react-slot': specifier: ^1.2.4 version: 1.2.4(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-tooltip': + specifier: ^1.2.8 + version: 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@ricky0123/vad-web': specifier: ^0.0.30 version: 0.0.30 @@ -869,6 +872,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-arrow@1.1.7': + resolution: {integrity: sha512-F+M1tLhO+mlQaOWspE8Wstg+z6PwxwRd8oQ8IXceWz92kfAmalTRf0EjrouQeo7QssEPfCn05B4Ihs1K9WQ/7w==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-compose-refs@1.1.2': resolution: {integrity: sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg==} peerDependencies: @@ -944,6 +960,19 @@ packages: '@types/react': optional: true + '@radix-ui/react-popper@1.2.8': + resolution: {integrity: sha512-0NJQ4LFFUuWkE7Oxf0htBKS6zLkkjBH+hM1uk7Ng705ReR8m/uelduy1DBo0PyBXPKVnBA6YBlU94MBGXrSBCw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-portal@1.1.9': resolution: {integrity: sha512-bpIxvq03if6UNwXZ+HTK71JLh4APvnXntDc6XOX8UVq4XQOVl7lwok0AvIl+b8zgCw3fSaVTZMpAPPagXbKmHQ==} peerDependencies: @@ -1001,6 +1030,19 @@ packages: '@types/react': optional: true + '@radix-ui/react-tooltip@1.2.8': + resolution: {integrity: sha512-tY7sVt1yL9ozIxvmbtN5qtmH2krXcBCfjEiCgKGLqunJHvgvZG2Pcl2oQ3kbcZARb1BGEHdkLzcYGO8ynVlieg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-use-callback-ref@1.1.1': resolution: {integrity: sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg==} peerDependencies: @@ -1046,6 +1088,40 @@ packages: '@types/react': optional: true + '@radix-ui/react-use-rect@1.1.1': + resolution: {integrity: sha512-QTYuDesS0VtuHNNvMh+CjlKJ4LJickCMUAqjlE3+j8w+RlRpwyX3apEQKGFzbZGdo7XNG1tXa+bQqIE7HIXT2w==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-size@1.1.1': + resolution: {integrity: sha512-ewrXRDTAqAXlkl6t/fkXWNAhFX9I+CkKlw6zjEwk86RSPKwZr3xpBRso655aqYafwtnbpHLj6toFzmd6xdVptQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-visually-hidden@1.2.3': + resolution: {integrity: sha512-pzJq12tEaaIhqjbzpCuv/OypJY/BPavOofm+dbab+MHLajy277+1lLm6JFcGgF5eskJ6mquGirhXY2GD/8u8Ug==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/rect@1.1.1': + resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==} + '@ricky0123/vad-web@0.0.30': resolution: {integrity: sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==} @@ -4419,6 +4495,15 @@ snapshots: '@types/react': 19.2.7 '@types/react-dom': 19.2.3(@types/react@19.2.7) + '@radix-ui/react-arrow@1.1.7(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': + dependencies: + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) + optionalDependencies: + '@types/react': 19.2.7 + '@types/react-dom': 19.2.3(@types/react@19.2.7) + '@radix-ui/react-compose-refs@1.1.2(@types/react@19.2.7)(react@19.2.3)': dependencies: react: 19.2.3 @@ -4490,6 +4575,24 @@ snapshots: optionalDependencies: '@types/react': 19.2.7 + '@radix-ui/react-popper@1.2.8(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': + dependencies: + '@floating-ui/react-dom': 2.1.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-arrow': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-use-rect': 1.1.1(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-use-size': 1.1.1(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/rect': 1.1.1 + react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) + optionalDependencies: + '@types/react': 19.2.7 + '@types/react-dom': 19.2.3(@types/react@19.2.7) + '@radix-ui/react-portal@1.1.9(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -4533,6 +4636,26 @@ snapshots: optionalDependencies: '@types/react': 19.2.7 + '@radix-ui/react-tooltip@1.2.8(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-dismissable-layer': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-popper': 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-slot': 1.2.3(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.7)(react@19.2.3) + '@radix-ui/react-visually-hidden': 1.2.3(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) + optionalDependencies: + '@types/react': 19.2.7 + '@types/react-dom': 19.2.3(@types/react@19.2.7) + '@radix-ui/react-use-callback-ref@1.1.1(@types/react@19.2.7)(react@19.2.3)': dependencies: react: 19.2.3 @@ -4567,6 +4690,31 @@ snapshots: optionalDependencies: '@types/react': 19.2.7 + '@radix-ui/react-use-rect@1.1.1(@types/react@19.2.7)(react@19.2.3)': + dependencies: + '@radix-ui/rect': 1.1.1 + react: 19.2.3 + optionalDependencies: + '@types/react': 19.2.7 + + '@radix-ui/react-use-size@1.1.1(@types/react@19.2.7)(react@19.2.3)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.7)(react@19.2.3) + react: 19.2.3 + optionalDependencies: + '@types/react': 19.2.7 + + '@radix-ui/react-visually-hidden@1.2.3(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': + dependencies: + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) + optionalDependencies: + '@types/react': 19.2.7 + '@types/react-dom': 19.2.3(@types/react@19.2.7) + + '@radix-ui/rect@1.1.1': {} + '@ricky0123/vad-web@0.0.30': dependencies: onnxruntime-web: 1.23.0 diff --git a/web/package.json b/web/package.json index 0052f1e..579a671 100644 --- a/web/package.json +++ b/web/package.json @@ -30,6 +30,7 @@ "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-slot": "^1.2.4", + "@radix-ui/react-tooltip": "^1.2.8", "@ricky0123/vad-web": "^0.0.30", "@szhsin/react-menu": "^4.5.1", "@tailwindcss/vite": "^4.1.18", diff --git a/web/src/apps/muse/voice/modules/AudioRecorder.ts b/web/src/apps/muse/voice/modules/AudioRecorder.ts index 57d6b23..141e221 100644 --- a/web/src/apps/muse/voice/modules/AudioRecorder.ts +++ b/web/src/apps/muse/voice/modules/AudioRecorder.ts @@ -35,7 +35,9 @@ export class AudioRecorder { public onAudioData(callback: AudioDataCallback): void { this.onAudioDataCallback = callback; } - + getMediaStream(): MediaStream | null { + return this.mediaStream; + } /** * 开始录制 */ @@ -47,17 +49,17 @@ export class AudioRecorder { try { // 获取麦克风权限 - this.mediaStream = await navigator.mediaDevices.getUserMedia({ + this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true, - } + } }); // 创建音频上下文 - this.audioContext = new AudioContext({ - sampleRate: this.config.sampleRate + this.audioContext = new AudioContext({ + sampleRate: this.config.sampleRate }); // 加载AudioWorklet处理器 @@ -68,7 +70,7 @@ export class AudioRecorder { // 创建AudioWorklet节点 this.workletNode = new AudioWorkletNode( - this.audioContext, + this.audioContext, 'audio-recorder-processor', { processorOptions: { @@ -228,7 +230,7 @@ export class AudioRecorder { for (let i = 0; i < binary.length; i++) { binaryString += String.fromCharCode(binary[i]); } - return typeof window !== 'undefined' && window.btoa + return typeof window !== 'undefined' && window.btoa ? window.btoa(binaryString) : Buffer.from(binaryString, 'binary').toString('base64'); } diff --git a/web/src/apps/muse/voice/modules/ShowText.tsx b/web/src/apps/muse/voice/modules/ShowText.tsx new file mode 100644 index 0000000..2347f6a --- /dev/null +++ b/web/src/apps/muse/voice/modules/ShowText.tsx @@ -0,0 +1,40 @@ +import { + Tooltip, TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { Copy } from "lucide-react"; +import { useMemo } from "react"; +import { toast } from "react-toastify"; + + +export const ShowText = (props: { text: string, icon?: any, title?: string }) => { + const title = props.title || ''; + const icon = props.icon || '🕘'; + return ( +
+ + + {icon} {title ? `${title}:` : ''} {props.text} + + {props.text && + {props.text} + + } + +
{ + // copy + navigator.clipboard.writeText(props.text).then(() => { + toast.success('已复制', { + autoClose: 500, + position: 'top-center' + }); + }).catch((error) => { + console.error('复制失败:', error); + toast.error('复制失败，请手动选择文字复制'); + }); + }}> + +
+
+ ) +} \ No newline at end of file diff --git a/web/src/apps/muse/voice/modules/VadVoice.tsx b/web/src/apps/muse/voice/modules/VadVoice.tsx index 5d5f4ba..85aef53 100644 --- a/web/src/apps/muse/voice/modules/VadVoice.tsx +++ b/web/src/apps/muse/voice/modules/VadVoice.tsx @@ -10,7 +10,7 @@ import { Speak } from "./speak-db/speak"; import { useVoiceStore } from "../store/voiceStore"; import { useSettingStore } from "../store/settingStore"; import { SettingModal } from "./SettingModal"; - +import { AudioRecorder } from "./AudioRecorder"; import { AlertDialog, AlertDialogAction, @@ -22,6 +22,7 @@ import { AlertDialogTitle, AlertDialogTrigger, } from "../../../../components/ui/alert-dialog"; +import { ShowText } from "./ShowText"; type VadVoiceProps = { data: Speak; @@ -381,6 +382,7 @@ export const VadVoice = () => { const [errorMessage, setErrorMessage] = useState(''); const [userInteracted, setUserInteracted] = useState(false); const ref = useRef(null); + const audioRecorderRef = useRef(null); const initializingRef = useRef(false); async function initializeVAD(ls: boolean = true) { @@ -397,11 +399,26 @@ export const VadVoice = () => { await new Promise((resolve) => setTimeout(resolve, 500)); const myvad = await MicVAD.new({ + getStream: async () => { + const audioRecorder = audioRecorderRef.current || new AudioRecorder({ + sampleRate: 16000, + bufferSize: 4096, + }); + await audioRecorder.start(); + // 设置音频数据回调 + audioRecorder.onAudioData((audioData) => { + const base64 = AudioRecorder.float32ArrayToBase64(audioData); + const relatime = useVoiceStore.getState().relatime; + relatime?.sendBase64(base64, { isRelatime: true }); + }); + audioRecorderRef.current = audioRecorder; + return audioRecorder.getMediaStream()!; + }, onSpeechEnd: async (audio) => { try { const wavBuffer = utils.encodeWAV(audio) const relatime = useVoiceStore.getState().relatime; - relatime?.sendBase64?.(utils.arrayBufferToBase64(wavBuffer)); + relatime?.sendBase64?.(utils.arrayBufferToBase64(wavBuffer), { isRelatime: false }); const audioBlob = new Blob([wavBuffer], { type: 'audio/wav' }) const tempUrl = URL.createObjectURL(audioBlob) @@ -474,8 +491,13 @@ export const VadVoice = () => { document.removeEventListener('click', handleFirstClick); } }; - - document.addEventListener('click', handleFirstClick); + // 如果是tauri的环境，不需要用户交互 + // @ts-ignore + if (window.__TAURI_INTERNALS__) { + handleUserInteraction(); + } else { + document.addEventListener('click', handleFirstClick); + } return () => { document.removeEventListener('click', handleFirstClick); }; @@ -499,6 +521,9 @@ export const VadVoice = () => { ref.current = null; setListen(false); setVadStatus('idle'); + audioRecorderRef.current?.stop(); + audioRecorderRef.current = null; + console.log('VAD closed'); } } @@ -547,10 +572,10 @@ export const VadVoice = () => { {/* Voice Control Bottom Section */} -
-
-
-
+
+
+
+
{
)}
-
+
{vadStatus === 'initializing' || storeLoading ? 'Initializing...' : vadStatus === 'error' || storeError ? 'Error' : @@ -578,53 +603,13 @@ export const VadVoice = () => { `${voiceList.length} recording${voiceList.length !== 1 ? 's' : ''}` )}
-
- {lastRecognizedText && ( -
-
- 🕘 上次识别: {lastRecognizedText} -
-
{ - // copy - navigator.clipboard.writeText(lastRecognizedText).then(() => { - toast.success('已复制', { - autoClose: 500, - position: 'top-center' - }); - }).catch((error) => { - console.error('复制失败:', error); - toast.error('复制失败，请手动选择文字复制'); - }); - }}> - -
-
- )} - {showText && ( -
-
- 📝 {showText} -
-
{ - // copy - navigator.clipboard.writeText(showText).then(() => { - toast.success('已复制', { - autoClose: 500, - position: 'top-center' - }); - }).catch((error) => { - console.error('复制失败:', error); - toast.error('复制失败，请手动选择文字复制'); - }); - }}> - -
-
- )} +
+ {lastRecognizedText && } +
-
+
{vadStatus === 'error' && (