feat: update dependencies and add video-tools package

- Added "@kevisual/video-tools" dependency to server/package.json. - Updated VITE_API_URL in astro.config.mjs to point to new local server. - Modified proxy settings in astro.config.mjs to include WebSocket support. - Updated various dependencies in web/package.json for improved functionality and security. - Refactored MuseApp component to use new resizable panel components. - Implemented real-time text display and copy functionality in VadVoice component. - Created a new Relatime class for handling WebSocket connections and real-time updates. - Added new board and muse pages to render the Record component.
2025-12-23 00:40:52 +08:00
parent 6a1e648747
commit 4407c6157f
14 changed files with 1448 additions and 939 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -4,8 +4,4 @@ dist

 .turbo

-filebrowser
-
-filebrowser.db
-
 /data
--- a/package.json
+++ b/package.json
@@ -15,9 +15,9 @@
  "keywords": [],
  "author": "abearxiong <xiongxiao@xiongxiao.me> (https://www.xiongxiao.me)",
  "license": "MIT",
-  "packageManager": "pnpm@10.18.3",
+  "packageManager": "pnpm@10.26.1",
  "type": "module",
  "dependencies": {
-    "turbo": "^2.5.8"
+    "turbo": "^2.7.1"
  }
 }
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/server/package.json
+++ b/server/package.json
@@ -24,6 +24,7 @@
    "@kevisual/noco": "^0.0.1",
    "@kevisual/query": "^0.0.29",
    "@kevisual/router": "^0.0.29",
+    "@kevisual/video-tools": "^0.0.10",
    "fast-glob": "^3.3.3",
    "pocketbase": "^0.26.2",
    "unstorage": "^1.17.1"
--- a/web/astro.config.mjs
+++ b/web/astro.config.mjs
@@ -7,13 +7,13 @@ import tailwindcss from '@tailwindcss/vite';

 const isDev = process.env.NODE_ENV === 'development';

-let target = process.env.VITE_API_URL || 'http://localhost:4005';
+let target = process.env.VITE_API_URL || 'http://localhost:51015';
 const apiProxy = { target: target, changeOrigin: true, ws: true, rewriteWsOrigin: true, secure: false, cookieDomainRewrite: 'localhost' };
+
 let proxy = {
-  '/root/': {
-    target: `${target}/root/`,
-  },
+  '/root': apiProxy,
  '/api': apiProxy,
+  '/ws': apiProxy,
 };

 const basename = isDev ? undefined : `${pkgs.basename}`;
@@ -25,15 +25,18 @@ export default defineConfig({
    // sitemap(), // sitemap must be site has a domain
  ],

+  server: {
+    port: 4321,
+    host: '0.0.0.0',
+    allowedHosts: true,
+  },
+
  vite: {
    plugins: [tailwindcss()],
    define: {
      basename: JSON.stringify(basename || ''),
    },
    server: {
-      port: 7008,
-      host: '0.0.0.0',
-      allowedHosts: true,
      proxy,
    },
  },
--- a/web/package.json
+++ b/web/package.json
@@ -17,49 +17,50 @@
  "license": "MIT",
  "type": "module",
  "dependencies": {
-    "@astrojs/mdx": "^4.3.7",
-    "@astrojs/react": "^4.4.0",
+    "@astrojs/mdx": "^4.3.13",
+    "@astrojs/react": "^4.4.2",
    "@astrojs/sitemap": "^3.6.0",
    "@faker-js/faker": "^10.1.0",
    "@floating-ui/react": "^0.27.16",
-    "@kevisual/noco": "^0.0.1",
-    "@kevisual/query": "^0.0.29",
-    "@kevisual/query-login": "^0.0.6",
+    "@kevisual/noco": "^0.0.10",
+    "@kevisual/query": "^0.0.33",
+    "@kevisual/query-login": "^0.0.7",
    "@kevisual/registry": "^0.0.1",
+    "@kevisual/video-tools": "^0.0.11",
    "@radix-ui/react-alert-dialog": "^1.1.15",
    "@radix-ui/react-dialog": "^1.1.15",
-    "@radix-ui/react-slot": "^1.2.3",
-    "@ricky0123/vad-web": "^0.0.28",
-    "@szhsin/react-menu": "^4.5.0",
-    "@tailwindcss/vite": "^4.1.14",
-    "@tanstack/react-form": "^1.23.7",
-    "@tanstack/react-query": "^5.90.5",
-    "astro": "^5.14.4",
+    "@radix-ui/react-slot": "^1.2.4",
+    "@ricky0123/vad-web": "^0.0.30",
+    "@szhsin/react-menu": "^4.5.1",
+    "@tailwindcss/vite": "^4.1.18",
+    "@tanstack/react-form": "^1.27.5",
+    "@tanstack/react-query": "^5.90.12",
+    "astro": "^5.16.6",
    "class-variance-authority": "^0.7.1",
    "clsx": "^2.1.1",
-    "dayjs": "^1.11.18",
-    "es-toolkit": "^1.40.0",
+    "dayjs": "^1.11.19",
+    "es-toolkit": "^1.43.0",
    "events": "^3.3.0",
    "graphology": "^0.26.0",
    "highlight.js": "^11.11.1",
-    "lodash-es": "^4.17.21",
-    "lucide-react": "^0.545.0",
-    "marked": "^16.4.1",
+    "lodash-es": "^4.17.22",
+    "lucide-react": "^0.562.0",
+    "marked": "^17.0.1",
    "nanoid": "^5.1.6",
-    "pocketbase": "^0.26.2",
+    "pocketbase": "^0.26.5",
    "pouchdb-adapter-memory": "^9.0.0",
    "pouchdb-browser": "^9.0.0",
-    "react": "^19.2.0",
-    "react-dom": "^19.2.0",
+    "react": "^19.2.3",
+    "react-dom": "^19.2.3",
    "react-modal": "^3.16.3",
-    "react-resizable-panels": "^3.0.6",
+    "react-resizable-panels": "^4.0.13",
    "react-toastify": "^11.0.5",
    "react-virtualized": "^9.22.6",
    "sigma": "^3.0.2",
-    "tailwind-merge": "^3.3.1",
-    "three": "^0.180.0",
-    "wavesurfer.js": "^7.11.0",
-    "zustand": "^5.0.8"
+    "tailwind-merge": "^3.4.0",
+    "three": "^0.182.0",
+    "wavesurfer.js": "^7.12.1",
+    "zustand": "^5.0.9"
  },
  "publishConfig": {
    "access": "public"
@@ -68,18 +69,18 @@
    "@kevisual/types": "^0.0.10",
    "@types/pouchdb": "^6.4.2",
    "@types/pouchdb-browser": "^6.1.5",
-    "@types/react": "^19.2.2",
-    "@types/react-dom": "^19.2.2",
+    "@types/react": "^19.2.7",
+    "@types/react-dom": "^19.2.3",
    "@types/react-modal": "^3.16.3",
    "@types/react-virtualized": "^9.22.3",
-    "@types/three": "^0.180.0",
+    "@types/three": "^0.182.0",
    "@vitejs/plugin-basic-ssl": "^2.1.0",
    "dotenv": "^17.2.3",
    "pouchdb": "^9.0.0",
-    "tailwindcss": "^4.1.14",
+    "tailwindcss": "^4.1.18",
    "tw-animate-css": "^1.4.0"
  },
-  "packageManager": "pnpm@10.18.3",
+  "packageManager": "pnpm@10.26.1",
  "onlyBuiltDependencies": [
    "@tailwindcss/oxide",
    "esbuild",
--- a/web/src/apps/muse/index.tsx
+++ b/web/src/apps/muse/index.tsx
@@ -1,6 +1,6 @@
 import { ToastContainer, toast } from 'react-toastify';
 import { AuthProvider } from '../login/AuthProvider';
-import { Panel, PanelGroup, PanelResizeHandle } from 'react-resizable-panels';
+import { Panel, Group, Separator } from 'react-resizable-panels';
 import { useRef } from 'react';

 import { App as Voice } from './voice/index.tsx';
@@ -149,20 +149,20 @@ export const MuseApp = () => {
          }}>
            初始化DB
          </button>
-          <button 
-            className="px-3 py-1 rounded text-sm bg-blue-500 text-white hover:bg-blue-600" 
+          <button
+            className="px-3 py-1 rounded text-sm bg-blue-500 text-white hover:bg-blue-600"
            onClick={handleExportDB}
          >
            导出DB
          </button>
-          <button 
-            className="px-3 py-1 rounded text-sm bg-purple-500 text-white hover:bg-purple-600" 
+          <button
+            className="px-3 py-1 rounded text-sm bg-purple-500 text-white hover:bg-purple-600"
            onClick={handleImportDB}
          >
            导入DB
          </button>
-          <button 
-            className="px-3 py-1 rounded text-sm bg-red-500 text-white hover:bg-red-600" 
+          <button
+            className="px-3 py-1 rounded text-sm bg-red-500 text-white hover:bg-red-600"
            onClick={handleDeleteDB}
          >
            删除DB
@@ -180,21 +180,21 @@ export const MuseApp = () => {

      {/* Resizable Panels */}
      <div className="flex-1 overflow-hidden">
-        <PanelGroup direction="horizontal">
+        <Group orientation="horizontal">
          {showLeftPanel && <LeftPanel />}

          {showLeftPanel && showCenterPanel && (
-            <PanelResizeHandle className="w-1 bg-gray-300 hover:bg-gray-400 cursor-col-resize transition-colors" />
+            <Separator className="w-1 bg-gray-300 hover:bg-gray-400 cursor-col-resize transition-colors" />
          )}

          {showCenterPanel && <CenterPanel />}

          {showCenterPanel && showRightPanel && (
-            <PanelResizeHandle className="w-1 bg-gray-300 hover:bg-gray-400 cursor-col-resize transition-colors" />
+            <Separator className="w-1 bg-gray-300 hover:bg-gray-400 cursor-col-resize transition-colors" />
          )}

          {showRightPanel && <RightPanel isVisible={showRightPanel} />}
-        </PanelGroup>
+        </Group>
      </div>
    </div>
  );
@@ -219,4 +219,24 @@ export const App: React.FC = () => {
      />
    </AuthProvider>
  );
-};
+};
+
+export const Record: React.FC = () => {
+  return (
+    <AuthProvider>
+      <Voice />
+      <ToastContainer
+        position="top-right"
+        autoClose={1000}
+        hideProgressBar={false}
+        newestOnTop={false}
+        closeOnClick
+        rtl={false}
+        pauseOnFocusLoss
+        draggable
+        pauseOnHover
+        theme="light"
+      />
+    </AuthProvider>
+  );
+}
--- a/web/src/apps/muse/voice/modules/VadVoice.tsx
+++ b/web/src/apps/muse/voice/modules/VadVoice.tsx
@@ -2,7 +2,7 @@ import { MicVAD, utils } from "@ricky0123/vad-web"
 import clsx from "clsx";
 import { useState, useEffect, useRef } from "react";
 import './style.css'
-import { MoreHorizontal, Play, Pause, Settings, FileAudio, StopCircle, Loader } from "lucide-react";
+import { MoreHorizontal, Play, Pause, Settings, FileAudio, StopCircle, Loader, Copy } from "lucide-react";
 import { Menu, MenuItem, MenuButton, } from '@szhsin/react-menu';
 import '@szhsin/react-menu/dist/index.css';
 import { toast } from 'react-toastify';
@@ -223,6 +223,12 @@ const VoicePlayer = ({ data }: VadVoiceProps) => {
  );
 }
 export const ShowVoicePlayer = ({ data }: { data: Speak[] }) => {
+  useEffect(() => {
+    const bottomElement = document.getElementById('voice-list-bottom');
+    if (bottomElement) {
+      bottomElement.scrollIntoView({ behavior: 'smooth' });
+    }
+  }, [data]);
  return (<ul className="space-y-2 max-h-full">
    {data.map((item, index) => (
      <li key={index} className="bg-white rounded-lg border border-gray-200 p-2 hover:shadow-sm transition-shadow">
@@ -343,6 +349,8 @@ export const ShowVoicePlayer = ({ data }: { data: Speak[] }) => {
        </div>
      </li>
    ))}
+
+    <div id="voice-list-bottom" />
  </ul>)
 }
 export const VadVoice = () => {
@@ -353,9 +361,11 @@ export const VadVoice = () => {
    error: storeError,
    initialize: initializeStore,
    addVoice,
-    setError: setStoreError
+    setError: setStoreError,
+    relatimeParialText,
+    relatimeFinalText
  } = useVoiceStore();
-
+  const showText = relatimeFinalText || relatimeParialText;
  // 使用设置 store
  const {
    openModal: openSettingModal,
@@ -391,7 +401,8 @@ export const VadVoice = () => {
            const wavBuffer = utils.encodeWAV(audio)
            const audioBlob = new Blob([wavBuffer], { type: 'audio/wav' })
            const tempUrl = URL.createObjectURL(audioBlob)
-
+            const relatime = useVoiceStore.getState().relatime;
+            relatime?.sendBase64?.(utils.arrayBufferToBase64(wavBuffer));
            // 从实际音频文件获取准确时长
            const getDuration = (): Promise<number> => {
              return new Promise((resolve) => {
@@ -508,7 +519,7 @@ export const VadVoice = () => {
  };
  return <div className="h-full flex flex-col">
    {/* Audio Recordings List */}
-    <div className="flex-1 overflow-y-auto px-2 py-3 min-h-0 max-h-200">
+    <div className="flex-1 overflow-y-auto px-2 py-3 min-h-0 max-h-76 overflow-hidden">
      {
        voiceList.length === 0 ? (
          <div className="text-center text-gray-400 text-sm py-8">
@@ -564,6 +575,29 @@ export const VadVoice = () => {
                `${voiceList.length} recording${voiceList.length !== 1 ? 's' : ''}`
              )}
            </div>
+            <div className="  ">
+              {showText && (
+                <div className="flex">
+                  <div className="text-xs text-gray-600 mt-1 truncate">
+                    📝 {showText}
+                  </div>
+                  <div className="cursor-pointer" onClick={() => {
+                    // copy
+                    navigator.clipboard.writeText(showText).then(() => {
+                      toast.success('已复制', {
+                        autoClose: 500,
+                        position: 'top-center'
+                      });
+                    }).catch((error) => {
+                      console.error('复制失败:', error);
+                      toast.error('复制失败，请手动选择文字复制');
+                    });
+                  }}>
+                    <Copy />
+                  </div>
+                </div>
+              )}
+            </div>
          </div>
        </div>
        <div className="flex items-center space-x-2">
@@ -572,7 +606,7 @@ export const VadVoice = () => {
              onClick={retryInitialization}
              className="px-2 py-1 text-xs font-medium text-blue-700 bg-blue-100 hover:bg-blue-200 rounded-md transition-colors"
            >
-              Retry
+              重试
            </button>
          )}
          <button
--- a/web/src/apps/muse/voice/modules/speak-db/index.ts
+++ b/web/src/apps/muse/voice/modules/speak-db/index.ts
@@ -1,26 +1,10 @@
 // Speak 数据库和服务的统一导出文件

 // 类型定义
-export {
-  Speak,
-  SpeakType,
-  CreateSpeakData,
-  UpdateSpeakData,
-  getDayOfYear
-} from './speak';
+export * from './speak.ts';

 // 数据库操作
-export {
-  SpeakDB,
-  SpeakDocument,
-  speakDB,
-  initSpeakDB,
-  createSpeakDB
-} from './speak-db';
+export * from './speak-db';

 // 服务层
-export {
-  SpeakService,
-  speakService,
-  exampleUsage
-} from './speak-service';
+export * from './speak-service';
--- a/web/src/apps/muse/voice/store/relatime.ts
+++ b/web/src/apps/muse/voice/store/relatime.ts
@@ -0,0 +1,65 @@
+import { WSServer } from '@kevisual/video-tools/src/asr/ws.ts'
+import { useVoiceStore } from './voiceStore'
+
+export class Relatime {
+  asr: WSServer
+  ready = false
+  timeoutHandle: NodeJS.Timeout | null = null
+  constructor() {
+    // const url = new URL('/ws/asr', "http://localhost:51015")
+    const url = new URL('/ws/asr', window.location.origin)
+    url.searchParams.set('id', 'muse-voice-relatime')
+    const ws = new WSServer({
+      url: url.toString(),
+      onConnect: () => {
+        console.log('WebSocket connected');
+        ws.emitter.on("message", (data) => {
+          // console.log("Received message:", data.data);
+          const json = JSON.parse(data.data);
+          console.log('json', json);
+          if (json && json.type === 'connected') {
+            ws.ws.send(JSON.stringify({ type: 'init' }));
+          }
+          if (json && json.type === 'asr' && json.code === 200) {
+            ws.emitter.emit('asr');
+          }
+          if (json && json.type === 'partial' || json.type === 'result') {
+            const text = json.text || '';
+            const isPartial = json.type === 'partial';
+            const isResult = json.type === 'result';
+            if (isPartial) {
+              // 部分结果
+              useVoiceStore.getState().setRelatimeParialText(text);
+            } else {
+              // 最终结果
+              useVoiceStore.getState().setRelatimeFinalText(text);
+            }
+          }
+        });
+        ws.emitter.once('asr', async () => {
+          console.log('ASR ready');
+          this.ready = true;
+        });
+      }
+    })
+    this.asr = ws
+  }
+  send(data: Buffer) {
+    if (!this.ready) return;
+    const voice = data.toString('base64');
+    this.asr.ws.send(JSON.stringify({ voice }));
+  }
+  sendBase64(data: string) {
+    if (!this.ready) return;
+    this.asr.ws.send(JSON.stringify({ voice: data, format: 'float32' }));
+    if (this.timeoutHandle) {
+      clearTimeout(this.timeoutHandle);
+    }
+    this.timeoutHandle = setTimeout(() => {
+      this.asr.sendBlankJson()
+      this.timeoutHandle = null;
+    }, 10000); // 5秒钟没有数据则发送空JSON保持连接
+
+  }
+
+}
--- a/web/src/apps/muse/voice/store/voiceStore.ts
+++ b/web/src/apps/muse/voice/store/voiceStore.ts
@@ -4,6 +4,7 @@ import { Speak, getDayOfYear, CreateSpeakData } from '../modules/speak-db/speak'
 import { speakService } from '../modules/speak-db/speak-service';
 import { getText } from '../modules/text';
 import { useSettingStore } from './settingStore';
+import { Relatime } from './relatime';

 interface VoiceState {
  // 状态数据
@@ -23,19 +24,29 @@ interface VoiceState {
  refreshList: () => Promise<void>;
  setError: (error: string | null) => void;
  setLoading: (loading: boolean) => void;
+  relatime: Relatime;
+  relatimeParialText: string;
+  relatimeFinalText: string;
+  setRelatimeParialText: (text: string) => void;
+  setRelatimeFinalText: (text: string) => void;
 }

-// 辅助函数：将 Blob 转换为 base64 字符串
+// 辅助函数：将 Blob 转换为 base64 字符串（兼容 Node.js）
 const blobToBase64 = (blob: Blob): Promise<string> => {
  return new Promise((resolve, reject) => {
    const reader = new FileReader();
    reader.onload = () => {
-      const result = reader.result as string;
-      // 移除 data URL 前缀，只保留 base64 数据
-      resolve(result.split(',')[1] || result);
+      const arrayBuffer = reader.result as ArrayBuffer;
+      // 转换为 Uint8Array 再转为 base64，与 Node.js Buffer.toString('base64') 一致
+      const bytes = new Uint8Array(arrayBuffer);
+      let binary = '';
+      for (let i = 0; i < bytes.length; i++) {
+        binary += String.fromCharCode(bytes[i]);
+      }
+      resolve(window.btoa(binary));
    };
    reader.onerror = reject;
-    reader.readAsDataURL(blob);
+    reader.readAsArrayBuffer(blob); // 使用 ArrayBuffer 而不是 DataURL
  });
 };

@@ -75,7 +86,8 @@ export const useVoiceStore = create<VoiceState>()(
      // 初始化：从 IndexedDB 获取当天的记录
      initialize: async () => {
        const { setLoading, setError, generateAudioUrls } = get();
-
+        const relatime = new Relatime();
+        set({ relatime });
        try {
          setLoading(true);
          setError(null);
@@ -130,7 +142,6 @@ export const useVoiceStore = create<VoiceState>()(
          if (autoRecognize) {
            speakData.text = await getText(fileData || '').then(res => res.text);
          }
-
          // 保存到 IndexedDB（不包含 url）
          const newSpeak = await speakService.createSpeakAuto(speakData);

@@ -323,7 +334,16 @@ export const useVoiceStore = create<VoiceState>()(
      // 设置加载状态
      setLoading: (loading: boolean) => {
        set({ isLoading: loading });
-      }
+      },
+
+      relatimeFinalText: '',
+      setRelatimeFinalText: (text: string) => {
+        set({ relatimeFinalText: text, relatimeParialText: '' });
+      },
+      setRelatimeParialText: (text: string) => {
+        set({ relatimeFinalText: '', relatimeParialText: text });
+      },
+      relatimeParialText: '',
    }),
    {
      name: 'voice-store', // persist key
--- a/web/src/pages/board.astro
+++ b/web/src/pages/board.astro
@@ -0,0 +1,8 @@
+---
+import Html from '../components/html.astro';
+import { App } from '@/apps/muse/index.tsx';
+---
+
+<Html>
+  <App client:only />
+</Html>
--- a/web/src/pages/index.astro
+++ b/web/src/pages/index.astro
@@ -1,8 +1,8 @@
 ---
 import Html from '../components/html.astro';
-import { App } from '@/apps/muse/index.tsx';
+import { Record } from '@/apps/muse/index.tsx';
 ---

 <Html>
-  <App client:only />
+  <Record client:only />
 </Html>
--- a/web/src/pages/muse.astro
+++ b/web/src/pages/muse.astro
@@ -1,8 +1,8 @@
 ---
 import Html from '../components/html.astro';
-import { App } from '@/apps/muse/index.tsx';
+import { Record } from '@/apps/muse/index.tsx';
 ---

 <Html>
-  <App client:only />
+  <Record client:only />
 </Html>