add funasr demo

src/app.ts

@@ -1,8 +0,0 @@
-import { App } from '@kevisual/router';
-import { useContextKey } from '@kevisual/use-config/context';
-
-const init = () => {
-  return new App();
-};
-
-export const app = useContextKey('app', init);

src/asr/provider/funasr/test/get-text.ts

@@ -0,0 +1,42 @@
+import { VideoWS } from '../ws.ts';
+import net from 'net';
+import path from 'path';
+import fs from 'fs';
+
+const videoTestPath = path.join(process.cwd(), 'videos/asr_example.wav');
+const ws = new VideoWS({
+  // url: 'wss://192.168.31.220:10095',
+  url: 'wss://funasr.xiongxiao.me',
+  isFile: true,
+  onConnect: async () => {
+    console.log('onConnect');
+    const data = fs.readFileSync(videoTestPath);
+    let sampleBuf = new Uint8Array(data);
+
+    var chunk_size = 960; // for asr chunk_size [5, 10, 5]
+    let totalsend = 0;
+    let len = 0;
+    ws.start();
+    while (sampleBuf.length >= chunk_size) {
+      const sendBuf = sampleBuf.slice(0, chunk_size);
+      totalsend = totalsend + sampleBuf.length;
+      sampleBuf = sampleBuf.slice(chunk_size, sampleBuf.length);
+      if (len === 100) {
+        // ws.stop();
+        // ws.start();
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+      }
+      ws.send(sendBuf);
+      len++;
+    }
+    ws.stop();
+    console.log('len', len);
+  },
+});
+
+const server = net.createServer((socket) => {
+  socket.on('data', (data) => {
+    console.log('data', data);
+  });
+});
+server.listen(10096);

src/asr/provider/funasr/test/recorder.ts

@@ -0,0 +1,41 @@
+import { VideoWS } from '../ws.ts';
+import net from 'net';
+import { Recording } from '../../../../recorder/index.ts';
+import Stream from 'stream';
+
+const recorder = new Recording();
+const writeStream = new Stream.Writable();
+const ws = new VideoWS({
+  url: 'wss://192.168.31.220:10095',
+  isFile: false,
+  onConnect: async () => {
+    console.log('onConnect');
+    let chunks: Buffer = Buffer.alloc(0);
+    var chunk_size = 960; // for asr chunk_size [5, 10, 5]
+    let totalsend = 0;
+    let len = 0;
+    recorder.stream().on('data', (chunk) => {
+      chunks = Buffer.concat([chunks, chunk]);
+      if (chunks.length > chunk_size) {
+        ws.send(chunks);
+        totalsend += chunks.length;
+        chunks = Buffer.alloc(0);
+      }
+    });
+    ws.start();
+    setTimeout(() => {
+      ws.stop();
+      setTimeout(() => {
+        process.exit(0);
+      }, 1000);
+      console.log('len', len);
+    }, 20000);
+  },
+});
+
+const server = net.createServer((socket) => {
+  socket.on('data', (data) => {
+    console.log('data', data);
+  });
+});
+server.listen(10096);

src/asr/provider/funasr/ws.ts

@@ -0,0 +1,114 @@
+import WebSocket from 'ws';
+
+type VideoWSOptions = {
+  url?: string;
+  ws?: WebSocket;
+  itn?: boolean;
+  mode?: string;
+  isFile?: boolean;
+  onConnect?: () => void;
+};
+export const VideoWsMode = ['2pass', 'online', 'offline'];
+type VideoWsMode = (typeof VideoWsMode)[number];
+
+export type VideoWsResult = {
+  isFinal: boolean;
+  mode: VideoWsMode;
+  stamp_sents: { end: number; punc: string; start: number; text_seg: string; tsList: [][] }[];
+  text: string;
+  timestamp: string;
+  wav_name: string;
+};
+
+export class VideoWS {
+  ws: WebSocket;
+  itn?: boolean;
+  mode?: VideoWsMode;
+  isFile?: boolean;
+  onConnect?: () => void;
+  constructor(options?: VideoWSOptions) {
+    this.ws =
+      options?.ws ||
+      new WebSocket(options.url, {
+        rejectUnauthorized: false,
+      });
+    this.itn = options?.itn || false;
+    this.mode = options?.mode || 'online';
+    this.isFile = options?.isFile || false;
+
+    this.onConnect = options?.onConnect || (() => {});
+    this.ws.onopen = this.onOpen.bind(this);
+    this.ws.onmessage = this.onMessage.bind(this);
+    this.ws.onerror = this.onError.bind(this);
+    this.ws.onclose = this.onClose.bind(this);
+  }
+
+  async onOpen() {
+    this.onConnect();
+  }
+  async start() {
+    let isFileMode = this.isFile;
+    const chunk_size = new Array(5, 10, 5);
+    type OpenRequest = {
+      chunk_size: number[];
+      wav_name: string;
+      is_speaking: boolean;
+      chunk_interval: number;
+      itn: boolean;
+      mode: VideoWsMode;
+      wav_format?: string;
+      audio_fs?: number;
+      hotwords?: string;
+    };
+    const request: OpenRequest = {
+      chunk_size: chunk_size,
+      wav_name: 'h5', //
+      is_speaking: true,
+      chunk_interval: 10,
+      itn: this.itn,
+      mode: this.mode || 'online',
+    };
+    console.log('request', request);
+    if (isFileMode) {
+      const file_ext = 'wav';
+      const file_sample_rate = 16000;
+      request.wav_format = file_ext;
+      if (file_ext == 'wav') {
+        request.wav_format = 'PCM';
+        request.audio_fs = file_sample_rate;
+      }
+    }
+    this.ws.send(JSON.stringify(request));
+  }
+  async stop() {
+    var chunk_size = new Array(5, 10, 5);
+    var request = {
+      chunk_size: chunk_size,
+      wav_name: 'h5',
+      is_speaking: false,
+      chunk_interval: 10,
+      mode: this.mode,
+    };
+    this.ws.send(JSON.stringify(request));
+  }
+  async send(data: any) {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+      this.ws.send(data);
+    }
+  }
+  async onMessage(event: MessageEvent) {
+    const data = event.data;
+    try {
+      const result = JSON.parse(data.toString());
+      console.log('result', result);
+    } catch (error) {
+      console.log('error', error);
+    }
+  }
+  async onError(event: Event) {
+    console.log('onError', event);
+  }
+  async onClose(event: CloseEvent) {
+    console.log('onClose', event);
+  }
+}

src/dev.ts

@@ -1,48 +1,20 @@
-import { app } from './index.ts';
-import { useConfig } from '@kevisual/use-config/env';
+import { Recording } from './recorder/index.ts';
+import fs from 'fs';
 
-app
-  .route({
-    path: 'auth',
-    id: 'auth',
-  })
-  .define(async (ctx) => {
-    ctx.query.token = '123';
-    ctx.state.tokenUser = {
-      id: '123',
-      username: 'admin',
-    };
-  })
-  .addTo(app);
+const file = fs.createWriteStream('test.wav', { encoding: 'binary' });
 
-app
-  .route({
-    path: 'auth-admin',
-    id: 'auth-admin',
-  })
-  .define(async (ctx) => {
-    ctx.body = '123';
-    ctx.state.tokenUser = {
-      id: '123',
-      username: 'admin',
-    };
-  })
-  .addTo(app);
-app
-  .route({
-    path: 'demo',
-    key: 'demo',
-  })
-  .define(async (ctx) => {
-    ctx.body = '123';
-  })
-  .addTo(app);
-
-const config = useConfig();
-const port = config.PORT || 4000;
-
-console.log('run demo: http://localhost:' + port + '/api/router?path=demo&key=demo');
-
-app.listen(port, () => {
-  console.log(`server is running at http://localhost:${port}`);
+const record = new Recording({
+  sampleRate: 16000,
+  channels: 1,
+  audioType: 'wav',
+  threshold: 0,
+  recorder: 'rec',
+  silence: '1.0',
+  endOnSilence: true,
 });
+record.stream().pipe(file);
+
+setTimeout(() => {
+  record.stop();
+  process.exit(0);
+}, 5000);

src/index.ts

@@ -1,3 +1 @@
-import { app } from './app.ts';
-
-export { app };
+export const test = 'test';

src/logger/index.ts

@@ -18,10 +18,10 @@ export const logger = pino({
     req: pino.stdSerializers.req,
     res: pino.stdSerializers.res,
   },
-  base: {
-    app: 'ai-chat',
-    env: process.env.NODE_ENV || 'development',
-  },
+  // base: {
+  //   app: 'ai-videos',
+  //   env: process.env.NODE_ENV || 'development',
+  // },
 });
 
 export const logError = (message: string, data?: any) => logger.error({ data }, message);

src/main.ts

@@ -1,9 +1,3 @@
-// 单应用实例启动
-import { useConfig } from '@kevisual/use-config/env';
-import { app } from './index.ts';
-
-const config = useConfig();
-const port = config.PORT || 4000;
-app.listen(port, () => {
-  console.log(`server is running at http://localhost:${port}`);
-});
+export const main = () => {
+  console.log('main');
+};

src/modules/redis.ts

@@ -1,28 +0,0 @@
-import { Redis } from 'ioredis';
-
-// 配置 Redis 连接
-export const redis = new Redis({
-  host: 'localhost', // Redis 服务器的主机名或 IP 地址
-  port: 6379, // Redis 服务器的端口号
-  // password: 'your_password', // Redis 的密码 (如果有)
-  db: 0, // 要使用的 Redis 数据库索引 (0-15)
-  keyPrefix: '', // key 前缀
-  retryStrategy(times) {
-    // 连接重试策略
-    return Math.min(times * 50, 2000); // 每次重试时延迟增加
-  },
-  maxRetriesPerRequest: null, // 允许请求重试的次数 (如果需要无限次重试)
-});
-
-// 监听连接事件
-redis.on('connect', () => {
-  console.log('Redis 连接成功');
-});
-
-redis.on('error', (err) => {
-  console.error('Redis 连接错误', err);
-});
-
-// 初始化 Redis 客户端
-export const redisPublisher = new Redis(); // 用于发布消息
-export const redisSubscriber = new Redis(); // 用于订阅消息

src/modules/sequelize.ts

@@ -1,31 +0,0 @@
-import { Sequelize } from 'sequelize';
-import { useConfig } from '@kevisual/use-config/env';
-
-const config = useConfig();
-
-export type PostgresConfig = {
-  postgres: {
-    username: string;
-    password: string;
-    host: string;
-    port: number;
-    database: string;
-  };
-};
-if (!config.POSTGRES_PASSWORD || !config.POSTGRES_USER) {
-  console.error('postgres config is required password and user');
-  process.exit(1);
-}
-const postgresConfig = {
-  username: config.POSTGRES_USER,
-  password: config.POSTGRES_PASSWORD,
-  host: config.POSTGRES_HOST || 'localhost',
-  port: parseInt(config.POSTGRES_PORT || '5432'),
-  database: config.POSTGRES_DB || 'postgres',
-};
-// connect to db
-export const sequelize = new Sequelize({
-  dialect: 'postgres',
-  ...postgresConfig,
-  // logging: false,
-});

src/modules/user.ts

@@ -1,9 +0,0 @@
-import { sequelize, User, UserInit, Org, OrgInit } from '@kevisual/code-center-module';
-
-export { sequelize, User, UserInit, Org, OrgInit };
-
-export const init = () => {
-  UserInit();
-  OrgInit();
-};
-init();

src/recorder/index.ts

@@ -0,0 +1,144 @@
+import assert from 'assert';
+import { logDebug, logInfo } from '../logger/index.ts';
+import { ChildProcessWithoutNullStreams, spawn } from 'child_process';
+import recorders from './recorders/index.ts';
+import Stream from 'stream';
+
+export type RecordingOptions = {
+  /* 采样率，默认为16000 */
+  sampleRate?: number;
+  /* 声道数，默认为1 */
+  channels?: number;
+  /* 是否压缩音频，默认为false */
+  compress?: boolean;
+  /* 录音开始的音量阈值，默认为0.5 */
+  threshold?: number;
+  /* 开始录音的音量阈值 */
+  thresholdStart?: number;
+  /* 结束录音的音量阈值 */
+  thresholdEnd?: number;
+  /* 录音结束的静默时间，默认为'1.0'秒 */
+  silence?: string;
+  /* 使用的录音器，默认为'sox' */
+  recorder?: string;
+  /* 是否在静默时结束录音，默认为false */
+  endOnSilence?: boolean;
+  /* 音频类型，默认为'wav' */
+  audioType?: string;
+};
+/**
+ * node-record-lpcm16
+ * https://github.com/gillesdemey/node-record-lpcm16
+ */
+export class Recording {
+  options: RecordingOptions;
+  cmd: string;
+  args: string[];
+  cmdOptions: any;
+  process: ChildProcessWithoutNullStreams;
+  _stream: Stream.Readable;
+
+  constructor(options?: RecordingOptions) {
+    const defaults = {
+      sampleRate: 16000,
+      channels: 1,
+      compress: false,
+      threshold: 0.5,
+      thresholdStart: null,
+      thresholdEnd: null,
+      silence: '1.0',
+      recorder: 'sox',
+      endOnSilence: false,
+      audioType: 'wav',
+    };
+
+    this.options = Object.assign(defaults, options);
+
+    const recorder = recorders[this.options.recorder];
+    if (!recorder) {
+      throw new Error(`No such recorder found: ${this.options.recorder}`);
+    }
+    const { cmd, args, spawnOptions = {} } = recorder(this.options);
+
+    this.cmd = cmd;
+    this.args = args;
+    this.cmdOptions = Object.assign({ encoding: 'binary', stdio: 'pipe' }, spawnOptions);
+
+    logDebug(`Started recording`);
+    logDebug('options', this.options);
+    logDebug(` ${this.cmd} ${this.args.join(' ')}`);
+
+    return this.start();
+  }
+
+  start() {
+    const { cmd, args, cmdOptions } = this;
+
+    const cp = spawn(cmd, args, cmdOptions);
+    const rec = cp.stdout;
+    const err = cp.stderr;
+
+    this.process = cp; // expose child process
+    this._stream = rec; // expose output stream
+
+    cp.on('close', (code) => {
+      if (code === 0) return;
+      rec.emit(
+        'error',
+        `${this.cmd} has exited with error code ${code}.
+
+Enable debugging with the environment variable DEBUG=record.`,
+      );
+    });
+
+    err.on('data', (chunk) => {
+      logDebug(`STDERR: ${chunk}`);
+    });
+
+    rec.on('data', (chunk) => {
+      logDebug(`Recording ${chunk.length} bytes`);
+    });
+
+    rec.on('end', () => {
+      logDebug('Recording ended');
+    });
+
+    return this;
+  }
+
+  stop() {
+    assert(this.process, 'Recording not yet started');
+
+    this.process.kill();
+  }
+
+  pause() {
+    assert(this.process, 'Recording not yet started');
+
+    this.process.kill('SIGSTOP');
+    this._stream.pause();
+    logDebug('Paused recording');
+  }
+
+  resume() {
+    assert(this.process, 'Recording not yet started');
+
+    this.process.kill('SIGCONT');
+    this._stream.resume();
+    logDebug('Resumed recording');
+  }
+
+  isPaused() {
+    assert(this.process, 'Recording not yet started');
+
+    return this._stream.isPaused();
+  }
+
+  stream() {
+    assert(this._stream, 'Recording not yet started');
+
+    return this._stream;
+  }
+}
+
+export const record = (...args) => new Recording(...args);

src/recorder/recorders/arecord.ts

@@ -0,0 +1,23 @@
+// On some systems (RasPi), arecord is the prefered recording binary
+export default (options: any) => {
+  const cmd = 'arecord';
+
+  const args = [
+    '-q', // show no progress
+    '-r',
+    options.sampleRate, // sample rate
+    '-c',
+    options.channels, // channels
+    '-t',
+    options.audioType, // audio type
+    '-f',
+    'S16_LE', // Sample format
+    '-', // pipe
+  ];
+
+  if (options.device) {
+    args.unshift('-D', options.device);
+  }
+
+  return { cmd, args };
+};

src/recorder/recorders/index.ts

@@ -0,0 +1,11 @@
+import path from 'path';
+
+import sox from './sox.ts';
+import rec from './rec.ts';
+import arecord from './arecord.ts';
+
+export default {
+  sox,
+  rec,
+  arecord,
+};

src/recorder/recorders/rec.ts

@@ -0,0 +1,32 @@
+export default (options: any) => {
+  const cmd = 'rec';
+
+  let args = [
+    '-q', // show no progress
+    '-r',
+    options.sampleRate, // sample rate
+    '-c',
+    options.channels, // channels
+    '-e',
+    'signed-integer', // sample encoding
+    '-b',
+    '16', // precision (bits)
+    '-t',
+    options.audioType, // audio type
+    '-', // pipe
+  ];
+
+  if (options.endOnSilence) {
+    args = args.concat([
+      'silence',
+      '1',
+      '0.1',
+      options.thresholdStart || options.threshold + '%',
+      '1',
+      options.silence,
+      options.thresholdEnd || options.threshold + '%',
+    ]);
+  }
+
+  return { cmd, args };
+};

src/recorder/recorders/sox.ts

@@ -0,0 +1,39 @@
+export default (options: any) => {
+  const cmd = 'sox';
+
+  let args = [
+    '--default-device',
+    '--no-show-progress', // show no progress
+    '--rate',
+    options.sampleRate, // sample rate
+    '--channels',
+    options.channels, // channels
+    '--encoding',
+    'signed-integer', // sample encoding
+    '--bits',
+    '16', // precision (bits)
+    '--type',
+    options.audioType, // audio type
+    '-', // pipe
+  ];
+
+  if (options.endOnSilence) {
+    args = args.concat([
+      'silence',
+      '1',
+      '0.1',
+      options.thresholdStart || options.threshold + '%',
+      '1',
+      options.silence,
+      options.thresholdEnd || options.threshold + '%',
+    ]);
+  }
+
+  const spawnOptions: any = {};
+
+  if (options.device) {
+    spawnOptions.env = { ...process.env, AUDIODEV: options.device };
+  }
+
+  return { cmd, args, spawnOptions };
+};

src/routes/app-demo/index.ts

@@ -1,119 +0,0 @@
-import { Op } from 'sequelize';
-import { AppDemoModel } from './models/index.ts';
-import { app } from '@/app.ts';
-
-app
-  .route({
-    path: 'app-demo',
-    key: 'list',
-    middleware: ['auth'],
-  })
-  .define(async (ctx) => {
-    const tokenUser = ctx.state.tokenUser;
-    const { page = 1, pageSize = 20, search, sort = 'DESC' } = ctx.query;
-    const searchWhere = search
-      ? {
-          [Op.or]: [{ title: { [Op.like]: `%${search}%` } }, { summary: { [Op.like]: `%${search}%` } }],
-        }
-      : {};
-
-    const { rows: appDemo, count } = await AppDemoModel.findAndCountAll({
-      where: {
-        uid: tokenUser.uid,
-        ...searchWhere,
-      },
-      offset: (page - 1) * pageSize,
-      limit: pageSize,
-      order: [['updatedAt', sort]],
-    });
-
-    ctx.body = {
-      list: appDemo,
-      pagination: {
-        page,
-        current: page,
-        pageSize,
-        total: count,
-      },
-    };
-  })
-  .addTo(app);
-
-app
-  .route({
-    path: 'app-demo',
-    key: 'update',
-    middleware: ['auth'],
-  })
-  .define(async (ctx) => {
-    const tokenUser = ctx.state.tokenUser;
-    const { id, data, updatedAt: _clear, createdAt: _clear2, ...rest } = ctx.query.data;
-    let appDemo: AppDemoModel;
-    let isNew = false;
-    if (id) {
-      const appDemo = await AppDemoModel.findByPk(id);
-      if (appDemo.uid !== tokenUser.uid) {
-        ctx.throw(403, 'No permission');
-      }
-    } else {
-      appDemo = await AppDemoModel.create({
-        data: data,
-        ...rest,
-        uid: tokenUser.uid,
-      });
-      isNew = true;
-    }
-    if (!appDemo) {
-      ctx.throw(404, 'AppDemo not found');
-    }
-    if (!isNew) {
-      appDemo = await appDemo.update({
-        data: { ...appDemo.data, ...data },
-        ...rest,
-      });
-    }
-
-    ctx.body = appDemo;
-  })
-  .addTo(app);
-
-app
-  .route({
-    path: 'app-demo',
-    key: 'delete',
-    middleware: ['auth'],
-  })
-  .define(async (ctx) => {
-    const tokenUser = ctx.state.tokenUser;
-    const { id, force = false } = ctx.query.data || {};
-    if (!id) {
-      ctx.throw(400, 'id is required');
-    }
-    const appDemo = await AppDemoModel.findByPk(id);
-    if (appDemo.uid !== tokenUser.uid) {
-      ctx.throw(403, 'No permission');
-    }
-    await appDemo.destroy({ force });
-    ctx.body = appDemo;
-  })
-  .addTo(app);
-
-app
-  .route({
-    path: 'app-demo',
-    key: 'get',
-    middleware: ['auth'],
-  })
-  .define(async (ctx) => {
-    const tokenUser = ctx.state.tokenUser;
-    const { id } = ctx.query.data || {};
-    if (!id) {
-      ctx.throw(400, 'id is required');
-    }
-    const appDemo = await AppDemoModel.findByPk(id);
-    if (appDemo.uid !== tokenUser.uid) {
-      ctx.throw(403, 'No permission');
-    }
-    ctx.body = appDemo;
-  })
-  .addTo(app);

src/routes/app-demo/models/index.ts

@@ -1,71 +0,0 @@
-import { sequelize } from '@/modules/sequelize.ts';
-import { DataTypes, Model } from 'sequelize';
-
-export interface AppDemoData {
-  [key: string]: any;
-}
-
-export type AppDemo = Partial<InstanceType<typeof AppDemoModel>>;
-
-export class AppDemoModel extends Model {
-  declare id: string;
-  declare title: string;
-  declare description: string;
-  declare summary: string;
-
-  declare data: AppDemoData;
-  declare tags: string[];
-  declare version: string;
-
-  declare uid: string;
-
-  declare createdAt: Date;
-  declare updatedAt: Date;
-}
-
-AppDemoModel.init(
-  {
-    id: {
-      type: DataTypes.UUID,
-      primaryKey: true,
-      defaultValue: DataTypes.UUIDV4,
-    },
-    title: {
-      type: DataTypes.TEXT,
-      defaultValue: '',
-    },
-    description: {
-      type: DataTypes.TEXT,
-      defaultValue: '',
-    },
-    summary: {
-      type: DataTypes.TEXT,
-      defaultValue: '',
-    },
-    tags: {
-      type: DataTypes.JSONB,
-      defaultValue: [],
-    },
-    version: {
-      type: DataTypes.INTEGER,
-      defaultValue: 0,
-    },
-    data: {
-      type: DataTypes.JSONB,
-      defaultValue: {},
-    },
-    uid: {
-      type: DataTypes.UUID,
-      allowNull: false,
-    },
-  },
-  {
-    sequelize,
-    tableName: 'kv_app_demo',
-    paranoid: true,
-  },
-);
-
-AppDemoModel.sync({ alter: true, logging: false }).catch((e) => {
-  console.error('AppDemoModel sync', e);
-});