generated from tailored/router-template
temp: add test
This commit is contained in:
parent
d92b93c6f9
commit
fdc3985b93
4
.gitignore
vendored
4
.gitignore
vendored
@ -14,4 +14,6 @@ cache-file
|
|||||||
logs
|
logs
|
||||||
|
|
||||||
.env*
|
.env*
|
||||||
!.env.example
|
!.env.example
|
||||||
|
|
||||||
|
models
|
17
package.json
17
package.json
@ -36,16 +36,17 @@
|
|||||||
"access": "public"
|
"access": "public"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@kevisual/code-center-module": "0.0.18",
|
|
||||||
"@kevisual/mark": "0.0.7",
|
|
||||||
"@kevisual/router": "0.0.10",
|
"@kevisual/router": "0.0.10",
|
||||||
"@kevisual/use-config": "^1.0.10",
|
"@kevisual/use-config": "^1.0.10",
|
||||||
|
"@kevisual/video": "^0.0.1",
|
||||||
|
"@picovoice/porcupine-node": "^3.0.6",
|
||||||
"cookie": "^1.0.2",
|
"cookie": "^1.0.2",
|
||||||
"dayjs": "^1.11.13",
|
"dayjs": "^1.11.13",
|
||||||
|
"eventemitter3": "^5.0.1",
|
||||||
"formidable": "^3.5.2",
|
"formidable": "^3.5.2",
|
||||||
"lodash-es": "^4.17.21",
|
"lodash-es": "^4.17.21",
|
||||||
"nanoid": "^5.1.5",
|
"nanoid": "^5.1.5",
|
||||||
"node-record-lpcm16": "^1.0.1"
|
"vosk": "^0.3.39"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@kevisual/types": "^0.0.6",
|
"@kevisual/types": "^0.0.6",
|
||||||
@ -59,22 +60,22 @@
|
|||||||
"@types/crypto-js": "^4.2.2",
|
"@types/crypto-js": "^4.2.2",
|
||||||
"@types/formidable": "^3.4.5",
|
"@types/formidable": "^3.4.5",
|
||||||
"@types/lodash-es": "^4.17.12",
|
"@types/lodash-es": "^4.17.12",
|
||||||
"@types/node": "^22.14.0",
|
"@types/node": "^22.14.1",
|
||||||
|
"@types/vosk": "^0.3.1",
|
||||||
"@types/ws": "^8.18.1",
|
"@types/ws": "^8.18.1",
|
||||||
"commander": "^13.1.0",
|
"commander": "^13.1.0",
|
||||||
"concurrently": "^9.1.2",
|
"concurrently": "^9.1.2",
|
||||||
"cross-env": "^7.0.3",
|
"cross-env": "^7.0.3",
|
||||||
"dotenv": "^16.5.0",
|
"dotenv": "^16.5.0",
|
||||||
"inquire": "^0.4.8",
|
"inquire": "^0.4.8",
|
||||||
"ioredis": "^5.6.0",
|
"ioredis": "^5.6.1",
|
||||||
"jsrepo": "^1.45.3",
|
|
||||||
"nodemon": "^3.1.9",
|
"nodemon": "^3.1.9",
|
||||||
"pg": "^8.14.1",
|
"pg": "^8.14.1",
|
||||||
"pino": "^9.6.0",
|
"pino": "^9.6.0",
|
||||||
"pino-pretty": "^13.0.0",
|
"pino-pretty": "^13.0.0",
|
||||||
"pm2": "^6.0.5",
|
"pm2": "^6.0.5",
|
||||||
"rimraf": "^6.0.1",
|
"rimraf": "^6.0.1",
|
||||||
"rollup": "^4.39.0",
|
"rollup": "^4.40.0",
|
||||||
"rollup-plugin-copy": "^3.5.0",
|
"rollup-plugin-copy": "^3.5.0",
|
||||||
"rollup-plugin-dts": "^6.2.1",
|
"rollup-plugin-dts": "^6.2.1",
|
||||||
"rollup-plugin-esbuild": "^6.2.1",
|
"rollup-plugin-esbuild": "^6.2.1",
|
||||||
@ -85,5 +86,5 @@
|
|||||||
"typescript": "^5.8.3",
|
"typescript": "^5.8.3",
|
||||||
"ws": "^8.18.1"
|
"ws": "^8.18.1"
|
||||||
},
|
},
|
||||||
"packageManager": "pnpm@10.7.1"
|
"packageManager": "pnpm@10.8.1"
|
||||||
}
|
}
|
2166
pnpm-lock.yaml
generated
2166
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@ -1,16 +1,16 @@
|
|||||||
// import WebSocket from 'ws';
|
// import WebSocket from 'ws';
|
||||||
import { initWs } from '../../ws/index.ts';
|
import { initWs } from '../../ws-adapter/index.ts';
|
||||||
|
|
||||||
type VideoWSOptions = {
|
export type VideoWSOptions = {
|
||||||
url?: string;
|
url?: string;
|
||||||
ws?: WebSocket;
|
ws?: WebSocket;
|
||||||
itn?: boolean;
|
itn?: boolean;
|
||||||
mode?: string;
|
mode?: VideoWsMode;
|
||||||
isFile?: boolean;
|
isFile?: boolean;
|
||||||
onConnect?: () => void;
|
onConnect?: () => void;
|
||||||
};
|
};
|
||||||
export const VideoWsMode = ['2pass', 'online', 'offline'];
|
export const videoWsMode = ['2pass', 'online', 'offline'] as const;
|
||||||
type VideoWsMode = (typeof VideoWsMode)[number];
|
type VideoWsMode = (typeof videoWsMode)[number];
|
||||||
|
|
||||||
export type VideoWsResult = {
|
export type VideoWsResult = {
|
||||||
isFinal: boolean;
|
isFinal: boolean;
|
||||||
|
0
src/asr/provider/provider.ts
Normal file
0
src/asr/provider/provider.ts
Normal file
@ -1,8 +1,4 @@
|
|||||||
import * as fs from 'fs/promises';
|
|
||||||
import * as path from 'path';
|
|
||||||
import * as zlib from 'zlib';
|
import * as zlib from 'zlib';
|
||||||
import * as util from 'util';
|
|
||||||
import { Readable } from 'stream';
|
|
||||||
import { promisify } from 'util';
|
import { promisify } from 'util';
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import { VolcEngineBase, uuid } from './base.ts';
|
import { VolcEngineBase, uuid } from './base.ts';
|
||||||
@ -254,7 +250,6 @@ interface AudioItem {
|
|||||||
* ASR WebSocket Client
|
* ASR WebSocket Client
|
||||||
*/
|
*/
|
||||||
export class AsrWsClient extends VolcEngineBase {
|
export class AsrWsClient extends VolcEngineBase {
|
||||||
private audioPath: string;
|
|
||||||
private successCode: number = 1000;
|
private successCode: number = 1000;
|
||||||
private segDuration: number;
|
private segDuration: number;
|
||||||
private format: string;
|
private format: string;
|
||||||
@ -262,16 +257,14 @@ export class AsrWsClient extends VolcEngineBase {
|
|||||||
private bits: number;
|
private bits: number;
|
||||||
private channel: number;
|
private channel: number;
|
||||||
private codec: string;
|
private codec: string;
|
||||||
private authMethod: string;
|
|
||||||
private hotWords: string[] | null;
|
private hotWords: string[] | null;
|
||||||
private streaming: boolean;
|
|
||||||
private mp3SegSize: number;
|
private mp3SegSize: number;
|
||||||
private reqEvent: number = 1;
|
private reqEvent: number = 1;
|
||||||
private uid: string;
|
private uid: string;
|
||||||
private seq: number = 1;
|
private seq: number = 1;
|
||||||
private hasSendFullClientRequest: boolean = false;
|
private hasSendFullClientRequest: boolean = false;
|
||||||
|
|
||||||
constructor(audioPath: string, options: AsrClientOptions = {}) {
|
constructor(options: AsrClientOptions = {}) {
|
||||||
super({
|
super({
|
||||||
url: options.wsUrl || 'wss://openspeech.bytedance.com/api/v3/sauc/bigmodel',
|
url: options.wsUrl || 'wss://openspeech.bytedance.com/api/v3/sauc/bigmodel',
|
||||||
onConnect: () => this.onWsConnect(),
|
onConnect: () => this.onWsConnect(),
|
||||||
@ -285,22 +278,19 @@ export class AsrWsClient extends VolcEngineBase {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
this.audioPath = audioPath;
|
this.segDuration = options.segDuration || 200;
|
||||||
this.segDuration = options.segDuration || 100;
|
|
||||||
this.uid = options.uid || 'test';
|
this.uid = options.uid || 'test';
|
||||||
this.format = options.format || 'wav';
|
this.format = options.format || 'wav';
|
||||||
this.rate = options.rate || 16000;
|
this.rate = options.rate || 16000;
|
||||||
this.bits = options.bits || 16;
|
this.bits = options.bits || 16;
|
||||||
this.channel = options.channel || 1;
|
this.channel = options.channel || 1;
|
||||||
this.codec = options.codec || 'raw';
|
this.codec = options.codec || 'raw';
|
||||||
this.authMethod = options.authMethod || 'none';
|
|
||||||
this.hotWords = options.hotWords || null;
|
this.hotWords = options.hotWords || null;
|
||||||
this.streaming = options.streaming !== undefined ? options.streaming : true;
|
|
||||||
this.mp3SegSize = options.mp3SegSize || 1000;
|
this.mp3SegSize = options.mp3SegSize || 1000;
|
||||||
}
|
}
|
||||||
|
|
||||||
private onWsConnect() {
|
private onWsConnect() {
|
||||||
console.log('ASR WebSocket connected');
|
console.log('ASR Big Model WebSocket connected');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -321,6 +311,7 @@ export class AsrWsClient extends VolcEngineBase {
|
|||||||
request: {
|
request: {
|
||||||
model_name: 'bigmodel',
|
model_name: 'bigmodel',
|
||||||
enable_punc: true,
|
enable_punc: true,
|
||||||
|
result_type: 'single', // all, single
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@ -357,90 +348,59 @@ export class AsrWsClient extends VolcEngineBase {
|
|||||||
private async segmentDataProcessor(audioData: Buffer, segmentSize: number): Promise<any> {
|
private async segmentDataProcessor(audioData: Buffer, segmentSize: number): Promise<any> {
|
||||||
await this.sendFullClientRequest();
|
await this.sendFullClientRequest();
|
||||||
const that = this;
|
const that = this;
|
||||||
// Wait for response
|
const sendVoice = async (audioData: Buffer, segmentSize: number) => {
|
||||||
const result = await new Promise<any>((resolve, reject) => {
|
that.setCanSend(false);
|
||||||
const onMessage = async (event: MessageEvent) => {
|
for (const [chunk, last] of sliceData(audioData, segmentSize)) {
|
||||||
try {
|
that.seq += 1;
|
||||||
const response = parseResponse(Buffer.from(event.data as ArrayBuffer));
|
const isEnd = that.isEnd && last; // 结束了,而且是语音的最后一包
|
||||||
console.log('Initial response:', response);
|
if (isEnd) {
|
||||||
|
that.seq = -that.seq;
|
||||||
// Process audio chunks
|
|
||||||
for (const [chunk, last] of sliceData(audioData, segmentSize)) {
|
|
||||||
that.seq += 1;
|
|
||||||
if (last) {
|
|
||||||
that.seq = -that.seq;
|
|
||||||
}
|
|
||||||
const seq = that.seq;
|
|
||||||
|
|
||||||
const start = Date.now();
|
|
||||||
const compressedChunk = await gzipPromise(chunk);
|
|
||||||
|
|
||||||
const messageType = AUDIO_ONLY_REQUEST;
|
|
||||||
const flags = last ? NEG_WITH_SEQUENCE : POS_SEQUENCE;
|
|
||||||
|
|
||||||
const audioRequest = Buffer.concat([generateHeader(messageType, flags), generateBeforePayload(seq), Buffer.alloc(4), compressedChunk]);
|
|
||||||
|
|
||||||
// Set payload size
|
|
||||||
audioRequest.writeUInt32BE(compressedChunk.length, 8);
|
|
||||||
|
|
||||||
// Send audio chunk
|
|
||||||
(this as any).ws.send(audioRequest);
|
|
||||||
|
|
||||||
// Wait for each response
|
|
||||||
const chunkResponse = await new Promise<any>((resolveChunk) => {
|
|
||||||
const onChunkMessage = (chunkEvent: MessageEvent) => {
|
|
||||||
(this as any).ws.removeEventListener('message', onChunkMessage);
|
|
||||||
const parsed = parseResponse(Buffer.from(chunkEvent.data as ArrayBuffer));
|
|
||||||
console.log(`Seq ${seq} response:`, parsed);
|
|
||||||
resolveChunk(parsed);
|
|
||||||
};
|
|
||||||
|
|
||||||
(this as any).ws.addEventListener('message', onChunkMessage, { once: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
// If streaming, add delay to simulate real-time
|
|
||||||
if (this.streaming) {
|
|
||||||
const elapsed = Date.now() - start;
|
|
||||||
const sleepTime = Math.max(0, this.segDuration - elapsed);
|
|
||||||
await new Promise((r) => setTimeout(r, sleepTime));
|
|
||||||
}
|
|
||||||
|
|
||||||
// If this is the last chunk, resolve with final result
|
|
||||||
if (last) {
|
|
||||||
resolve(chunkResponse);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
(this as any).ws.removeEventListener('message', onMessage);
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error processing response:', error);
|
|
||||||
reject(error);
|
|
||||||
}
|
}
|
||||||
};
|
const seq = that.seq;
|
||||||
|
const compressedChunk = await gzipPromise(chunk);
|
||||||
|
|
||||||
(this as any).ws.addEventListener('message', onMessage, { once: true });
|
const messageType = AUDIO_ONLY_REQUEST;
|
||||||
|
const flags = isEnd ? NEG_WITH_SEQUENCE : POS_SEQUENCE;
|
||||||
|
|
||||||
(this as any).ws.addEventListener(
|
const audioRequest = Buffer.concat([generateHeader(messageType, flags), generateBeforePayload(seq), Buffer.alloc(4), compressedChunk]);
|
||||||
'error',
|
|
||||||
(error) => {
|
|
||||||
console.error('WebSocket error:', error);
|
|
||||||
reject(error);
|
|
||||||
},
|
|
||||||
{ once: true },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
return result;
|
// Set payload size
|
||||||
|
audioRequest.writeUInt32BE(compressedChunk.length, 8);
|
||||||
|
// Send audio chunk
|
||||||
|
this.ws.send(audioRequest);
|
||||||
|
// 待测试, 是否需要等待
|
||||||
|
// const chunkResponse = await new Promise<any>((resolveChunk) => {
|
||||||
|
// const onChunkMessage = (chunkEvent: MessageEvent) => {
|
||||||
|
// (this as any).ws.removeEventListener('message', onChunkMessage);
|
||||||
|
// const parsed = parseResponse(Buffer.from(chunkEvent.data as ArrayBuffer));
|
||||||
|
// resolveChunk(parsed);
|
||||||
|
// };
|
||||||
|
// (this as any).ws.addEventListener('message', onChunkMessage, { once: true });
|
||||||
|
// });
|
||||||
|
// if (last) {
|
||||||
|
// console.log('last', JSON.stringify(chunkResponse));
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
that.setCanSend(true);
|
||||||
|
};
|
||||||
|
// Wait for response
|
||||||
|
await sendVoice(audioData, segmentSize);
|
||||||
}
|
}
|
||||||
|
async onMessage(event: MessageEvent) {
|
||||||
/**
|
try {
|
||||||
* Execute ASR on the audio file
|
const parsed = parseResponse(Buffer.from(event.data as ArrayBuffer));
|
||||||
*/
|
console.log(`Seq ${parsed.payloadSequence} response:`, parsed);
|
||||||
public async execute(): Promise<any> {
|
// console.log('parsed', parsed.payloadSequence, parsed.payloadMsg.result.text);
|
||||||
|
if (parsed.isLastPackage) {
|
||||||
|
this.emitter.emit('end', parsed);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error processing response:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public async sendVoiceFile(data: Buffer) {
|
||||||
try {
|
try {
|
||||||
const data = await fs.readFile(this.audioPath);
|
|
||||||
|
|
||||||
if (this.format === 'mp3') {
|
if (this.format === 'mp3') {
|
||||||
const segmentSize = this.mp3SegSize;
|
const segmentSize = this.mp3SegSize;
|
||||||
return await this.segmentDataProcessor(data, segmentSize);
|
return await this.segmentDataProcessor(data, segmentSize);
|
||||||
@ -465,65 +425,9 @@ export class AsrWsClient extends VolcEngineBase {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
public async sendVoiceStream(data: Buffer) {
|
||||||
/**
|
const segmentSize = Buffer.byteLength(data);
|
||||||
* Send OPUS data for processing
|
console.log('segmentSize', segmentSize);
|
||||||
*/
|
return await this.segmentDataProcessor(data, segmentSize);
|
||||||
public async sendOpusData(audioData: Buffer): Promise<any> {
|
|
||||||
const segmentSize = Math.floor((this.rate * 2 * this.channel * this.segDuration) / 500);
|
|
||||||
return await this.segmentDataProcessor(audioData, segmentSize);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute ASR on a single audio file
|
|
||||||
*/
|
|
||||||
export async function executeOne(audioItem: AudioItem, options: AsrClientOptions = {}): Promise<any> {
|
|
||||||
if (!audioItem.id || !audioItem.path) {
|
|
||||||
throw new Error('Audio item must have id and path properties');
|
|
||||||
}
|
|
||||||
|
|
||||||
const audioId = audioItem.id;
|
|
||||||
const audioPath = path.resolve(process.cwd(), audioItem.path);
|
|
||||||
|
|
||||||
const asrClient = new AsrWsClient(audioPath, options);
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
||||||
|
|
||||||
return asrClient.execute().then((result) => {
|
|
||||||
return {
|
|
||||||
id: audioId,
|
|
||||||
path: audioPath,
|
|
||||||
result: result,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Test stream processing
|
|
||||||
*/
|
|
||||||
export const testStream = async () => {
|
|
||||||
console.log('测试流式');
|
|
||||||
const audioPath = 'videos/asr_example.wav';
|
|
||||||
|
|
||||||
const res = await executeOne({
|
|
||||||
id: 1,
|
|
||||||
path: audioPath,
|
|
||||||
})
|
|
||||||
.then((result) => {
|
|
||||||
console.log('====end test=====');
|
|
||||||
console.log(result);
|
|
||||||
return result;
|
|
||||||
})
|
|
||||||
.catch((error) => {
|
|
||||||
console.error('Test error:', error);
|
|
||||||
return '';
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle audio data directly
|
|
||||||
*/
|
|
||||||
export async function handleAudioData(audioData: Buffer, options: AsrClientOptions = {}): Promise<any> {
|
|
||||||
const asrClient = new AsrWsClient('', options);
|
|
||||||
return await asrClient.sendOpusData(audioData);
|
|
||||||
}
|
|
||||||
|
@ -265,7 +265,6 @@ export class AsrWsClient extends VolcEngineBase {
|
|||||||
super({
|
super({
|
||||||
url: options.wsUrl || 'wss://openspeech.bytedance.com/api/v2/asr',
|
url: options.wsUrl || 'wss://openspeech.bytedance.com/api/v2/asr',
|
||||||
onConnect: () => this.onWsConnect(),
|
onConnect: () => this.onWsConnect(),
|
||||||
enabled: false,
|
|
||||||
wsOptions: {
|
wsOptions: {
|
||||||
headers: {
|
headers: {
|
||||||
Authorization: `Bearer; ${options.token}`,
|
Authorization: `Bearer; ${options.token}`,
|
||||||
@ -329,13 +328,6 @@ export class AsrWsClient extends VolcEngineBase {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
/**
|
|
||||||
* Generate headers for authentication
|
|
||||||
*/
|
|
||||||
private tokenAuth(): Record<string, string> {
|
|
||||||
return { Authorization: `Bearer; ${this.token}` };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Process audio data in segments
|
* Process audio data in segments
|
||||||
*/
|
*/
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
import { initWs } from '../../ws/index.ts';
|
import { initWs } from '../../ws-adapter/index.ts';
|
||||||
import { WSServer } from '../ws-server.ts';
|
import { WSServer } from '../../provider/ws-server.ts';
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
|
|
||||||
export const uuid = () => nanoid(16);
|
export const uuid = () => nanoid(16);
|
||||||
@ -7,7 +7,6 @@ export const uuid = () => nanoid(16);
|
|||||||
type VolcEngineBaseOptions = {
|
type VolcEngineBaseOptions = {
|
||||||
url?: string;
|
url?: string;
|
||||||
ws?: WebSocket;
|
ws?: WebSocket;
|
||||||
enabled?: boolean;
|
|
||||||
onConnect?: () => void;
|
onConnect?: () => void;
|
||||||
wsOptions?: {
|
wsOptions?: {
|
||||||
headers?: {
|
headers?: {
|
||||||
@ -20,17 +19,38 @@ type VolcEngineBaseOptions = {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
export class VolcEngineBase extends WSServer {
|
export class VolcEngineBase extends WSServer {
|
||||||
|
canSend = false;
|
||||||
|
isEnd: boolean = false;
|
||||||
|
|
||||||
constructor(opts: VolcEngineBaseOptions) {
|
constructor(opts: VolcEngineBaseOptions) {
|
||||||
super({
|
super({
|
||||||
url: opts.url,
|
url: opts.url,
|
||||||
ws: opts.ws,
|
ws: opts.ws,
|
||||||
onConnect: opts.onConnect,
|
onConnect: opts.onConnect,
|
||||||
wsOptions: opts.wsOptions,
|
wsOptions: opts.wsOptions,
|
||||||
enabled: opts.enabled,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
async onOpen() {
|
async onOpen() {
|
||||||
console.log('VolcEngineBase onOpen');
|
console.log('VolcEngineBase onOpen');
|
||||||
// 发送认证信息
|
// 发送认证信息
|
||||||
}
|
}
|
||||||
|
async isCanSend() {
|
||||||
|
if (this.canSend) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
this.emitter.once('canSend', () => {
|
||||||
|
resolve(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async setCanSend(canSend: boolean) {
|
||||||
|
this.canSend = canSend;
|
||||||
|
if (canSend) {
|
||||||
|
this.emitter.emit('canSend', canSend);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async setIsEnd(isEnd: boolean) {
|
||||||
|
this.isEnd = isEnd;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,28 +1,21 @@
|
|||||||
import { AsrWsClient, testStream } from '../asr-ws-big-model-client.ts';
|
import { AsrWsClient } from '../asr-ws-big-model-client.ts';
|
||||||
import { audioPath, config } from './common.ts';
|
import { audioPath, audioPath2, blankAudioPath, config } from './common.ts';
|
||||||
|
import fs from 'fs';
|
||||||
// const asr = new AsrWsClient('videos/asr_example.wav');
|
// const asr = new AsrWsClient('videos/asr_example.wav');
|
||||||
|
|
||||||
// tsx src/asr/provider/volcengine/test/asr-bigmodel.ts
|
// tsx src/asr/provider/volcengine/test/asr-bigmodel.ts
|
||||||
const main = async () => {
|
const main = async () => {
|
||||||
const audioId = '123';
|
const audioId = '123';
|
||||||
const asrClient = new AsrWsClient(audioPath, {
|
const asrClient = new AsrWsClient({
|
||||||
appid: config.APP_ID,
|
appid: config.APP_ID,
|
||||||
token: config.TOKEN,
|
token: config.TOKEN,
|
||||||
streaming: false,
|
|
||||||
});
|
});
|
||||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||||
|
const data = fs.readFileSync(audioPath);
|
||||||
return asrClient.execute().then((result) => {
|
await asrClient.sendVoiceFile(data);
|
||||||
return {
|
await asrClient.sendVoiceFile(fs.readFileSync(blankAudioPath));
|
||||||
id: audioId,
|
asrClient.setIsEnd(true);
|
||||||
path: audioPath,
|
await asrClient.sendVoiceFile(fs.readFileSync(audioPath2));
|
||||||
result: result,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
};
|
|
||||||
const main2 = async () => {
|
|
||||||
testStream();
|
|
||||||
};
|
};
|
||||||
|
|
||||||
main();
|
main();
|
||||||
|
@ -6,5 +6,7 @@ export const config = dotenv.config({
|
|||||||
}).parsed;
|
}).parsed;
|
||||||
|
|
||||||
export const audioPath = path.join(process.cwd(), 'videos/asr_example.wav');
|
export const audioPath = path.join(process.cwd(), 'videos/asr_example.wav');
|
||||||
|
export const audioPath2 = path.join(process.cwd(), 'videos/asr_example2.wav');
|
||||||
|
export const blankAudioPath = path.join(process.cwd(), 'videos/blank.wav');
|
||||||
|
|
||||||
export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
39
src/asr/provider/volcengine/test/recorder.ts
Normal file
39
src/asr/provider/volcengine/test/recorder.ts
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
import { AsrWsClient } from '../asr-ws-big-model-client.ts';
|
||||||
|
|
||||||
|
import { audioPath, config, sleep } from '../test/common.ts';
|
||||||
|
|
||||||
|
import net from 'net';
|
||||||
|
import { Recording } from '../../../../recorder/index.ts';
|
||||||
|
import Stream from 'stream';
|
||||||
|
|
||||||
|
const recorder = new Recording();
|
||||||
|
|
||||||
|
const asrClient = new AsrWsClient({
|
||||||
|
appid: config.APP_ID,
|
||||||
|
token: config.TOKEN,
|
||||||
|
});
|
||||||
|
// tsx src/asr/provider/volcengine/test/recorder.ts
|
||||||
|
|
||||||
|
const main = async () => {
|
||||||
|
// await asrClient.sendVoiceFile(fs.readFileSync(audioPath));
|
||||||
|
const send = (data: Buffer) => {
|
||||||
|
asrClient.sendVoiceStream(data);
|
||||||
|
};
|
||||||
|
let chunks: Buffer = Buffer.alloc(0);
|
||||||
|
var chunk_size = 960; // for asr chunk_size [5, 10, 5]
|
||||||
|
let totalsend = 0;
|
||||||
|
recorder.stream().on('data', (chunk) => {
|
||||||
|
chunks = Buffer.concat([chunks, chunk]);
|
||||||
|
const chunkSize = Buffer.byteLength(chunks);
|
||||||
|
if (chunkSize > chunk_size) {
|
||||||
|
send(chunks);
|
||||||
|
totalsend += chunks.length;
|
||||||
|
chunks = Buffer.alloc(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// setTimeout(() => {
|
||||||
|
// recorder.stop();
|
||||||
|
// }, 10000);
|
||||||
|
};
|
||||||
|
|
||||||
|
main();
|
@ -1,41 +1,77 @@
|
|||||||
import { initWs } from '../ws/index.ts';
|
import { EventEmitter } from 'eventemitter3';
|
||||||
|
import { initWs } from '../ws-adapter/index.ts';
|
||||||
import type { ClientOptions } from 'ws';
|
import type { ClientOptions } from 'ws';
|
||||||
type WSSOptions = {
|
type WSSOptions = {
|
||||||
url: string;
|
url: string;
|
||||||
ws?: WebSocket;
|
ws?: WebSocket;
|
||||||
onConnect?: () => void;
|
onConnect?: () => void;
|
||||||
wsOptions?: ClientOptions;
|
wsOptions?: ClientOptions;
|
||||||
enabled?: boolean;
|
emitter?: EventEmitter;
|
||||||
};
|
};
|
||||||
export class WSServer {
|
export class WSServer {
|
||||||
ws: WebSocket;
|
ws: WebSocket;
|
||||||
onConnect?: () => void;
|
onConnect?: () => void;
|
||||||
|
connected: boolean;
|
||||||
|
emitter: EventEmitter;
|
||||||
constructor(opts: WSSOptions) {
|
constructor(opts: WSSOptions) {
|
||||||
this.initWs(opts);
|
this.initWs(opts);
|
||||||
}
|
}
|
||||||
async initWs(opts: WSSOptions) {
|
async initWs(opts: WSSOptions) {
|
||||||
const enabled = opts.enabled || true;
|
|
||||||
if (opts.ws) {
|
if (opts.ws) {
|
||||||
this.ws = opts.ws;
|
this.ws = opts.ws;
|
||||||
} else if (enabled) {
|
|
||||||
this.ws = await initWs(opts.url, opts.wsOptions);
|
|
||||||
}
|
}
|
||||||
|
this.emitter = opts.emitter || new EventEmitter();
|
||||||
|
this.ws = await initWs(opts.url, opts.wsOptions);
|
||||||
this.onConnect = opts?.onConnect || (() => {});
|
this.onConnect = opts?.onConnect || (() => {});
|
||||||
this.ws.onopen = this.onOpen.bind(this);
|
this.ws.onopen = this.onOpen.bind(this);
|
||||||
this.ws.onmessage = this.onMessage.bind(this);
|
this.ws.onmessage = this.onMessage.bind(this);
|
||||||
this.ws.onerror = this.onError.bind(this);
|
this.ws.onerror = this.onError.bind(this);
|
||||||
this.ws.onclose = this.onClose.bind(this);
|
this.ws.onclose = this.onClose.bind(this);
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* 连接成功 ws 事件
|
||||||
|
*/
|
||||||
async onOpen() {
|
async onOpen() {
|
||||||
|
this.connected = true;
|
||||||
this.onConnect();
|
this.onConnect();
|
||||||
|
this.emitter.emit('open');
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* 检查是否连接
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
async isConnected() {
|
||||||
|
if (this.connected) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
this.emitter.once('open', () => {
|
||||||
|
resolve(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 收到消息 ws 事件
|
||||||
|
* @param event
|
||||||
|
*/
|
||||||
async onMessage(event: MessageEvent) {
|
async onMessage(event: MessageEvent) {
|
||||||
// console.log('WSS onMessage', event);
|
// console.log('WSS onMessage', event);
|
||||||
|
this.emitter.emit('message', event);
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* ws 错误事件
|
||||||
|
* @param event
|
||||||
|
*/
|
||||||
async onError(event: Event) {
|
async onError(event: Event) {
|
||||||
console.error('WSS onError');
|
console.error('WSS onError');
|
||||||
|
this.emitter.emit('error', event);
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* ws 关闭事件
|
||||||
|
* @param event
|
||||||
|
*/
|
||||||
async onClose(event: CloseEvent) {
|
async onClose(event: CloseEvent) {
|
||||||
console.error('WSS onClose');
|
console.error('WSS onClose');
|
||||||
|
this.emitter.emit('close', event);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
const isBrowser = process?.env?.BROWSER === 'true';
|
const isBrowser = process?.env?.BROWSER === 'true';
|
||||||
|
import { EventEmitter } from 'events';
|
||||||
|
|
||||||
type WebSocketOptions = {
|
type WebSocketOptions = {
|
||||||
/**
|
/**
|
||||||
@ -23,3 +24,18 @@ export const initWs = async (url: string, options?: WebSocketOptions) => {
|
|||||||
}
|
}
|
||||||
return ws;
|
return ws;
|
||||||
};
|
};
|
||||||
|
interface EventEmitterOptions {
|
||||||
|
/**
|
||||||
|
* Enables automatic capturing of promise rejection.
|
||||||
|
*/
|
||||||
|
captureRejections?: boolean | undefined;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* 初始化一个事件发射器
|
||||||
|
* @param opts 事件发射器选项
|
||||||
|
* @returns 事件发射器
|
||||||
|
*/
|
||||||
|
export const initEmitter = (opts?: EventEmitterOptions) => {
|
||||||
|
const emitter = new EventEmitter(opts);
|
||||||
|
return emitter;
|
||||||
|
};
|
@ -1,7 +1,7 @@
|
|||||||
import assert from 'assert';
|
import assert from 'assert';
|
||||||
import { logDebug, logInfo } from '../logger/index.ts';
|
import { logDebug, logInfo } from '../logger/index.ts';
|
||||||
import { ChildProcessWithoutNullStreams, spawn } from 'child_process';
|
import { ChildProcessWithoutNullStreams, spawn } from 'child_process';
|
||||||
import recorders from './recorders/index.ts';
|
import recorders from '../recorder/recorders/index.ts';
|
||||||
import Stream from 'stream';
|
import Stream from 'stream';
|
||||||
|
|
||||||
export type RecordingOptions = {
|
export type RecordingOptions = {
|
||||||
|
69
src/utils/convert.ts
Normal file
69
src/utils/convert.ts
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
type EncodeWavOptions = {
|
||||||
|
numChannels?: number;
|
||||||
|
sampleRate?: number;
|
||||||
|
byteRate?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 编码pcm文件为wav文件
|
||||||
|
* @param rawPCM
|
||||||
|
* @param options
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
export function encodeWav(rawPCM: Buffer | string, options?: EncodeWavOptions) {
|
||||||
|
if (typeof rawPCM === 'string') {
|
||||||
|
rawPCM = Buffer.from(rawPCM, 'binary');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Buffer.isBuffer(rawPCM)) {
|
||||||
|
throw new TypeError('pcm data must be Buffer or string');
|
||||||
|
}
|
||||||
|
const opt = options || {};
|
||||||
|
const sampleRate = opt.sampleRate || 16000;
|
||||||
|
const numChannels = opt.numChannels || 1;
|
||||||
|
const byteRate = opt.byteRate || 16;
|
||||||
|
|
||||||
|
const buf = rawPCM;
|
||||||
|
const header = Buffer.alloc(44);
|
||||||
|
|
||||||
|
header.write('RIFF', 0);
|
||||||
|
header.writeUInt32LE(buf.length, 4);
|
||||||
|
header.write('WAVE', 8);
|
||||||
|
header.write('fmt ', 12);
|
||||||
|
header.writeUInt8(16, 16);
|
||||||
|
header.writeUInt8(1, 20);
|
||||||
|
header.writeUInt8(numChannels, 22);
|
||||||
|
header.writeUInt32LE(sampleRate, 24);
|
||||||
|
header.writeUInt32LE(byteRate, 28);
|
||||||
|
header.writeUInt8(4, 32);
|
||||||
|
header.writeUInt8(16, 34);
|
||||||
|
header.write('data', 36);
|
||||||
|
header.writeUInt32LE(buf.length + 44 - 8, 40);
|
||||||
|
|
||||||
|
return Buffer.concat([header, buf]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解码wav文件
|
||||||
|
* @param rawWav
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
export function decodeWav(rawWav: Buffer | string) {
|
||||||
|
if (typeof rawWav === 'string') {
|
||||||
|
rawWav = Buffer.from(rawWav, 'binary');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Buffer.isBuffer(rawWav)) {
|
||||||
|
throw new TypeError('pcm data must be Buffer or string');
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove the header of pcm format
|
||||||
|
rawWav = rawWav.subarray(44);
|
||||||
|
|
||||||
|
return rawWav;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const converter = {
|
||||||
|
encodeWav,
|
||||||
|
decodeWav,
|
||||||
|
};
|
49
src/wake/test/build.ts
Normal file
49
src/wake/test/build.ts
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
import vosk from 'vosk';
|
||||||
|
import { Recording } from '../../recorder/index.ts';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
import { audioPath, sleep } from './common.ts';
|
||||||
|
import { encodeWav, decodeWav } from '../../utils/convert.ts';
|
||||||
|
// 需要先下载Vosk模型
|
||||||
|
// const MODEL_PATH = 'vosk-model-small-en-us-0.15';
|
||||||
|
const MODEL_PATH = path.join(process.cwd(), 'models/vosk-model-small-cn-0.22');
|
||||||
|
|
||||||
|
async function detectWithVosk(audioFilePath) {
|
||||||
|
if (!fs.existsSync(MODEL_PATH)) {
|
||||||
|
console.error('请先下载Vosk模型');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const model = new vosk.Model(MODEL_PATH);
|
||||||
|
const rec = new vosk.Recognizer({ model: model, sampleRate: 16000 });
|
||||||
|
|
||||||
|
// const wakeWords = ['hey computer', 'okay jarvis']; // 自定义唤醒词列表
|
||||||
|
const wakeWords = ['欢迎']; // 自定义唤醒词列表
|
||||||
|
const audioBuffer = fs.readFileSync(audioFilePath);
|
||||||
|
const pcmBuffer = decodeWav(audioBuffer);
|
||||||
|
const result = await rec.acceptWaveformAsync(pcmBuffer);
|
||||||
|
console.log('result', result, rec.result());
|
||||||
|
// return new Promise((resolve) => {
|
||||||
|
// const pcmBufferLength = Buffer.byteLength(pcmBuffer);
|
||||||
|
// console.log('pcmBufferLength', pcmBufferLength);
|
||||||
|
// const bufferLength = 1024 * 8;
|
||||||
|
// let index = 0;
|
||||||
|
// for (let i = 0; i < pcmBufferLength; i += bufferLength) {
|
||||||
|
// const chunk = pcmBuffer.subarray(i, i + bufferLength);
|
||||||
|
// index++;
|
||||||
|
// if (rec.acceptWaveform(chunk)) {
|
||||||
|
// const result = rec.result();
|
||||||
|
// console.log('=========result', result, index);
|
||||||
|
// const text = result.text.toLowerCase();
|
||||||
|
// if (wakeWords.some((word) => text.includes(word))) {
|
||||||
|
// resolve(true);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// resolve(false);
|
||||||
|
// });
|
||||||
|
}
|
||||||
|
|
||||||
|
detectWithVosk(audioPath).then((result) => {
|
||||||
|
console.log('result', result);
|
||||||
|
});
|
12
src/wake/test/common.ts
Normal file
12
src/wake/test/common.ts
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
import path from 'path';
|
||||||
|
import dotenv from 'dotenv';
|
||||||
|
|
||||||
|
export const config = dotenv.config({
|
||||||
|
path: path.join(process.cwd(), '.env'),
|
||||||
|
}).parsed;
|
||||||
|
|
||||||
|
export const audioPath = path.join(process.cwd(), 'videos/asr_example.wav');
|
||||||
|
export const audioPath2 = path.join(process.cwd(), 'videos/asr_example2.wav');
|
||||||
|
export const blankAudioPath = path.join(process.cwd(), 'videos/blank.wav');
|
||||||
|
|
||||||
|
export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
@ -1,23 +1,41 @@
|
|||||||
import { defineConfig } from 'tsup';
|
import { defineConfig } from 'tsup';
|
||||||
// import glob from 'fast-glob';
|
// import glob from 'fast-glob';
|
||||||
// const services = glob.sync('src/services/*.ts');
|
// const services = glob.sync('src/services/*.ts');
|
||||||
|
import fs from 'fs';
|
||||||
|
|
||||||
|
const clean = () => {
|
||||||
|
const distDir = 'dist';
|
||||||
|
if (fs.existsSync(distDir)) {
|
||||||
|
fs.rmSync(distDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
clean();
|
||||||
|
|
||||||
const entrys = ['src/index.ts'];
|
const entrys = ['src/index.ts'];
|
||||||
|
const nodeEntrys = ['src/dev.ts'];
|
||||||
|
|
||||||
export default defineConfig({
|
const getCommonConfig = (opts = {}) => {
|
||||||
entry: entrys,
|
return {
|
||||||
outExtension: ({ format }) => ({
|
entry: opts.entry,
|
||||||
js: format === 'esm' ? '.mjs' : '.js',
|
outExtension: ({ format }) => ({
|
||||||
}),
|
js: format === 'esm' ? '.mjs' : '.js',
|
||||||
define: {
|
}),
|
||||||
'process.env.IS_BROWSER': JSON.stringify(process.env.BROWSER || false),
|
splitting: false,
|
||||||
},
|
sourcemap: false,
|
||||||
splitting: false,
|
// clean: true,
|
||||||
sourcemap: false,
|
format: 'esm',
|
||||||
clean: true,
|
external: ['dotenv'],
|
||||||
format: 'esm',
|
dts: true,
|
||||||
external: ['dotenv'],
|
outDir: 'dist',
|
||||||
dts: true,
|
tsconfig: 'tsconfig.json',
|
||||||
outDir: 'dist',
|
...opts,
|
||||||
tsconfig: 'tsconfig.json',
|
define: {
|
||||||
});
|
'process.env.IS_BROWSER': JSON.stringify(process.env.BROWSER || false),
|
||||||
|
...opts.define,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
export default defineConfig([
|
||||||
|
// getCommonConfig({ entry: entrys, define: { 'process.env.IS_BROWSER': JSON.stringify(true) } }), // 浏览器
|
||||||
|
getCommonConfig({ entry: nodeEntrys, define: { 'process.env.IS_BROWSER': JSON.stringify(false) } }), // node
|
||||||
|
]);
|
||||||
|
BIN
videos/asr_example2.wav
Normal file
BIN
videos/asr_example2.wav
Normal file
Binary file not shown.
BIN
videos/blank.wav
Normal file
BIN
videos/blank.wav
Normal file
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user