kevisual/kevisual-query-upload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init upload module

Browse Source
This commit is contained in:
xion 2025-03-31 21:17:41 +08:00
parent 64ca348277
commit a7faaca418
16 changed files with 633 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored Normal file
View File

@ -0,0 +1,7 @@
dist
node_modules
.DS_Store
.env*
!.env.example
.turbo

2
.npmrc Normal file
View File

@ -0,0 +1,2 @@
//npm.xiongxiao.me/:_authToken=${ME_NPM_TOKEN}
//registry.npmjs.org/:_authToken=${NPM_TOKEN}

11
index.html Normal file
View File

@ -0,0 +1,11 @@
<html>
<head>
<title>Query Upload</title>
</head>
<body>
<script src="test/upload.ts" type="module"></script>
</body>
</html>

36
package.json Normal file
View File

@ -0,0 +1,36 @@
{
"name": "@kevisual/query-upload",
"version": "0.0.1",
"description": "",
"main": "dist/query-upload.js",
"types": "dist/query-upload.d.ts",
"scripts": {
"build": "tsup",
"watch": "tsup --watch",
"dev": "tsup --watch",
"dev:lib": "pnpm run dev"
},
"files": [
"dist",
"react",
"src"
],
"keywords": [],
"author": "abearxiong <xiongxiao@xiongxiao.me>",
"license": "MIT",
"packageManager": "pnpm@10.6.5",
"type": "module",
"devDependencies": {
"@types/node": "^22.13.14",
"eventsource": "^3.0.6",
"tsup": "^8.4.0"
},
"publishConfig": {
"access": "public"
},
"exports": {
".": "./dist/query-upload.js",
"./query-upload": "./dist/query-upload.js",
"./query-upload-node": "./dist/query-upload-node.js"
}
}

134
src/core/upload-chunk.ts Normal file
View File

@ -0,0 +1,134 @@
import { randomId } from '../utils/random-id.ts';
import { UploadProgress } from './upload-progress.ts';
export type ConvertOpts = {
appKey?: string;
version?: string;
username?: string;
directory?: string;
isPublic?: boolean;
filename?: string;
/**
* 是否不检查应用文件, 默认 true，默认不检测
*/
noCheckAppFiles?: boolean;
};
// createEventSource: (baseUrl: string, searchParams: URLSearchParams) => {
// return new EventSource(baseUrl + '/api/s1/events?' + searchParams.toString());
// },
export type UploadOpts = {
uploadProgress: UploadProgress;
/**
* 创建 EventSource 兼容 nodejs
* @param baseUrl 基础 URL
* @param searchParams 查询参数
* @returns EventSource
*/
createEventSource: (baseUrl: string, searchParams: URLSearchParams) => EventSource;
baseUrl?: string;
token: string;
FormDataFn: any;
};
export const uploadFileChunked = async (file: File, opts: ConvertOpts, opts2: UploadOpts) => {
const { directory, appKey, version, username, isPublic, noCheckAppFiles = true } = opts;
const { uploadProgress, createEventSource, baseUrl = '', token, FormDataFn } = opts2 || {};
return new Promise(async (resolve, reject) => {
const taskId = randomId();
const filename = opts.filename || file.name;
uploadProgress?.start(`${filename} 上传中...`);
const searchParams = new URLSearchParams();
searchParams.set('taskId', taskId);
if (isPublic) {
searchParams.set('public', 'true');
}
if (noCheckAppFiles) {
searchParams.set('noCheckAppFiles', '1');
}
const eventSource = createEventSource(baseUrl + '/api/s1/events', searchParams);
let isError = false;
// 监听服务器推送的进度更新
eventSource.onmessage = function (event) {
console.log('Progress update:', event.data);
const parseIfJson = (data: string) => {
try {
return JSON.parse(data);
} catch (e) {
return data;
}
};
const receivedData = parseIfJson(event.data);
if (typeof receivedData === 'string') return;
const progress = Number(receivedData.progress);
const progressFixed = progress.toFixed(2);
uploadProgress?.set(progress, { ...receivedData, progressFixed, filename, taskId });
};
eventSource.onerror = function (event) {
console.log('eventSource.onerror', event);
isError = true;
reject(event);
};
const chunkSize = 1 * 1024 * 1024; // 1MB
const totalChunks = Math.ceil(file.size / chunkSize);
for (let currentChunk = 0; currentChunk < totalChunks; currentChunk++) {
const start = currentChunk * chunkSize;
const end = Math.min(start + chunkSize, file.size);
const chunk = file.slice(start, end);
const formData = new FormDataFn();
formData.append('file', chunk, filename);
formData.append('chunkIndex', currentChunk.toString());
formData.append('totalChunks', totalChunks.toString());
const isLast = currentChunk === totalChunks - 1;
if (directory) {
formData.append('directory', directory);
}
if (appKey && version) {
formData.append('appKey', appKey);
formData.append('version', version);
}
if (username) {
formData.append('username', username);
}
try {
const res = await fetch(baseUrl + '/api/s1/resources/upload/chunk?taskId=' + taskId, {
method: 'POST',
body: formData,
headers: {
'task-id': taskId,
Authorization: `Bearer ${token}`,
},
}).then((response) => response.json());
if (res?.code !== 200) {
console.log('uploadChunk error', res);
uploadProgress?.error(res?.message || '上传失败');
isError = true;
eventSource.close();
uploadProgress?.done();
reject(new Error(res?.message || '上传失败'));
return;
}
if (isLast) {
fetch(baseUrl + '/api/s1/events/close?taskId=' + taskId);
eventSource.close();
uploadProgress?.done();
resolve(res);
}
// console.log(`Chunk ${currentChunk + 1}/${totalChunks} uploaded`, res);
} catch (error) {
console.log('Error uploading chunk', error);
fetch(baseUrl + '/api/s1/events/close?taskId=' + taskId);
reject(error);
return;
}
}
// 循环结束
if (!uploadProgress?.end) {
uploadProgress?.done();
}
});
};

103
src/core/upload-progress.ts Normal file
View File

@ -0,0 +1,103 @@
interface UploadNProgress {
start: (msg?: string) => void;
done: () => void;
set: (progress: number) => void;
}
type UploadProgressData = {
progress: number;
progressFixed: number;
filename?: string;
taskId?: string;
};
type UploadProgressOpts = {
onStart?: () => void;
onDone?: () => void;
onProgress?: (progress: number, data?: UploadProgressData) => void;
};
export class UploadProgress implements UploadNProgress {
/**
* 进度
*/
progress: number;
/**
* 开始回调
*/
onStart: (() => void) | undefined;
/**
* 结束回调
*/
onDone: (() => void) | undefined;
/**
* 消息回调
*/
onProgress: ((progress: number, data?: UploadProgressData) => void) | undefined;
/**
* 数据
*/
data: any;
/**
* 是否结束
*/
end: boolean;
constructor(uploadOpts: UploadProgressOpts) {
this.progress = 0;
this.end = false;
const mockFn = () => {};
this.onStart = uploadOpts.onStart || mockFn;
this.onDone = uploadOpts.onDone || mockFn;
this.onProgress = uploadOpts.onProgress || mockFn;
}
start(msg?: string) {
this.progress = 0;
msg && this.info(msg);
this.end = false;
this.onStart?.();
}
done() {
this.progress = 100;
this.end = true;
this.onDone?.();
}
set(progress: number, data?: UploadProgressData) {
this.progress = progress;
this.data = data;
this.onProgress?.(progress, data);
console.log('uploadProgress set', progress, data);
}
/**
* 开始回调
*/
setOnStart(callback: () => void) {
this.onStart = callback;
}
/**
* 结束回调
*/
setOnDone(callback: () => void) {
this.onDone = callback;
}
/**
* 消息回调
*/
setOnProgress(callback: (progress: number, data?: UploadProgressData) => void) {
this.onProgress = callback;
}
/**
* 打印信息
*/
info(msg: string) {
console.log(msg);
}
/**
* 打印错误
*/
error(msg: string) {
console.error(msg);
}
/**
* 打印警告
*/
warn(msg: string) {
console.warn(msg);
}
}

113
src/core/upload.ts Normal file
View File

@ -0,0 +1,113 @@
import { randomId } from '../utils/random-id.ts';
import type { UploadOpts } from './upload-chunk.ts';
type ConvertOpts = {
appKey?: string;
version?: string;
username?: string;
directory?: string;
/**
* 文件大小限制
*/
maxSize?: number;
/**
* 文件数量限制
*/
maxCount?: number;
/**
* 是否不检查应用文件, 默认 true，默认不检测
*/
noCheckAppFiles?: boolean;
};
export const uploadFiles = async (files: File[], opts: ConvertOpts, opts2: UploadOpts) => {
const { directory, appKey, version, username, noCheckAppFiles = true } = opts;
const { uploadProgress, createEventSource, baseUrl = '', token, FormDataFn } = opts2 || {};
const length = files.length;
const maxSize = opts.maxSize || 20 * 1024 * 1024; // 20MB
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
if (totalSize > maxSize) {
const maxSizeMB = maxSize / 1024 / 1024;
uploadProgress?.error('有文件大小不能超过' + maxSizeMB + 'MB');
return;
}
const maxCount = opts.maxCount || 10;
if (length > maxCount) {
uploadProgress?.error(`最多只能上传${maxCount}个文件`);
return;
}
uploadProgress?.info(`上传中，共${length}个文件`);
return new Promise((resolve, reject) => {
const formData = new FormDataFn();
const webkitRelativePath = files[0]?.webkitRelativePath;
const keepDirectory = webkitRelativePath !== '';
const root = keepDirectory ? webkitRelativePath.split('/')[0] : '';
for (let i = 0; i < files.length; i++) {
const file = files[i];
if (keepDirectory) {
// relativePath 去除第一级
const webkitRelativePath = file.webkitRelativePath.replace(root + '/', '');
formData.append('file', file, webkitRelativePath); // 保留文件夹路径
} else {
formData.append('file', files[i], files[i].name);
}
}
if (directory) {
formData.append('directory', directory);
}
if (appKey && version) {
formData.append('appKey', appKey);
formData.append('version', version);
}
if (username) {
formData.append('username', username);
}
const searchParams = new URLSearchParams();
const taskId = randomId();
searchParams.set('taskId', taskId);
if (noCheckAppFiles) {
searchParams.set('noCheckAppFiles', '1');
}
const eventSource = new EventSource('/api/s1/events?taskId=' + taskId);
uploadProgress?.start('上传中...');
eventSource.onopen = async function (event) {
const res = await fetch('/api/s1/resources/upload?' + searchParams.toString(), {
method: 'POST',
body: formData,
headers: {
'task-id': taskId,
Authorization: `Bearer ${token}`,
},
}).then((response) => response.json());
console.log('upload success', res);
fetch('/api/s1/events/close?taskId=' + taskId);
eventSource.close();
uploadProgress?.done();
resolve(res);
};
// 监听服务器推送的进度更新
eventSource.onmessage = function (event) {
console.log('Progress update:', event.data);
const parseIfJson = (data: string) => {
try {
return JSON.parse(data);
} catch (e) {
return data;
}
};
const receivedData = parseIfJson(event.data);
if (typeof receivedData === 'string') return;
const progress = Number(receivedData.progress);
const progressFixed = progress.toFixed(2);
console.log('progress', progress);
uploadProgress?.set(progress, { ...receivedData, taskId, progressFixed });
};
eventSource.onerror = function (event) {
console.log('eventSource.onerror', event);
reject(event);
};
});
};

0
src/query-upload-node.ts Normal file
View File

6
src/query-upload.ts Normal file
View File

@ -0,0 +1,6 @@
import { uploadFiles } from './core/upload.ts';
import { uploadFileChunked } from './core/upload-chunk.ts';
import { UploadProgress } from './core/upload-progress.ts';
export { uploadFiles, uploadFileChunked, UploadProgress };

23
src/utils/filter-files.ts Normal file
View File

@ -0,0 +1,23 @@
/**
* 过滤文件, 过滤 .DS_Store, node_modules, 以.开头的文件， 过滤 __开头的文件
* @param files
* @returns
*/
export const filterFiles = (files: File[]) => {
files = files.filter((file) => {
if (file.webkitRelativePath.startsWith('__MACOSX')) {
return false;
}
// 过滤node_modules
if (file.webkitRelativePath.includes('node_modules')) {
return false;
}
// 过滤文件 .DS_Store
if (file.name === '.DS_Store') {
return false;
}
// 过滤以.开头的文件
return !file.name.startsWith('.');
});
return files;
};

3
src/utils/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * from './to-file.ts';
export * from './filter-files.ts';
export * from './random-id.ts';

3
src/utils/random-id.ts Normal file
View File

@ -0,0 +1,3 @@
export const randomId = () => {
return Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
};

105
src/utils/to-file.ts Normal file
View File

@ -0,0 +1,105 @@
const getFileExtension = (filename: string) => {
return filename.split('.').pop();
};
const getFileType = (extension: string) => {
switch (extension) {
case 'js':
return 'text/javascript';
case 'css':
return 'text/css';
case 'html':
return 'text/html';
case 'json':
return 'application/json';
case 'png':
return 'image/png';
case 'jpg':
return 'image/jpeg';
case 'jpeg':
return 'image/jpeg';
case 'gif':
return 'image/gif';
case 'svg':
return 'image/svg+xml';
case 'webp':
return 'image/webp';
case 'ico':
return 'image/x-icon';
default:
return 'text/plain';
}
};
const checkIsBase64 = (content: string) => {
return content.startsWith('data:');
};
/**
* 获取文件的目录和文件名
* @param filename 文件名
* @returns 目录和文件名
*/
export const getDirectoryAndName = (filename: string) => {
if (!filename) {
return null;
}
if (filename.startsWith('.')) {
return null;
} else {
filename = filename.replace(/^\/+/, ''); // Remove all leading slashes
}
const hasDirectory = filename.includes('/');
if (!hasDirectory) {
return { directory: '', name: filename };
}
const parts = filename.split('/');
const name = parts.pop()!; // Get the last part as the file name
const directory = parts.join('/'); // Join the remaining parts as the directory
return { directory, name };
};
/**
* 把字符串转为文件流，并返回文件流，根据filename的扩展名，自动设置文件类型.
* 当不是文本类型，自动需要把base64的字符串转为blob
* @param content 字符串
* @param filename 文件名
* @returns 文件流
*/
export const toFile = (content: string, filename: string) => {
// 如果文件名是 a/d/a.js 格式的，则需要把d作为目录，a.js作为文件名
const directoryAndName = getDirectoryAndName(filename);
if (!directoryAndName) {
throw new Error('Invalid filename');
}
const { name } = directoryAndName;
const extension = getFileExtension(name);
if (!extension) {
throw new Error('Invalid filename');
}
const isBase64 = checkIsBase64(content);
const type = getFileType(extension);
if (isBase64) {
// Decode base64 string
const base64Data = content.split(',')[1]; // Remove the data URL prefix
const byteCharacters = atob(base64Data);
const byteNumbers = new Array(byteCharacters.length);
for (let i = 0; i < byteCharacters.length; i++) {
byteNumbers[i] = byteCharacters.charCodeAt(i);
}
const byteArray = new Uint8Array(byteNumbers);
const blob = new Blob([byteArray], { type });
return new File([blob], filename, { type });
} else {
const blob = new Blob([content], { type });
return new File([blob], filename, { type });
}
};
/**
* 把字符串转为文本文件
* @param content 字符串
* @param filename 文件名
* @returns 文件流
*/
export const toTextFile = (content: string = 'keep directory exist', filename: string = 'keep.txt') => {
const file = toFile(content, filename);
return file;
};

43
test/upload.ts Normal file
View File

@ -0,0 +1,43 @@
import { uploadFiles, UploadProgress } from '../src/query-upload.ts';
const div = document.createElement('div');
div.innerHTML = `
<input type="file" id="file" multiple />
<button id="upload">上传</button>
`;
document.body.appendChild(div);
const upload = document.getElementById('upload');
upload.addEventListener('click', () => {
const file = document.getElementById('file') as HTMLInputElement;
const files = file.files;
// console.log(files);
const uploadProgress = new UploadProgress({
onStart: () => {
console.log('开始上传');
},
onDone: () => {
console.log('上传完成');
},
onProgress: (progress: number, data: any) => {
console.log('上传进度', progress);
},
});
uploadFiles(
Array.from(files),
{
appKey: 'test',
version: '1.0.0',
username: 'test',
directory: 'test',
},
{
uploadProgress,
token: 'test',
createEventSource: (baseUrl: string, searchParams: URLSearchParams) => {
return new EventSource(baseUrl + '/api/s1/events?' + searchParams.toString());
},
FormDataFn: FormData,
},
);
});

28
tsconfig.json Normal file
View File

@ -0,0 +1,28 @@
{
"compilerOptions": {
"module": "nodenext",
"target": "esnext",
"noImplicitAny": false,
"outDir": "./dist",
"sourceMap": false,
"allowJs": true,
"newLine": "LF",
"baseUrl": "./",
"typeRoots": [
"node_modules/@types",
],
"declaration": true,
"noEmit": false,
"allowImportingTsExtensions": true,
"emitDeclarationOnly": true,
"moduleResolution": "NodeNext",
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"esModuleInterop": true,
"paths": {
"@/*": [
"src/*"
]
}
},
}

16
tsup.config.ts Normal file
View File

@ -0,0 +1,16 @@
import { defineConfig } from 'tsup';
export default defineConfig({
entry: ['src/query-upload.ts'],
splitting: false,
sourcemap: false,
clean: true,
format: 'esm',
dts: true,
outDir: 'dist',
tsconfig: 'tsconfig.json',
define: {
IS_BROWSER: 'false',
},
});