Compare commits

...

9 Commits

Author SHA1 Message Date
7e3b748d30 fix bugs 2025-06-19 00:19:01 +08:00
cd43d66265 update getOBjectname 2025-06-07 12:09:09 +08:00
0dae4872b0 remove: check zero size 2025-05-30 19:47:45 +08:00
ab29a1eb0b update hash and put object 2025-05-12 17:37:30 +08:00
1c4e9d9bb9 fix 2025-05-12 16:23:51 +08:00
2dc567d8d7 fix 2025-05-12 04:29:29 +08:00
bcb8ea1f30 perf 2025-05-12 02:55:53 +08:00
1a2201cdc6 update npm registry 2025-03-28 07:53:03 +08:00
68332c9c8d add oss for download 2025-03-23 16:47:25 +08:00
13 changed files with 591 additions and 37 deletions

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
node_modules
dist
.turbo

View File

@@ -1,8 +1,7 @@
{
"name": "@kevisual/oss",
"version": "0.0.1",
"description": "",
"main": "index.js",
"version": "0.0.12",
"main": "dist/index.js",
"scripts": {
"build": "tsup",
"dev": "tsup --watch",
@@ -16,7 +15,25 @@
"license": "MIT",
"type": "module",
"devDependencies": {
"dotenv": "^16.5.0",
"minio": "^8.0.5",
"tsup": "^8.4.0"
},
"exports": {
".": {
"import": "./dist/index.js",
"types": "./dist/index.d.ts"
},
"./config": {
"import": "./dist/services/config.js",
"types": "./dist/services/config.d.ts"
},
"./services": {
"import": "./dist/services/index.js",
"types": "./dist/services/index.d.ts"
}
},
"publishConfig": {
"access": "public"
}
}

View File

@@ -3,7 +3,7 @@ import { Client, CopyDestinationOptions, CopySourceOptions } from 'minio';
type CopyObjectOpts = {
bucketName: string;
newMetadata: Record<string, string>;
prefix: string;
objectName: string;
client: Client;
};
/**
@@ -11,12 +11,14 @@ type CopyObjectOpts = {
* @param param0
* @returns
*/
export const copyObject = async ({ bucketName, newMetadata, prefix, client }: CopyObjectOpts) => {
const source = new CopySourceOptions({ Bucket: bucketName, Object: prefix });
export const copyObject = async ({ bucketName, newMetadata, objectName, client }: CopyObjectOpts) => {
const source = new CopySourceOptions({ Bucket: bucketName, Object: objectName });
const stat = await client.statObject(bucketName, objectName);
const sourceMetadata = stat.metaData;
const destination = new CopyDestinationOptions({
Bucket: bucketName,
Object: prefix,
UserMetadata: newMetadata,
Object: objectName,
UserMetadata: { ...sourceMetadata, ...newMetadata },
MetadataDirective: 'REPLACE',
});
const copyResult = await client.copyObject(source, destination);

View File

@@ -1,13 +1,32 @@
import { ItemBucketMetadata, Client } from 'minio';
type UploadedObjectInfo = {
export type UploadedObjectInfo = {
etag: string;
lastModified?: Date;
size?: number;
versionId: string;
metadata?: ItemBucketMetadata;
};
export type StatObjectResult = {
size: number;
etag: string;
lastModified: Date;
size: number;
versionId: string;
metadata: ItemBucketMetadata;
metaData: ItemBucketMetadata;
versionId?: string | null;
};
export type ListFileObject = {
name: string;
size: number;
lastModified: Date;
etag: string;
};
export type ListDirectoryObject = {
prefix: string;
size: number;
};
export type ListObjectResult = ListFileObject | ListDirectoryObject;
export interface OssBaseOperation {
prefix: string;
setPrefix(prefix: string): void;
/**
* 获取对象
* @param objectName 对象名
@@ -29,7 +48,7 @@ export interface OssBaseOperation {
* 获取对象信息
* @param objectName 对象名
*/
statObject(objectName: string): Promise<any>;
statObject(objectName: string): Promise<StatObjectResult>;
/**
* 删除对象
* @param objectName 对象名
@@ -37,9 +56,24 @@ export interface OssBaseOperation {
deleteObject(objectName: string): Promise<any>;
/**
* 列出对象
* @param prefix 前缀
* @param objectName 对象名
* @param opts 选项
* @param opts.recursive 是否递归
* @param opts.startAfter 开始位置
*/
listObjects(prefix: string, opts?: { recursive?: boolean }): Promise<any>;
listObjects(
objectName: string,
opts?: {
/**
* 是否递归
*/
recursive?: boolean;
/**
* 开始位置
*/
startAfter?: string;
},
): Promise<ListObjectResult[]>;
/**
* 复制对象
* @param sourceObject 源对象
@@ -47,3 +81,7 @@ export interface OssBaseOperation {
*/
copyObject: Client['copyObject'];
}
export interface OssService extends OssBaseOperation {
owner: string;
}

View File

@@ -1,59 +1,242 @@
import { Client, ItemBucketMetadata } from 'minio';
import { OssBaseOperation } from './core/type.ts';
type OssBaseOptions = {
import { ListFileObject, ListObjectResult, OssBaseOperation } from './core/type.ts';
import { hash } from './util/hash.ts';
import { copyObject } from './core/copy-object.ts';
import omit from 'lodash/omit.js';
import { getContentType } from './util/get-content-type.ts';
export type OssBaseOptions<T = { [key: string]: any }> = {
/**
* 已经初始化好的minio client
*/
client?: Client;
client: Client;
/**
* 桶名
*/
bucketName: string;
};
/**
* 前缀
*/
prefix?: string;
} & T;
export class OssBase implements OssBaseOperation {
client?: Client;
bucketName: string;
prefix = '';
/**
* 计算字符串或者对象的的md5值
*/
hash = hash;
constructor(opts: OssBaseOptions) {
if (!opts.client) {
throw new Error('client is required');
}
this.bucketName = opts.bucketName;
this.client = opts.client;
this.prefix = opts?.prefix ?? '';
}
setPrefix(prefix: string) {
this.prefix = prefix;
}
async getObject(objectName: string) {
const bucketName = this.bucketName;
const obj = await this.client.getObject(bucketName, objectName);
const obj = await this.client.getObject(bucketName, `${this.prefix}${objectName}`);
return obj;
}
async putObject(objectName: string, data: Buffer | string, metaData?: ItemBucketMetadata) {
const size = data.length;
const bucketName = this.bucketName;
const obj = await this.client.putObject(bucketName, objectName, data, size, metaData);
return obj as any;
async getJson(objectName: string): Promise<Record<string, any>> {
const obj = await this.getObject(objectName);
return new Promise((resolve, reject) => {
let data = '';
obj.on('data', (chunk) => {
data += chunk;
});
obj.on('end', () => {
try {
const jsonData = JSON.parse(data);
resolve(jsonData);
} catch (error) {
reject(new Error('Failed to parse JSON'));
}
});
obj.on('error', (err) => {
reject(err);
});
});
}
/**
* 上传文件, 当是流的时候中断之后的etag会变所以传递的时候不要嵌套async await例如 busboy 监听文件流内部的时候不要用check
* @param objectName
* @param data
* @param metaData
* @param options 如果文件本身存在则复制原有的meta的内容
* @returns
*/
async putObject(
objectName: string,
data: Buffer | string | Object,
metaData: ItemBucketMetadata = {},
opts?: { check?: boolean; isStream?: boolean; size?: number },
) {
let putData: Buffer | string;
let size: number = opts?.size;
const isStream = opts?.isStream;
if (!isStream) {
if (data instanceof Object) {
putData = JSON.stringify(data);
size = putData.length;
} else if (typeof data === 'string') {
putData = data;
size = putData.length;
} else {
putData = data;
}
} else {
putData = data as any;
size = null;
}
if (opts?.check) {
const obj = await this.statObject(objectName, true);
if (obj) {
const omitMeda = ['size', 'content-type', 'cache-control', 'app-source'];
const objMeta = JSON.parse(JSON.stringify(omit(obj.metaData, omitMeda)));
metaData = {
...objMeta,
...metaData,
};
}
}
const bucketName = this.bucketName;
const obj = await this.client.putObject(bucketName, `${this.prefix}${objectName}`, putData, size, metaData);
return obj;
}
async deleteObject(objectName: string) {
const bucketName = this.bucketName;
const obj = await this.client.removeObject(bucketName, objectName);
const obj = await this.client.removeObject(bucketName, `${this.prefix}${objectName}`);
return obj;
}
async listObjects(prefix: string) {
async listObjects<IS_FILE = false>(objectName: string, opts?: { recursive?: boolean; startAfter?: string }) {
const bucketName = this.bucketName;
const obj = this.client.listObjects(bucketName, prefix);
return obj;
const prefix = `${this.prefix}${objectName}`;
const res = await new Promise((resolve, reject) => {
let res: any[] = [];
let hasError = false;
this.client
.listObjectsV2(bucketName, prefix, opts?.recursive ?? false, opts?.startAfter)
.on('data', (data) => {
res.push(data);
})
.on('error', (err) => {
console.error('minio error', prefix, err);
hasError = true;
})
.on('end', () => {
if (hasError) {
reject();
return;
} else {
resolve(res);
}
});
});
return res as IS_FILE extends true ? ListFileObject[] : ListObjectResult[];
}
async fPutObject(objectName: string, filePath: string, metaData?: ItemBucketMetadata) {
const bucketName = this.bucketName;
const obj = await this.client.fPutObject(bucketName, objectName, filePath, metaData);
const obj = await this.client.fPutObject(bucketName, `${this.prefix}${objectName}`, filePath, metaData);
return obj as any;
}
async statObject(objectName: string) {
const bucketName = this.bucketName;
const obj = await this.client.statObject(bucketName, objectName);
return obj;
/**
* 获取完整的对象名称
* @param objectName
* @returns
*/
async getObjectName(objectName: string) {
return `${this.prefix}${objectName}`;
}
async statObject(objectName: string, checkFile = true) {
const bucketName = this.bucketName;
try {
const obj = await this.client.statObject(bucketName, `${this.prefix}${objectName}`);
return obj;
} catch (e) {
if (e.code === 'NotFound') {
return null;
}
throw e;
}
}
/**
* 检查文件hash是否一致
* @param objectName
* @param hash
* @returns
*/
async checkObjectHash(
objectName: string,
hash: string,
meta?: ItemBucketMetadata,
): Promise<{ success: boolean; metaData: ItemBucketMetadata | null; obj: any; equalMeta?: boolean }> {
const obj = await this.statObject(`${this.prefix}${objectName}`, true);
if (!obj) {
return { success: false, metaData: null, obj: null, equalMeta: false };
}
let metaData: ItemBucketMetadata = {};
const omitMeda = ['content-type', 'cache-control', 'app-source'];
const objMeta = omit(obj.metaData, omitMeda);
metaData = {
...objMeta,
};
let equalMeta = false;
if (meta) {
equalMeta = JSON.stringify(metaData) === JSON.stringify(meta);
}
return { success: obj.etag === hash, metaData, obj, equalMeta };
}
getMetadata(pathname: string, meta: ItemBucketMetadata = { 'app-source': 'user-app' }) {
const isHtml = pathname.endsWith('.html');
if (isHtml) {
meta = {
...meta,
'content-type': 'text/html; charset=utf-8',
'cache-control': 'no-cache',
};
} else {
meta = {
...meta,
'content-type': getContentType(pathname),
'cache-control': 'max-age=31536000, immutable',
};
}
return meta;
}
async copyObject(sourceObject: any, targetObject: any) {
const bucketName = this.bucketName;
const obj = await this.client.copyObject(bucketName, sourceObject, targetObject);
return obj;
}
async replaceObject(objectName: string, meta: { [key: string]: string }) {
const { bucketName, client } = this;
return copyObject({ bucketName, client, objectName: `${this.prefix}${objectName}`, newMetadata: meta });
}
static create<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, opts: OssBaseOptions<U>): T {
return new this(opts);
}
static fromBase<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, createOpts: { oss: OssBase; opts: Partial<OssBaseOptions<U>> }): T {
const base = createOpts.oss;
const opts = createOpts.opts as any;
return new this({
client: base.client,
bucketName: base.bucketName,
...opts,
});
}
}

67
src/services/config.ts Normal file
View File

@@ -0,0 +1,67 @@
import { OssBase, OssBaseOptions } from '../index.ts';
import { OssService } from '../core/type.ts';
import * as util from '../util/index.ts';
export class ConfigOssService extends OssBase implements OssService {
owner: string;
constructor(opts: OssBaseOptions<{ owner: string }>) {
super(opts);
this.owner = opts.owner;
this.setPrefix(`${this.owner}/config/1.0.0/`);
}
async listAllFile() {
const list = await this.listObjects<true>('');
return list.filter((item) => item.size > 0 && this.isEndWithJson(item.name));
}
async listAll() {
return await this.listObjects<false>('');
}
configMap = new Map<string, any>();
keys: string[] = [];
async getAllConfigJson() {
const list = await this.listAllFile();
return list;
}
isEndWithJson(string: string) {
return string?.endsWith?.(`.json`);
}
putJsonObject(key: string, data: any) {
const json = util.hashSringify(data);
return this.putObject(key, json, {
'Content-Type': 'application/json',
'app-source': 'user-config-app',
'Cache-Control': 'max-age=31536000, immutable',
});
}
async getObjectList(objectNameList: string[]) {
const jsonMap = new Map<string, Record<string, any>>();
for (const objectName of objectNameList) {
try {
const json = await this.getJson(objectName);
jsonMap.set(objectName, json);
} catch (error) {
console.error(error);
continue;
}
}
return jsonMap;
}
async getList() {
const list = await this.listAllFile();
/**
* key -> etag
*/
const keyEtagMap = new Map<string, Etag>();
const listKeys = list.map((item) => {
const key = item.name.replace(this.prefix, '');
keyEtagMap.set(key, item.etag);
return {
...item,
key,
};
});
type Etag = string;
const keys = Array.from(keyEtagMap.keys());
return { list: listKeys, keys, keyEtagMap };
}
}

9
src/services/index.ts Normal file
View File

@@ -0,0 +1,9 @@
export { OssBase } from '../index.ts';
export type { OssBaseOptions } from '../index.ts';
export { ConfigOssService } from './config.ts';
export * from '../util/download.ts';
export * from '../util/index.ts';
export * from '../core/type.ts';

84
src/test/config-admin.ts Normal file
View File

@@ -0,0 +1,84 @@
import dotenv from 'dotenv';
import { ConfigOssService } from '../services/index.ts';
import { Client } from 'minio';
import path from 'path';
import { downloadObject } from '../util/download.ts';
const cwd = process.cwd();
dotenv.config({ path: path.resolve(cwd, '..', '..', '.env.dev') });
console.log(
'config',
process.env.MINIO_ENDPOINT,
process.env.MINIO_USE_SSL,
process.env.MINIO_ACCESS_KEY,
process.env.MINIO_SECRET_KEY,
process.env.MINIO_BUCKET_NAME,
);
const client = new Client({
endPoint: process.env.MINIO_ENDPOINT,
useSSL: process.env.MINIO_USE_SSL === 'true',
accessKey: process.env.MINIO_ACCESS_KEY,
secretKey: process.env.MINIO_SECRET_KEY,
});
const configOssService = new ConfigOssService({
client,
bucketName: process.env.MINIO_BUCKET_NAME,
owner: 'admin',
});
const main = async () => {
configOssService.setPrefix('root');
const config = await configOssService.statObject('avatar.png');
console.log(config);
};
main();
const putJson = async () => {
const config = await configOssService.putObject(
'a.json',
{
a: 'a',
},
{
'content-type': 'application/json',
'cache-control': 'no-cache',
},
);
console.log(config);
};
// putJson();
const downloadMain = async () => {
const stat = await configOssService.statObject('a.json'); // 582af9ef5cdc53d6628f45cb842f874a
console.log(stat);
const objectStream = await downloadObject({
objectName: 'a.json',
client: configOssService,
filePath: path.resolve(cwd, 'a.json'),
});
// console.log(objectStream);
};
// downloadMain();
const statInfo = async () => {
try {
const stat = await configOssService.statObject('a.json');
// configOssService.setPrefix('root/');
// const stat = await configOssService.statObject('avatar.png');
// const stat = await configOssService.statObject('center/0.0.1/index.html');
console.log(stat);
} catch (e) {
if (e.code === 'NotFound') {
console.log('not found');
} else {
console.log('error', e);
}
}
};
// statInfo();
const listObjects = async () => {
configOssService.setPrefix('root/avatar.png');
const list = await configOssService.listObjects('');
console.log(list);
};
// listObjects();

73
src/util/download.ts Normal file
View File

@@ -0,0 +1,73 @@
import { ServerResponse } from 'node:http';
import { BucketItemStat } from 'minio';
import fs from 'node:fs';
import path from 'node:path';
const viewableExtensions = ['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp', 'mp4', 'webm', 'mp3', 'wav', 'ogg', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx'];
import { OssBase } from '../index.ts';
/**
* 过滤 metaData 中的 key, 去除 password, accesskey, secretkey
* 并返回过滤后的 metaData
* @param metaData
* @returns
*/
export const filterMetaDataKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
const keys = Object.keys(metaData);
// remove X-Amz- meta data
const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
return filteredKeys.reduce((acc, key) => {
acc[key] = metaData[key];
return acc;
}, {} as Record<string, string>);
};
type SendObjectOptions = {
res: ServerResponse;
client: OssBase;
objectName: string;
isDownload?: boolean;
};
export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
res.writeHead(code, { 'Content-Type': 'text/plain' });
res.end(msg || 'Not Found File');
return;
};
export const sendObject = async ({ res, objectName, client, isDownload = false }: SendObjectOptions) => {
let stat: BucketItemStat;
try {
stat = await client.statObject(objectName);
} catch (e) {
} finally {
if (!stat || stat.size === 0) {
return NotFoundFile(res);
}
const contentLength = stat.size;
const etag = stat.etag;
const lastModified = stat.lastModified.toISOString();
const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
const filteredMetaData = filterMetaDataKeys(stat.metaData, ['size', 'etag', 'last-modified']);
const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
res.writeHead(200, {
'Content-Length': contentLength,
etag,
'last-modified': lastModified,
'Content-Disposition': contentDisposition,
...filteredMetaData,
});
const objectStream = await client.getObject(objectName);
objectStream.pipe(res, { end: true });
}
};
export const downloadObject = async ({ objectName, client, filePath }: Pick<SendObjectOptions, 'objectName' | 'client'> & { filePath: string }) => {
const objectStream = await client.getObject(objectName);
const dir = path.dirname(filePath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
objectStream.pipe(fs.createWriteStream(filePath));
return objectStream;
};

View File

@@ -0,0 +1,49 @@
import path from 'node:path';
// 获取文件的 content-type
export const getContentType = (filePath: string) => {
const extname = path.extname(filePath);
const contentType = {
'.html': 'text/html; charset=utf-8',
'.js': 'text/javascript; charset=utf-8',
'.css': 'text/css; charset=utf-8',
'.txt': 'text/plain; charset=utf-8',
'.json': 'application/json; charset=utf-8',
'.png': 'image/png',
'.jpg': 'image/jpg',
'.gif': 'image/gif',
'.svg': 'image/svg+xml',
'.wav': 'audio/wav',
'.mp4': 'video/mp4',
'.md': 'text/markdown; charset=utf-8', // utf-8配置
'.ico': 'image/x-icon', // Favicon 图标
'.webp': 'image/webp', // WebP 图像格式
'.webm': 'video/webm', // WebM 视频格式
'.ogg': 'audio/ogg', // Ogg 音频格式
'.mp3': 'audio/mpeg', // MP3 音频格式
'.m4a': 'audio/mp4', // M4A 音频格式
'.m3u8': 'application/vnd.apple.mpegurl', // HLS 播放列表
'.ts': 'video/mp2t', // MPEG Transport Stream
'.pdf': 'application/pdf', // PDF 文档
'.doc': 'application/msword', // Word 文档
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', // Word 文档 (新版)
'.ppt': 'application/vnd.ms-powerpoint', // PowerPoint 演示文稿
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', // PowerPoint (新版)
'.xls': 'application/vnd.ms-excel', // Excel 表格
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // Excel 表格 (新版)
'.csv': 'text/csv; charset=utf-8', // CSV 文件
'.xml': 'application/xml; charset=utf-8', // XML 文件
'.rtf': 'application/rtf', // RTF 文本文件
'.eot': 'application/vnd.ms-fontobject', // Embedded OpenType 字体
'.ttf': 'font/ttf', // TrueType 字体
'.woff': 'font/woff', // Web Open Font Format 1.0
'.woff2': 'font/woff2', // Web Open Font Format 2.0
'.otf': 'font/otf', // OpenType 字体
'.wasm': 'application/wasm', // WebAssembly 文件
'.pem': 'application/x-pem-file', // PEM 证书文件
'.crt': 'application/x-x509-ca-cert', // CRT 证书文件
'.yaml': 'application/x-yaml; charset=utf-8', // YAML 文件
'.yml': 'application/x-yaml; charset=utf-8', // YAML 文件(别名)
'.zip': 'application/octet-stream',
};
return contentType[extname] || 'application/octet-stream';
};

26
src/util/hash.ts Normal file
View File

@@ -0,0 +1,26 @@
import crypto from 'node:crypto';
// 582af9ef5cdc53d6628f45cb842f874a
// const hashStr = '{"a":"a"}';
// const hash = crypto.createHash('md5').update(hashStr).digest('hex');
// console.log(hash);
/**
* 计算字符串的md5值
* @param str
* @returns
*/
export const hash = (str: string | Buffer | Object) => {
let hashStr: string | Buffer;
if (str instanceof Buffer) {
hashStr = str;
} else if (str instanceof Object) {
hashStr = JSON.stringify(str, null, 2);
} else {
hashStr = str;
}
return crypto.createHash('md5').update(hashStr).digest('hex');
};
export const hashSringify = (str: Object) => {
return JSON.stringify(str, null, 2);
};

3
src/util/index.ts Normal file
View File

@@ -0,0 +1,3 @@
export * from './hash.ts';
export * from './get-content-type.ts';

View File

@@ -1,8 +1,10 @@
import { defineConfig } from 'tsup';
import glob from 'fast-glob';
const services = glob.sync('src/services/*.ts');
export default defineConfig({
entry: ['src/index.ts'],
entry: ['src/index.ts', ...services],
target: 'node22',
splitting: false,
sourcemap: false,
clean: true,