From 68332c9c8dd74751ee3b1a6944984ee02313f929 Mon Sep 17 00:00:00 2001 From: xion Date: Sun, 23 Mar 2025 16:47:25 +0800 Subject: [PATCH] add oss for download --- .gitignore | 1 + package.json | 7 ++- src/core/type.ts | 54 ++++++++++++++--- src/index.ts | 125 ++++++++++++++++++++++++++++++++++----- src/services/config.ts | 67 +++++++++++++++++++++ src/services/index.ts | 2 + src/test/config-admin.ts | 77 ++++++++++++++++++++++++ src/util/download.ts | 74 +++++++++++++++++++++++ src/util/hash.ts | 24 ++++++++ src/util/index.ts | 1 + tsup.config.ts | 4 +- 11 files changed, 410 insertions(+), 26 deletions(-) create mode 100644 src/services/config.ts create mode 100644 src/services/index.ts create mode 100644 src/test/config-admin.ts create mode 100644 src/util/download.ts create mode 100644 src/util/hash.ts create mode 100644 src/util/index.ts diff --git a/.gitignore b/.gitignore index f06235c..604e896 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules dist +.turbo \ No newline at end of file diff --git a/package.json b/package.json index ec6b0f7..6ab0aca 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@kevisual/oss", "version": "0.0.1", "description": "", - "main": "index.js", + "main": "dist/index.js", "scripts": { "build": "tsup", "dev": "tsup --watch", @@ -16,7 +16,12 @@ "license": "MIT", "type": "module", "devDependencies": { + "dotenv": "^16.4.7", "minio": "^8.0.5", "tsup": "^8.4.0" + }, + "exports": { + "./*": "./dist/*.js", + "./services": "./dist/services/index.js" } } \ No newline at end of file diff --git a/src/core/type.ts b/src/core/type.ts index dd9fe0b..83a0cff 100644 --- a/src/core/type.ts +++ b/src/core/type.ts @@ -1,13 +1,32 @@ import { ItemBucketMetadata, Client } from 'minio'; -type UploadedObjectInfo = { +export type UploadedObjectInfo = { + etag: string; + lastModified?: Date; + size?: number; + versionId: string; + metadata?: ItemBucketMetadata; +}; +export type StatObjectResult = { + size: number; etag: string; lastModified: Date; - size: number; - versionId: string; - metadata: ItemBucketMetadata; + metaData: ItemBucketMetadata; + versionId?: string | null; }; - +export type ListFileObject = { + name: string; + size: number; + lastModified: Date; + etag: string; +}; +export type ListDirectoryObject = { + prefix: string; + size: number; +}; +export type ListObjectResult = ListFileObject | ListDirectoryObject; export interface OssBaseOperation { + prefix: string; + setPrefix(prefix: string): void; /** * 获取对象 * @param objectName 对象名 @@ -29,7 +48,7 @@ export interface OssBaseOperation { * 获取对象信息 * @param objectName 对象名 */ - statObject(objectName: string): Promise; + statObject(objectName: string): Promise; /** * 删除对象 * @param objectName 对象名 @@ -37,9 +56,24 @@ export interface OssBaseOperation { deleteObject(objectName: string): Promise; /** * 列出对象 - * @param prefix 前缀 + * @param objectName 对象名 + * @param opts 选项 + * @param opts.recursive 是否递归 + * @param opts.startAfter 开始位置 */ - listObjects(prefix: string, opts?: { recursive?: boolean }): Promise; + listObjects( + objectName: string, + opts?: { + /** + * 是否递归 + */ + recursive?: boolean; + /** + * 开始位置 + */ + startAfter?: string; + }, + ): Promise; /** * 复制对象 * @param sourceObject 源对象 @@ -47,3 +81,7 @@ export interface OssBaseOperation { */ copyObject: Client['copyObject']; } + +export interface OssService extends OssBaseOperation { + owner: string; +} diff --git a/src/index.ts b/src/index.ts index 7f123c9..b562cb8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,8 @@ import { Client, ItemBucketMetadata } from 'minio'; -import { OssBaseOperation } from './core/type.ts'; -type OssBaseOptions = { +import { ListFileObject, ListObjectResult, OssBaseOperation } from './core/type.ts'; +import { hash } from './util/hash.ts'; + +export type OssBaseOptions = { /** * 已经初始化好的minio client */ @@ -9,51 +11,142 @@ type OssBaseOptions = { * 桶名 */ bucketName: string; -}; + /** + * 前缀 + */ + prefix?: string; +} & T; + export class OssBase implements OssBaseOperation { client?: Client; bucketName: string; + prefix = ''; + /** + * 计算字符串或者对象的的md5值 + */ + hash = hash; constructor(opts: OssBaseOptions) { if (!opts.client) { throw new Error('client is required'); } this.bucketName = opts.bucketName; this.client = opts.client; + this.prefix = opts.prefix; } + + setPrefix(prefix: string) { + this.prefix = prefix; + } + async getObject(objectName: string) { const bucketName = this.bucketName; - const obj = await this.client.getObject(bucketName, objectName); + const obj = await this.client.getObject(bucketName, `${this.prefix}${objectName}`); return obj; } - async putObject(objectName: string, data: Buffer | string, metaData?: ItemBucketMetadata) { - const size = data.length; - const bucketName = this.bucketName; - const obj = await this.client.putObject(bucketName, objectName, data, size, metaData); - return obj as any; + + async getJson(objectName: string): Promise> { + const obj = await this.getObject(objectName); + return new Promise((resolve, reject) => { + let data = ''; + obj.on('data', (chunk) => { + data += chunk; + }); + obj.on('end', () => { + try { + const jsonData = JSON.parse(data); + resolve(jsonData); + } catch (error) { + reject(new Error('Failed to parse JSON')); + } + }); + obj.on('error', (err) => { + reject(err); + }); + }); } + + async putObject(objectName: string, data: Buffer | string | Object, metaData?: ItemBucketMetadata) { + let putData: Buffer | string; + if (typeof data === 'object') { + putData = JSON.stringify(data); + } else { + putData = data; + } + const size = putData.length; + const bucketName = this.bucketName; + const obj = await this.client.putObject(bucketName, `${this.prefix}${objectName}`, putData, size, metaData); + return obj; + } + async deleteObject(objectName: string) { const bucketName = this.bucketName; - const obj = await this.client.removeObject(bucketName, objectName); + const obj = await this.client.removeObject(bucketName, `${this.prefix}${objectName}`); return obj; } - async listObjects(prefix: string) { + + async listObjects(objectName: string, opts?: { recursive?: boolean; startAfter?: string }) { const bucketName = this.bucketName; - const obj = this.client.listObjects(bucketName, prefix); - return obj; + const prefix = `${this.prefix}${objectName}`; + const res = await new Promise((resolve, reject) => { + let res: any[] = []; + let hasError = false; + this.client + .listObjectsV2(bucketName, prefix, opts?.recursive ?? false, opts?.startAfter) + .on('data', (data) => { + res.push(data); + }) + .on('error', (err) => { + console.error('minio error', prefix, err); + hasError = true; + }) + .on('end', () => { + if (hasError) { + reject(); + return; + } else { + resolve(res); + } + }); + }); + return res as IS_FILE extends true ? ListFileObject[] : ListObjectResult[]; } + async fPutObject(objectName: string, filePath: string, metaData?: ItemBucketMetadata) { const bucketName = this.bucketName; - const obj = await this.client.fPutObject(bucketName, objectName, filePath, metaData); + const obj = await this.client.fPutObject(bucketName, `${this.prefix}${objectName}`, filePath, metaData); return obj as any; } + async statObject(objectName: string) { const bucketName = this.bucketName; - const obj = await this.client.statObject(bucketName, objectName); - return obj; + try { + const obj = await this.client.statObject(bucketName, `${this.prefix}${objectName}`); + return obj; + } catch (e) { + if (e.code === 'NotFound') { + return null; + } + throw e; + } } + async copyObject(sourceObject: any, targetObject: any) { const bucketName = this.bucketName; const obj = await this.client.copyObject(bucketName, sourceObject, targetObject); return obj; } + + static create(this: new (opts: OssBaseOptions) => T, opts: OssBaseOptions): T { + return new this(opts); + } + + static fromBase(this: new (opts: OssBaseOptions) => T, createOpts: { oss: OssBase; opts: Partial> }): T { + const base = createOpts.oss; + const opts = createOpts.opts as any; + return new this({ + client: base.client, + bucketName: base.bucketName, + ...opts, + }); + } } diff --git a/src/services/config.ts b/src/services/config.ts new file mode 100644 index 0000000..e3fcf85 --- /dev/null +++ b/src/services/config.ts @@ -0,0 +1,67 @@ +import { OssBase, OssBaseOptions } from '../index.ts'; +import { OssService } from '../core/type.ts'; +import * as util from '../util/index.ts'; + +export class ConfigOssService extends OssBase implements OssService { + owner: string; + constructor(opts: OssBaseOptions<{ owner: string }>) { + super(opts); + this.owner = opts.owner; + this.setPrefix(`${this.owner}/config/1.0.0/`); + } + async listAllFile() { + const list = await this.listObjects(''); + return list.filter((item) => item.size > 0 && this.isEndWithJson(item.name)); + } + async listAll() { + return await this.listObjects(''); + } + configMap = new Map(); + keys: string[] = []; + async getAllConfigJson() { + const list = await this.listAllFile(); + return list; + } + isEndWithJson(string: string) { + return string?.endsWith?.(`.json`); + } + putJsonObject(key: string, data: any) { + const json = util.hashSringify(data); + return this.putObject(key, json, { + 'Content-Type': 'application/json', + 'app-source': 'user-config-app', + 'Cache-Control': 'max-age=31536000, immutable', + }); + } + async getObjectList(objectNameList: string[]) { + const jsonMap = new Map>(); + for (const objectName of objectNameList) { + try { + const json = await this.getJson(objectName); + jsonMap.set(objectName, json); + } catch (error) { + console.error(error); + continue; + } + } + return jsonMap; + } + async getList() { + const list = await this.listAllFile(); + /** + * key -> etag + */ + const keyEtagMap = new Map(); + const listKeys = list.map((item) => { + const key = item.name.replace(this.prefix, ''); + keyEtagMap.set(key, item.etag); + return { + ...item, + key, + }; + }); + type Etag = string; + const keys = Array.from(keyEtagMap.keys()); + return { list: listKeys, keys, keyEtagMap }; + } +} diff --git a/src/services/index.ts b/src/services/index.ts new file mode 100644 index 0000000..8354a2a --- /dev/null +++ b/src/services/index.ts @@ -0,0 +1,2 @@ +export { OssBase, OssBaseOptions } from '../index.ts'; +export { ConfigOssService } from './config.ts'; diff --git a/src/test/config-admin.ts b/src/test/config-admin.ts new file mode 100644 index 0000000..4e93c08 --- /dev/null +++ b/src/test/config-admin.ts @@ -0,0 +1,77 @@ +import dotenv from 'dotenv'; +import { ConfigOssService } from '../services/index.ts'; +import { Client } from 'minio'; +import path from 'path'; +import { downloadObject } from '../util/download.ts'; +const cwd = process.cwd(); +dotenv.config({ path: path.resolve(cwd, '..', '..', '.env') }); + +console.log(process.env.MINIO_ENDPOINT, process.env.MINIO_USE_SSL, process.env.MINIO_ACCESS_KEY, process.env.MINIO_SECRET_KEY, process.env.MINIO_BUCKET_NAME); +const client = new Client({ + endPoint: process.env.MINIO_ENDPOINT, + useSSL: process.env.MINIO_USE_SSL === 'true', + accessKey: process.env.MINIO_ACCESS_KEY, + secretKey: process.env.MINIO_SECRET_KEY, +}); +const configOssService = new ConfigOssService({ + client, + bucketName: process.env.MINIO_BUCKET_NAME, + owner: 'admin', +}); + +const main = async () => { + configOssService.setPrefix('root'); + const config = await configOssService.statObject('avatar.png'); + console.log(config); +}; + +// main(); +const putJson = async () => { + const config = await configOssService.putObject( + 'a.json', + { + a: 'a', + }, + { + 'content-type': 'application/json', + 'cache-control': 'no-cache', + }, + ); + console.log(config); +}; +// putJson(); +const downloadMain = async () => { + const stat = await configOssService.statObject('a.json'); // 582af9ef5cdc53d6628f45cb842f874a + console.log(stat); + const objectStream = await downloadObject({ + objectName: 'a.json', + client: configOssService, + filePath: path.resolve(cwd, 'a.json'), + }); + // console.log(objectStream); +}; +downloadMain(); + +const statInfo = async () => { + try { + const stat = await configOssService.statObject('a.json'); + // configOssService.setPrefix('root/'); + // const stat = await configOssService.statObject('avatar.png'); + // const stat = await configOssService.statObject('center/0.0.1/index.html'); + console.log(stat); + } catch (e) { + if (e.code === 'NotFound') { + console.log('not found'); + } else { + console.log('error', e); + } + } +}; +// statInfo(); + +const listObjects = async () => { + configOssService.setPrefix('root/avatar.png'); + const list = await configOssService.listObjects(''); + console.log(list); +}; +// listObjects(); diff --git a/src/util/download.ts b/src/util/download.ts new file mode 100644 index 0000000..82dd36e --- /dev/null +++ b/src/util/download.ts @@ -0,0 +1,74 @@ +import { ServerResponse } from 'http'; +import { BucketItemStat } from 'minio'; +import fs from 'fs'; +import path from 'path'; + +const viewableExtensions = ['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp', 'mp4', 'webm', 'mp3', 'wav', 'ogg', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx']; +import { OssBase } from '../index.ts'; +/** + * 过滤 metaData 中的 key, 去除 password, accesskey, secretkey， + * 并返回过滤后的 metaData + * @param metaData + * @returns + */ +export const filterMetaDataKeys = (metaData: Record, clearKeys: string[] = []) => { + const keys = Object.keys(metaData); + // remove X-Amz- meta data + const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys]; + const filteredKeys = keys.filter((key) => !removeKeys.includes(key)); + return filteredKeys.reduce((acc, key) => { + acc[key] = metaData[key]; + return acc; + }, {} as Record); +}; +type SendObjectOptions = { + res: ServerResponse; + client: OssBase; + objectName: string; + isDownload?: boolean; +}; +export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => { + res.writeHead(code, { 'Content-Type': 'text/plain' }); + res.end(msg || 'Not Found File'); + return; +}; +export const sendObject = async ({ res, objectName, client, isDownload = false }: SendObjectOptions) => { + let stat: BucketItemStat; + try { + stat = await client.statObject(objectName); + } catch (e) { + } finally { + if (!stat || stat.size === 0) { + return NotFoundFile(res); + } + const contentLength = stat.size; + const etag = stat.etag; + const lastModified = stat.lastModified.toISOString(); + const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName + const fileExtension = filename.split('.').pop()?.toLowerCase() || ''; + const filteredMetaData = filterMetaDataKeys(stat.metaData, ['size', 'etag', 'last-modified']); + const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`; + + res.writeHead(200, { + 'Content-Length': contentLength, + etag, + 'last-modified': lastModified, + 'Content-Disposition': contentDisposition, + 'file-name': filename, + ...filteredMetaData, + }); + const objectStream = await client.getObject(objectName); + + objectStream.pipe(res, { end: true }); + } +}; + +export const downloadObject = async ({ objectName, client, filePath }: Pick & { filePath: string }) => { + const objectStream = await client.getObject(objectName); + const dir = path.dirname(filePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + objectStream.pipe(fs.createWriteStream(filePath)); + return objectStream; +}; diff --git a/src/util/hash.ts b/src/util/hash.ts new file mode 100644 index 0000000..321db88 --- /dev/null +++ b/src/util/hash.ts @@ -0,0 +1,24 @@ +import crypto from 'crypto'; +// 582af9ef5cdc53d6628f45cb842f874a +// const hashStr = '{"a":"a"}'; +// const hash = crypto.createHash('md5').update(hashStr).digest('hex'); +// console.log(hash); + +/** + * 计算字符串的md5值 + * @param str + * @returns + */ +export const hash = (str: string | Buffer | Object) => { + let hashStr: string | Buffer; + if (typeof str === 'object') { + hashStr = JSON.stringify(str, null, 2); + } else { + hashStr = str; + } + return crypto.createHash('md5').update(hashStr).digest('hex'); +}; + +export const hashSringify = (str: Object) => { + return JSON.stringify(str, null, 2); +}; diff --git a/src/util/index.ts b/src/util/index.ts new file mode 100644 index 0000000..0f4b53d --- /dev/null +++ b/src/util/index.ts @@ -0,0 +1 @@ +export * from './hash.ts'; diff --git a/tsup.config.ts b/tsup.config.ts index a277145..fed798e 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -1,7 +1,9 @@ import { defineConfig } from 'tsup'; +import glob from 'fast-glob'; +const services = glob.sync('src/services/*.ts'); export default defineConfig({ - entry: ['src/index.ts'], + entry: ['src/index.ts', ...services], splitting: false, sourcemap: false,