add oss for download

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 dist
+.turbo

package.json

@@ -2,7 +2,7 @@
   "name": "@kevisual/oss",
   "version": "0.0.1",
   "description": "",
-  "main": "index.js",
+  "main": "dist/index.js",
   "scripts": {
     "build": "tsup",
     "dev": "tsup --watch",
@@ -16,7 +16,12 @@
   "license": "MIT",
   "type": "module",
   "devDependencies": {
+    "dotenv": "^16.4.7",
     "minio": "^8.0.5",
     "tsup": "^8.4.0"
+  },
+  "exports": {
+    "./*": "./dist/*.js",
+    "./services": "./dist/services/index.js"
   }
 }

src/core/type.ts

@@ -1,13 +1,32 @@
 import { ItemBucketMetadata, Client } from 'minio';
-type UploadedObjectInfo = {
+export type UploadedObjectInfo = {
+  etag: string;
+  lastModified?: Date;
+  size?: number;
+  versionId: string;
+  metadata?: ItemBucketMetadata;
+};
+export type StatObjectResult = {
+  size: number;
   etag: string;
   lastModified: Date;
-  size: number;
+  metaData: ItemBucketMetadata;
-  versionId: string;
+  versionId?: string | null;
-  metadata: ItemBucketMetadata;
 };
-
+export type ListFileObject = {
+  name: string;
+  size: number;
+  lastModified: Date;
+  etag: string;
+};
+export type ListDirectoryObject = {
+  prefix: string;
+  size: number;
+};
+export type ListObjectResult = ListFileObject | ListDirectoryObject;
 export interface OssBaseOperation {
+  prefix: string;
+  setPrefix(prefix: string): void;
   /**
    * 获取对象
    * @param objectName 对象名
@@ -29,7 +48,7 @@ export interface OssBaseOperation {
    * 获取对象信息
    * @param objectName 对象名
    */
-  statObject(objectName: string): Promise<any>;
+  statObject(objectName: string): Promise<StatObjectResult>;
   /**
    * 删除对象
    * @param objectName 对象名
@@ -37,9 +56,24 @@ export interface OssBaseOperation {
   deleteObject(objectName: string): Promise<any>;
   /**
    * 列出对象
-   * @param prefix 前缀
+   * @param objectName 对象名
+   * @param opts 选项
+   * @param opts.recursive 是否递归
+   * @param opts.startAfter 开始位置
    */
-  listObjects(prefix: string, opts?: { recursive?: boolean }): Promise<any>;
+  listObjects(
+    objectName: string,
+    opts?: {
+      /**
+       * 是否递归
+       */
+      recursive?: boolean;
+      /**
+       * 开始位置
+       */
+      startAfter?: string;
+    },
+  ): Promise<ListObjectResult[]>;
   /**
    * 复制对象
    * @param sourceObject 源对象
@@ -47,3 +81,7 @@ export interface OssBaseOperation {
    */
   copyObject: Client['copyObject'];
 }
+
+export interface OssService extends OssBaseOperation {
+  owner: string;
+}

src/index.ts

@@ -1,6 +1,8 @@
 import { Client, ItemBucketMetadata } from 'minio';
-import { OssBaseOperation } from './core/type.ts';
+import { ListFileObject, ListObjectResult, OssBaseOperation } from './core/type.ts';
-type OssBaseOptions = {
+import { hash } from './util/hash.ts';
+
+export type OssBaseOptions<T = { [key: string]: any }> = {
   /**
    * 已经初始化好的minio client
    */
@@ -9,51 +11,142 @@ type OssBaseOptions = {
    * 桶名
    */
   bucketName: string;
-};
+  /**
+   * 前缀
+   */
+  prefix?: string;
+} & T;
+
 export class OssBase implements OssBaseOperation {
   client?: Client;
   bucketName: string;
+  prefix = '';
+  /**
+   * 计算字符串或者对象的的md5值
+   */
+  hash = hash;
   constructor(opts: OssBaseOptions) {
     if (!opts.client) {
       throw new Error('client is required');
     }
     this.bucketName = opts.bucketName;
     this.client = opts.client;
+    this.prefix = opts.prefix;
   }
+
+  setPrefix(prefix: string) {
+    this.prefix = prefix;
+  }
+
   async getObject(objectName: string) {
     const bucketName = this.bucketName;
-    const obj = await this.client.getObject(bucketName, objectName);
+    const obj = await this.client.getObject(bucketName, `${this.prefix}${objectName}`);
     return obj;
   }
-  async putObject(objectName: string, data: Buffer | string, metaData?: ItemBucketMetadata) {
+
-    const size = data.length;
+  async getJson(objectName: string): Promise<Record<string, any>> {
-    const bucketName = this.bucketName;
+    const obj = await this.getObject(objectName);
-    const obj = await this.client.putObject(bucketName, objectName, data, size, metaData);
+    return new Promise((resolve, reject) => {
-    return obj as any;
+      let data = '';
+      obj.on('data', (chunk) => {
+        data += chunk;
+      });
+      obj.on('end', () => {
+        try {
+          const jsonData = JSON.parse(data);
+          resolve(jsonData);
+        } catch (error) {
+          reject(new Error('Failed to parse JSON'));
         }
+      });
+      obj.on('error', (err) => {
+        reject(err);
+      });
+    });
+  }
+
+  async putObject(objectName: string, data: Buffer | string | Object, metaData?: ItemBucketMetadata) {
+    let putData: Buffer | string;
+    if (typeof data === 'object') {
+      putData = JSON.stringify(data);
+    } else {
+      putData = data;
+    }
+    const size = putData.length;
+    const bucketName = this.bucketName;
+    const obj = await this.client.putObject(bucketName, `${this.prefix}${objectName}`, putData, size, metaData);
+    return obj;
+  }
+
   async deleteObject(objectName: string) {
     const bucketName = this.bucketName;
-    const obj = await this.client.removeObject(bucketName, objectName);
+    const obj = await this.client.removeObject(bucketName, `${this.prefix}${objectName}`);
     return obj;
   }
-  async listObjects(prefix: string) {
+
+  async listObjects<IS_FILE = false>(objectName: string, opts?: { recursive?: boolean; startAfter?: string }) {
     const bucketName = this.bucketName;
-    const obj = this.client.listObjects(bucketName, prefix);
+    const prefix = `${this.prefix}${objectName}`;
-    return obj;
+    const res = await new Promise((resolve, reject) => {
+      let res: any[] = [];
+      let hasError = false;
+      this.client
+        .listObjectsV2(bucketName, prefix, opts?.recursive ?? false, opts?.startAfter)
+        .on('data', (data) => {
+          res.push(data);
+        })
+        .on('error', (err) => {
+          console.error('minio error', prefix, err);
+          hasError = true;
+        })
+        .on('end', () => {
+          if (hasError) {
+            reject();
+            return;
+          } else {
+            resolve(res);
           }
+        });
+    });
+    return res as IS_FILE extends true ? ListFileObject[] : ListObjectResult[];
+  }
+
   async fPutObject(objectName: string, filePath: string, metaData?: ItemBucketMetadata) {
     const bucketName = this.bucketName;
-    const obj = await this.client.fPutObject(bucketName, objectName, filePath, metaData);
+    const obj = await this.client.fPutObject(bucketName, `${this.prefix}${objectName}`, filePath, metaData);
     return obj as any;
   }
+
   async statObject(objectName: string) {
     const bucketName = this.bucketName;
-    const obj = await this.client.statObject(bucketName, objectName);
+    try {
+      const obj = await this.client.statObject(bucketName, `${this.prefix}${objectName}`);
       return obj;
+    } catch (e) {
+      if (e.code === 'NotFound') {
+        return null;
       }
+      throw e;
+    }
+  }
+
   async copyObject(sourceObject: any, targetObject: any) {
     const bucketName = this.bucketName;
     const obj = await this.client.copyObject(bucketName, sourceObject, targetObject);
     return obj;
   }
+
+  static create<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, opts: OssBaseOptions<U>): T {
+    return new this(opts);
+  }
+
+  static fromBase<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, createOpts: { oss: OssBase; opts: Partial<OssBaseOptions<U>> }): T {
+    const base = createOpts.oss;
+    const opts = createOpts.opts as any;
+    return new this({
+      client: base.client,
+      bucketName: base.bucketName,
+      ...opts,
+    });
+  }
 }

src/services/config.ts

@@ -0,0 +1,67 @@
+import { OssBase, OssBaseOptions } from '../index.ts';
+import { OssService } from '../core/type.ts';
+import * as util from '../util/index.ts';
+
+export class ConfigOssService extends OssBase implements OssService {
+  owner: string;
+  constructor(opts: OssBaseOptions<{ owner: string }>) {
+    super(opts);
+    this.owner = opts.owner;
+    this.setPrefix(`${this.owner}/config/1.0.0/`);
+  }
+  async listAllFile() {
+    const list = await this.listObjects<true>('');
+    return list.filter((item) => item.size > 0 && this.isEndWithJson(item.name));
+  }
+  async listAll() {
+    return await this.listObjects<false>('');
+  }
+  configMap = new Map<string, any>();
+  keys: string[] = [];
+  async getAllConfigJson() {
+    const list = await this.listAllFile();
+    return list;
+  }
+  isEndWithJson(string: string) {
+    return string?.endsWith?.(`.json`);
+  }
+  putJsonObject(key: string, data: any) {
+    const json = util.hashSringify(data);
+    return this.putObject(key, json, {
+      'Content-Type': 'application/json',
+      'app-source': 'user-config-app',
+      'Cache-Control': 'max-age=31536000, immutable',
+    });
+  }
+  async getObjectList(objectNameList: string[]) {
+    const jsonMap = new Map<string, Record<string, any>>();
+    for (const objectName of objectNameList) {
+      try {
+        const json = await this.getJson(objectName);
+        jsonMap.set(objectName, json);
+      } catch (error) {
+        console.error(error);
+        continue;
+      }
+    }
+    return jsonMap;
+  }
+  async getList() {
+    const list = await this.listAllFile();
+    /**
+     * key -> etag
+     */
+    const keyEtagMap = new Map<string, Etag>();
+    const listKeys = list.map((item) => {
+      const key = item.name.replace(this.prefix, '');
+      keyEtagMap.set(key, item.etag);
+      return {
+        ...item,
+        key,
+      };
+    });
+    type Etag = string;
+    const keys = Array.from(keyEtagMap.keys());
+    return { list: listKeys, keys, keyEtagMap };
+  }
+}

src/services/index.ts

@@ -0,0 +1,2 @@
+export { OssBase, OssBaseOptions } from '../index.ts';
+export { ConfigOssService } from './config.ts';

src/test/config-admin.ts

@@ -0,0 +1,77 @@
+import dotenv from 'dotenv';
+import { ConfigOssService } from '../services/index.ts';
+import { Client } from 'minio';
+import path from 'path';
+import { downloadObject } from '../util/download.ts';
+const cwd = process.cwd();
+dotenv.config({ path: path.resolve(cwd, '..', '..', '.env') });
+
+console.log(process.env.MINIO_ENDPOINT, process.env.MINIO_USE_SSL, process.env.MINIO_ACCESS_KEY, process.env.MINIO_SECRET_KEY, process.env.MINIO_BUCKET_NAME);
+const client = new Client({
+  endPoint: process.env.MINIO_ENDPOINT,
+  useSSL: process.env.MINIO_USE_SSL === 'true',
+  accessKey: process.env.MINIO_ACCESS_KEY,
+  secretKey: process.env.MINIO_SECRET_KEY,
+});
+const configOssService = new ConfigOssService({
+  client,
+  bucketName: process.env.MINIO_BUCKET_NAME,
+  owner: 'admin',
+});
+
+const main = async () => {
+  configOssService.setPrefix('root');
+  const config = await configOssService.statObject('avatar.png');
+  console.log(config);
+};
+
+// main();
+const putJson = async () => {
+  const config = await configOssService.putObject(
+    'a.json',
+    {
+      a: 'a',
+    },
+    {
+      'content-type': 'application/json',
+      'cache-control': 'no-cache',
+    },
+  );
+  console.log(config);
+};
+// putJson();
+const downloadMain = async () => {
+  const stat = await configOssService.statObject('a.json'); // 582af9ef5cdc53d6628f45cb842f874a
+  console.log(stat);
+  const objectStream = await downloadObject({
+    objectName: 'a.json',
+    client: configOssService,
+    filePath: path.resolve(cwd, 'a.json'),
+  });
+  // console.log(objectStream);
+};
+downloadMain();
+
+const statInfo = async () => {
+  try {
+    const stat = await configOssService.statObject('a.json');
+    // configOssService.setPrefix('root/');
+    // const stat = await configOssService.statObject('avatar.png');
+    // const stat = await configOssService.statObject('center/0.0.1/index.html');
+    console.log(stat);
+  } catch (e) {
+    if (e.code === 'NotFound') {
+      console.log('not found');
+    } else {
+      console.log('error', e);
+    }
+  }
+};
+// statInfo();
+
+const listObjects = async () => {
+  configOssService.setPrefix('root/avatar.png');
+  const list = await configOssService.listObjects('');
+  console.log(list);
+};
+// listObjects();

src/util/download.ts

@@ -0,0 +1,74 @@
+import { ServerResponse } from 'http';
+import { BucketItemStat } from 'minio';
+import fs from 'fs';
+import path from 'path';
+
+const viewableExtensions = ['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp', 'mp4', 'webm', 'mp3', 'wav', 'ogg', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx'];
+import { OssBase } from '../index.ts';
+/**
+ * 过滤 metaData 中的 key, 去除 password, accesskey, secretkey，
+ * 并返回过滤后的 metaData
+ * @param metaData
+ * @returns
+ */
+export const filterMetaDataKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
+  const keys = Object.keys(metaData);
+  // remove X-Amz- meta data
+  const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
+  const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
+  return filteredKeys.reduce((acc, key) => {
+    acc[key] = metaData[key];
+    return acc;
+  }, {} as Record<string, string>);
+};
+type SendObjectOptions = {
+  res: ServerResponse;
+  client: OssBase;
+  objectName: string;
+  isDownload?: boolean;
+};
+export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
+  res.writeHead(code, { 'Content-Type': 'text/plain' });
+  res.end(msg || 'Not Found File');
+  return;
+};
+export const sendObject = async ({ res, objectName, client, isDownload = false }: SendObjectOptions) => {
+  let stat: BucketItemStat;
+  try {
+    stat = await client.statObject(objectName);
+  } catch (e) {
+  } finally {
+    if (!stat || stat.size === 0) {
+      return NotFoundFile(res);
+    }
+    const contentLength = stat.size;
+    const etag = stat.etag;
+    const lastModified = stat.lastModified.toISOString();
+    const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
+    const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
+    const filteredMetaData = filterMetaDataKeys(stat.metaData, ['size', 'etag', 'last-modified']);
+    const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
+
+    res.writeHead(200, {
+      'Content-Length': contentLength,
+      etag,
+      'last-modified': lastModified,
+      'Content-Disposition': contentDisposition,
+      'file-name': filename,
+      ...filteredMetaData,
+    });
+    const objectStream = await client.getObject(objectName);
+
+    objectStream.pipe(res, { end: true });
+  }
+};
+
+export const downloadObject = async ({ objectName, client, filePath }: Pick<SendObjectOptions, 'objectName' | 'client'> & { filePath: string }) => {
+  const objectStream = await client.getObject(objectName);
+  const dir = path.dirname(filePath);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+  objectStream.pipe(fs.createWriteStream(filePath));
+  return objectStream;
+};

src/util/hash.ts

@@ -0,0 +1,24 @@
+import crypto from 'crypto';
+// 582af9ef5cdc53d6628f45cb842f874a
+// const hashStr = '{"a":"a"}';
+// const hash = crypto.createHash('md5').update(hashStr).digest('hex');
+// console.log(hash);
+
+/**
+ * 计算字符串的md5值
+ * @param str
+ * @returns
+ */
+export const hash = (str: string | Buffer | Object) => {
+  let hashStr: string | Buffer;
+  if (typeof str === 'object') {
+    hashStr = JSON.stringify(str, null, 2);
+  } else {
+    hashStr = str;
+  }
+  return crypto.createHash('md5').update(hashStr).digest('hex');
+};
+
+export const hashSringify = (str: Object) => {
+  return JSON.stringify(str, null, 2);
+};

src/util/index.ts

@@ -0,0 +1 @@
+export * from './hash.ts';

tsup.config.ts

@@ -1,7 +1,9 @@
 import { defineConfig } from 'tsup';
+import glob from 'fast-glob';
+const services = glob.sync('src/services/*.ts');
 
 export default defineConfig({
-  entry: ['src/index.ts'],
+  entry: ['src/index.ts', ...services],
 
   splitting: false,
   sourcemap: false,