Refactor storage integration from MinIO to S3

- Removed MinIO client and related imports from various modules.
- Introduced S3 client and OSS integration for object storage.
- Updated all references to MinIO methods with corresponding S3 methods.
- Added new flowme table schema to the database.
- Adjusted upload and download routes to utilize S3 for file operations.
- Removed obsolete MinIO-related files and routes.
- Ensured compatibility with existing application logic while transitioning to S3.

src/routes-simple/code/upload.ts

@@ -4,8 +4,7 @@ import { router, clients, writeEvents } from '../router.ts';
 import { error } from '../middleware/auth.ts';
 import fs from 'fs';
 import { useFileStore } from '@kevisual/use-config';
-import { app, minioClient } from '@/app.ts';
-import { bucketName } from '@/modules/minio.ts';
+import { app, oss } from '@/app.ts';
 import { getContentType } from '@/utils/get-content-type.ts';
 import path from 'path';
 import { createWriteStream } from 'fs';
@@ -123,7 +122,7 @@ router.post('/api/micro-app/upload', async (req, res) => {
     const minioPath = `private/${tokenUser.username}/${appKey}/${relativePath}`;
     // 上传到 MinIO 并保留文件夹结构
     const isHTML = relativePath.endsWith('.html');
-    await minioClient.fPutObject(bucketName, minioPath, tempPath, {
+    await oss.fPutObject(minioPath, tempPath, {
       'Content-Type': getContentType(relativePath),
       'app-source': 'user-micro-app',
       'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年

src/routes-simple/handle-request.ts

@@ -2,9 +2,8 @@ import { useFileStore } from '@kevisual/use-config';
 import http from 'node:http';
 import fs from 'fs';
 import Busboy from 'busboy';
-import { app, minioClient } from '@/app.ts';
+import { app, oss } from '@/app.ts';
 
-import { bucketName } from '@/modules/minio.ts';
 import { getContentType } from '@/utils/get-content-type.ts';
 import { User } from '@/models/user.ts';
 import { router, error, checkAuth, writeEvents } from './router.ts';
@@ -151,7 +150,7 @@ router.post('/api/app/upload', async (req, res) => {
       const minioPath = `${username || tokenUser.username}/${appKey}/${version}/${relativePath}`;
       // 上传到 MinIO 并保留文件夹结构
       const isHTML = relativePath.endsWith('.html');
-      await minioClient.fPutObject(bucketName, minioPath, tempPath, {
+      await oss.fPutObject(minioPath, tempPath, {
         'Content-Type': getContentType(relativePath),
         'app-source': 'user-app',
         'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年

src/routes-simple/minio/download.ts

@@ -1,18 +0,0 @@
-import { minioClient } from '@/app.ts';
-import { bucketName } from '@/modules/minio.ts';
-
-import { router } from '../router.ts';
-
-router.post('/api/minio', async (ctx) => {
-  let { username, appKey } = { username: '', appKey: '' };
-  const path = `${username}/${appKey}`;
-  const res = await minioClient.listObjectsV2(bucketName, path, true);
-  const file = res.filter((item) => item.isFile);
-  const fileList = file.map((item) => {
-    return {
-      name: item.name,
-      size: item.size,
-    };
-  });
-  // ctx.body = fileList;
-});

src/routes-simple/minio/get-minio-resource.ts

@@ -1,107 +0,0 @@
-/**
- * 更新时间：2025-03-17
- * 第二次更新：2025-03-22
- */
-import { minioClient } from '@/app.ts';
-import { IncomingMessage, ServerResponse } from 'http';
-import { bucketName } from '@/modules/minio.ts';
-import { getLoginUser } from '../middleware/auth.ts';
-import { BucketItemStat } from 'minio';
-import { UserPermission, Permission } from '@kevisual/permission';
-import { pipeMinioStream } from '@/modules/fm-manager/index.ts';
-
-/**
- * 过滤 metaData 中的 key, 去除 password, accesskey, secretkey，
- * 并返回过滤后的 metaData
- * @param metaData
- * @returns
- */
-const filterKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
-  const keys = Object.keys(metaData);
-  // remove X-Amz- meta data
-  const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
-  const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
-  return filteredKeys.reduce((acc, key) => {
-    acc[key] = metaData[key];
-    return acc;
-  }, {} as Record<string, string>);
-};
-
-export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
-  res.writeHead(code, { 'Content-Type': 'text/plain' });
-  res.end(msg || 'Not Found File');
-  return;
-};
-export const shareType = ['public', 'private', 'protected'] as const;
-export type ShareType = (typeof shareType)[number];
-
-export const authMinio = async (req: IncomingMessage, res: ServerResponse, objectName: string) => {
-  let stat: BucketItemStat;
-  try {
-    stat = await minioClient.statObject(bucketName, objectName);
-  } catch (e) {
-    return NotFoundFile(res);
-  }
-  const [userKey, ...rest] = objectName.split('/');
-  const _url = new URL(req.url || '', 'http://localhost');
-  const password = _url.searchParams.get('p') || '';
-  const isDownload = !!_url.searchParams.get('download');
-  const metaData = stat.metaData || {};
-  const filteredMetaData = filterKeys(metaData, ['size', 'etag', 'last-modified']);
-  if (stat.size === 0) {
-    return NotFoundFile(res);
-  }
-  const { tokenUser } = await getLoginUser(req);
-  const username = tokenUser?.username;
-  const owner = userKey;
-  const permission = new UserPermission({
-    permission: metaData as Permission,
-    owner,
-  });
-  const checkPermissionResult = permission.checkPermissionSuccess({
-    username,
-    password,
-  });
-  if (!checkPermissionResult.success) {
-    return NotFoundFile(res, checkPermissionResult.message, checkPermissionResult.code);
-  }
-
-  const contentLength = stat.size;
-  const etag = stat.etag;
-  const lastModified = stat.lastModified.toISOString();
-  const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
-  const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
-  const viewableExtensions = [
-    'jpg',
-    'jpeg',
-    'png',
-    'gif',
-    'svg',
-    'webp',
-    'mp4',
-    'webm',
-    'mp3',
-    'wav',
-    'ogg',
-    'pdf',
-    'doc',
-    'docx',
-    'xls',
-    'xlsx',
-    'ppt',
-    'pptx',
-  ];
-  const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
-
-  res.writeHead(200, {
-    'Content-Length': contentLength,
-    etag,
-    'last-modified': lastModified,
-    'Content-Disposition': contentDisposition,
-    ...filteredMetaData,
-  });
-  const objectStream = await minioClient.getObject(bucketName, objectName);
-
-  // objectStream.pipe(res, { end: true });
-  pipeMinioStream(objectStream, res);
-};

src/routes-simple/resources/chunk.ts

@@ -161,9 +161,8 @@ router.post('/api/s1/resources/upload/chunk', async (req, res) => {
         if (share) {
           metadata.share = 'public';
         }
-        const bucketName = oss.bucketName;
         // All chunks uploaded, now upload to MinIO
-        await oss.client.fPutObject(bucketName, minioPath, finalFilePath, {
+        await oss.fPutObject(minioPath, finalFilePath, {
           'Content-Type': getContentType(relativePath),
           'app-source': 'user-app',
           'Cache-Control': relativePath.endsWith('.html') ? 'no-cache' : 'max-age=31536000, immutable',

src/routes-simple/resources/get-resources.ts

@@ -1,19 +0,0 @@
-import { router } from '@/app.ts';
-
-import { authMinio } from '../minio/get-minio-resource.ts';
-
-// 功能可以抽离为某一个dns请求的服务
-
-router.all('/api/s1/share/*splat', async (req, res) => {
-  try {
-    const url = req.url;
-    const _url = new URL(url || '', 'http://localhost');
-    let objectName = _url.pathname.replace('/api/s1/share/', '');
-    objectName = decodeURIComponent(objectName); 
-    await authMinio(req, res, objectName);
-  } catch (e) {
-    console.log('get share resource error url', req.url);
-    console.error('get share resource is error.', e.message);
-    res.end('get share resource is error.');
-  }
-});

src/routes-simple/resources/upload.ts

@@ -1,9 +1,8 @@
 import { useFileStore } from '@kevisual/use-config';
 import { checkAuth, error, router, writeEvents, getKey } from '../router.ts';
 import Busboy from 'busboy';
-import { app, minioClient } from '@/app.ts';
+import { app } from '@/app.ts';
 
-import { bucketName } from '@/modules/minio.ts';
 import { getContentType } from '@/utils/get-content-type.ts';
 import { User } from '@/models/user.ts';
 import fs from 'fs';
@@ -15,6 +14,7 @@ import { validateDirectory } from './util.ts';
 import { pick } from 'es-toolkit';
 import { getFileStat } from '@/routes/file/index.ts';
 import { logger } from '@/modules/logger.ts';
+import { oss } from '@/modules/s3.ts';
 
 const cacheFilePath = useFileStore('cache-file', { needExists: true });
 
@@ -233,7 +233,7 @@ router.post('/api/s1/resources/upload', async (req, res) => {
         metadata.share = 'public';
       }
       Object.assign(metadata, meta);
-      await minioClient.fPutObject(bucketName, minioPath, tempPath, {
+      await oss.fPutObject(minioPath, tempPath, {
         'Content-Type': getContentType(relativePath),
         'app-source': 'user-app',
         'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年