Refactor storage integration from MinIO to S3

- Removed MinIO client and related imports from various modules.
- Introduced S3 client and OSS integration for object storage.
- Updated all references to MinIO methods with corresponding S3 methods.
- Added new flowme table schema to the database.
- Adjusted upload and download routes to utilize S3 for file operations.
- Removed obsolete MinIO-related files and routes.
- Ensured compatibility with existing application logic while transitioning to S3.

src/modules/fm-manager/proxy/http-proxy.ts

@@ -1,6 +1,6 @@
-import { pipeline, Readable } from 'node:stream';
-import { promisify } from 'node:util';
-import { bucketName, minioClient, minioResources } from '@/modules/minio.ts';
+import { Readable } from 'node:stream';
+import { minioResources } from '@/modules/s3.ts';
+import { oss } from '@/app.ts';
 import fs from 'node:fs';
 import { IncomingMessage, ServerResponse } from 'node:http';
 import http from 'node:http';
@@ -11,14 +11,18 @@ import path from 'path';
 import { getTextContentType } from '@/modules/fm-manager/index.ts';
 import { logger } from '@/modules/logger.ts';
 import { pipeStream } from '../pipe.ts';
-const pipelineAsync = promisify(pipeline);
+import { GetObjectCommandOutput } from '@aws-sdk/client-s3';
 
 export async function downloadFileFromMinio(fileUrl: string, destFile: string) {
   const objectName = fileUrl.replace(minioResources + '/', '');
-  const objectStream = await minioClient.getObject(bucketName, objectName);
-  const destStream = fs.createWriteStream(destFile);
-  await pipelineAsync(objectStream, destStream);
-  console.log(`minio File downloaded to ${minioResources}/${objectName} \n ${destFile}`);
+  const objectStream = await oss.getObject(objectName) as GetObjectCommandOutput;
+  const body = objectStream.Body as Readable;
+
+  const chunks: Uint8Array[] = [];
+  for await (const chunk of body) {
+    chunks.push(chunk);
+  }
+  fs.writeFileSync(destFile, Buffer.concat(chunks));
 }
 export const filterKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
   const keys = Object.keys(metaData);
@@ -43,8 +47,8 @@ export async function minioProxy(
   const { createNotFoundPage, isDownload = false } = opts;
   const objectName = fileUrl.replace(minioResources + '/', '');
   try {
-    const stat = await minioClient.statObject(bucketName, objectName);
-    if (stat.size === 0) {
+    const stat = await oss.statObject(objectName);
+    if (stat?.size === 0) {
       createNotFoundPage('Invalid proxy url');
       return true;
     }
@@ -54,7 +58,8 @@ export async function minioProxy(
     const lastModified = stat.lastModified.toISOString();
     const fileName = objectName.split('/').pop();
     const ext = path.extname(fileName || '');
-    const objectStream = await minioClient.getObject(bucketName, objectName);
+    const objectStreamOutput = await oss.getObject(objectName);
+    const objectStream = objectStreamOutput.Body as Readable;
     const headers = {
       'Content-Length': contentLength,
       etag,
@@ -151,6 +156,7 @@ export const httpProxy = async (
         return createNotFoundPage('Invalid proxy url:' + error.message);
       }
     } else {
+      console.log('Proxying file: headers', headers);
       res.writeHead(proxyRes.statusCode, {
         ...headers,
       });