Refactor storage integration from MinIO to S3
- Removed MinIO client and related imports from various modules. - Introduced S3 client and OSS integration for object storage. - Updated all references to MinIO methods with corresponding S3 methods. - Added new flowme table schema to the database. - Adjusted upload and download routes to utilize S3 for file operations. - Removed obsolete MinIO-related files and routes. - Ensured compatibility with existing application logic while transitioning to S3.
This commit is contained in:
@@ -3,4 +3,11 @@ code center
|
||||
|
||||
```
|
||||
unzip -x app.config.json5
|
||||
```
|
||||
|
||||
```
|
||||
"@kevisual/oss": "file:/home/ubuntu/kevisual/dev3/kevisual-oss",
|
||||
```
|
||||
```
|
||||
TODO: /home/ubuntu/kevisual/code-center/src/routes/file/module/get-minio-list.ts
|
||||
```
|
||||
@@ -71,12 +71,13 @@
|
||||
"zod-to-json-schema": "^3.25.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@aws-sdk/client-s3": "^3.978.0",
|
||||
"@kevisual/oss": "file:/home/ubuntu/kevisual/dev3/kevisual-oss",
|
||||
"@kevisual/code-center-module": "0.0.24",
|
||||
"@kevisual/context": "^0.0.4",
|
||||
"@kevisual/file-listener": "^0.0.2",
|
||||
"@kevisual/local-app-manager": "0.1.32",
|
||||
"@kevisual/logger": "^0.0.4",
|
||||
"@kevisual/oss": "0.0.16",
|
||||
"@kevisual/permission": "^0.0.3",
|
||||
"@kevisual/router": "0.0.60",
|
||||
"@kevisual/types": "^0.0.12",
|
||||
|
||||
1301
pnpm-lock.yaml
generated
1301
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
12
src/app.ts
12
src/app.ts
@@ -1,10 +1,9 @@
|
||||
import { App } from '@kevisual/router';
|
||||
import * as redisLib from './modules/redis.ts';
|
||||
import * as minioLib from './modules/minio.ts';
|
||||
import * as sequelizeLib from './modules/sequelize.ts';
|
||||
import { useContextKey } from '@kevisual/context';
|
||||
import { SimpleRouter } from '@kevisual/router/simple';
|
||||
import { OssBase } from '@kevisual/oss/services';
|
||||
import { s3Client, oss as s3Oss } from './modules/s3.ts';
|
||||
import { BailianProvider } from '@kevisual/ai';
|
||||
import * as schema from './db/schema.ts';
|
||||
import { drizzle } from 'drizzle-orm/node-postgres';
|
||||
@@ -18,16 +17,11 @@ export const runtime = useContextKey('runtime', () => {
|
||||
});
|
||||
export const oss = useContextKey(
|
||||
'oss',
|
||||
() =>
|
||||
new OssBase({
|
||||
client: minioLib.minioClient,
|
||||
bucketName: minioLib.bucketName,
|
||||
prefix: '',
|
||||
}),
|
||||
() => s3Oss,
|
||||
);
|
||||
export { s3Client }
|
||||
export const redis = useContextKey('redis', () => redisLib.redis);
|
||||
export const subscriber = useContextKey('subscriber', () => redisLib.subscriber);
|
||||
export const minioClient = useContextKey('minioClient', () => minioLib.minioClient);
|
||||
export const sequelize = useContextKey('sequelize', () => sequelizeLib.sequelize);
|
||||
export const db = useContextKey('db', () => {
|
||||
const db = drizzle(config.DATABASE_URL || '');
|
||||
|
||||
@@ -492,4 +492,22 @@ export const queryViews = pgTable("query_views", {
|
||||
}, (table) => [
|
||||
index('query_views_uid_idx').using('btree', table.uid.asc().nullsLast()),
|
||||
index('query_title_idx').using('btree', table.title.asc().nullsLast()),
|
||||
]);
|
||||
|
||||
export const flowme = pgTable("flowme", {
|
||||
id: uuid().primaryKey().notNull().defaultRandom(),
|
||||
uid: uuid(),
|
||||
title: text('title').default(''),
|
||||
description: text('description').default(''),
|
||||
tags: jsonb().default([]),
|
||||
link: text('link').default(''),
|
||||
data: jsonb().default({}),
|
||||
|
||||
type: text('type').default(''),
|
||||
|
||||
createdAt: timestamp('createdAt').notNull().defaultNow(),
|
||||
updatedAt: timestamp('updatedAt').notNull().defaultNow(),
|
||||
}, (table) => [
|
||||
index('flowme_uid_idx').using('btree', table.uid.asc().nullsLast()),
|
||||
index('flowme_title_idx').using('btree', table.title.asc().nullsLast()),
|
||||
]);
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useConfig } from '@kevisual/use-config';
|
||||
import { useFileStore } from '@kevisual/use-config';
|
||||
import { minioResources } from './minio.ts';
|
||||
import { minioResources } from './s3.ts';
|
||||
|
||||
export const config = useConfig() as any;
|
||||
export const port = config.PORT ? Number(config.PORT) : 4005;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { bucketName, minioClient } from '@/modules/minio.ts';
|
||||
import { oss } from '@/app.ts';
|
||||
import { IncomingMessage, ServerResponse } from 'http';
|
||||
import { filterKeys } from './http-proxy.ts';
|
||||
import { getUserFromRequest } from '../utils.ts';
|
||||
@@ -11,6 +11,7 @@ import { parseSearchValue } from '@kevisual/router/src/server/parse-body.ts';
|
||||
import { logger } from '@/modules/logger.ts';
|
||||
import { pipeBusboy } from '../pipe-busboy.ts';
|
||||
import { pipeMinioStream } from '../pipe.ts';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
type FileList = {
|
||||
name: string;
|
||||
@@ -112,7 +113,8 @@ const getAiProxy = async (req: IncomingMessage, res: ServerResponse, opts: Proxy
|
||||
const etag = stat.etag;
|
||||
const lastModified = stat.lastModified.toISOString();
|
||||
|
||||
const objectStream = await minioClient.getObject(bucketName, objectName);
|
||||
const objectStream = await oss.getObject(objectName);
|
||||
// const objectStream = await minioClient.getObject(bucketName, objectName);
|
||||
const headers = {
|
||||
'Content-Length': contentLength,
|
||||
etag,
|
||||
@@ -124,8 +126,7 @@ const getAiProxy = async (req: IncomingMessage, res: ServerResponse, opts: Proxy
|
||||
...headers,
|
||||
});
|
||||
// objectStream.pipe(res, { end: true });
|
||||
// @ts-ignore
|
||||
pipeMinioStream(objectStream, res);
|
||||
pipeMinioStream(objectStream.Body as Readable, res);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(`Proxy request error: ${error.message}`);
|
||||
@@ -164,9 +165,6 @@ export const postProxy = async (req: IncomingMessage, res: ServerResponse, opts:
|
||||
let fileSize: number | undefined = undefined;
|
||||
if (_fileSize) {
|
||||
fileSize = parseInt(_fileSize, 10);
|
||||
} else if (req.headers['content-length']) {
|
||||
// 如果 URL 参数没有 size,尝试从请求头获取
|
||||
fileSize = parseInt(req.headers['content-length'], 10);
|
||||
}
|
||||
let meta = parseSearchValue(params.get('meta'), { decode: true });
|
||||
if (!hash && !force) {
|
||||
@@ -214,7 +212,7 @@ export const postProxy = async (req: IncomingMessage, res: ServerResponse, opts:
|
||||
...getMetadata(pathname),
|
||||
...meta,
|
||||
},
|
||||
{ check: false, isStream: true, size: fileSize },
|
||||
{ check: false, isStream: true },
|
||||
);
|
||||
end({ success: true, name, info, isNew: true, hash, meta: meta?.metaData, statMeta }, '上传成功', 200);
|
||||
|
||||
@@ -301,8 +299,9 @@ export const deleteProxy = async (req: IncomingMessage, res: ServerResponse, opt
|
||||
if (objectName.endsWith('/')) {
|
||||
const objects = await oss.listObjects<true>(objectName, { recursive: true });
|
||||
if (objects.length > 0) {
|
||||
const objectNames = objects.map((obj: any) => obj.name);
|
||||
await minioClient.removeObjects(bucketName, objectNames);
|
||||
for (const obj of objects) {
|
||||
await oss.deleteObject(obj.name);
|
||||
}
|
||||
}
|
||||
end({ success: true, objectName, deletedCount: objects.length }, 'delete success', 200);
|
||||
} else {
|
||||
@@ -357,17 +356,16 @@ export const renameProxy = async (req: IncomingMessage, res: ServerResponse, opt
|
||||
const oldKey = obj.name;
|
||||
const newKey = oldKey.replace(objectName, newObjectName);
|
||||
console.log('rename dir object', oldKey, newKey);
|
||||
await minioClient.copyObject(bucketName, newKey, `/${bucketName}/${oldKey}`);
|
||||
await oss.copyObject(oldKey, newKey);
|
||||
copiedCount++;
|
||||
}
|
||||
// 删除原对象
|
||||
const objectNames = objects.map((obj: any) => obj.name);
|
||||
if (objectNames.length > 0) {
|
||||
await minioClient.removeObjects(bucketName, objectNames);
|
||||
for (const obj of objects) {
|
||||
console.log('deleted object', obj.name);
|
||||
await oss.deleteObject(obj.name);
|
||||
}
|
||||
} else {
|
||||
// 重命名文件
|
||||
await minioClient.copyObject(bucketName, newObjectName, `/${bucketName}/${objectName}`);
|
||||
await oss.copyObject(objectName, newObjectName);
|
||||
await oss.deleteObject(objectName);
|
||||
copiedCount = 1;
|
||||
}
|
||||
@@ -384,7 +382,6 @@ type ProxyOptions = {
|
||||
oss?: OssBase;
|
||||
};
|
||||
export const aiProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => {
|
||||
const oss = new OssBase({ bucketName, client: minioClient });
|
||||
if (!opts.oss) {
|
||||
opts.oss = oss;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { pipeline, Readable } from 'node:stream';
|
||||
import { promisify } from 'node:util';
|
||||
import { bucketName, minioClient, minioResources } from '@/modules/minio.ts';
|
||||
import { Readable } from 'node:stream';
|
||||
import { minioResources } from '@/modules/s3.ts';
|
||||
import { oss } from '@/app.ts';
|
||||
import fs from 'node:fs';
|
||||
import { IncomingMessage, ServerResponse } from 'node:http';
|
||||
import http from 'node:http';
|
||||
@@ -11,14 +11,18 @@ import path from 'path';
|
||||
import { getTextContentType } from '@/modules/fm-manager/index.ts';
|
||||
import { logger } from '@/modules/logger.ts';
|
||||
import { pipeStream } from '../pipe.ts';
|
||||
const pipelineAsync = promisify(pipeline);
|
||||
import { GetObjectCommandOutput } from '@aws-sdk/client-s3';
|
||||
|
||||
export async function downloadFileFromMinio(fileUrl: string, destFile: string) {
|
||||
const objectName = fileUrl.replace(minioResources + '/', '');
|
||||
const objectStream = await minioClient.getObject(bucketName, objectName);
|
||||
const destStream = fs.createWriteStream(destFile);
|
||||
await pipelineAsync(objectStream, destStream);
|
||||
console.log(`minio File downloaded to ${minioResources}/${objectName} \n ${destFile}`);
|
||||
const objectStream = await oss.getObject(objectName) as GetObjectCommandOutput;
|
||||
const body = objectStream.Body as Readable;
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
for await (const chunk of body) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
fs.writeFileSync(destFile, Buffer.concat(chunks));
|
||||
}
|
||||
export const filterKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
|
||||
const keys = Object.keys(metaData);
|
||||
@@ -43,8 +47,8 @@ export async function minioProxy(
|
||||
const { createNotFoundPage, isDownload = false } = opts;
|
||||
const objectName = fileUrl.replace(minioResources + '/', '');
|
||||
try {
|
||||
const stat = await minioClient.statObject(bucketName, objectName);
|
||||
if (stat.size === 0) {
|
||||
const stat = await oss.statObject(objectName);
|
||||
if (stat?.size === 0) {
|
||||
createNotFoundPage('Invalid proxy url');
|
||||
return true;
|
||||
}
|
||||
@@ -54,7 +58,8 @@ export async function minioProxy(
|
||||
const lastModified = stat.lastModified.toISOString();
|
||||
const fileName = objectName.split('/').pop();
|
||||
const ext = path.extname(fileName || '');
|
||||
const objectStream = await minioClient.getObject(bucketName, objectName);
|
||||
const objectStreamOutput = await oss.getObject(objectName);
|
||||
const objectStream = objectStreamOutput.Body as Readable;
|
||||
const headers = {
|
||||
'Content-Length': contentLength,
|
||||
etag,
|
||||
@@ -151,6 +156,7 @@ export const httpProxy = async (
|
||||
return createNotFoundPage('Invalid proxy url:' + error.message);
|
||||
}
|
||||
} else {
|
||||
console.log('Proxying file: headers', headers);
|
||||
res.writeHead(proxyRes.statusCode, {
|
||||
...headers,
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import http from 'node:http';
|
||||
import { minioClient } from '@/modules/minio.ts';
|
||||
import { pipeMinioStream } from '../pipe.ts';
|
||||
import { oss } from '@/app.ts';
|
||||
import { Readable } from 'node:stream';
|
||||
type ProxyInfo = {
|
||||
path?: string;
|
||||
target: string;
|
||||
@@ -15,9 +16,8 @@ export const minioProxyOrigin = async (req: http.IncomingMessage, res: http.Serv
|
||||
if (objectName.startsWith(bucketName)) {
|
||||
objectName = objectName.slice(bucketName.length);
|
||||
}
|
||||
const objectStream = await minioClient.getObject(bucketName, objectName);
|
||||
// objectStream.pipe(res);
|
||||
pipeMinioStream(objectStream, res);
|
||||
const objectStream = await oss.getObject(objectName);
|
||||
pipeMinioStream(objectStream.Body as Readable, res);
|
||||
} catch (error) {
|
||||
console.error('Error fetching object from MinIO:', error);
|
||||
res.statusCode = 500;
|
||||
|
||||
@@ -1,38 +1,13 @@
|
||||
import { Client, ClientOptions } from 'minio';
|
||||
import { Client, } from 'minio';
|
||||
import { useConfig } from '@kevisual/use-config';
|
||||
const config = useConfig();
|
||||
import { OssBase } from '@kevisual/oss/services';
|
||||
const minioConfig = {
|
||||
endPoint: config.MINIO_ENDPOINT || 'localhost',
|
||||
// @ts-ignore
|
||||
port: parseInt(config.MINIO_PORT || '9000'),
|
||||
useSSL: config.MINIO_USE_SSL === 'true',
|
||||
accessKey: config.MINIO_ACCESS_KEY,
|
||||
secretKey: config.MINIO_SECRET_KEY,
|
||||
};
|
||||
const { port, endPoint, useSSL } = minioConfig;
|
||||
// console.log('minioConfig', minioConfig);
|
||||
export const minioClient = new Client(minioConfig);
|
||||
export const bucketName = config.MINIO_BUCKET_NAME || 'resources';
|
||||
|
||||
export const minioUrl = `http${useSSL ? 's' : ''}://${endPoint}:${port || 9000}`;
|
||||
export const minioResources = `${minioUrl}/resources`;
|
||||
|
||||
if (!minioClient) {
|
||||
throw new Error('Minio client not initialized');
|
||||
}
|
||||
// 验证权限
|
||||
(async () => {
|
||||
const bucketExists = await minioClient.bucketExists(bucketName);
|
||||
if (!bucketExists) {
|
||||
await minioClient.makeBucket(bucketName);
|
||||
}
|
||||
console.log('bucketExists', bucketExists);
|
||||
// const res = await minioClient.putObject(bucketName, 'root/test/0.0.1/a.txt', 'test');
|
||||
// console.log('minio putObject', res);
|
||||
})();
|
||||
|
||||
export const oss = new OssBase({
|
||||
client: minioClient,
|
||||
bucketName: bucketName,
|
||||
prefix: '',
|
||||
});
|
||||
|
||||
44
src/modules/s3.ts
Normal file
44
src/modules/s3.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import { CreateBucketCommand, HeadObjectCommand, S3Client, } from '@aws-sdk/client-s3';
|
||||
import { OssBase } from '@kevisual/oss/s3.ts';
|
||||
import { useConfig } from '@kevisual/use-config';
|
||||
const config = useConfig();
|
||||
|
||||
export const bucketName = config.S3_BUCKET_NAME || 'resources';
|
||||
|
||||
export const s3Client = new S3Client({
|
||||
credentials: {
|
||||
accessKeyId: config.S3_ACCESS_KEY_ID || '',
|
||||
secretAccessKey: config.S3_SECRET_ACCESS_KEY || '',
|
||||
},
|
||||
region: config.S3_REGION,
|
||||
endpoint: config.S3_ENDPOINT,
|
||||
// minio配置
|
||||
forcePathStyle: true,
|
||||
});
|
||||
|
||||
// 判断 bucketName 是否存在,不存在则创建
|
||||
(async () => {
|
||||
try {
|
||||
await s3Client.send(new HeadObjectCommand({ Bucket: bucketName, Key: '' }));
|
||||
console.log(`Bucket ${bucketName} exists.`);
|
||||
} catch (error) {
|
||||
console.log(`Bucket ${bucketName} does not exist. Creating...`);
|
||||
if (config.S3_ENDPOINT?.includes?.('9000')) {
|
||||
// 创建 bucket
|
||||
await s3Client.send(new CreateBucketCommand({ Bucket: bucketName }));
|
||||
console.log(`Bucket ${bucketName} created.`);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
if (!s3Client) {
|
||||
throw new Error('S3 client not initialized');
|
||||
}
|
||||
|
||||
export const oss = new OssBase({
|
||||
client: s3Client,
|
||||
bucketName: bucketName,
|
||||
prefix: '',
|
||||
})
|
||||
|
||||
export const minioResources = `${config.S3_ENDPOINT}/${bucketName}`;
|
||||
@@ -7,7 +7,7 @@ import { nanoid } from 'nanoid';
|
||||
import { pipeline } from 'stream';
|
||||
import { promisify } from 'util';
|
||||
import { getAppLoadStatus, setAppLoadStatus } from './get-app-status.ts';
|
||||
import { minioResources } from '../minio.ts';
|
||||
import { minioResources } from '../s3.ts';
|
||||
import { downloadFileFromMinio, fetchApp, fetchDomain, fetchTest } from '@/modules/fm-manager/index.ts';
|
||||
import { logger } from '../logger.ts';
|
||||
export * from './get-app-status.ts';
|
||||
|
||||
@@ -4,8 +4,7 @@ import { router, clients, writeEvents } from '../router.ts';
|
||||
import { error } from '../middleware/auth.ts';
|
||||
import fs from 'fs';
|
||||
import { useFileStore } from '@kevisual/use-config';
|
||||
import { app, minioClient } from '@/app.ts';
|
||||
import { bucketName } from '@/modules/minio.ts';
|
||||
import { app, oss } from '@/app.ts';
|
||||
import { getContentType } from '@/utils/get-content-type.ts';
|
||||
import path from 'path';
|
||||
import { createWriteStream } from 'fs';
|
||||
@@ -123,7 +122,7 @@ router.post('/api/micro-app/upload', async (req, res) => {
|
||||
const minioPath = `private/${tokenUser.username}/${appKey}/${relativePath}`;
|
||||
// 上传到 MinIO 并保留文件夹结构
|
||||
const isHTML = relativePath.endsWith('.html');
|
||||
await minioClient.fPutObject(bucketName, minioPath, tempPath, {
|
||||
await oss.fPutObject(minioPath, tempPath, {
|
||||
'Content-Type': getContentType(relativePath),
|
||||
'app-source': 'user-micro-app',
|
||||
'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年
|
||||
|
||||
@@ -2,9 +2,8 @@ import { useFileStore } from '@kevisual/use-config';
|
||||
import http from 'node:http';
|
||||
import fs from 'fs';
|
||||
import Busboy from 'busboy';
|
||||
import { app, minioClient } from '@/app.ts';
|
||||
import { app, oss } from '@/app.ts';
|
||||
|
||||
import { bucketName } from '@/modules/minio.ts';
|
||||
import { getContentType } from '@/utils/get-content-type.ts';
|
||||
import { User } from '@/models/user.ts';
|
||||
import { router, error, checkAuth, writeEvents } from './router.ts';
|
||||
@@ -151,7 +150,7 @@ router.post('/api/app/upload', async (req, res) => {
|
||||
const minioPath = `${username || tokenUser.username}/${appKey}/${version}/${relativePath}`;
|
||||
// 上传到 MinIO 并保留文件夹结构
|
||||
const isHTML = relativePath.endsWith('.html');
|
||||
await minioClient.fPutObject(bucketName, minioPath, tempPath, {
|
||||
await oss.fPutObject(minioPath, tempPath, {
|
||||
'Content-Type': getContentType(relativePath),
|
||||
'app-source': 'user-app',
|
||||
'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
import { minioClient } from '@/app.ts';
|
||||
import { bucketName } from '@/modules/minio.ts';
|
||||
|
||||
import { router } from '../router.ts';
|
||||
|
||||
router.post('/api/minio', async (ctx) => {
|
||||
let { username, appKey } = { username: '', appKey: '' };
|
||||
const path = `${username}/${appKey}`;
|
||||
const res = await minioClient.listObjectsV2(bucketName, path, true);
|
||||
const file = res.filter((item) => item.isFile);
|
||||
const fileList = file.map((item) => {
|
||||
return {
|
||||
name: item.name,
|
||||
size: item.size,
|
||||
};
|
||||
});
|
||||
// ctx.body = fileList;
|
||||
});
|
||||
@@ -1,107 +0,0 @@
|
||||
/**
|
||||
* 更新时间:2025-03-17
|
||||
* 第二次更新:2025-03-22
|
||||
*/
|
||||
import { minioClient } from '@/app.ts';
|
||||
import { IncomingMessage, ServerResponse } from 'http';
|
||||
import { bucketName } from '@/modules/minio.ts';
|
||||
import { getLoginUser } from '../middleware/auth.ts';
|
||||
import { BucketItemStat } from 'minio';
|
||||
import { UserPermission, Permission } from '@kevisual/permission';
|
||||
import { pipeMinioStream } from '@/modules/fm-manager/index.ts';
|
||||
|
||||
/**
|
||||
* 过滤 metaData 中的 key, 去除 password, accesskey, secretkey,
|
||||
* 并返回过滤后的 metaData
|
||||
* @param metaData
|
||||
* @returns
|
||||
*/
|
||||
const filterKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
|
||||
const keys = Object.keys(metaData);
|
||||
// remove X-Amz- meta data
|
||||
const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
|
||||
const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
|
||||
return filteredKeys.reduce((acc, key) => {
|
||||
acc[key] = metaData[key];
|
||||
return acc;
|
||||
}, {} as Record<string, string>);
|
||||
};
|
||||
|
||||
export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
|
||||
res.writeHead(code, { 'Content-Type': 'text/plain' });
|
||||
res.end(msg || 'Not Found File');
|
||||
return;
|
||||
};
|
||||
export const shareType = ['public', 'private', 'protected'] as const;
|
||||
export type ShareType = (typeof shareType)[number];
|
||||
|
||||
export const authMinio = async (req: IncomingMessage, res: ServerResponse, objectName: string) => {
|
||||
let stat: BucketItemStat;
|
||||
try {
|
||||
stat = await minioClient.statObject(bucketName, objectName);
|
||||
} catch (e) {
|
||||
return NotFoundFile(res);
|
||||
}
|
||||
const [userKey, ...rest] = objectName.split('/');
|
||||
const _url = new URL(req.url || '', 'http://localhost');
|
||||
const password = _url.searchParams.get('p') || '';
|
||||
const isDownload = !!_url.searchParams.get('download');
|
||||
const metaData = stat.metaData || {};
|
||||
const filteredMetaData = filterKeys(metaData, ['size', 'etag', 'last-modified']);
|
||||
if (stat.size === 0) {
|
||||
return NotFoundFile(res);
|
||||
}
|
||||
const { tokenUser } = await getLoginUser(req);
|
||||
const username = tokenUser?.username;
|
||||
const owner = userKey;
|
||||
const permission = new UserPermission({
|
||||
permission: metaData as Permission,
|
||||
owner,
|
||||
});
|
||||
const checkPermissionResult = permission.checkPermissionSuccess({
|
||||
username,
|
||||
password,
|
||||
});
|
||||
if (!checkPermissionResult.success) {
|
||||
return NotFoundFile(res, checkPermissionResult.message, checkPermissionResult.code);
|
||||
}
|
||||
|
||||
const contentLength = stat.size;
|
||||
const etag = stat.etag;
|
||||
const lastModified = stat.lastModified.toISOString();
|
||||
const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
|
||||
const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
|
||||
const viewableExtensions = [
|
||||
'jpg',
|
||||
'jpeg',
|
||||
'png',
|
||||
'gif',
|
||||
'svg',
|
||||
'webp',
|
||||
'mp4',
|
||||
'webm',
|
||||
'mp3',
|
||||
'wav',
|
||||
'ogg',
|
||||
'pdf',
|
||||
'doc',
|
||||
'docx',
|
||||
'xls',
|
||||
'xlsx',
|
||||
'ppt',
|
||||
'pptx',
|
||||
];
|
||||
const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
|
||||
|
||||
res.writeHead(200, {
|
||||
'Content-Length': contentLength,
|
||||
etag,
|
||||
'last-modified': lastModified,
|
||||
'Content-Disposition': contentDisposition,
|
||||
...filteredMetaData,
|
||||
});
|
||||
const objectStream = await minioClient.getObject(bucketName, objectName);
|
||||
|
||||
// objectStream.pipe(res, { end: true });
|
||||
pipeMinioStream(objectStream, res);
|
||||
};
|
||||
@@ -161,9 +161,8 @@ router.post('/api/s1/resources/upload/chunk', async (req, res) => {
|
||||
if (share) {
|
||||
metadata.share = 'public';
|
||||
}
|
||||
const bucketName = oss.bucketName;
|
||||
// All chunks uploaded, now upload to MinIO
|
||||
await oss.client.fPutObject(bucketName, minioPath, finalFilePath, {
|
||||
await oss.fPutObject(minioPath, finalFilePath, {
|
||||
'Content-Type': getContentType(relativePath),
|
||||
'app-source': 'user-app',
|
||||
'Cache-Control': relativePath.endsWith('.html') ? 'no-cache' : 'max-age=31536000, immutable',
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
import { router } from '@/app.ts';
|
||||
|
||||
import { authMinio } from '../minio/get-minio-resource.ts';
|
||||
|
||||
// 功能可以抽离为某一个dns请求的服务
|
||||
|
||||
router.all('/api/s1/share/*splat', async (req, res) => {
|
||||
try {
|
||||
const url = req.url;
|
||||
const _url = new URL(url || '', 'http://localhost');
|
||||
let objectName = _url.pathname.replace('/api/s1/share/', '');
|
||||
objectName = decodeURIComponent(objectName);
|
||||
await authMinio(req, res, objectName);
|
||||
} catch (e) {
|
||||
console.log('get share resource error url', req.url);
|
||||
console.error('get share resource is error.', e.message);
|
||||
res.end('get share resource is error.');
|
||||
}
|
||||
});
|
||||
@@ -1,9 +1,8 @@
|
||||
import { useFileStore } from '@kevisual/use-config';
|
||||
import { checkAuth, error, router, writeEvents, getKey } from '../router.ts';
|
||||
import Busboy from 'busboy';
|
||||
import { app, minioClient } from '@/app.ts';
|
||||
import { app } from '@/app.ts';
|
||||
|
||||
import { bucketName } from '@/modules/minio.ts';
|
||||
import { getContentType } from '@/utils/get-content-type.ts';
|
||||
import { User } from '@/models/user.ts';
|
||||
import fs from 'fs';
|
||||
@@ -15,6 +14,7 @@ import { validateDirectory } from './util.ts';
|
||||
import { pick } from 'es-toolkit';
|
||||
import { getFileStat } from '@/routes/file/index.ts';
|
||||
import { logger } from '@/modules/logger.ts';
|
||||
import { oss } from '@/modules/s3.ts';
|
||||
|
||||
const cacheFilePath = useFileStore('cache-file', { needExists: true });
|
||||
|
||||
@@ -233,7 +233,7 @@ router.post('/api/s1/resources/upload', async (req, res) => {
|
||||
metadata.share = 'public';
|
||||
}
|
||||
Object.assign(metadata, meta);
|
||||
await minioClient.fPutObject(bucketName, minioPath, tempPath, {
|
||||
await oss.fPutObject(minioPath, tempPath, {
|
||||
'Content-Type': getContentType(relativePath),
|
||||
'app-source': 'user-app',
|
||||
'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年
|
||||
|
||||
@@ -2,7 +2,6 @@ import { app } from '@/app.ts';
|
||||
import { getFileStat, getMinioList, deleteFile, updateFileStat, deleteFiles } from './module/get-minio-list.ts';
|
||||
import path from 'path';
|
||||
import { CustomError } from '@kevisual/router';
|
||||
import { get } from 'http';
|
||||
import { callDetectAppVersion } from '../app-manager/export.ts';
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import dayjs from 'dayjs';
|
||||
import { minioClient } from '../../../modules/minio.ts';
|
||||
import { bucketName } from '../../../modules/minio.ts';
|
||||
import { BucketItemStat, CopyDestinationOptions, CopySourceOptions } from 'minio';
|
||||
import { oss } from '@/app.ts';
|
||||
import { StatObjectResult } from '@kevisual/oss';
|
||||
type MinioListOpt = {
|
||||
prefix: string;
|
||||
recursive?: boolean;
|
||||
@@ -20,33 +19,13 @@ export type MinioList = (MinioFile | MinioDirectory)[];
|
||||
export const getMinioList = async <IS_FILE extends boolean>(opts: MinioListOpt): Promise<IS_FILE extends true ? MinioFile[] : MinioDirectory[]> => {
|
||||
const prefix = opts.prefix;
|
||||
const recursive = opts.recursive ?? false;
|
||||
const res = await new Promise((resolve, reject) => {
|
||||
let res: any[] = [];
|
||||
let hasError = false;
|
||||
minioClient
|
||||
.listObjectsV2(bucketName, prefix, recursive)
|
||||
.on('data', (data) => {
|
||||
res.push(data);
|
||||
})
|
||||
.on('error', (err) => {
|
||||
console.error('minio error', opts.prefix, err);
|
||||
hasError = true;
|
||||
})
|
||||
.on('end', () => {
|
||||
if (hasError) {
|
||||
reject();
|
||||
return;
|
||||
} else {
|
||||
resolve(res);
|
||||
}
|
||||
});
|
||||
});
|
||||
const res = await oss.listObjects(prefix, { recursive });
|
||||
return res as IS_FILE extends true ? MinioFile[] : MinioDirectory[];
|
||||
};
|
||||
export const getFileStat = async (prefix: string, isFile?: boolean): Promise<BucketItemStat | null> => {
|
||||
export const getFileStat = async (prefix: string, isFile?: boolean): Promise<StatObjectResult | null> => {
|
||||
try {
|
||||
const obj = await minioClient.statObject(bucketName, prefix);
|
||||
if (isFile && obj.size === 0) {
|
||||
const obj = await oss.statObject(prefix);
|
||||
if (isFile && obj?.size === 0) {
|
||||
return null;
|
||||
}
|
||||
return obj;
|
||||
@@ -69,10 +48,7 @@ export const deleteFile = async (prefix: string): Promise<{ code: number; messag
|
||||
message: 'file not found',
|
||||
};
|
||||
}
|
||||
await minioClient.removeObject(bucketName, prefix, {
|
||||
versionId: 'null',
|
||||
forceDelete: true, // 强制删除
|
||||
});
|
||||
await oss.deleteObject(prefix);
|
||||
return {
|
||||
code: 200,
|
||||
message: 'delete success',
|
||||
@@ -89,7 +65,9 @@ export const deleteFile = async (prefix: string): Promise<{ code: number; messag
|
||||
// 批量删除文件
|
||||
export const deleteFiles = async (prefixs: string[]): Promise<any> => {
|
||||
try {
|
||||
await minioClient.removeObjects(bucketName, prefixs);
|
||||
for (const prefix of prefixs) {
|
||||
await oss.deleteObject(prefix);
|
||||
}
|
||||
return true;
|
||||
} catch (e) {
|
||||
console.error('delete Files Error not handle', e);
|
||||
@@ -135,14 +113,9 @@ export const updateFileStat = async (
|
||||
message?: string;
|
||||
}> => {
|
||||
try {
|
||||
const source = new CopySourceOptions({ Bucket: bucketName, Object: prefix });
|
||||
const destination = new CopyDestinationOptions({
|
||||
Bucket: bucketName,
|
||||
Object: prefix,
|
||||
UserMetadata: newMetadata,
|
||||
MetadataDirective: 'REPLACE',
|
||||
const copyResult = await oss.replaceObject(prefix, {
|
||||
...newMetadata
|
||||
});
|
||||
const copyResult = await minioClient.copyObject(source, destination);
|
||||
console.log('copyResult', copyResult);
|
||||
console.log(`Metadata for ${prefix} updated successfully.`);
|
||||
return {
|
||||
@@ -171,23 +144,13 @@ export const mvUserAToUserB = async (usernameA: string, usernameB: string, clear
|
||||
const newPrefix = `${usernameB}/`;
|
||||
const listSource = await getMinioList<true>({ prefix: oldPrefix, recursive: true });
|
||||
for (const item of listSource) {
|
||||
const source = new CopySourceOptions({ Bucket: bucketName, Object: item.name });
|
||||
const stat = await getFileStat(item.name);
|
||||
const newName = item.name.slice(oldPrefix.length);
|
||||
// @ts-ignore
|
||||
const metadata = stat?.userMetadata || stat.metaData;
|
||||
const destination = new CopyDestinationOptions({
|
||||
Bucket: bucketName,
|
||||
Object: `${newPrefix}${newName}`,
|
||||
UserMetadata: metadata,
|
||||
MetadataDirective: 'COPY',
|
||||
});
|
||||
await minioClient.copyObject(source, destination);
|
||||
await oss.copyObject(item.name, `${newPrefix}${newName}`);
|
||||
}
|
||||
if (clearOldUser) {
|
||||
const files = await getMinioList<true>({ prefix: oldPrefix, recursive: true });
|
||||
for (const file of files) {
|
||||
await minioClient.removeObject(bucketName, file.name);
|
||||
await oss.deleteObject(file.name);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -202,7 +165,7 @@ export const backupUserA = async (usernameA: string, id: string, backName?: stri
|
||||
for (const item of deleteBackup) {
|
||||
const files = await getMinioList<true>({ prefix: item.prefix, recursive: true });
|
||||
for (const file of files) {
|
||||
await minioClient.removeObject(bucketName, file.name);
|
||||
await oss.deleteObject(file.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -215,6 +178,6 @@ export const backupUserA = async (usernameA: string, id: string, backName?: stri
|
||||
export const deleteUser = async (username: string) => {
|
||||
const list = await getMinioList<true>({ prefix: `${username}/`, recursive: true });
|
||||
for (const item of list) {
|
||||
await minioClient.removeObject(bucketName, item.name);
|
||||
await oss.deleteObject(item.name);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { minioClient } from '@/app.ts';
|
||||
import { bucketName } from '@/modules/minio.ts';
|
||||
import { oss } from '@/app.ts';
|
||||
import { fileIsExist } from '@kevisual/use-config';
|
||||
import { spawn, spawnSync } from 'child_process';
|
||||
import { getFileStat, getMinioList, MinioFile } from '@/routes/file/index.ts';
|
||||
@@ -8,6 +7,7 @@ import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { appsPath } from '../lib/index.ts';
|
||||
import { installAppFromKey } from './manager.ts';
|
||||
import { Readable } from 'stream';
|
||||
export type InstallAppOpts = {
|
||||
needInstallDeps?: boolean;
|
||||
// minio中
|
||||
@@ -46,7 +46,7 @@ export const installApp = async (opts: InstallAppOpts) => {
|
||||
if (!fileIsExist(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
const fileStream = await minioClient.getObject(bucketName, `${name}`);
|
||||
const fileStream = (await oss.getObject(`${name}`)).Body as Readable;
|
||||
const writeStream = fs.createWriteStream(outputPath);
|
||||
fileStream.pipe(writeStream);
|
||||
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
import { bucketName, minioClient } from '@/modules/minio.ts';
|
||||
import { S3Error } from 'minio';
|
||||
const main = async () => {
|
||||
const res = await new Promise((resolve, reject) => {
|
||||
let res: any[] = [];
|
||||
let hasError = false;
|
||||
minioClient
|
||||
.listObjectsV2(bucketName, 'root/codeflow/0.0.1/')
|
||||
.on('data', (data) => {
|
||||
res.push(data);
|
||||
})
|
||||
.on('error', (err) => {
|
||||
console.error('error', err);
|
||||
hasError = true;
|
||||
})
|
||||
.on('end', () => {
|
||||
if (hasError) {
|
||||
reject();
|
||||
return;
|
||||
} else {
|
||||
resolve(res);
|
||||
}
|
||||
});
|
||||
});
|
||||
console.log(res);
|
||||
};
|
||||
// main();
|
||||
|
||||
const main2 = async () => {
|
||||
try {
|
||||
const obj = await minioClient.statObject(bucketName, 'root/codeflow/0.0.1/README.md');
|
||||
|
||||
console.log(obj);
|
||||
} catch (e) {
|
||||
console.log('', e.message, '\n\r', e.code);
|
||||
// console.error(e);
|
||||
}
|
||||
};
|
||||
|
||||
main2();
|
||||
4
src/test/s3-stat.ts
Normal file
4
src/test/s3-stat.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
import { oss } from '@/modules/s3.ts'
|
||||
|
||||
const stat = await oss.statObject('root/codepod/0.0.3/index.html');
|
||||
console.log('Object Stat:', stat);
|
||||
Reference in New Issue
Block a user