From 0179fe73a3b8ce75572370fa55e28a426747c181 Mon Sep 17 00:00:00 2001 From: xion Date: Thu, 20 Mar 2025 02:29:26 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=B8=8A=E4=BC=A0=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E5=92=8C=E4=B8=8B=E8=BD=BD=E8=B5=84=E6=BA=90=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 8 +- src/routes-simple/event.ts | 35 ++- src/routes-simple/index.ts | 4 + src/routes-simple/middleware/auth.ts | 6 +- src/routes-simple/minio/get-minio-resource.ts | 152 +++++++++++++ src/routes-simple/resources/chunk.ts | 178 +++++++++++++++ src/routes-simple/resources/get-resources.ts | 15 ++ src/routes-simple/resources/upload.ts | 17 +- src/routes-simple/resources/util.ts | 29 +++ src/routes-simple/router.ts | 39 +++- src/routes-simple/upload.ts | 61 +---- src/routes/app-manager/export.ts | 12 + src/routes/app-manager/list.ts | 45 +++- src/routes/app-manager/module/app.ts | 16 +- src/routes/config/models/model.ts | 2 +- src/routes/config/upload-config.ts | 21 +- src/routes/container/list.ts | 208 +++++++----------- src/routes/file/list.ts | 55 ++++- src/routes/file/module/get-minio-list.ts | 69 +++++- 19 files changed, 747 insertions(+), 225 deletions(-) create mode 100644 src/routes-simple/minio/get-minio-resource.ts create mode 100644 src/routes-simple/resources/chunk.ts create mode 100644 src/routes-simple/resources/get-resources.ts create mode 100644 src/routes-simple/resources/util.ts create mode 100644 src/routes/app-manager/export.ts diff --git a/package.json b/package.json index ba77f6b..c04ad65 100644 --- a/package.json +++ b/package.json @@ -45,12 +45,12 @@ "json5": "^2.2.3", "jsonwebtoken": "^9.0.2", "lodash-es": "^4.17.21", - "minio": "^8.0.4", + "minio": "^8.0.5", "nanoid": "^5.1.3", "node-fetch": "^3.3.2", "p-queue": "^8.1.0", - "pg": "^8.13.3", - "pm2": "^5.4.3", + "pg": "^8.14.0", + "pm2": "^6.0.5", "rollup-plugin-esbuild": "^6.2.1", "semver": "^7.7.1", "sequelize": "^6.37.6", @@ -66,7 +66,7 @@ "@rollup/plugin-alias": "^5.1.1", "@rollup/plugin-commonjs": "^28.0.3", "@rollup/plugin-json": "^6.1.0", - "@rollup/plugin-node-resolve": "^16.0.0", + "@rollup/plugin-node-resolve": "^16.0.1", "@rollup/plugin-replace": "^6.0.2", "@rollup/plugin-typescript": "^12.1.2", "@types/archiver": "^6.0.3", diff --git a/src/routes-simple/event.ts b/src/routes-simple/event.ts index 29cfef7..12dec63 100644 --- a/src/routes-simple/event.ts +++ b/src/routes-simple/event.ts @@ -1,4 +1,4 @@ -import { router, error, checkAuth, clients, getTaskId } from './router.ts'; +import { router, error, checkAuth, clients, getTaskId, writeEvents, deleteOldClients } from './router.ts'; router.get('/api/events', async (req, res) => { res.writeHead(200, { @@ -6,17 +6,44 @@ router.get('/api/events', async (req, res) => { 'Cache-Control': 'no-cache', Connection: 'keep-alive', }); - const tokenUser = await checkAuth(req, res); - if (!tokenUser) return; const taskId = getTaskId(req); if (!taskId) { res.end(error('task-id is required')); return; } // 将客户端连接推送到 clients 数组 - clients.set(taskId, { client: res, tokenUser }); + clients.set(taskId, { client: res, createTime: Date.now() }); // 移除客户端连接 req.on('close', () => { clients.delete(taskId); }); }); + +router.get('/api/s1/events', async (req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); + const taskId = getTaskId(req); + if (!taskId) { + res.end(error('task-id is required')); + return; + } + // 将客户端连接推送到 clients 数组 + clients.set(taskId, { client: res, createTime: Date.now() }); + writeEvents(req, { progress: 0, message: 'start' }); + // 不自动关闭连接 + // res.end('ok'); +}); + +router.get('/api/s1/events/close', async (req, res) => { + const taskId = getTaskId(req); + if (!taskId) { + res.end(error('task-id is required')); + return; + } + deleteOldClients(); + clients.delete(taskId); + res.end('ok'); +}); diff --git a/src/routes-simple/index.ts b/src/routes-simple/index.ts index ce1b11c..c508b96 100644 --- a/src/routes-simple/index.ts +++ b/src/routes-simple/index.ts @@ -1,2 +1,6 @@ import './code/upload.ts'; +import './event.ts'; + import './resources/upload.ts'; +import './resources/chunk.ts'; +import './resources/get-resources.ts'; diff --git a/src/routes-simple/middleware/auth.ts b/src/routes-simple/middleware/auth.ts index d027104..b6839ad 100644 --- a/src/routes-simple/middleware/auth.ts +++ b/src/routes-simple/middleware/auth.ts @@ -5,7 +5,7 @@ export const error = (msg: string, code = 500) => { return JSON.stringify({ code, message: msg }); }; export const checkAuth = async (req: http.IncomingMessage, res: http.ServerResponse) => { - let token = (req.headers?.['authorization'] as string) || ''; + let token = (req.headers?.['authorization'] as string) || (req.headers?.['Authorization'] as string) || ''; const url = new URL(req.url || '', 'http://localhost'); const resNoPermission = () => { res.statusCode = 401; @@ -22,10 +22,14 @@ export const checkAuth = async (req: http.IncomingMessage, res: http.ServerRespo if (!token) { return resNoPermission(); } + if (token) { + token = token.replace('Bearer ', ''); + } let tokenUser; try { tokenUser = await User.verifyToken(token); } catch (e) { + console.log('checkAuth error', e); res.statusCode = 401; res.end(error('Invalid token')); return { tokenUser: null, token: null }; diff --git a/src/routes-simple/minio/get-minio-resource.ts b/src/routes-simple/minio/get-minio-resource.ts new file mode 100644 index 0000000..893cf80 --- /dev/null +++ b/src/routes-simple/minio/get-minio-resource.ts @@ -0,0 +1,152 @@ +/** + * 更新时间：2025-03-17 + */ +import { minioClient } from '@/app.ts'; +import { IncomingMessage, ServerResponse } from 'http'; +import { bucketName } from '@/modules/minio.ts'; +import { checkAuth } from '../middleware/auth.ts'; +import { BucketItemStat } from 'minio'; + +/** + * 过滤 metaData 中的 key, 去除 password, accesskey, secretkey， + * 并返回过滤后的 metaData + * @param metaData + * @returns + */ +const filterKeys = (metaData: Record, clearKeys: string[] = []) => { + const keys = Object.keys(metaData); + // remove X-Amz- meta data + const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys]; + const filteredKeys = keys.filter((key) => !removeKeys.includes(key)); + return filteredKeys.reduce((acc, key) => { + acc[key] = metaData[key]; + return acc; + }, {} as Record); +}; +export const checkMetaAuth = async ( + metaData: Record, + { tokenUser, token, share, userKey, password }: { tokenUser: any; share: ShareType; token: string; userKey: string; password: string }, +) => { + const tokenUsername = tokenUser?.username; + if (share === 'public') { + return { + code: 20000, + msg: '资源是公开的', + }; + } + if (tokenUsername === userKey) { + return { + code: 20001, + msg: '用户是资源所有者', + }; + } + // 1. 检查资源是否过期（有，则检查） + if (metaData['expiration-time']) { + const expirationTime = new Date(metaData['expiration-time']); + const currentTime = new Date(); + if (expirationTime < currentTime) { + return { + code: 20100, + msg: '资源已过期', + }; + } + } + // 2. 检查密码是否正确（可选，password存在的情况） + if (password && metaData.password && password === metaData.password) { + return { + code: 20002, + msg: '用户通过密码正确访问', + }; + } + const usernames = metaData['usernames'] || ''; + if (usernames && usernames.includes(tokenUsername)) { + // TODO: 可以检查用户的orgs 是否在 metaData['orgs'] 中 + return { + code: 20003, + msg: '用户在usernames列表中', + }; + } + return { + code: 20101, + msg: '用户没有权限访问', + }; +}; +export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => { + res.writeHead(code, { 'Content-Type': 'text/plain' }); + res.end(msg || 'Not Found File'); + return; +}; +export const shareType = ['public', 'private', 'protected'] as const; +export type ShareType = (typeof shareType)[number]; + +export const authMinio = async (req: IncomingMessage, res: ServerResponse, objectName: string) => { + let stat: BucketItemStat; + try { + stat = await minioClient.statObject(bucketName, objectName); + } catch (e) { + return NotFoundFile(res); + } + const [userKey, ...rest] = objectName.split('/'); + const _url = new URL(req.url || '', 'http://localhost'); + const password = _url.searchParams.get('p') || ''; + const isDownload = !!_url.searchParams.get('download'); + const metaData = stat.metaData || {}; + const filteredMetaData = filterKeys(metaData, ['size', 'etag', 'last-modified']); + if (stat.size === 0) { + return NotFoundFile(res); + } + const share = (metaData.share as ShareType) || 'private'; // 默认是 private + let tokenUser: any = null; + let token: string | null = null; + if (password && metaData.password && password === metaData.password) { + // 密码正确，直接返回 + } else if (share !== 'public') { + ({ tokenUser, token } = await checkAuth(req, res)); + if (!tokenUser) { + return; + } + const checkMetaAuthResult = await checkMetaAuth(metaData, { tokenUser, token, share, userKey, password }); + const { code } = checkMetaAuthResult; + if (code >= 20100) { + return NotFoundFile(res); + } + } + const contentLength = stat.size; + const etag = stat.etag; + const lastModified = stat.lastModified.toISOString(); + const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName + const fileExtension = filename.split('.').pop()?.toLowerCase() || ''; + const viewableExtensions = [ + 'jpg', + 'jpeg', + 'png', + 'gif', + 'svg', + 'webp', + 'mp4', + 'webm', + 'mp3', + 'wav', + 'ogg', + 'pdf', + 'doc', + 'docx', + 'xls', + 'xlsx', + 'ppt', + 'pptx', + ]; + const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`; + + res.writeHead(200, { + 'Content-Length': contentLength, + etag, + 'last-modified': lastModified, + 'Content-Disposition': contentDisposition, + 'file-name': filename, + ...filteredMetaData, + }); + const objectStream = await minioClient.getObject(bucketName, objectName); + + objectStream.pipe(res, { end: true }); +}; diff --git a/src/routes-simple/resources/chunk.ts b/src/routes-simple/resources/chunk.ts new file mode 100644 index 0000000..dda93a5 --- /dev/null +++ b/src/routes-simple/resources/chunk.ts @@ -0,0 +1,178 @@ +import { useFileStore } from '@kevisual/use-config/file-store'; +import { checkAuth, error, router, writeEvents, getKey, getTaskId } from '../router.ts'; +import { IncomingForm } from 'formidable'; +import { app, minioClient } from '@/app.ts'; + +import { bucketName } from '@/modules/minio.ts'; +import { getContentType } from '@/utils/get-content-type.ts'; +import { User } from '@/models/user.ts'; +import fs from 'fs'; +import { ConfigModel } from '@/routes/config/models/model.ts'; +import { validateDirectory } from './util.ts'; + +const cacheFilePath = useFileStore('cache-file', { needExists: true }); + +router.get('/api/s1/resources/upload/chunk', async (req, res) => { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('Upload API is ready'); +}); + +// /api/s1/resources/upload +router.post('/api/s1/resources/upload/chunk', async (req, res) => { + const { tokenUser, token } = await checkAuth(req, res); + if (!tokenUser) return; + const url = new URL(req.url || '', 'http://localhost'); + const share = !!url.searchParams.get('public'); + // 使用 formidable 解析 multipart/form-data + const form = new IncomingForm({ + multiples: false, // 改为单文件上传 + uploadDir: cacheFilePath, // 上传文件存储目录 + allowEmptyFiles: true, // 允许空 + minFileSize: 0, // 最小文件大小 + createDirsFromUploads: false, // 根据上传的文件夹结构创建目录 + keepExtensions: true, // 保留文件拓展名 + hashAlgorithm: 'md5', // 文件哈希算法 + }); + const taskId = getTaskId(req); + const finalFilePath = `${cacheFilePath}/${taskId}`; + if (!taskId) { + res.end(error('taskId is required')); + return; + } + // 解析上传的文件 + form.parse(req, async (err, fields, files) => { + const file = Array.isArray(files.file) ? files.file[0] : files.file; + const clearFiles = () => { + if (file) { + fs.unlinkSync(file.filepath); + fs.unlinkSync(finalFilePath); + } + }; + + if (err) { + res.end(error(`Upload error: ${err.message}`)); + clearFiles(); + return; + } + + // Handle chunked upload logic here + let { chunkIndex, totalChunks, appKey, version, username, directory } = getKey(fields, [ + 'chunkIndex', + 'totalChunks', + 'appKey', + 'version', + 'username', + 'directory', + ]); + if (!chunkIndex || !totalChunks) { + res.end(error('chunkIndex, totalChunks is required')); + clearFiles(); + return; + } + const tempPath = file.filepath; + const relativePath = file.originalFilename; + // Append chunk to the final file + + const writeStream = fs.createWriteStream(finalFilePath, { flags: 'a' }); + const readStream = fs.createReadStream(tempPath); + readStream.pipe(writeStream); + + writeStream.on('finish', async () => { + fs.unlinkSync(tempPath); // 删除临时文件 + + // Write event for progress tracking + const progress = ((parseInt(chunkIndex) + 1) / parseInt(totalChunks)) * 100; + writeEvents(req, { + progress, + message: `Upload progress: ${progress}%`, + }); + + if (parseInt(chunkIndex) + 1 === parseInt(totalChunks)) { + let uid = tokenUser.id; + if (username) { + const user = await User.getUserByToken(token); + const has = await user.hasUser(username, true); + if (!has) { + res.end(error('username is not found')); + clearFiles(); + return; + } + const _user = await User.findOne({ where: { username } }); + uid = _user?.id || ''; + } + if (!appKey || !version) { + const config = await ConfigModel.getUploadConfig({ uid }); + if (config) { + appKey = config.config?.data?.key || ''; + version = config.config?.data?.version || ''; + } + } + if (!appKey || !version) { + res.end(error('appKey or version is not found, please check the upload config.')); + clearFiles(); + return; + } + const { code, message } = validateDirectory(directory); + if (code !== 200) { + res.end(error(message)); + clearFiles(); + return; + } + const minioPath = `${username || tokenUser.username}/${appKey}/${version}${directory ? `/${directory}` : ''}/${relativePath}`; + const metadata: any = {}; + if (share) { + metadata.share = 'public'; + } + // All chunks uploaded, now upload to MinIO + await minioClient.fPutObject(bucketName, minioPath, finalFilePath, { + 'Content-Type': getContentType(relativePath), + 'app-source': 'user-app', + 'Cache-Control': relativePath.endsWith('.html') ? 'no-cache' : 'max-age=31536000, immutable', + ...metadata, + }); + + // Clean up the final file + fs.unlinkSync(finalFilePath); + + // Notify the app + const r = await app.call({ + path: 'app', + key: 'detect-version-list', + payload: { + token: token, + data: { + appKey, + version, + username, + }, + }, + }); + const downloadBase = '/api/s1/share'; + const data: any = { + code: r.code, + data: { + app: r.body, + resource: `${downloadBase}/${minioPath}`, + }, + }; + if (r.message) { + data.message = r.message; + } + console.log('upload data', data); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(data)); + } else { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + message: 'Chunk uploaded successfully', + data: { + chunkIndex, + totalChunks, + }, + }), + ); + } + }); + }); +}); diff --git a/src/routes-simple/resources/get-resources.ts b/src/routes-simple/resources/get-resources.ts new file mode 100644 index 0000000..bda6b8f --- /dev/null +++ b/src/routes-simple/resources/get-resources.ts @@ -0,0 +1,15 @@ +import { router } from '@/app.ts'; + +import { authMinio } from '../minio/get-minio-resource.ts'; +router.all('/api/s1/share/*splat', async (req, res) => { + try { + const url = req.url; + const _url = new URL(url || '', 'http://localhost'); + const objectName = _url.pathname.replace('/api/s1/share/', ''); + await authMinio(req, res, objectName); + } catch (e) { + console.log('get share resource error url', req.url); + console.error('get share resource is error.', e.message); + res.end('get share resource is error.'); + } +}); diff --git a/src/routes-simple/resources/upload.ts b/src/routes-simple/resources/upload.ts index 1307d31..ef3cf87 100644 --- a/src/routes-simple/resources/upload.ts +++ b/src/routes-simple/resources/upload.ts @@ -8,6 +8,7 @@ import { getContentType } from '@/utils/get-content-type.ts'; import { User } from '@/models/user.ts'; import fs from 'fs'; import { ConfigModel } from '@/routes/config/models/model.ts'; +import { validateDirectory } from './util.ts'; const cacheFilePath = useFileStore('cache-file', { needExists: true }); @@ -36,6 +37,7 @@ router.post('/api/s1/resources/upload', async (req, res) => { progress: progress.toFixed(2), message: `Upload progress: ${progress.toFixed(2)}%`, }; + console.log('progress-upload', data); writeEvents(req, data); }); // 解析上传的文件 @@ -51,7 +53,7 @@ router.post('/api/s1/resources/upload', async (req, res) => { clearFiles(); return; } - let { appKey, version, username } = getKey(fields, ['appKey', 'version', 'username']); + let { appKey, version, username, directory } = getKey(fields, ['appKey', 'version', 'username', 'directory']); let uid = tokenUser.id; if (username) { const user = await User.getUserByToken(token); @@ -76,7 +78,12 @@ router.post('/api/s1/resources/upload', async (req, res) => { clearFiles(); return; } - + const { code, message } = validateDirectory(directory); + if (code !== 200) { + res.end(error(message)); + clearFiles(); + return; + } // 逐个处理每个上传的文件 const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file]; const uploadResults = []; @@ -86,7 +93,7 @@ router.post('/api/s1/resources/upload', async (req, res) => { const tempPath = file.filepath; // 文件上传时的临时路径 const relativePath = file.originalFilename; // 保留表单中上传的文件名 (包含文件夹结构) // 比如 child2/b.txt - const minioPath = `${username || tokenUser.username}/${appKey}/${version}/${relativePath}`; + const minioPath = `${username || tokenUser.username}/${appKey}/${version}${directory ? `/${directory}` : ''}/${relativePath}`; // 上传到 MinIO 并保留文件夹结构 const isHTML = relativePath.endsWith('.html'); await minioClient.fPutObject(bucketName, minioPath, tempPath, { @@ -99,7 +106,7 @@ router.post('/api/s1/resources/upload', async (req, res) => { path: minioPath, }); fs.unlinkSync(tempPath); // 删除临时文件 - } // 受控 + } // 受控 const r = await app.call({ path: 'app', key: 'uploadFiles', @@ -120,6 +127,8 @@ router.post('/api/s1/resources/upload', async (req, res) => { if (r.message) { data.message = r.message; } + console.log('upload data', data); + res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(data)); }); }); diff --git a/src/routes-simple/resources/util.ts b/src/routes-simple/resources/util.ts new file mode 100644 index 0000000..db5e743 --- /dev/null +++ b/src/routes-simple/resources/util.ts @@ -0,0 +1,29 @@ +/** + * 校验directory是否合法, 合法返回200, 不合法返回500 + * + * directory 不能以/开头，不能以/结尾。不能以.开头，不能以.结尾。 + * 把directory的/替换掉后，只能包含数字、字母、下划线、中划线 + * @param directory 目录 + * @returns + */ +export const validateDirectory = (directory?: string) => { + // 对directory进行校验，不能以/开头，不能以/结尾。不能以.开头，不能以.结尾。 + if (directory && (directory.startsWith('/') || directory.endsWith('/') || directory.startsWith('.') || directory.endsWith('.'))) { + return { + code: 500, + message: 'directory is invalid', + }; + } + // 把directory的/替换掉后，只能包含数字、字母、下划线、中划线 + let _directory = directory?.replace(/\//g, ''); + if (_directory && !/^[a-zA-Z0-9_-]+$/.test(_directory)) { + return { + code: 500, + message: 'directory is invalid, only number, letter, underline and hyphen are allowed', + }; + } + return { + code: 200, + message: 'directory is valid', + }; +}; diff --git a/src/routes-simple/router.ts b/src/routes-simple/router.ts index 7540b1b..a4ed7dd 100644 --- a/src/routes-simple/router.ts +++ b/src/routes-simple/router.ts @@ -9,7 +9,7 @@ export { router, checkAuth, error }; * 事件客户端 */ const eventClientsInit = () => { - const clients = new Map(); + const clients = new Map(); return clients; }; export const clients = useContextKey('event-clients', () => eventClientsInit()); @@ -19,18 +19,49 @@ export const clients = useContextKey('event-clients', () => eventClientsInit()); * @returns */ export const getTaskId = (req: http.IncomingMessage) => { + const url = new URL(req.url || '', 'http://localhost'); + const taskId = url.searchParams.get('taskId'); + if (taskId) { + return taskId; + } return req.headers['task-id'] as string; }; +type EventData = { + progress: number | string; + message: string; +}; + /** * 写入事件 * @param req * @param data */ -export const writeEvents = (req: http.IncomingMessage, data: any) => { +export const writeEvents = (req: http.IncomingMessage, data: EventData) => { const taskId = getTaskId(req); - taskId && clients.get(taskId)?.client?.write?.(`${JSON.stringify(data)}\n`); + if (taskId) { + const client = clients.get(taskId)?.client; + if (client) { + client.write(`data: ${JSON.stringify(data)}\n\n`); + } + if (Number(data.progress) === 100) { + clients.delete(taskId); + } + } else { + console.log('taskId is remove.', taskId); + } +}; +/** + * 查找超出2个小时的clients，都删除了 + */ +export const deleteOldClients = () => { + const now = Date.now(); + for (const [taskId, client] of clients) { + // 如果创建时间超过2个小时，则删除 + if (now - client.createTime > 1000 * 60 * 60 * 2) { + clients.delete(taskId); + } + } }; - /** * 解析表单数据, 如果表单数据是数组, 则取第一个，appKey, version, username 等 * @param fields 表单数据 diff --git a/src/routes-simple/upload.ts b/src/routes-simple/upload.ts index 462d84b..cbfdb7a 100644 --- a/src/routes-simple/upload.ts +++ b/src/routes-simple/upload.ts @@ -25,64 +25,7 @@ router.get('/api/app/upload', async (req, res) => { res.writeHead(200, { 'Content-Type': 'text/plain' }); res.end('Upload API is ready'); }); -router.post('/api/upload', async (req, res) => { - if (res.headersSent) return; // 如果响应已发送，不再处理 - res.writeHead(200, { 'Content-Type': 'application/json' }); - const { tokenUser } = await checkAuth(req, res); - if (!tokenUser) return; - // 使用 formidable 解析 multipart/form-data - const form = new IncomingForm({ - multiples: true, // 支持多文件上传 - uploadDir: filePath, // 上传文件存储目录 - allowEmptyFiles: true, // 允许空文件 - }); - form.on('progress', (bytesReceived, bytesExpected) => { - const progress = (bytesReceived / bytesExpected) * 100; - console.log(`Upload progress: ${progress.toFixed(2)}%`); - const data = { - progress: progress.toFixed(2), - message: `Upload progress: ${progress.toFixed(2)}%`, - }; - writeEvents(req, data); - }); - // 解析上传的文件 - form.parse(req, async (err, fields, files) => { - if (err) { - res.end(error(`Upload error: ${err.message}`)); - // 删除临时文件 - const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file]; - uploadedFiles.forEach((file) => { - fs.unlinkSync(file.filepath); - }); - return; - } - // 逐个处理每个上传的文件 - const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file]; - const uploadResults = []; - for (let i = 0; i < uploadedFiles.length; i++) { - const file = uploadedFiles[i]; - // @ts-ignore - const tempPath = file.filepath; // 文件上传时的临时路径 - const relativePath = file.originalFilename; // 保留表单中上传的文件名 (包含文件夹结构) - // 比如 child2/b.txt - const minioPath = `${tokenUser.username}/${relativePath}`; - // 上传到 MinIO 并保留文件夹结构 - const isHTML = relativePath.endsWith('.html'); - await minioClient.fPutObject(bucketName, minioPath, tempPath, { - 'Content-Type': getContentType(relativePath), - 'app-source': 'user-files', - 'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年 - }); - uploadResults.push({ - name: relativePath, - path: minioPath, - }); - fs.unlinkSync(tempPath); // 删除临时文件 - } - res.end(JSON.stringify({ code: 200, data: uploadResults })); - }); -}); router.post('/api/app/upload', async (req, res) => { if (res.headersSent) return; // 如果响应已发送，不再处理 res.writeHead(200, { 'Content-Type': 'application/json' }); @@ -247,5 +190,9 @@ export const uploadMiddleware = async (req: http.IncomingMessage, res: http.Serv if (req.url?.startsWith('/api/router')) { return; } + // 设置跨域 + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST'); + res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization'); return router.parse(req, res); }; diff --git a/src/routes/app-manager/export.ts b/src/routes/app-manager/export.ts new file mode 100644 index 0000000..5c8e428 --- /dev/null +++ b/src/routes/app-manager/export.ts @@ -0,0 +1,12 @@ +import { app } from '@/app.ts'; +export const callDetectAppVersion = async ({ appKey, version, username }: { appKey: string; version: string; username: string }, token: string) => { + const res = await app.call({ + path: 'app', + key: 'detect-version-list', + payload: { + token: token, + data: { appKey, version, username }, + }, + }); + return res; +}; diff --git a/src/routes/app-manager/list.ts b/src/routes/app-manager/list.ts index 57a250d..0ee7a6f 100644 --- a/src/routes/app-manager/list.ts +++ b/src/routes/app-manager/list.ts @@ -309,20 +309,27 @@ app }) .define(async (ctx) => { const tokenUser = ctx.state.tokenUser; - let { key, version, username } = ctx.query?.data || {}; - if (!key || !version) { - throw new CustomError('key and version are required'); + let { appKey, version, username } = ctx.query?.data || {}; + if (!appKey || !version) { + throw new CustomError('appKey and version are required'); } const uid = await getUidByUsername(app, ctx, username); - const appList = await AppListModel.findOne({ where: { key, version, uid: uid } }); + let appList = await AppListModel.findOne({ where: { key: appKey, version, uid } }); if (!appList) { - throw new CustomError('app not found'); + appList = await AppListModel.create({ + key: appKey, + version, + uid, + data: { + files: [], + }, + }); } const checkUsername = username || tokenUser.username; - const files = await getMinioListAndSetToAppList({ username: checkUsername, appKey: key, version }); + const files = await getMinioListAndSetToAppList({ username: checkUsername, appKey, version }); const newFiles = files.map((item) => { return { - name: item.name.replace(`${checkUsername}/${key}/${version}/`, ''), + name: item.name.replace(`${checkUsername}/${appKey}/${version}/`, ''), path: item.name, }; }); @@ -330,17 +337,31 @@ app const needAddFiles = newFiles.map((item) => { const findFile = appListFiles.find((appListFile) => appListFile.name === item.name); if (findFile && findFile.path === item.path) { - return findFile; + return { ...findFile, ...item }; } return item; }); await appList.update({ data: { files: needAddFiles } }); setExpire(appList.id, 'test'); - const appModel = await AppModel.findOne({ where: { key, version, uid: uid } }); - if (appModel) { - await appModel.update({ data: { files: needAddFiles } }); - setExpire(appModel.key, appModel.user); + let am = await AppModel.findOne({ where: { key: appKey, uid } }); + if (!am) { + am = await AppModel.create({ + title: appKey, + key: appKey, + version: version || '0.0.0', + user: checkUsername, + uid, + data: { files: needAddFiles }, + proxy: true, + }); + } else { + const appModel = await AppModel.findOne({ where: { key: appKey, version, uid } }); + if (appModel) { + await appModel.update({ data: { files: needAddFiles } }); + setExpire(appModel.key, appModel.user); + } } + ctx.body = appList; }) .addTo(app); diff --git a/src/routes/app-manager/module/app.ts b/src/routes/app-manager/module/app.ts index 46beeb5..fba65a3 100644 --- a/src/routes/app-manager/module/app.ts +++ b/src/routes/app-manager/module/app.ts @@ -2,13 +2,21 @@ import { sequelize } from '../../../modules/sequelize.ts'; import { DataTypes, Model } from 'sequelize'; type AppPermissionType = 'public' | 'private' | 'protected'; +/** + * 共享设置 + * 1. 设置公共可以直接访问 + * 2. 设置受保护需要登录后访问 + * 3. 设置私有只有自己可以访问。\n + * 受保护可以设置密码，设置访问的用户名。切换共享状态后，需要重新设置密码和用户名。 + */ export interface AppData { files: { name: string; path: string }[]; permission?: { - // 访问权限 - type: AppPermissionType; // public, private(Only Self), protected(protected, 通过配置访问) - users?: string[]; - orgs?: string[]; + // 访问权限, 字段和minio的权限配置一致 + share: AppPermissionType; // public, private(Only Self), protected(protected, 通过配置访问) + usernames?: string; // 受保护的访问用户名，多个用逗号分隔 + password?: string; // 受保护的访问密码 + 'expiration-time'?: string; // 受保护的访问过期时间 }; } export type AppType = 'web-single' | 'web-module'; // 可以做到网页代理 diff --git a/src/routes/config/models/model.ts b/src/routes/config/models/model.ts index 0b99d4b..65762bb 100644 --- a/src/routes/config/models/model.ts +++ b/src/routes/config/models/model.ts @@ -85,7 +85,7 @@ export class ConfigModel extends Model { prefix, }; } - static async setUploadConfig(opts: { uid: string; data: any }) { + static async setUploadConfig(opts: { uid: string; data: { key?: string; version?: string } }) { const config = await ConfigModel.setConfig('upload', { uid: opts.uid, data: opts.data, diff --git a/src/routes/config/upload-config.ts b/src/routes/config/upload-config.ts index 4e0dc60..8dc2b1f 100644 --- a/src/routes/config/upload-config.ts +++ b/src/routes/config/upload-config.ts @@ -8,23 +8,36 @@ app middleware: ['auth'], }) .define(async (ctx) => { - const { id } = ctx.state.tokenUser; + const tokenUser = ctx.state.tokenUser; const config = await ConfigModel.getUploadConfig({ - uid: id, + uid: tokenUser.id, }); - ctx.body = config; + const key = config?.config?.data?.key || ''; + const version = config?.config?.data?.version || ''; + const username = tokenUser.username; + const prefix = `${key}/${version}/`; + ctx.body = { + key, + version, + username, + prefix, + }; }) .addTo(app); app .route({ path: 'config', - key: 'setUploadConfig', + key: 'updateUploadConfig', middleware: ['auth'], }) .define(async (ctx) => { const { id } = ctx.state.tokenUser; const data = ctx.query.data || {}; + const { key, version } = data; + if (!key && !version) { + ctx.throw(400, 'key or version is required'); + } const config = await ConfigModel.setUploadConfig({ uid: id, data, diff --git a/src/routes/container/list.ts b/src/routes/container/list.ts index f9a3860..afa5b06 100644 --- a/src/routes/container/list.ts +++ b/src/routes/container/list.ts @@ -1,109 +1,36 @@ import { CustomError } from '@kevisual/router'; import { app } from '../../app.ts'; import { ContainerModel, ContainerData, Container } from './models/index.ts'; -import { uploadMinioContainer } from '../page/module/cache-file.ts'; -const list = app.route({ - path: 'container', - key: 'list', - middleware: ['auth'], -}); -list.run = async (ctx) => { - const tokenUser = ctx.state.tokenUser; - const list = await ContainerModel.findAll({ - order: [['updatedAt', 'DESC']], - where: { - uid: tokenUser.id, - }, - }); - ctx.body = list; - return ctx; -}; - -list.addTo(app); +app + .route({ + path: 'container', + key: 'list', + middleware: ['auth'], + }) + .define(async (ctx) => { + const tokenUser = ctx.state.tokenUser; + const list = await ContainerModel.findAll({ + order: [['updatedAt', 'DESC']], + where: { + uid: tokenUser.id, + }, + attributes: { exclude: ['code'] }, + }); + ctx.body = list; + return ctx; + }) + .addTo(app); app .route({ path: 'container', key: 'get', - }) - .define(async (ctx) => { - const id = ctx.query.id; - if (!id) { - throw new CustomError('id is required'); - } - ctx.body = await ContainerModel.findByPk(id); - return ctx; - }) - .addTo(app); - -const add = app.route({ - path: 'container', - key: 'update', - middleware: ['auth'], -}); -add.run = async (ctx) => { - const tokenUser = ctx.state.tokenUser; - const data = ctx.query.data; - const container = { - ...data, - }; - let containerModel: ContainerModel | null = null; - if (container.id) { - containerModel = await ContainerModel.findByPk(container.id); - if (containerModel) { - containerModel.update({ - ...container, - publish: { - ...containerModel.publish, - ...container.publish, - }, - }); - await containerModel.save(); - } - } else { - try { - containerModel = await ContainerModel.create({ - ...container, - uid: tokenUser.id, - }); - } catch (e) { - console.log('error', e); - } - console.log('containerModel', container); - } - - ctx.body = containerModel; - return ctx; -}; -add.addTo(app); - -const deleteRoute = app.route({ - path: 'container', - key: 'delete', -}); -deleteRoute.run = async (ctx) => { - const id = ctx.query.id; - const container = await ContainerModel.findByPk(id); - if (container) { - await container.destroy(); - } - ctx.body = container; - return ctx; -}; -deleteRoute.addTo(app); - -app - .route({ - path: 'container', - key: 'publish', - // nextRoute: { path: 'resource', key: 'publishContainer' }, middleware: ['auth'], }) .define(async (ctx) => { const tokenUser = ctx.state.tokenUser; - const { data, token } = ctx.query; - const { id, publish } = data; + const id = ctx.query.id; if (!id) { throw new CustomError('id is required'); } @@ -111,36 +38,71 @@ app if (!container) { throw new CustomError('container not found'); } - container.publish = publish; - await container.save(); - const { title, description, key, version, fileName, saveHTML } = publish; + if (container.uid !== tokenUser.id) { + throw new CustomError('container not found'); + } ctx.body = container; - if (!key || !version || !fileName) { - return; - } - if (container.type === 'render-js') { - const uploadResult = await uploadMinioContainer({ - key, - tokenUser: ctx.state.tokenUser, - version: version, - code: container.code, - filePath: fileName, - saveHTML, - }); - await ctx.call({ - path: 'app', - key: 'uploadFiles', - payload: { - token, - data: { - appKey: key, - version, - files: uploadResult, - }, - }, - }); - } else { - ctx.throw(500, 'container type not supported:' + container.type); - } + return ctx; + }) + .addTo(app); + +app + .route({ + path: 'container', + key: 'update', + middleware: ['auth'], + }) + .define(async (ctx) => { + const tokenUser = ctx.state.tokenUser; + const data = ctx.query.data; + const { id, ...container } = data; + let containerModel: ContainerModel | null = null; + if (id) { + containerModel = await ContainerModel.findByPk(id); + if (containerModel) { + containerModel.update({ + ...container, + publish: { + ...containerModel.publish, + ...container.publish, + }, + }); + await containerModel.save(); + } + } else { + try { + containerModel = await ContainerModel.create({ + ...container, + uid: tokenUser.id, + }); + } catch (e) { + console.log('error', e); + } + console.log('containerModel', container); + } + ctx.body = containerModel; + return ctx; + }) + .addTo(app); + +app + .route({ + path: 'container', + key: 'delete', + middleware: ['auth'], + }) + .define(async (ctx) => { + const tokenUser = ctx.state.tokenUser; + const id = ctx.query.id; + const container = await ContainerModel.findByPk(id); + if (!container) { + throw new CustomError('container not found'); + } + if (container.uid !== tokenUser.id) { + throw new CustomError('container not found'); + } + await container.destroy(); + ctx.body = container; + return ctx; }) .addTo(app); diff --git a/src/routes/file/list.ts b/src/routes/file/list.ts index 8f982db..5b87aa1 100644 --- a/src/routes/file/list.ts +++ b/src/routes/file/list.ts @@ -1,9 +1,15 @@ import { app } from '@/app.ts'; -import { getFileStat, getMinioList } from './module/get-minio-list.ts'; +import { getFileStat, getMinioList, deleteFile, updateFileStat } from './module/get-minio-list.ts'; import path from 'path'; import { CustomError } from '@kevisual/router'; import { get } from 'http'; +import { callDetectAppVersion } from '../app-manager/export.ts'; +/** + * 清理prefix中的'..' + * @param prefix + * @returns + */ const handlePrefix = (prefix: string) => { // 清理所有的 '..' if (!prefix) return ''; @@ -94,3 +100,50 @@ app }; }) .addTo(app); + +app + .route({ + path: 'file', + key: 'delete', + middleware: ['auth'], + }) + .define(async (ctx) => { + const tokenUser = ctx.state.tokenUser; + const data = ctx.query.data || {}; + const { prefix } = getPrefixByUser(data, tokenUser); + const [username, appKey, version] = prefix.slice(1).split('/'); + const res = await deleteFile(prefix.slice(1)); + if (res.code === 200) { + ctx.body = 'delete success'; + } else { + ctx.throw(500, res.message || 'delete failed'); + } + const r = await callDetectAppVersion({ appKey, version, username }, ctx.query.token); + if (r.code !== 200) { + console.error('callDetectAppVersion failed', r, prefix); + } + }) + .addTo(app); + +app + .route({ + path: 'file', + key: 'update-metadata', + middleware: ['auth'], + }) + .define(async (ctx) => { + const tokenUser = ctx.state.tokenUser; + const data = ctx.query.data || {}; + if (!data.metadata || JSON.stringify(data.metadata) === '{}') { + ctx.throw(400, 'metadata is required'); + } + const { prefix } = getPrefixByUser(data, tokenUser); + const res = await updateFileStat(prefix.slice(1), data.metadata); + if (res.code === 200) { + ctx.body = 'update metadata success'; + } else { + ctx.throw(500, res.message || 'update metadata failed'); + } + return ctx; + }) + .addTo(app); diff --git a/src/routes/file/module/get-minio-list.ts b/src/routes/file/module/get-minio-list.ts index 96a4bd6..625e0df 100644 --- a/src/routes/file/module/get-minio-list.ts +++ b/src/routes/file/module/get-minio-list.ts @@ -1,6 +1,6 @@ import { minioClient } from '@/app.ts'; import { bucketName } from '@/modules/minio.ts'; - +import { CopyDestinationOptions, CopySourceOptions } from 'minio'; type MinioListOpt = { prefix: string; recursive?: boolean; @@ -41,9 +41,12 @@ export const getMinioList = async (opts: MinioListOpt): Promise => { }); }); }; -export const getFileStat = async (prefix: string): Promise => { +export const getFileStat = async (prefix: string, isFile?: boolean): Promise => { try { const obj = await minioClient.statObject(bucketName, prefix); + if (isFile && obj.size === 0) { + return null; + } return obj; } catch (e) { if (e.code === 'NotFound') { @@ -54,16 +57,30 @@ export const getFileStat = async (prefix: string): Promise => { } }; -export const deleteFile = async (prefix: string): Promise => { +export const deleteFile = async (prefix: string): Promise<{ code: number; message: string }> => { try { + const fileStat = await getFileStat(prefix); + if (!fileStat) { + console.warn(`File not found: ${prefix}`); + return { + code: 404, + message: 'file not found', + }; + } await minioClient.removeObject(bucketName, prefix, { versionId: 'null', - forceDelete: true, + forceDelete: true, // 强制删除 }); - return true; + return { + code: 200, + message: 'delete success', + }; } catch (e) { console.error('delete File Error not handle', e); - return false; + return { + code: 500, + message: 'delete failed', + }; } }; @@ -90,3 +107,43 @@ export const getMinioListAndSetToAppList = async (opts: GetMinioListAndSetToAppL const files = minioList; return files as MinioFile[]; }; + +/** + * 更新文件的元数据 + * @param prefix 文件前缀 + * @param newMetadata 新的元数据 + * @returns + */ +export const updateFileStat = async ( + prefix: string, + newMetadata: Record, +): Promise<{ + code: number; + data: any; + message?: string; +}> => { + try { + const source = new CopySourceOptions({ Bucket: bucketName, Object: prefix }); + const destination = new CopyDestinationOptions({ + Bucket: bucketName, + Object: prefix, + UserMetadata: newMetadata, + MetadataDirective: 'REPLACE', + }); + const copyResult = await minioClient.copyObject(source, destination); + console.log('copyResult', copyResult); + console.log(`Metadata for ${prefix} updated successfully.`); + return { + code: 200, + data: copyResult, + message: 'update metadata success', + }; + } catch (e) { + console.error('Error updating file stat', e); + return { + code: 500, + data: null, + message: `update metadata failed. ${e.message}`, + }; + } +};