feat: 上传资源和下载资源更新

src/routes-simple/event.ts

@@ -1,4 +1,4 @@
-import { router, error, checkAuth, clients, getTaskId } from './router.ts';
+import { router, error, checkAuth, clients, getTaskId, writeEvents, deleteOldClients } from './router.ts';
 
 router.get('/api/events', async (req, res) => {
   res.writeHead(200, {
@@ -6,17 +6,44 @@ router.get('/api/events', async (req, res) => {
     'Cache-Control': 'no-cache',
     Connection: 'keep-alive',
   });
-  const tokenUser = await checkAuth(req, res);
-  if (!tokenUser) return;
   const taskId = getTaskId(req);
   if (!taskId) {
     res.end(error('task-id is required'));
     return;
   }
   // 将客户端连接推送到 clients 数组
-  clients.set(taskId, { client: res, tokenUser });
+  clients.set(taskId, { client: res, createTime: Date.now() });
   // 移除客户端连接
   req.on('close', () => {
     clients.delete(taskId);
   });
 });
+
+router.get('/api/s1/events', async (req, res) => {
+  res.writeHead(200, {
+    'Content-Type': 'text/event-stream',
+    'Cache-Control': 'no-cache',
+    Connection: 'keep-alive',
+  });
+  const taskId = getTaskId(req);
+  if (!taskId) {
+    res.end(error('task-id is required'));
+    return;
+  }
+  // 将客户端连接推送到 clients 数组
+  clients.set(taskId, { client: res, createTime: Date.now() });
+  writeEvents(req, { progress: 0, message: 'start' });
+  // 不自动关闭连接
+  // res.end('ok');
+});
+
+router.get('/api/s1/events/close', async (req, res) => {
+  const taskId = getTaskId(req);
+  if (!taskId) {
+    res.end(error('task-id is required'));
+    return;
+  }
+  deleteOldClients();
+  clients.delete(taskId);
+  res.end('ok');
+});

src/routes-simple/index.ts

@@ -1,2 +1,6 @@
 import './code/upload.ts';
+import './event.ts';
+
 import './resources/upload.ts';
+import './resources/chunk.ts';
+import './resources/get-resources.ts';

src/routes-simple/middleware/auth.ts

@@ -5,7 +5,7 @@ export const error = (msg: string, code = 500) => {
   return JSON.stringify({ code, message: msg });
 };
 export const checkAuth = async (req: http.IncomingMessage, res: http.ServerResponse) => {
-  let token = (req.headers?.['authorization'] as string) || '';
+  let token = (req.headers?.['authorization'] as string) || (req.headers?.['Authorization'] as string) || '';
   const url = new URL(req.url || '', 'http://localhost');
   const resNoPermission = () => {
     res.statusCode = 401;
@@ -22,10 +22,14 @@ export const checkAuth = async (req: http.IncomingMessage, res: http.ServerRespo
   if (!token) {
     return resNoPermission();
   }
+  if (token) {
+    token = token.replace('Bearer ', '');
+  }
   let tokenUser;
   try {
     tokenUser = await User.verifyToken(token);
   } catch (e) {
+    console.log('checkAuth error', e);
     res.statusCode = 401;
     res.end(error('Invalid token'));
     return { tokenUser: null, token: null };

src/routes-simple/minio/get-minio-resource.ts

@@ -0,0 +1,152 @@
+/**
+ * 更新时间：2025-03-17
+ */
+import { minioClient } from '@/app.ts';
+import { IncomingMessage, ServerResponse } from 'http';
+import { bucketName } from '@/modules/minio.ts';
+import { checkAuth } from '../middleware/auth.ts';
+import { BucketItemStat } from 'minio';
+
+/**
+ * 过滤 metaData 中的 key, 去除 password, accesskey, secretkey，
+ * 并返回过滤后的 metaData
+ * @param metaData
+ * @returns
+ */
+const filterKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
+  const keys = Object.keys(metaData);
+  // remove X-Amz- meta data
+  const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
+  const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
+  return filteredKeys.reduce((acc, key) => {
+    acc[key] = metaData[key];
+    return acc;
+  }, {} as Record<string, string>);
+};
+export const checkMetaAuth = async (
+  metaData: Record<string, string>,
+  { tokenUser, token, share, userKey, password }: { tokenUser: any; share: ShareType; token: string; userKey: string; password: string },
+) => {
+  const tokenUsername = tokenUser?.username;
+  if (share === 'public') {
+    return {
+      code: 20000,
+      msg: '资源是公开的',
+    };
+  }
+  if (tokenUsername === userKey) {
+    return {
+      code: 20001,
+      msg: '用户是资源所有者',
+    };
+  }
+  // 1. 检查资源是否过期（有，则检查）
+  if (metaData['expiration-time']) {
+    const expirationTime = new Date(metaData['expiration-time']);
+    const currentTime = new Date();
+    if (expirationTime < currentTime) {
+      return {
+        code: 20100,
+        msg: '资源已过期',
+      };
+    }
+  }
+  // 2. 检查密码是否正确（可选，password存在的情况）
+  if (password && metaData.password && password === metaData.password) {
+    return {
+      code: 20002,
+      msg: '用户通过密码正确访问',
+    };
+  }
+  const usernames = metaData['usernames'] || '';
+  if (usernames && usernames.includes(tokenUsername)) {
+    // TODO: 可以检查用户的orgs 是否在 metaData['orgs'] 中
+    return {
+      code: 20003,
+      msg: '用户在usernames列表中',
+    };
+  }
+  return {
+    code: 20101,
+    msg: '用户没有权限访问',
+  };
+};
+export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
+  res.writeHead(code, { 'Content-Type': 'text/plain' });
+  res.end(msg || 'Not Found File');
+  return;
+};
+export const shareType = ['public', 'private', 'protected'] as const;
+export type ShareType = (typeof shareType)[number];
+
+export const authMinio = async (req: IncomingMessage, res: ServerResponse, objectName: string) => {
+  let stat: BucketItemStat;
+  try {
+    stat = await minioClient.statObject(bucketName, objectName);
+  } catch (e) {
+    return NotFoundFile(res);
+  }
+  const [userKey, ...rest] = objectName.split('/');
+  const _url = new URL(req.url || '', 'http://localhost');
+  const password = _url.searchParams.get('p') || '';
+  const isDownload = !!_url.searchParams.get('download');
+  const metaData = stat.metaData || {};
+  const filteredMetaData = filterKeys(metaData, ['size', 'etag', 'last-modified']);
+  if (stat.size === 0) {
+    return NotFoundFile(res);
+  }
+  const share = (metaData.share as ShareType) || 'private'; // 默认是 private
+  let tokenUser: any = null;
+  let token: string | null = null;
+  if (password && metaData.password && password === metaData.password) {
+    // 密码正确，直接返回
+  } else if (share !== 'public') {
+    ({ tokenUser, token } = await checkAuth(req, res));
+    if (!tokenUser) {
+      return;
+    }
+    const checkMetaAuthResult = await checkMetaAuth(metaData, { tokenUser, token, share, userKey, password });
+    const { code } = checkMetaAuthResult;
+    if (code >= 20100) {
+      return NotFoundFile(res);
+    }
+  }
+  const contentLength = stat.size;
+  const etag = stat.etag;
+  const lastModified = stat.lastModified.toISOString();
+  const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
+  const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
+  const viewableExtensions = [
+    'jpg',
+    'jpeg',
+    'png',
+    'gif',
+    'svg',
+    'webp',
+    'mp4',
+    'webm',
+    'mp3',
+    'wav',
+    'ogg',
+    'pdf',
+    'doc',
+    'docx',
+    'xls',
+    'xlsx',
+    'ppt',
+    'pptx',
+  ];
+  const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
+
+  res.writeHead(200, {
+    'Content-Length': contentLength,
+    etag,
+    'last-modified': lastModified,
+    'Content-Disposition': contentDisposition,
+    'file-name': filename,
+    ...filteredMetaData,
+  });
+  const objectStream = await minioClient.getObject(bucketName, objectName);
+
+  objectStream.pipe(res, { end: true });
+};

src/routes-simple/resources/chunk.ts

@@ -0,0 +1,178 @@
+import { useFileStore } from '@kevisual/use-config/file-store';
+import { checkAuth, error, router, writeEvents, getKey, getTaskId } from '../router.ts';
+import { IncomingForm } from 'formidable';
+import { app, minioClient } from '@/app.ts';
+
+import { bucketName } from '@/modules/minio.ts';
+import { getContentType } from '@/utils/get-content-type.ts';
+import { User } from '@/models/user.ts';
+import fs from 'fs';
+import { ConfigModel } from '@/routes/config/models/model.ts';
+import { validateDirectory } from './util.ts';
+
+const cacheFilePath = useFileStore('cache-file', { needExists: true });
+
+router.get('/api/s1/resources/upload/chunk', async (req, res) => {
+  res.writeHead(200, { 'Content-Type': 'text/plain' });
+  res.end('Upload API is ready');
+});
+
+// /api/s1/resources/upload
+router.post('/api/s1/resources/upload/chunk', async (req, res) => {
+  const { tokenUser, token } = await checkAuth(req, res);
+  if (!tokenUser) return;
+  const url = new URL(req.url || '', 'http://localhost');
+  const share = !!url.searchParams.get('public');
+  // 使用 formidable 解析 multipart/form-data
+  const form = new IncomingForm({
+    multiples: false, // 改为单文件上传
+    uploadDir: cacheFilePath, // 上传文件存储目录
+    allowEmptyFiles: true, // 允许空
+    minFileSize: 0, // 最小文件大小
+    createDirsFromUploads: false, // 根据上传的文件夹结构创建目录
+    keepExtensions: true, // 保留文件拓展名
+    hashAlgorithm: 'md5', // 文件哈希算法
+  });
+  const taskId = getTaskId(req);
+  const finalFilePath = `${cacheFilePath}/${taskId}`;
+  if (!taskId) {
+    res.end(error('taskId is required'));
+    return;
+  }
+  // 解析上传的文件
+  form.parse(req, async (err, fields, files) => {
+    const file = Array.isArray(files.file) ? files.file[0] : files.file;
+    const clearFiles = () => {
+      if (file) {
+        fs.unlinkSync(file.filepath);
+        fs.unlinkSync(finalFilePath);
+      }
+    };
+
+    if (err) {
+      res.end(error(`Upload error: ${err.message}`));
+      clearFiles();
+      return;
+    }
+
+    // Handle chunked upload logic here
+    let { chunkIndex, totalChunks, appKey, version, username, directory } = getKey(fields, [
+      'chunkIndex',
+      'totalChunks',
+      'appKey',
+      'version',
+      'username',
+      'directory',
+    ]);
+    if (!chunkIndex || !totalChunks) {
+      res.end(error('chunkIndex, totalChunks is required'));
+      clearFiles();
+      return;
+    }
+    const tempPath = file.filepath;
+    const relativePath = file.originalFilename;
+    // Append chunk to the final file
+
+    const writeStream = fs.createWriteStream(finalFilePath, { flags: 'a' });
+    const readStream = fs.createReadStream(tempPath);
+    readStream.pipe(writeStream);
+
+    writeStream.on('finish', async () => {
+      fs.unlinkSync(tempPath); // 删除临时文件
+
+      // Write event for progress tracking
+      const progress = ((parseInt(chunkIndex) + 1) / parseInt(totalChunks)) * 100;
+      writeEvents(req, {
+        progress,
+        message: `Upload progress: ${progress}%`,
+      });
+
+      if (parseInt(chunkIndex) + 1 === parseInt(totalChunks)) {
+        let uid = tokenUser.id;
+        if (username) {
+          const user = await User.getUserByToken(token);
+          const has = await user.hasUser(username, true);
+          if (!has) {
+            res.end(error('username is not found'));
+            clearFiles();
+            return;
+          }
+          const _user = await User.findOne({ where: { username } });
+          uid = _user?.id || '';
+        }
+        if (!appKey || !version) {
+          const config = await ConfigModel.getUploadConfig({ uid });
+          if (config) {
+            appKey = config.config?.data?.key || '';
+            version = config.config?.data?.version || '';
+          }
+        }
+        if (!appKey || !version) {
+          res.end(error('appKey or version is not found, please check the upload config.'));
+          clearFiles();
+          return;
+        }
+        const { code, message } = validateDirectory(directory);
+        if (code !== 200) {
+          res.end(error(message));
+          clearFiles();
+          return;
+        }
+        const minioPath = `${username || tokenUser.username}/${appKey}/${version}${directory ? `/${directory}` : ''}/${relativePath}`;
+        const metadata: any = {};
+        if (share) {
+          metadata.share = 'public';
+        }
+        // All chunks uploaded, now upload to MinIO
+        await minioClient.fPutObject(bucketName, minioPath, finalFilePath, {
+          'Content-Type': getContentType(relativePath),
+          'app-source': 'user-app',
+          'Cache-Control': relativePath.endsWith('.html') ? 'no-cache' : 'max-age=31536000, immutable',
+          ...metadata,
+        });
+
+        // Clean up the final file
+        fs.unlinkSync(finalFilePath);
+
+        // Notify the app
+        const r = await app.call({
+          path: 'app',
+          key: 'detect-version-list',
+          payload: {
+            token: token,
+            data: {
+              appKey,
+              version,
+              username,
+            },
+          },
+        });
+        const downloadBase = '/api/s1/share';
+        const data: any = {
+          code: r.code,
+          data: {
+            app: r.body,
+            resource: `${downloadBase}/${minioPath}`,
+          },
+        };
+        if (r.message) {
+          data.message = r.message;
+        }
+        console.log('upload data', data);
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify(data));
+      } else {
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(
+          JSON.stringify({
+            message: 'Chunk uploaded successfully',
+            data: {
+              chunkIndex,
+              totalChunks,
+            },
+          }),
+        );
+      }
+    });
+  });
+});

src/routes-simple/resources/get-resources.ts

@@ -0,0 +1,15 @@
+import { router } from '@/app.ts';
+
+import { authMinio } from '../minio/get-minio-resource.ts';
+router.all('/api/s1/share/*splat', async (req, res) => {
+  try {
+    const url = req.url;
+    const _url = new URL(url || '', 'http://localhost');
+    const objectName = _url.pathname.replace('/api/s1/share/', '');
+    await authMinio(req, res, objectName);
+  } catch (e) {
+    console.log('get share resource error url', req.url);
+    console.error('get share resource is error.', e.message);
+    res.end('get share resource is error.');
+  }
+});

src/routes-simple/resources/upload.ts

@@ -8,6 +8,7 @@ import { getContentType } from '@/utils/get-content-type.ts';
 import { User } from '@/models/user.ts';
 import fs from 'fs';
 import { ConfigModel } from '@/routes/config/models/model.ts';
+import { validateDirectory } from './util.ts';
 
 const cacheFilePath = useFileStore('cache-file', { needExists: true });
 
@@ -36,6 +37,7 @@ router.post('/api/s1/resources/upload', async (req, res) => {
       progress: progress.toFixed(2),
       message: `Upload progress: ${progress.toFixed(2)}%`,
     };
+    console.log('progress-upload', data);
     writeEvents(req, data);
   });
   // 解析上传的文件
@@ -51,7 +53,7 @@ router.post('/api/s1/resources/upload', async (req, res) => {
       clearFiles();
       return;
     }
-    let { appKey, version, username } = getKey(fields, ['appKey', 'version', 'username']);
+    let { appKey, version, username, directory } = getKey(fields, ['appKey', 'version', 'username', 'directory']);
     let uid = tokenUser.id;
     if (username) {
       const user = await User.getUserByToken(token);
@@ -76,7 +78,12 @@ router.post('/api/s1/resources/upload', async (req, res) => {
       clearFiles();
       return;
     }
-
+    const { code, message } = validateDirectory(directory);
+    if (code !== 200) {
+      res.end(error(message));
+      clearFiles();
+      return;
+    }
     // 逐个处理每个上传的文件
     const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
     const uploadResults = [];
@@ -86,7 +93,7 @@ router.post('/api/s1/resources/upload', async (req, res) => {
       const tempPath = file.filepath; // 文件上传时的临时路径
       const relativePath = file.originalFilename; // 保留表单中上传的文件名 (包含文件夹结构)
       // 比如 child2/b.txt
-      const minioPath = `${username || tokenUser.username}/${appKey}/${version}/${relativePath}`;
+      const minioPath = `${username || tokenUser.username}/${appKey}/${version}${directory ? `/${directory}` : ''}/${relativePath}`;
       // 上传到 MinIO 并保留文件夹结构
       const isHTML = relativePath.endsWith('.html');
       await minioClient.fPutObject(bucketName, minioPath, tempPath, {
@@ -99,7 +106,7 @@ router.post('/api/s1/resources/upload', async (req, res) => {
         path: minioPath,
       });
       fs.unlinkSync(tempPath); // 删除临时文件
-    }    // 受控
+    } // 受控
     const r = await app.call({
       path: 'app',
       key: 'uploadFiles',
@@ -120,6 +127,8 @@ router.post('/api/s1/resources/upload', async (req, res) => {
     if (r.message) {
       data.message = r.message;
     }
+    console.log('upload data', data);
+    res.writeHead(200, { 'Content-Type': 'application/json' });
     res.end(JSON.stringify(data));
   });
 });

src/routes-simple/resources/util.ts

@@ -0,0 +1,29 @@
+/**
+ * 校验directory是否合法, 合法返回200, 不合法返回500
+ *
+ * directory 不能以/开头，不能以/结尾。不能以.开头，不能以.结尾。
+ * 把directory的/替换掉后，只能包含数字、字母、下划线、中划线
+ * @param directory 目录
+ * @returns
+ */
+export const validateDirectory = (directory?: string) => {
+  // 对directory进行校验，不能以/开头，不能以/结尾。不能以.开头，不能以.结尾。
+  if (directory && (directory.startsWith('/') || directory.endsWith('/') || directory.startsWith('.') || directory.endsWith('.'))) {
+    return {
+      code: 500,
+      message: 'directory is invalid',
+    };
+  }
+  // 把directory的/替换掉后，只能包含数字、字母、下划线、中划线
+  let _directory = directory?.replace(/\//g, '');
+  if (_directory && !/^[a-zA-Z0-9_-]+$/.test(_directory)) {
+    return {
+      code: 500,
+      message: 'directory is invalid, only number, letter, underline and hyphen are allowed',
+    };
+  }
+  return {
+    code: 200,
+    message: 'directory is valid',
+  };
+};

src/routes-simple/router.ts

@@ -9,7 +9,7 @@ export { router, checkAuth, error };
  * 事件客户端
  */
 const eventClientsInit = () => {
-  const clients = new Map<string, { client?: http.ServerResponse; [key: string]: any }>();
+  const clients = new Map<string, { client?: http.ServerResponse; createTime?: number; [key: string]: any }>();
   return clients;
 };
 export const clients = useContextKey('event-clients', () => eventClientsInit());
@@ -19,18 +19,49 @@ export const clients = useContextKey('event-clients', () => eventClientsInit());
  * @returns
  */
 export const getTaskId = (req: http.IncomingMessage) => {
+  const url = new URL(req.url || '', 'http://localhost');
+  const taskId = url.searchParams.get('taskId');
+  if (taskId) {
+    return taskId;
+  }
   return req.headers['task-id'] as string;
 };
+type EventData = {
+  progress: number | string;
+  message: string;
+};
+
 /**
  * 写入事件
  * @param req
  * @param data
  */
-export const writeEvents = (req: http.IncomingMessage, data: any) => {
+export const writeEvents = (req: http.IncomingMessage, data: EventData) => {
   const taskId = getTaskId(req);
-  taskId && clients.get(taskId)?.client?.write?.(`${JSON.stringify(data)}\n`);
+  if (taskId) {
+    const client = clients.get(taskId)?.client;
+    if (client) {
+      client.write(`data: ${JSON.stringify(data)}\n\n`);
+    }
+    if (Number(data.progress) === 100) {
+      clients.delete(taskId);
+    }
+  } else {
+    console.log('taskId is remove.', taskId);
+  }
+};
+/**
+ * 查找超出2个小时的clients，都删除了
+ */
+export const deleteOldClients = () => {
+  const now = Date.now();
+  for (const [taskId, client] of clients) {
+    // 如果创建时间超过2个小时，则删除
+    if (now - client.createTime > 1000 * 60 * 60 * 2) {
+      clients.delete(taskId);
+    }
+  }
 };
-
 /**
  * 解析表单数据, 如果表单数据是数组, 则取第一个，appKey, version, username 等
  * @param fields 表单数据

src/routes-simple/upload.ts

@@ -25,64 +25,7 @@ router.get('/api/app/upload', async (req, res) => {
   res.writeHead(200, { 'Content-Type': 'text/plain' });
   res.end('Upload API is ready');
 });
-router.post('/api/upload', async (req, res) => {
-  if (res.headersSent) return; // 如果响应已发送，不再处理
-  res.writeHead(200, { 'Content-Type': 'application/json' });
-  const { tokenUser } = await checkAuth(req, res);
-  if (!tokenUser) return;
-  // 使用 formidable 解析 multipart/form-data
-  const form = new IncomingForm({
-    multiples: true, // 支持多文件上传
-    uploadDir: filePath, // 上传文件存储目录
-    allowEmptyFiles: true, // 允许空文件
-  });
-  form.on('progress', (bytesReceived, bytesExpected) => {
-    const progress = (bytesReceived / bytesExpected) * 100;
-    console.log(`Upload progress: ${progress.toFixed(2)}%`);
-    const data = {
-      progress: progress.toFixed(2),
-      message: `Upload progress: ${progress.toFixed(2)}%`,
-    };
-    writeEvents(req, data);
-  });
 
-  // 解析上传的文件
-  form.parse(req, async (err, fields, files) => {
-    if (err) {
-      res.end(error(`Upload error: ${err.message}`));
-      // 删除临时文件
-      const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
-      uploadedFiles.forEach((file) => {
-        fs.unlinkSync(file.filepath);
-      });
-      return;
-    }
-    // 逐个处理每个上传的文件
-    const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
-    const uploadResults = [];
-    for (let i = 0; i < uploadedFiles.length; i++) {
-      const file = uploadedFiles[i];
-      // @ts-ignore
-      const tempPath = file.filepath; // 文件上传时的临时路径
-      const relativePath = file.originalFilename; // 保留表单中上传的文件名 (包含文件夹结构)
-      // 比如 child2/b.txt
-      const minioPath = `${tokenUser.username}/${relativePath}`;
-      // 上传到 MinIO 并保留文件夹结构
-      const isHTML = relativePath.endsWith('.html');
-      await minioClient.fPutObject(bucketName, minioPath, tempPath, {
-        'Content-Type': getContentType(relativePath),
-        'app-source': 'user-files',
-        'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年
-      });
-      uploadResults.push({
-        name: relativePath,
-        path: minioPath,
-      });
-      fs.unlinkSync(tempPath); // 删除临时文件
-    }
-    res.end(JSON.stringify({ code: 200, data: uploadResults }));
-  });
-});
 router.post('/api/app/upload', async (req, res) => {
   if (res.headersSent) return; // 如果响应已发送，不再处理
   res.writeHead(200, { 'Content-Type': 'application/json' });
@@ -247,5 +190,9 @@ export const uploadMiddleware = async (req: http.IncomingMessage, res: http.Serv
   if (req.url?.startsWith('/api/router')) {
     return;
   }
+  // 设置跨域
+  res.setHeader('Access-Control-Allow-Origin', '*');
+  res.setHeader('Access-Control-Allow-Methods', 'GET, POST');
+  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
   return router.parse(req, res);
 };