diff --git a/.gitignore b/.gitignore index 58768e5..e5ebd65 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ release/* pack-dist app.config.json5.envision + +/pages \ No newline at end of file diff --git a/config/pacage6/package.json b/config/pacage6/package.json deleted file mode 100644 index 9ad7406..0000000 --- a/config/pacage6/package.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "codecenter", - "version": "1.0.0", - "author": "abearxiong", - "basename": "/root/code-center", - "app": { - "type": "pm2-system-app", - "key": "code-center", - "entry": "./dist/app.mjs" - }, - "scripts": { - "start": "pm2 start apps/code-center/dist/app.mjs --name code-center" - }, - "dependencies": { - "@kevisual/router": "^0.0.20", - "@kevisual/use-config": "^1.0.17", - "ioredis": "^5.6.1", - "minio": "^8.0.5", - "pg": "^8.16.0", - "sequelize": "^6.37.7", - "sqlite3": "^5.1.7", - "socket.io": "^4.8.1", - "pm2": "^6.0.6", - "dotenv": "^16.5.0" - } -} \ No newline at end of file diff --git a/config/release/generate-token.mjs b/config/release/generate-token.mjs deleted file mode 100644 index 5b10d51..0000000 --- a/config/release/generate-token.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as nanoid from 'nanoid'; - -const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'; - -//HMAC-256：推荐 32 字节。 -// HMAC-512：推荐 64 字节。 - -// jsonwentoken 默认使用 HMAC-256 算法，生成 32 字节的 token。 - -const v = nanoid.customAlphabet(alphabet, 32); -console.log('v', v()); diff --git a/config/release/index.mjs b/config/release/index.mjs deleted file mode 100644 index c61307f..0000000 --- a/config/release/index.mjs +++ /dev/null @@ -1,121 +0,0 @@ -import fs from 'fs'; -import path from 'path'; -import archiver from 'archiver'; -import { exec } from 'child_process'; -import { nanoid } from 'nanoid'; - -const cwd = process.cwd(); -const pkgPath = path.join(cwd, 'package.json'); - -export const checkFileExistsSync = (filePath) => { - try { - // 使用 F_OK 检查文件或目录是否存在 - fs.accessSync(filePath, fs.constants.F_OK); - return true; - } catch (err) { - return false; - } -}; - -const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8')); - -const releasePath = path.join(cwd, 'release'); - -const distPath = path.join(cwd, 'dist'); -const scriptPath = path.join(cwd, 'script'); -const zip = archiver('zip', { - zlib: { level: 9 }, -}); -const zipName = `code-center-${pkg.version}.zip`; -const zipCache = path.join(releasePath, `code-center-${pkg.version}.zip`); - -const getZip = async () => { - return new Promise((resolve, reject) => { - const output = fs.createWriteStream(zipCache); - const startTime = (new Date().getTime() / 1000).toFixed(0); - // 监听事件 - output.on('close', async () => { - const bytes = zip.pointer(); - const size = bytes < 1024 ? `${bytes} bytes` : `${(bytes / 1024).toFixed(2)} KB`; - console.log(`Zip file has been created successfully. Total size: ${size} bytes.`); - let time = (new Date().getTime() / 1000).toFixed(0); - console.log('time', time - startTime); - resolve(); - }); - - output.on('end', () => { - console.log('Data has been drained.'); // 数据已被耗尽 - throw new CustomError('Data has been drained.'); - }); - - zip.on('warning', (err) => { - if (err.code === 'ENOENT') { - console.warn('File not found:', err); - } else { - throw err; - } - }); - - zip.on('error', (err) => { - throw err; - }); - - // 通过管道将 zip 数据流输出到指定文件 - zip.pipe(output); - - // 添加 sh 字符串作为文件到 zip 中 - const sh = `#!/bin/bash - npm i -g pnpm - pnpm install --prod - `; - zip.append(sh, { - name: 'start.sh', - //类型是可执行文件 - mode: 0o755, - }); - // 把dist目录下的文件添加到zip中 - zip.directory(distPath, 'dist'); - zip.directory(scriptPath, 'script'); - - // 把README.md添加到zip中 - zip.file(path.join(cwd, 'README.md'), { name: 'README.md' }); - // 把package.json添加到zip中 - zip.file(pkgPath, { name: 'package.json' }); - const ecosystemContent = `module.exports = { - apps: [ - { - name: 'codecenter', // 应用名称 - script: './dist/app.mjs', // 入口文件 - // cwd: '.', // 设置当前工作目录 - output: './logs/codecenter.log', - error: './logs/codecenter.log', - log_date_format: 'YYYY-MM-DD HH:mm:ss', - // watch: true, // 自动监控文件变化 - watch: ['dist'], // 监控的文件夹 - ignore_watch: ['node_modules', 'logs'], // 忽略的文件夹 - } - ] -} -`; - zip.append(ecosystemContent, { name: 'ecosystem.config.cjs' }); - let json5Content = fs.readFileSync(path.join(cwd, 'app.config.json5.example'), 'utf8'); - // tokenSecret 是一个随机字符串，用于生成 token - let tokenSecret = 'XX' + nanoid(39); - json5Content = json5Content.replace('', tokenSecret); - // tokenSecret - // 把app.config.json5.example添加到zip中 - // zip.file(path.join(cwd, 'app.config.json5.example'), { name: 'app.config.json5.example' }); - zip.append(json5Content, { name: 'app.config.json5.example' }); - zip.append('version-check', { name: `version-check-${pkg.version}` }); - // 结束归档（必须调用，否则 zip 文件无法完成） - zip.finalize(); - }); -}; - -getZip().then(() => { - console.log('zip success'); - // 固定上传位置在1.0.0版本，不需要更新，因为是zip文件 - console.log(`envision switch system && envision deploy ./release/${zipName} -v 1.0.0 -k code-center -y y -u`); - - console.log(`download zip:\n\ncurl -O https://kevisual.xiongxiao.me/system/code-center/${zipName} && unzip ${zipName}`); -}); diff --git a/package.json b/package.json index be076e6..0d044ff 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "start": "pm2 start dist/app.js --name code-center", "client:start": "pm2 start apps/code-center/dist/app.js --name code-center", "ssl": "ssh -L 5432:localhost:5432 light", + "ssl:redis": "ssh -L 6379:localhost:6379 light", "pub": "envision pack -p -u -c" }, "keywords": [], @@ -44,12 +45,17 @@ ], "license": "UNLICENSED", "dependencies": { + "@types/busboy": "^1.5.4", + "@types/send": "^1.2.1", + "@types/ws": "^8.18.1", + "busboy": "^1.6.0", "commander": "^14.0.2", "cookie": "^1.1.1", "ioredis": "^5.8.2", "minio": "^8.0.6", "pg": "^8.16.3", "pm2": "^6.0.14", + "send": "^1.2.0", "sequelize": "^6.37.7" }, "devDependencies": { @@ -90,6 +96,7 @@ "strip-ansi": "^7.1.2", "tar": "^7.5.2", "typescript": "^5.9.3", + "ws": "npm:@kevisual/ws", "zod": "^4.1.13" }, "resolutions": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da67e95..453f14c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,18 @@ importers: .: dependencies: + '@types/busboy': + specifier: ^1.5.4 + version: 1.5.4 + '@types/send': + specifier: ^1.2.1 + version: 1.2.1 + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 + busboy: + specifier: ^1.6.0 + version: 1.6.0 commander: specifier: ^14.0.2 version: 14.0.2 @@ -30,6 +42,9 @@ importers: pm2: specifier: ^6.0.14 version: 6.0.14 + send: + specifier: ^1.2.0 + version: 1.2.0 sequelize: specifier: ^6.37.7 version: 6.37.7(pg@8.16.3) @@ -130,6 +145,9 @@ importers: typescript: specifier: ^5.9.3 version: 5.9.3 + ws: + specifier: npm:@kevisual/ws + version: '@kevisual/ws@8.0.0' zod: specifier: ^4.1.13 version: 4.1.13 @@ -197,6 +215,10 @@ packages: peerDependencies: dotenv: ^17 + '@kevisual/ws@8.0.0': + resolution: {integrity: sha512-jlFxSlXUEz93cFW+UYT5BXv/rFVgiMQnIfqRYZ0gj1hSP8PMGRqMqUoHSLfKvfRRS4jseLSvTTeEKSQpZJtURg==} + engines: {node: '>=10.0.0'} + '@noble/hashes@1.8.0': resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} engines: {node: ^14.21.3 || >=16} @@ -236,6 +258,9 @@ packages: '@types/archiver@7.0.0': resolution: {integrity: sha512-/3vwGwx9n+mCQdYZ2IKGGHEFL30I96UgBlk8EtRDDFQ9uxM1l4O5Ci6r00EMAkiDaTqD9DQ6nVrWRICnBPtzzg==} + '@types/busboy@1.5.4': + resolution: {integrity: sha512-kG7WrUuAKK0NoyxfQHsVE6j1m01s6kMma64E+OZenQABMQyTJop1DumUWcLwAQ2JzpefU7PDYoRDKl8uZosFjw==} + '@types/cookie@0.4.1': resolution: {integrity: sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==} @@ -275,9 +300,15 @@ packages: '@types/semver@7.7.1': resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==} + '@types/send@1.2.1': + resolution: {integrity: sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==} + '@types/validator@13.12.2': resolution: {integrity: sha512-6SlHBzUW8Jhf3liqrGGXyTJSIFe4nqlJ5A5KaMZ2l/vbM3Wh3KSybots/wfWVzNLK4D1NZluDlSQIbIEPx6oyA==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@zxing/text-encoding@0.9.0': resolution: {integrity: sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==} @@ -411,6 +442,10 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + call-bind@1.0.7: resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} engines: {node: '>= 0.4'} @@ -1424,6 +1459,10 @@ packages: stream-json@1.8.0: resolution: {integrity: sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==} + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + streamx@2.20.1: resolution: {integrity: sha512-uTa0mU6WUC65iUvzKH4X9hEdvSW7rbPxPtwfWiLMSj3qTdQbAiUboZTxauKfpFuGIGa1C2BYijZ7wgdUXICJhA==} @@ -1779,6 +1818,8 @@ snapshots: '@kevisual/load': 0.0.6 dotenv: 17.2.3 + '@kevisual/ws@8.0.0': {} + '@noble/hashes@1.8.0': {} '@paralleldrive/cuid2@2.2.2': @@ -1848,6 +1889,10 @@ snapshots: dependencies: '@types/readdir-glob': 1.1.5 + '@types/busboy@1.5.4': + dependencies: + '@types/node': 24.10.1 + '@types/cookie@0.4.1': {} '@types/cors@2.8.17': @@ -1891,8 +1936,16 @@ snapshots: '@types/semver@7.7.1': {} + '@types/send@1.2.1': + dependencies: + '@types/node': 24.10.1 + '@types/validator@13.12.2': {} + '@types/ws@8.18.1': + dependencies: + '@types/node': 24.10.1 + '@zxing/text-encoding@0.9.0': optional: true @@ -2021,6 +2074,10 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + call-bind@1.0.7: dependencies: es-define-property: 1.0.0 @@ -3068,6 +3125,8 @@ snapshots: dependencies: stream-chain: 2.2.5 + streamsearch@1.1.0: {} + streamx@2.20.1: dependencies: fast-fifo: 1.3.2 diff --git a/src/app.ts b/src/app.ts index f3ddee5..2873cee 100644 --- a/src/app.ts +++ b/src/app.ts @@ -22,8 +22,7 @@ export const oss = useContextKey( }), ); export const redis = useContextKey('redis', () => redisLib.redis); -export const redisPublisher = useContextKey('redisPublisher', () => redisLib.redisPublisher); -export const redisSubscriber = useContextKey('redisSubscriber', () => redisLib.redisSubscriber); +export const subscriber = useContextKey('subscriber', () => redisLib.subscriber); export const minioClient = useContextKey('minioClient', () => minioLib.minioClient); export const sequelize = useContextKey('sequelize', () => sequelizeLib.sequelize); diff --git a/src/demo/index.ts b/src/demo/index.ts deleted file mode 100644 index 4c1bbc0..0000000 --- a/src/demo/index.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { router } from '../modules/router.ts'; -import { Route } from '@kevisual/router'; - -const getList = new Route('test', 'getList'); -getList.run = async (ctx) => { - ctx.body = 'test'; - return ctx; -}; -router.add(getList); - -const codeRun = `async function run(ctx) { - ctx.body = 'test js'; - return ctx; -}`; -const fn: any = new Function( - 'ctx', - ` - ${codeRun} - return run(ctx); -`, -); -const codeRunRoute = new Route('test', 'run'); -codeRunRoute.run = fn; - -router.add(codeRunRoute); diff --git a/src/index.ts b/src/index.ts index 4853da4..1566f15 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,62 @@ -import { config } from './modules/config.ts'; +import { myConfig as config } from './modules/config.ts'; import { app } from './app.ts'; import './route.ts'; -import { uploadMiddleware } from './routes-simple/upload.ts'; +import { handleRequest } from './routes-simple/handle-request.ts'; import { port } from './modules/config.ts'; - +import { WssApp } from './modules/ws-proxy/index.ts'; +import net from 'node:net'; // if (import.meta.url === `file://${process.argv[1]}`) { app.listen(port, () => { console.log(`server is running at http://localhost:${port}`); }); -app.server.on(uploadMiddleware); +app.server.on(handleRequest); + +const wssApp = new WssApp(); +const main = () => { + console.log('Upgrade initialization started'); + + app.server.server.on('upgrade', async (req, socket, head) => { + const isUpgrade = wssApp.upgrade(req, socket, head); + if (isUpgrade) { + console.log('WebSocket upgrade successful for path:', req.url); + return; + } + const proxyApiList = config?.apiList || []; + const proxyApi = proxyApiList.find((item) => req.url.startsWith(item.path)); + + if (proxyApi) { + const _u = new URL(req.url, `${proxyApi.target}`); + const options = { + hostname: _u.hostname, + port: Number(_u.port) || 80, + path: _u.pathname, + headers: req.headers, + }; + + const proxySocket = net.connect(options.port, options.hostname, () => { + proxySocket.write( + `GET ${options.path} HTTP/1.1\r\n` + + `Host: ${options.hostname}\r\n` + + `Connection: Upgrade\r\n` + + `Upgrade: websocket\r\n` + + `Sec-WebSocket-Key: ${req.headers['sec-websocket-key']}\r\n` + + `Sec-WebSocket-Version: ${req.headers['sec-websocket-version']}\r\n` + + `\r\n`, + ); + proxySocket.pipe(socket); + socket.pipe(proxySocket); + }); + + proxySocket.on('error', (err) => { + console.error(`WebSocket proxy error: ${err.message}`); + socket.end(); + }); + } else { + socket.end(); + } + }); +}; + +setTimeout(() => { + main(); +}, 1200); \ No newline at end of file diff --git a/src/logger/index.ts b/src/logger/index.ts deleted file mode 100644 index 3cc9c4d..0000000 --- a/src/logger/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { useConfig } from '@kevisual/use-config/env'; -import { Logger } from '@kevisual/logger'; -const config = useConfig(); - -export const logger = new Logger({ - level: config.LOG_LEVEL || 'info', - showTime: true, -}); - -export const logError = (message: string, data?: any) => logger.error({ data }, message); -export const logWarning = (message: string, data?: any) => logger.warn({ data }, message); -export const logInfo = (message: string, data?: any) => logger.info({ data }, message); -export const logDebug = (message: string, data?: any) => logger.debug({ data }, message); - -export const log = { - error: logError, - warn: logWarning, - info: logInfo, - debug: logDebug, -}; diff --git a/src/modules/auth.ts b/src/modules/auth.ts new file mode 100644 index 0000000..b55753a --- /dev/null +++ b/src/modules/auth.ts @@ -0,0 +1,66 @@ +import { User } from '@/models/user.ts'; +import http from 'http'; +import cookie from 'cookie'; +import { logger } from './logger.ts'; +export const error = (msg: string, code = 500) => { + return JSON.stringify({ code, message: msg }); +}; +export const checkAuth = async (req: http.IncomingMessage, res: http.ServerResponse) => { + let token = (req.headers?.['authorization'] as string) || (req.headers?.['Authorization'] as string) || ''; + const url = new URL(req.url || '', 'http://localhost'); + const resNoPermission = () => { + res.statusCode = 401; + res.end(error('Invalid authorization')); + return { tokenUser: null, token: null }; + }; + if (!token) { + token = url.searchParams.get('token') || ''; + } + if (!token) { + const parsedCookies = cookie.parse(req.headers.cookie || ''); + token = parsedCookies.token || ''; + } + if (!token) { + return resNoPermission(); + } + if (token) { + token = token.replace('Bearer ', ''); + } + let tokenUser; + try { + tokenUser = await User.verifyToken(token); + } catch (e) { + console.log('checkAuth error', e); + res.statusCode = 401; + res.end(error('Invalid token')); + return { tokenUser: null, token: null }; + } + return { tokenUser, token }; +}; + +export const getLoginUser = async (req: http.IncomingMessage) => { + let token = (req.headers?.['authorization'] as string) || (req.headers?.['Authorization'] as string) || ''; + const url = new URL(req.url || '', 'http://localhost'); + if (!token) { + token = url.searchParams.get('token') || ''; + } + if (!token) { + const parsedCookies = cookie.parse(req.headers.cookie || ''); + token = parsedCookies.token || ''; + } + + if (token) { + token = token.replace('Bearer ', ''); + } + if (!token) { + return null; + } + let tokenUser; + logger.debug('getLoginUser', token); + try { + tokenUser = await User.verifyToken(token); + return { tokenUser, token }; + } catch (e) { + return null; + } +}; diff --git a/src/modules/config.ts b/src/modules/config.ts index ca6e0b7..8d31e39 100644 --- a/src/modules/config.ts +++ b/src/modules/config.ts @@ -1,17 +1,87 @@ -import path from 'path'; -import dotenv from 'dotenv'; -// import { useConfig } from '@kevisual/use-config/env'; +import { useConfig } from '@kevisual/use-config'; +import { useFileStore } from '@kevisual/use-config/file-store'; +import { minioResources } from './minio.ts'; -export const envFiles = [ - path.resolve(process.cwd(), process.env.NODE_ENV === 'development' ? '.env.dev' : '.env'), - // path.resolve(process.cwd(), '.env'), // -]; -console.log('envFiles', envFiles); -export const config = dotenv.config({ - path: envFiles, - override: true, -}).parsed; -// const config = useConfig(); -// export const config = process.env; -// console.log('config', config); +export const config = useConfig() as any; export const port = config.PORT || 4005; +export const fileStore = useFileStore('pages'); +type ConfigType = { + api: { + /** + * API host address + */ + host: string; + path?: string; + port?: number; + }; + apiList: { + path: string; + /** + * url或者相对路径 + */ + target: string; + /** + * 类型 + */ + type?: 'static' | 'dynamic' | 'minio'; + }[]; + proxy: { + port?: number; + /** + * self domain kevisual.xiongxiao.me + */ + domain: string; + /** + * resources path + * https://minio.xiongxiao.me/resources + */ + resources: string; + /** + * allow origin xiongxiao.me zxj.im silkyai.cn + * 允许跨域访问的地址 + */ + allowedOrigin: string[]; + }; + stat: { + /** + * 统计网站ID + */ + websiteId: string; + }; + redis?: { + host: string; + port: number; + password?: string; + }; +}; +export const myConfig: ConfigType = { + api: { + host: config.API_HOST, + path: config.API_PATH, + port: config.PROXY_PORT, + }, + apiList: [ + // { + // path: '/api', + // target: config.API_HOST, + // }, + { + path: '/client', + target: config.API_CLIENT_HOST || 'http://localhost:51015', + }, + ], + proxy: { + port: config.PROXY_PORT, + domain: config.PROXY_DOMAIN, + resources: minioResources, + allowedOrigin: (config.PROXY_ALLOWED_ORIGINS as string)?.split(',') || [], + }, + redis: { + host: config.REDIS_HOST, + port: config.REDIS_PORT, + password: config.REDIS_PASSWORD, + }, + stat: { + websiteId: config.DATA_WEBSITE_ID, + }, +}; diff --git a/src/modules/fm-manager/get-content-type.ts b/src/modules/fm-manager/get-content-type.ts new file mode 100644 index 0000000..8d794b5 --- /dev/null +++ b/src/modules/fm-manager/get-content-type.ts @@ -0,0 +1,90 @@ +import path from 'node:path'; +export const getTextContentType = (ext: string) => { + const textContentTypes = [ + '.tsx', + '.jsx', // + '.conf', + '.env', + '.example', + '.log', + '.mjs', + '.map', + '.json5', + '.pem', + '.crt', + ]; + const include = textContentTypes.includes(ext); + if (!include) { + return {}; + } + const contentType = getContentTypeCore(ext); + if (!contentType) { + return {}; + } + return { + 'Content-Type': contentType, + }; +}; +// 获取文件的 content-type +export const getContentTypeCore = (extname: string) => { + const contentType = { + '.html': 'text/html; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.txt': 'text/plain; charset=utf-8', + '.tsx': 'text/typescript; charset=utf-8', + '.ts': 'text/typescript; charset=utf-8', + '.jsx': 'text/javascript; charset=utf-8', + '.conf': 'text/plain; charset=utf-8', + '.env': 'text/plain; charset=utf-8', + '.example': 'text/plain; charset=utf-8', + '.log': 'text/plain; charset=utf-8', + '.mjs': 'text/javascript; charset=utf-8', + '.map': 'application/json; charset=utf-8', + + '.json5': 'application/json5; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.png': 'image/png', + '.jpg': 'image/jpg', + '.gif': 'image/gif', + '.svg': 'image/svg+xml', + '.wav': 'audio/wav', + '.mp4': 'video/mp4', + '.md': 'text/markdown; charset=utf-8', // utf-8配置 + '.ico': 'image/x-icon', // Favicon 图标 + '.webp': 'image/webp', // WebP 图像格式 + '.webm': 'video/webm', // WebM 视频格式 + '.ogg': 'audio/ogg', // Ogg 音频格式 + '.mp3': 'audio/mpeg', // MP3 音频格式 + '.m4a': 'audio/mp4', // M4A 音频格式 + '.m3u8': 'application/vnd.apple.mpegurl', // HLS 播放列表 + '.pdf': 'application/pdf', // PDF 文档 + '.doc': 'application/msword', // Word 文档 + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', // Word 文档 (新版) + '.ppt': 'application/vnd.ms-powerpoint', // PowerPoint 演示文稿 + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', // PowerPoint (新版) + '.xls': 'application/vnd.ms-excel', // Excel 表格 + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // Excel 表格 (新版) + '.csv': 'text/csv; charset=utf-8', // CSV 文件 + '.xml': 'application/xml; charset=utf-8', // XML 文件 + '.rtf': 'application/rtf', // RTF 文本文件 + '.eot': 'application/vnd.ms-fontobject', // Embedded OpenType 字体 + '.ttf': 'font/ttf', // TrueType 字体 + '.woff': 'font/woff', // Web Open Font Format 1.0 + '.woff2': 'font/woff2', // Web Open Font Format 2.0 + '.otf': 'font/otf', // OpenType 字体 + '.wasm': 'application/wasm', // WebAssembly 文件 + '.pem': 'application/x-pem-file', // PEM 证书文件 + '.crt': 'application/x-x509-ca-cert', // CRT 证书文件 + '.yaml': 'application/x-yaml; charset=utf-8', // YAML 文件 + '.yml': 'application/x-yaml; charset=utf-8', // YAML 文件（别名） + '.zip': 'application/octet-stream', + }; + return contentType[extname]; +}; + +export const getContentType = (filePath: string) => { + const extname = path.extname(filePath).toLowerCase(); + const contentType = getContentTypeCore(extname); + return contentType || 'application/octet-stream'; +}; diff --git a/src/modules/fm-manager/get-router.ts b/src/modules/fm-manager/get-router.ts new file mode 100644 index 0000000..9822a79 --- /dev/null +++ b/src/modules/fm-manager/get-router.ts @@ -0,0 +1,67 @@ +import { config } from '../config.ts'; +import { app } from '@/app.ts' +const api = config?.api || { host: 'https://kevisual.cn', path: '/api/router' }; +const apiPath = api.path || '/api/router'; +export const fetchTest = async (id: string) => { + const res = await app.call({ + path: 'user-app', + key: 'test', + payload: { + id, + }, + }); + return { + code: res.code, + data: res.body, + }; +}; + +export const fetchDomain = async (domain: string): Promise<{ code: number; data: any, message?: string }> => { + const res = await app.call({ + path: 'app', + key: 'getDomainApp', + payload: { + data: { + domain, + } + }, + }); + return { + code: res.code, + data: res.body as any, + }; +}; + +export const fetchApp = async (payload: { user: string; app: string }) => { + const res = await app.call({ + path: 'app', + key: 'getApp', + payload: { + data: { + user: payload.user, + key: payload.app, + } + }, + }); + return { + code: res.code, + data: res.body, + }; +}; + +export const getUserConfig = async (token: string) => { + // await queryConfig.getConfigByKey('user.json', { token }) + const res = await app.call({ + path: 'config', + key: 'defaultConfig', + payload: { + configKey: 'user.json', + token, + } + }); + const data = res.body; + return { + code: res.code, + data + } +} \ No newline at end of file diff --git a/src/modules/fm-manager/index.ts b/src/modules/fm-manager/index.ts new file mode 100644 index 0000000..b5a1170 --- /dev/null +++ b/src/modules/fm-manager/index.ts @@ -0,0 +1,10 @@ +export * from './proxy/http-proxy.ts' +export * from './proxy/file-proxy.ts' +export * from './proxy/minio-proxy.ts' +export * from './proxy/ai-proxy.ts' + +export * from './get-router.ts' + +export * from './get-content-type.ts' + +export * from './utils.ts' \ No newline at end of file diff --git a/src/modules/fm-manager/proxy/ai-proxy.ts b/src/modules/fm-manager/proxy/ai-proxy.ts new file mode 100644 index 0000000..362e9d5 --- /dev/null +++ b/src/modules/fm-manager/proxy/ai-proxy.ts @@ -0,0 +1,292 @@ +import { bucketName, minioClient } from '@/modules/minio.ts'; +import { IncomingMessage, ServerResponse } from 'http'; +import { filterKeys } from './http-proxy.ts'; +import { getUserFromRequest } from '../utils.ts'; +import { UserPermission, Permission } from '@kevisual/permission'; +import { getLoginUser } from '@/modules/auth.ts'; +import busboy from 'busboy'; +import { getContentType } from '../get-content-type.ts'; +import { OssBase } from '@kevisual/oss'; +import { parseSearchValue } from '@kevisual/router/browser'; +import { logger } from '@/modules/logger.ts'; + +type FileList = { + name: string; + prefix?: string; + size?: number; + etag?: string; + lastModified?: Date; + + path?: string; + url?: string; + pathname?: string; +}; +export const getFileList = async (list: any, opts?: { objectName: string; app: string; host?: string }) => { + const { app, host } = opts || {}; + const objectName = opts?.objectName || ''; + let newObjectName = objectName; + const [user] = objectName.split('/'); + let replaceUser = user + '/'; + if (app === 'resources') { + replaceUser = `${user}/resources/`; + newObjectName = objectName.replace(`${user}/`, replaceUser); + } + return list.map((item: FileList) => { + if (item.name) { + item.path = item.name?.replace?.(objectName, ''); + item.pathname = '/' + item.name.replace(`${user}/`, replaceUser); + } else { + item.path = item.prefix?.replace?.(objectName, ''); + item.pathname = '/' + item.prefix.replace(`${user}/`, replaceUser); + } + if (item.name && app === 'ai') { + const [_user, _app, _version, ...rest] = item.name.split('/'); + item.pathname = item.pathname.replace(`/${_user}/${_app}/${_version}/`, `/${_user}/${_app}/`); + } else if (app === 'ai') { + const [_user, _app, _version, ...rest] = item.prefix?.split('/'); + item.pathname = item.pathname.replace(`/${_user}/${_app}/${_version}/`, `/${_user}/${_app}/`); + } + item.url = new URL(item.pathname, `https://${host}`).toString(); + return item; + }); +}; +// import { logger } from '@/module/logger.ts'; +const getAiProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => { + const { createNotFoundPage } = opts; + const _u = new URL(req.url, 'http://localhost'); + const oss = opts.oss; + const params = _u.searchParams; + const password = params.get('p'); + const hash = params.get('hash'); + let dir = !!params.get('dir'); + const recursive = !!params.get('recursive'); + const { objectName, app, owner, loginUser, isOwner } = await getObjectName(req); + if (!dir && _u.pathname.endsWith('/')) { + dir = true; // 如果是目录请求，强制设置为true + } + logger.debug(`proxy request: ${objectName}`, dir); + try { + if (dir) { + if (!isOwner) { + return createNotFoundPage('no dir permission'); + } + const list = await oss.listObjects(objectName, { recursive: recursive }); + res.writeHead(200, { 'content-type': 'application/json' }); + const host = req.headers['host'] || 'localhost'; + res.end( + JSON.stringify({ + code: 200, + data: await getFileList(list, { + objectName: objectName, + app: app, + host, + }), + }), + ); + return true; + } + const stat = await oss.statObject(objectName); + if (!stat) { + createNotFoundPage('Invalid proxy url'); + logger.debug('no stat', objectName, owner, req.url); + return true; + } + const permissionInstance = new UserPermission({ permission: stat.metaData as Permission, owner: owner }); + const checkPermission = permissionInstance.checkPermissionSuccess({ + username: loginUser?.tokenUser?.username || '', + password: password, + }); + if (!checkPermission.success) { + logger.info('no permission', checkPermission, loginUser, owner); + return createNotFoundPage('no permission'); + } + if (hash && stat.etag === hash) { + res.writeHead(304); // not modified + res.end('not modified'); + return true; + } + const filterMetaData = filterKeys(stat.metaData, ['size', 'etag', 'last-modified']); + const contentLength = stat.size; + const etag = stat.etag; + const lastModified = stat.lastModified.toISOString(); + + const objectStream = await minioClient.getObject(bucketName, objectName); + const headers = { + 'Content-Length': contentLength, + etag, + 'last-modified': lastModified, + ...filterMetaData, + }; + + res.writeHead(200, { + ...headers, + }); + objectStream.pipe(res, { end: true }); + return true; + } catch (error) { + console.error(`Proxy request error: ${error.message}`); + createNotFoundPage('Invalid ai proxy url'); + return false; + } +}; +export const getMetadata = (pathname: string) => { + let meta: any = { 'app-source': 'user-app' }; + const isHtml = pathname.endsWith('.html'); + if (isHtml) { + meta = { + ...meta, + 'content-type': 'text/html; charset=utf-8', + 'cache-control': 'no-cache', + }; + } else { + meta = { + ...meta, + 'content-type': getContentType(pathname), + 'cache-control': 'max-age=31536000, immutable', + }; + } + return meta; +}; + +export const postProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => { + const _u = new URL(req.url, 'http://localhost'); + + const pathname = _u.pathname; + const oss = opts.oss; + const params = _u.searchParams; + const force = !!params.get('force'); + const hash = params.get('hash'); + let meta = parseSearchValue(params.get('meta'), { decode: true }); + if (!hash && !force) { + return opts?.createNotFoundPage?.('no hash'); + } + const { objectName, isOwner } = await getObjectName(req); + if (!isOwner) { + return opts?.createNotFoundPage?.('no permission'); + } + const end = (data: any, message?: string, code = 200) => { + res.writeHead(code, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ code: code, data: data, message: message || 'success' })); + }; + let statMeta: any = {}; + if (!force) { + const check = await oss.checkObjectHash(objectName, hash, meta); + statMeta = check?.metaData || {}; + let isNewMeta = false; + if (check.success && JSON.stringify(meta) !== '{}' && !check.equalMeta) { + meta = { ...statMeta, ...getMetadata(pathname), ...meta }; + isNewMeta = true; + await oss.replaceObject(objectName, { ...meta }); + } + if (check.success) { + return end({ success: true, hash, meta, isNewMeta, equalMeta: check.equalMeta }, '文件已存在'); + } + } + const bb = busboy({ + headers: req.headers, + limits: { + fileSize: 100 * 1024 * 1024, // 100MB + files: 1, + }, + }); + let fileProcessed = false; + bb.on('file', async (name, file, info) => { + fileProcessed = true; + try { + // console.log('file', stat?.metaData); + // await sleep(2000); + await oss.putObject( + objectName, + file, + { + ...statMeta, + ...getMetadata(pathname), + ...meta, + }, + { check: false, isStream: true }, + ); + end({ success: true, name, info, isNew: true, hash, meta: meta?.metaData, statMeta }, '上传成功', 200); + } catch (error) { + end({ error: error }, '上传失败', 500); + } + }); + + bb.on('finish', () => { + // 只有当没有文件被处理时才执行end + if (!fileProcessed) { + end({ success: false }, '没有接收到文件', 400); + } + }); + bb.on('error', (err) => { + console.error('Busboy 错误:', err); + end({ error: err }, '文件解析失败', 500); + }); + + req.pipe(bb); +}; +export const getObjectName = async (req: IncomingMessage, opts?: { checkOwner?: boolean }) => { + const _u = new URL(req.url, 'http://localhost'); + const pathname = decodeURIComponent(_u.pathname); + const params = _u.searchParams; + const { user, app } = getUserFromRequest(req); + const checkOwner = opts?.checkOwner ?? true; + let objectName = ''; + let owner = ''; + if (app === 'ai') { + const version = params.get('version') || '1.0.0'; // root/ai + objectName = pathname.replace(`/${user}/${app}/`, `${user}/${app}/${version}/`); + } else { + objectName = pathname.replace(`/${user}/${app}/`, `${user}/`); // root/resources + } + owner = user; + let isOwner = undefined; + let loginUser: Awaited> = null; + if (checkOwner) { + loginUser = await getLoginUser(req); + logger.debug('getObjectName', loginUser, user, app); + isOwner = loginUser?.tokenUser?.username === owner; + } + return { + objectName, + loginUser, + owner, + isOwner, + app, + user, + }; +}; +export const deleteProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => { + const { objectName, isOwner } = await getObjectName(req); + let oss = opts.oss; + if (!isOwner) { + return opts?.createNotFoundPage?.('no permission'); + } + try { + await oss.deleteObject(objectName); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: true, message: 'delete success', objectName })); + } catch (error) { + logger.error('deleteProxy error', error); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: false, error: error })); + } +}; + +type ProxyOptions = { + createNotFoundPage: (msg?: string) => any; + oss?: OssBase; +}; +export const aiProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => { + const oss = new OssBase({ bucketName, client: minioClient }); + if (!opts.oss) { + opts.oss = oss; + } + if (req.method === 'POST') { + return postProxy(req, res, opts); + } + if (req.method === 'DELETE') { + return deleteProxy(req, res, opts); + } + + return getAiProxy(req, res, opts); +}; diff --git a/src/modules/fm-manager/proxy/file-proxy.ts b/src/modules/fm-manager/proxy/file-proxy.ts new file mode 100644 index 0000000..091579c --- /dev/null +++ b/src/modules/fm-manager/proxy/file-proxy.ts @@ -0,0 +1,25 @@ +import http from 'node:http'; +import send from 'send'; +import { fileIsExist } from '@kevisual/use-config'; +import path from 'node:path'; +export type ProxyInfo = { + path?: string; + target: string; + type?: 'static' | 'dynamic' | 'minio'; +}; +export const fileProxy = (req: http.IncomingMessage, res: http.ServerResponse, proxyApi: ProxyInfo) => { + // url开头的文件 + const url = new URL(req.url, 'http://localhost'); + const pathname = url.pathname; + // 检测文件是否存在，如果文件不存在，则返回404 + const filePath = path.join(process.cwd(), proxyApi.target, pathname); + if (!fileIsExist(filePath)) { + res.statusCode = 404; + res.end('Not Found File'); + return; + } + const file = send(req, pathname, { + root: proxyApi.target, + }); + file.pipe(res); +}; diff --git a/src/modules/fm-manager/proxy/http-proxy.ts b/src/modules/fm-manager/proxy/http-proxy.ts new file mode 100644 index 0000000..b1445f0 --- /dev/null +++ b/src/modules/fm-manager/proxy/http-proxy.ts @@ -0,0 +1,165 @@ +import { pipeline, Readable } from 'node:stream'; +import { promisify } from 'node:util'; +import { bucketName, minioClient, minioResources } from '@/modules/minio.ts'; +import fs from 'node:fs'; +import { IncomingMessage, ServerResponse } from 'node:http'; +import http from 'node:http'; +import https from 'node:https'; +import { UserApp } from '@/modules/user-app/index.ts'; +import { addStat } from '@/modules/html/stat/index.ts'; +import path from 'path'; +import { getTextContentType } from '@/modules/fm-manager/index.ts'; +import { logger } from '@/modules/logger.ts'; + +const pipelineAsync = promisify(pipeline); + +export async function downloadFileFromMinio(fileUrl: string, destFile: string) { + const objectName = fileUrl.replace(minioResources + '/', ''); + const objectStream = await minioClient.getObject(bucketName, objectName); + const destStream = fs.createWriteStream(destFile); + await pipelineAsync(objectStream, destStream); + console.log(`minio File downloaded to ${minioResources}/${objectName} \n ${destFile}`); +} +export const filterKeys = (metaData: Record, clearKeys: string[] = []) => { + const keys = Object.keys(metaData); + // remove X-Amz- meta data + const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys]; + const filteredKeys = keys.filter((key) => !removeKeys.includes(key)); + return filteredKeys.reduce((acc, key) => { + acc[key] = metaData[key]; + return acc; + }, {} as Record); +}; +export async function minioProxy( + req: IncomingMessage, + res: ServerResponse, + opts: { + proxyUrl: string; + createNotFoundPage: (msg?: string) => any; + isDownload?: boolean; + }, +) { + const fileUrl = opts.proxyUrl; + const { createNotFoundPage, isDownload = false } = opts; + const objectName = fileUrl.replace(minioResources + '/', ''); + try { + const stat = await minioClient.statObject(bucketName, objectName); + if (stat.size === 0) { + createNotFoundPage('Invalid proxy url'); + return true; + } + const filterMetaData = filterKeys(stat.metaData, ['size', 'etag', 'last-modified']); + const contentLength = stat.size; + const etag = stat.etag; + const lastModified = stat.lastModified.toISOString(); + const fileName = objectName.split('/').pop(); + const ext = path.extname(fileName || ''); + const objectStream = await minioClient.getObject(bucketName, objectName); + const headers = { + 'Content-Length': contentLength, + etag, + 'last-modified': lastModified, + 'file-name': fileName, + ...filterMetaData, + ...getTextContentType(ext), + }; + if (objectName.endsWith('.html') && !isDownload) { + const { html, contentLength } = await getTextFromStreamAndAddStat(objectStream); + res.writeHead(200, { + ...headers, + 'Content-Length': contentLength, + }); + res.end(html); + } else { + res.writeHead(200, { + ...headers, + }); + objectStream.pipe(res, { end: true }); + } + return true; + } catch (error) { + console.error(`Proxy request error: ${error.message}`); + createNotFoundPage('Invalid proxy url'); + return false; + } +} + +// 添加一个辅助函数来从流中获取文本 +async function getTextFromStream(stream: Readable | IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let data = ''; + stream.on('data', (chunk) => { + data += chunk; + }); + stream.on('end', () => { + resolve(data); + }); + stream.on('error', (err) => { + reject(err); + }); + }); +} +export async function getTextFromStreamAndAddStat(stream: Readable | IncomingMessage): Promise<{ html: string; contentLength: number }> { + const html = await getTextFromStream(stream); + const newHtml = addStat(html); + const newContentLength = Buffer.byteLength(newHtml); + return { html: newHtml, contentLength: newContentLength }; +} +export const httpProxy = async ( + req: IncomingMessage, + res: ServerResponse, + opts: { + proxyUrl: string; + userApp: UserApp; + createNotFoundPage: (msg?: string) => any; + }, +) => { + const { proxyUrl, userApp, createNotFoundPage } = opts; + const _u = new URL(req.url, 'http://localhost'); + const params = _u.searchParams; + const isDownload = params.get('download') === 'true'; + if (proxyUrl.startsWith(minioResources)) { + const isOk = await minioProxy(req, res, { ...opts, isDownload }); + if (!isOk) { + userApp.clearCacheData(); + } + return; + } + let protocol = proxyUrl.startsWith('https') ? https : http; + // 代理 + const proxyReq = protocol.request(proxyUrl, async (proxyRes) => { + const headers = proxyRes.headers; + + if (proxyRes.statusCode === 404) { + userApp.clearCacheData(); + return createNotFoundPage('Invalid proxy url'); + } + if (proxyRes.statusCode === 302) { + res.writeHead(302, { Location: proxyRes.headers.location }); + return res.end(); + } + if (proxyUrl.endsWith('.html') && !isDownload) { + try { + const { html, contentLength } = await getTextFromStreamAndAddStat(proxyRes); + res.writeHead(200, { + ...headers, + 'Content-Length': contentLength, + }); + res.end(html); + } catch (error) { + console.error(`Proxy request error: ${error.message}`); + return createNotFoundPage('Invalid proxy url:' + error.message); + } + } else { + res.writeHead(proxyRes.statusCode, { + ...headers, + }); + proxyRes.pipe(res, { end: true }); + } + }); + proxyReq.on('error', (err) => { + console.error(`Proxy request error: ${err.message}`); + userApp.clearCacheData(); + }); + proxyReq.end(); +}; diff --git a/src/modules/fm-manager/proxy/minio-proxy.ts b/src/modules/fm-manager/proxy/minio-proxy.ts new file mode 100644 index 0000000..da7deca --- /dev/null +++ b/src/modules/fm-manager/proxy/minio-proxy.ts @@ -0,0 +1,24 @@ +import http from 'http'; +import { minioClient } from '@/modules/minio.ts'; +type ProxyInfo = { + path?: string; + target: string; + type?: 'static' | 'dynamic' | 'minio'; +}; +export const minioProxyOrigin = async (req: http.IncomingMessage, res: http.ServerResponse, proxyApi: ProxyInfo) => { + try { + const requestUrl = new URL(req.url, 'http://localhost'); + const objectPath = requestUrl.pathname; + const bucketName = proxyApi.target; + let objectName = objectPath.slice(1); + if (objectName.startsWith(bucketName)) { + objectName = objectName.slice(bucketName.length); + } + const objectStream = await minioClient.getObject(bucketName, objectName); + objectStream.pipe(res); + } catch (error) { + console.error('Error fetching object from MinIO:', error); + res.statusCode = 500; + res.end('Internal Server Error'); + } +}; diff --git a/src/modules/fm-manager/utils.ts b/src/modules/fm-manager/utils.ts new file mode 100644 index 0000000..bc564ad --- /dev/null +++ b/src/modules/fm-manager/utils.ts @@ -0,0 +1,31 @@ +import { IncomingMessage } from 'node:http'; +import http from 'node:http'; + +export const getUserFromRequest = (req: IncomingMessage) => { + const url = new URL(req.url, `http://${req.headers.host}`); + const pathname = url.pathname; + const keys = pathname.split('/'); + const [_, user, app] = keys; + return { + user, + app, + }; +}; + + +export const getDNS = (req: http.IncomingMessage) => { + const hostName = req.headers.host; + const ip = req.socket.remoteAddress; + return { hostName, ip }; +}; + +export const isLocalhost = (hostName: string) => { + return hostName.includes('localhost') || hostName.includes('192.168'); +}; + +export const isIpv4OrIpv6 = (hostName: string) => { + const ipv4 = /^(\d{1,3}\.){3}\d{1,3}$/; + const ipv6 = /^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$/; + return ipv4.test(hostName) || ipv6.test(hostName); +}; +export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/src/modules/html/create-refresh-html.ts b/src/modules/html/create-refresh-html.ts new file mode 100644 index 0000000..74e5220 --- /dev/null +++ b/src/modules/html/create-refresh-html.ts @@ -0,0 +1,199 @@ +/** + * 创建一个刷新页面，定时 + * fetch('/api/proxy/refresh?user=user&app=app'), 如果返回200，则刷新页面 + * @param {string} user - 用户名 + * @param {string} app - 应用名 + * @returns {string} - HTML字符串 + */ +export const createRefreshHtml = (user, app) => { + return ` + + + + + + App: ${user}/${app} + + + +
+

+ 📱 + ${user}/${app} +

+ +
+

正在加载应用...

+ +

应用正在启动中，请稍候

+
+ +

如果长时间没有加载出来，请手动 刷新页面

+ +
+ 检查次数: + 0 +
+ +
+ © ${new Date().getFullYear()} ${user}/${app} - 自动刷新页面 +
+
+ + + + + `; +}; \ No newline at end of file diff --git a/src/modules/html/favicon.ico b/src/modules/html/favicon.ico new file mode 100644 index 0000000..3b16294 Binary files /dev/null and b/src/modules/html/favicon.ico differ diff --git a/src/modules/html/stat/index.ts b/src/modules/html/stat/index.ts new file mode 100644 index 0000000..9869cde --- /dev/null +++ b/src/modules/html/stat/index.ts @@ -0,0 +1,17 @@ +import { config } from '../../config.ts'; + +/** + * 添加统计脚本 + * @param html + * @returns + */ +export const addStat = (html: string, addStat = true) => { + if (!addStat) { + return html; + } + const { websiteId } = config.stat || {}; + if (!websiteId) { + return html; + } + return html.replace('', ``); +}; diff --git a/src/modules/minio.ts b/src/modules/minio.ts index 4051858..994e0c9 100644 --- a/src/modules/minio.ts +++ b/src/modules/minio.ts @@ -1,5 +1,6 @@ import { Client, ClientOptions } from 'minio'; -import { config } from './config.ts'; +import { useConfig } from '@kevisual/use-config'; +const config = useConfig(); import { OssBase } from '@kevisual/oss/services'; const minioConfig = { endPoint: config.MINIO_ENDPOINT || 'localhost', @@ -8,10 +9,14 @@ const minioConfig = { accessKey: config.MINIO_ACCESS_KEY, secretKey: config.MINIO_SECRET_KEY, }; +const { port, endPoint, useSSL } = minioConfig; // console.log('minioConfig', minioConfig); export const minioClient = new Client(minioConfig); - export const bucketName = config.MINIO_BUCKET_NAME || 'resources'; + +export const minioUrl = `http${useSSL ? 's' : ''}://${endPoint}:${port || 9000}`; +export const minioResources = `${minioUrl}/resources`; + if (!minioClient) { throw new Error('Minio client not initialized'); } diff --git a/src/modules/redis.ts b/src/modules/redis.ts index 0e26954..f3fdc7f 100644 --- a/src/modules/redis.ts +++ b/src/modules/redis.ts @@ -34,6 +34,31 @@ redis.on('error', (err) => { console.error('Redis 连接错误', err); }); -// 初始化 Redis 客户端 -export const redisPublisher = createRedisClient(); // 用于发布消息 -export const redisSubscriber = createRedisClient(); // 用于订阅消息 +export const subscriber = redis.duplicate(); // 创建一个订阅者连接 + +async function ensureKeyspaceNotifications() { + try { + // 获取当前的 `notify-keyspace-events` 配置 + const currentConfig = (await redis.config('GET', 'notify-keyspace-events')) as string[]; + + // 检查返回的数组长度是否大于1，表示获取成功 + if (currentConfig && currentConfig.length > 1) { + const currentSetting = currentConfig[1]; // 值在数组的第二个元素 + // 检查当前配置是否包含 "Ex" + if (!currentSetting.includes('E') || !currentSetting.includes('x')) { + console.log('Keyspace notifications are not fully enabled. Setting correct value...'); + await redis.config('SET', 'notify-keyspace-events', 'Ex'); + console.log('Keyspace notifications enabled with setting "Ex".'); + } else { + // console.log('Keyspace notifications are already correctly configured.'); + } + } else { + console.error('Failed to get the current notify-keyspace-events setting.'); + } + } catch (err) { + console.error('Error while configuring Redis keyspace notifications:', err); + } +} + +// 确保键空间通知被正确设置 +ensureKeyspaceNotifications().catch(console.error); diff --git a/src/modules/user-app/get-app-status.ts b/src/modules/user-app/get-app-status.ts new file mode 100644 index 0000000..a7ad04f --- /dev/null +++ b/src/modules/user-app/get-app-status.ts @@ -0,0 +1,30 @@ +import { redis } from '../redis.ts'; + +export type AppLoadStatus = { + status: 'running' | 'loading' | 'error' | 'not-exist'; + message: string; +}; + +export const getAppLoadStatus = async (user: string, app: string): Promise => { + const key = 'user:app:status:' + app + ':' + user; + const value = await redis.get(key); + if (!value) { + return { + status: 'not-exist', + message: 'not-exist', + }; // 没有加载过 + } + try { + return JSON.parse(value); + } catch (err) { + return { + status: 'error', + message: 'error', + }; + } +}; +export const setAppLoadStatus = async (user: string, app: string, status: AppLoadStatus, exp = 3 * 60) => { + const key = 'user:app:status:' + app + ':' + user; + const value = JSON.stringify(status); + await redis.set(key, value, 'EX', exp); // 5分钟过期 +}; diff --git a/src/modules/user-app/index.ts b/src/modules/user-app/index.ts new file mode 100644 index 0000000..82c29cb --- /dev/null +++ b/src/modules/user-app/index.ts @@ -0,0 +1,464 @@ +import path from 'path'; +import { redis, subscriber } from '../redis.ts'; +import { myConfig as config, fileStore } from '../config.ts'; +import fs from 'fs'; +import crypto from 'crypto'; +import { nanoid } from 'nanoid'; +import { pipeline } from 'stream'; +import { promisify } from 'util'; +import { getAppLoadStatus, setAppLoadStatus } from './get-app-status.ts'; +import { minioResources } from '../minio.ts'; +import { downloadFileFromMinio, fetchApp, fetchDomain, fetchTest } from '@/modules/fm-manager/index.ts'; +import { logger } from '../logger.ts'; +export * from './get-app-status.ts'; +export * from './user-home.ts'; + +const pipelineAsync = promisify(pipeline); + +const { resources } = config?.proxy || { resources: minioResources }; +const wrapperResources = (resources: string, urlpath: string) => { + if (urlpath.startsWith('http')) { + return urlpath; + } + return `${resources}/${urlpath}`; +}; +const demoData = { + user: 'root', + key: 'codeflow', + appType: 'web-single', // + version: '1.0.0', + domain: null, + type: 'oss', // oss， 默认是oss + data: { + files: [ + { + name: 'index.html', + path: 'codeflow/index.html', + }, + { + name: 'assets/index-14y4J8dP.js', + path: 'codeflow/assets/index-14y4J8dP.js', + }, + { + name: 'assets/index-C-libw4a.css', + path: 'codeflow/assets/index-C-libw4a.css', + }, + ], + }, +}; +type UserAppOptions = { + user: string; + app: string; +}; + +export class UserApp { + user: string; + app: string; + isTest: boolean; + constructor(options: UserAppOptions) { + this.user = options.user; + this.app = options.app; + if (this.user === 'test') { + this.isTest = true; + } + } + /** + * 是否已经加载到本地了 + * @returns + */ + async getExist() { + const app = this.app; + const user = this.user; + const key = 'user:app:exist:' + app + ':' + user; + const permissionKey = 'user:app:permission:' + app + ':' + user; + const value = await redis.get(key); + const permission = await redis.get(permissionKey); + if (!value) { + return false; + } + const [indexFilePath, etag, proxy] = value.split('||'); + try { + return { + indexFilePath, + etag, + proxy: proxy === 'true', + permission: permission ? JSON.parse(permission) : { share: 'private' }, + }; + } catch (e) { + console.error('getExist error parse', e); + await this.clearCacheData(); + return false; + } + } + /** + * 获取缓存数据，不存在不会加载 + * @returns + */ + async getCache() { + const app = this.app; + const user = this.user; + const key = 'user:app:' + app + ':' + user; + const value = await redis.get(key); + if (!value) { + return null; + } + return JSON.parse(value); + } + async getFile(appFileUrl: string) { + const app = this.app; + const user = this.user; + const key = 'user:app:set:' + app + ':' + user; + const value = await redis.hget(key, appFileUrl); + // const values = await redis.hgetall(key); + // console.log('getFile', values); + return value; + } + static async getDomainApp(domain: string) { + const key = 'domain:' + domain; + const value = await redis.get(key); + if (value) { + const [_user, _app] = value.split(':'); + return { + user: _user, + app: _app, + }; + } + + // 获取域名对应的用户和应用 + const fetchRes = await fetchDomain(domain).catch((err) => { + return { + code: 500, + message: err, + }; + }); + if (fetchRes?.code !== 200) { + console.log('fetchRes is error', fetchRes); + return null; + } + const fetchData = fetchRes.data; + if (fetchData.status !== 'running') { + console.error('fetchData status is not running', fetchData.user, fetchData.key); + return null; + } + const data = { + user: fetchData.user, + app: fetchData.key, + }; + redis.set(key, data.user + ':' + data.app, 'EX', 60 * 60 * 24 * 7); // 7天 + + const userDomainApp = 'user:domain:app:' + data.user + ':' + data.app; + + const domainKeys = await redis.get(userDomainApp); + let domainKeysList = domainKeys ? JSON.parse(domainKeys) : []; + domainKeysList.push(domain); + const uniq = (arr: string[]) => { + return [...new Set(arr)]; + }; + domainKeysList = uniq(domainKeysList); + await redis.set(userDomainApp, JSON.stringify(domainKeysList), 'EX', 60 * 60 * 24 * 7); // 7天 + return data; + } + /** + * 加载结束 + * @param msg + */ + async setLoaded(status: 'running' | 'error' | 'loading', msg?: string) { + const app = this.app; + const user = this.user; + await setAppLoadStatus(user, app, { + status, + message: msg, + }); + } + /** + * 获取加载状态 + * @returns + */ + async getLoaded() { + const app = this.app; + const user = this.user; + const value = await getAppLoadStatus(user, app); + return value; + } + /** + * 设置缓存数据，当出问题了，就重新加载。 + * @returns + */ + async setCacheData() { + const app = this.app; + const user = this.user; + const isTest = this.isTest; + const key = 'user:app:' + app + ':' + user; + const fetchRes = isTest ? await fetchTest(app) : await fetchApp({ user, app }); + if (fetchRes?.code !== 200) { + console.log('获取缓存的cache错误', fetchRes, 'user', user, 'app', app); + return { code: 500, message: 'fetchRes is error' }; + } + + const loadStatus = await getAppLoadStatus(user, app); + logger.debug('loadStatus', loadStatus); + if (loadStatus.status === 'loading') { + // 其他情况，error或者running都可以重新加载 + return { + loading: true, + }; + } + + const fetchData = fetchRes.data; + if (!fetchData.type) { + // console.error('fetchData type is error', fetchData); + // return false; + fetchData.type = 'oss'; + } + if (fetchData.status !== 'running') { + console.error('fetchData status is not running', fetchData.user, fetchData.key); + return { code: 500, message: 'app status is not running' }; + } + // console.log('fetchData', JSON.stringify(fetchData.data.files, null, 2)); + // const getFileSize + this.setLoaded('loading', 'loading'); + const loadProxy = async () => { + const value = fetchData; + await redis.set(key, JSON.stringify(value)); + const version = value.version; + let indexHtml = resources + '/' + user + '/' + app + '/' + version + '/index.html'; + const files = value?.data?.files || []; + const permission = value?.data?.permission || { share: 'private' }; + const data = {}; + + // 将文件名和路径添加到 `data` 对象中 + files.forEach((file) => { + if (file.name === 'index.html') { + indexHtml = wrapperResources(resources, file.path); + } + data[file.name] = wrapperResources(resources, file.path); + }); + await redis.set('user:app:exist:' + app + ':' + user, indexHtml + '||etag||true', 'EX', 60 * 60 * 24 * 7); // 7天 + await redis.set('user:app:permission:' + app + ':' + user, JSON.stringify(permission), 'EX', 60 * 60 * 24 * 7); // 7天 + await redis.hset('user:app:set:' + app + ':' + user, data); + this.setLoaded('running', 'loaded'); + }; + const loadFilesFn = async () => { + const value = await downloadUserAppFiles(user, app, fetchData); + if (value.data.files.length === 0) { + console.error('root files length is zero', user, app); + this.setLoaded('running', 'root files length is zero'); + const mockPath = path.join(fileStore, user, app, 'index.html'); + value.data.files = [ + { + name: 'index.html', // 映射 + path: mockPath.replace(fileStore, ''), // 实际 + }, + ]; + if (!checkFileExistsSync(path.join(fileStore, user, app))) { + fs.mkdirSync(path.join(fileStore, user, app), { recursive: true }); + } + // 自己创建一个index.html + fs.writeFileSync(path.join(fileStore, user, app, 'index.html'), 'not has any app info', { + encoding: 'utf-8', + }); + } + await redis.set(key, JSON.stringify(value)); + const files = value.data.files; + const permission = fetchData?.data?.permission || { share: 'private' }; + const data = {}; + let indexHtml = path.join(fileStore, user, app, 'index.html') + '||etag||false'; + // 将文件名和路径添加到 `data` 对象中 + files.forEach((file) => { + data[file.name] = file.path; + if (file.name === 'index.html') { + indexHtml = file.path; + } + }); + await redis.set('user:app:exist:' + app + ':' + user, indexHtml, 'EX', 60 * 60 * 24 * 7); // 7天 + await redis.set('user:app:permission:' + app + ':' + user, JSON.stringify(permission), 'EX', 60 * 60 * 24 * 7); // 7天 + await redis.hset('user:app:set:' + app + ':' + user, data); + this.setLoaded('running', 'loaded'); + }; + logger.debug('loadFilesFn', fetchData.proxy); + try { + if (fetchData.proxy === true) { + await loadProxy(); + return { + code: 200, + data: 'loaded', + }; + } else { + loadFilesFn(); + } + } catch (e) { + console.error('loadFilesFn error', e); + this.setLoaded('error', 'loadFilesFn error'); + } + return { + code: 20000, + data: 'loading', + }; + } + async clearCacheData() { + const app = this.app; + const user = this.user; + const key = 'user:app:' + app + ':' + user; + await redis.del(key); + await redis.del('user:app:exist:' + app + ':' + user); + await redis.del('user:app:set:' + app + ':' + user); + await redis.del('user:app:status:' + app + ':' + user); + await redis.del('user:app:permission:' + app + ':' + user); + const userDomainApp = 'user:domain:app:' + user + ':' + app; + const domainKeys = await redis.get(userDomainApp); + if (domainKeys) { + const domainKeysList = JSON.parse(domainKeys); + domainKeysList.forEach(async (domain: string) => { + await redis.del('domain:' + domain); + }); + } + await redis.del(userDomainApp); + + // 删除所有文件 + deleteUserAppFiles(user, app); + } + fileCheck(file: string) { + return checkFileExistsSync(file); + } + async close() { + // 关闭连接 + await redis.quit(); + } +} +export const downloadUserAppFiles = async (user: string, app: string, data: typeof demoData) => { + const { + data: { files, ...dataRest }, + ...rest + } = data; + const uploadFiles = path.join(fileStore, user, app); + if (!checkFileExistsSync(uploadFiles)) { + fs.mkdirSync(uploadFiles, { recursive: true }); + } + const newFiles = []; + + if (data.type === 'oss') { + let serverPath = new URL(resources).href; + let hasIndexHtml = false; + // server download file + for (let i = 0; i < files.length; i++) { + const file = files[i]; + const destFile = path.join(uploadFiles, file.name); + const destDir = path.dirname(destFile); // 获取目标文件所在的目录路径 + if (file.name === 'index.html') { + hasIndexHtml = true; + } + // 检查目录是否存在，如果不存在则创建 + if (!checkFileExistsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); // 递归创建目录 + } + const downloadURL = wrapperResources(serverPath, file.path); + // 下载文件到 destFile + await downloadFile(downloadURL, destFile); + const etag = nanoid(); + newFiles.push({ + name: file.name, + path: destFile.replace(fileStore, '') + '||' + etag, + }); + } + if (!hasIndexHtml) { + newFiles.push({ + name: 'index.html', + path: path.join(uploadFiles, 'index.html'), + }); + fs.writeFileSync(path.join(uploadFiles, 'index.html'), JSON.stringify(files), { + encoding: 'utf-8', + }); + } + } + + return { + ...rest, + data: { + ...dataRest, + files: newFiles, + }, + }; +}; +export const checkFileExistsSync = (filePath: string) => { + try { + // 使用 F_OK 检查文件或目录是否存在 + fs.accessSync(filePath, fs.constants.F_OK); + return true; + } catch (err) { + return false; + } +}; +export const deleteUserAppFiles = async (user: string, app: string) => { + const uploadFiles = path.join(fileStore, user, app); + try { + fs.rmSync(uploadFiles, { recursive: true }); + } catch (err) { + if (err.code === 'ENOENT') { + // 文件不存在 + } else { + console.error('deleteUserAppFiles', err); + } + } + // console.log('deleteUserAppFiles', res); +}; + +async function downloadFile(fileUrl: string, destFile: string) { + if (fileUrl.startsWith(minioResources)) { + await downloadFileFromMinio(fileUrl, destFile); + return; + } + console.log('destFile', destFile, 'fileUrl', fileUrl); + const res = await fetch(fileUrl); + + if (!res.ok) { + throw new Error(`Failed to fetch ${fileUrl}: ${res.statusText}`); + } + const destStream = fs.createWriteStream(destFile); + + // 使用 `pipeline` 将 `res.body` 中的数据传递给 `destStream` + await pipelineAsync(res.body, destStream); + + console.log(`File downloaded to ${destFile}`); +} + +export const clearAllUserApp = async () => { + // redis 删除 所有的 user:app:* + const keys = await redis.keys('user:app:*'); + console.log('clearAllUserApp', keys); + if (keys.length > 0) { + const pipeline = redis.pipeline(); + keys.forEach((key) => pipeline.del(key)); // 将每个键的删除操作添加到 pipeline 中 + await pipeline.exec(); // 执行 pipeline 中的所有命令 + console.log('All keys deleted successfully using pipeline'); + } +}; +export const setEtag = async (fileContent: string) => { + const eTag = crypto.createHash('md5').update(fileContent).digest('hex'); + return eTag; +}; + +// redis 监听 user:app:exist:*的过期 +subscriber.on('ready', () => { + console.log('Subscriber is ready and connected.'); +}); + +// 订阅 Redis 频道 +subscriber.subscribe('__keyevent@0__:expired', (err, count) => { + if (err) { + console.error('Failed to subscribe: ', err); + } else { + console.log(`Subscribed to ${count} channel(s). Waiting for expired events...`); + } +}); + +// 监听消息事件 +subscriber.on('message', (channel, message) => { + // 检查是否匹配 user:app:exist:* 模式 + if (message.startsWith('user:app:exist:')) { + const [_user, _app, _exist, app, user] = message.split(':'); + // 在这里执行你的逻辑，例如清理缓存或通知用户 + console.log('User app exist key expired:', app, user); + const userApp = new UserApp({ user, app }); + userApp.clearCacheData(); + } +}); diff --git a/src/modules/user-app/user-home.ts b/src/modules/user-app/user-home.ts new file mode 100644 index 0000000..e905f4b --- /dev/null +++ b/src/modules/user-app/user-home.ts @@ -0,0 +1,31 @@ +import http from 'http'; +import { getLoginUser } from '@/modules/auth.ts'; +import { getUserConfig } from '@/modules/fm-manager/index.ts'; + +/** + * 重定向到用户首页 + * @param req + * @param res + */ +export const rediretHome = async (req: http.IncomingMessage, res: http.ServerResponse) => { + const user = await getLoginUser(req); + if (!user?.token) { + res.writeHead(302, { Location: '/root/login/' }); + res.end(); + return; + } + let redirectURL = '/root/center/'; + try { + const token = user.token; + const resConfig = await getUserConfig(token); + if (resConfig.code === 200) { + const configData = resConfig.data?.data as any; + redirectURL = configData?.redirectURL || redirectURL; + } + } catch (error) { + console.error('get resConfig user.json', error); + } finally { + res.writeHead(302, { Location: redirectURL }); + res.end(); + } +}; diff --git a/src/modules/ws-proxy/index.ts b/src/modules/ws-proxy/index.ts new file mode 100644 index 0000000..ace5eb8 --- /dev/null +++ b/src/modules/ws-proxy/index.ts @@ -0,0 +1,69 @@ +import { WebSocketServer } from 'ws'; +import { nanoid } from 'nanoid'; +import { WsProxyManager } from './manager.ts'; +import { getLoginUser } from '@/modules/auth.ts'; +import { logger } from '../logger.ts'; +export const wsProxyManager = new WsProxyManager(); + +export const upgrade = async (request: any, socket: any, head: any) => { + const req = request as any; + const url = new URL(req.url, 'http://localhost'); + const id = url.searchParams.get('id'); + if (url.pathname === '/ws/proxy') { + console.log('upgrade', request.url, id); + wss.handleUpgrade(req, socket, head, (ws) => { + // 这里手动触发 connection 事件 + // @ts-ignore + wss.emit('connection', ws, req); + }); + return true; + } + return false; +}; +export const wss = new WebSocketServer({ + noServer: true, + path: '/ws/proxy', +}); + +wss.on('connection', async (ws, req) => { + console.log('connected', req.url); + const url = new URL(req.url, 'http://localhost'); + const id = url?.searchParams?.get('id') || nanoid(); + const loginUser = await getLoginUser(req); + if (!loginUser) { + ws.send(JSON.stringify({ code: 401, message: 'No Login' })); + ws.close(); + return; + } + const user = loginUser.tokenUser?.username; + wsProxyManager.register(id, { user, ws }); + ws.send( + JSON.stringify({ + type: 'connected', + user: user, + id, + }), + ); + ws.on('message', async (event: Buffer) => { + const eventData = event.toString(); + if (!eventData) { + return; + } + const data = JSON.parse(eventData); + logger.debug('message', data); + }); + ws.on('close', () => { + logger.debug('ws closed'); + wsProxyManager.unregister(id, user); + }); +}); + +export class WssApp { + wss: WebSocketServer; + constructor() { + this.wss = wss; + } + upgrade(request: any, socket: any, head: any) { + return upgrade(request, socket, head); + } +} diff --git a/src/modules/ws-proxy/manager.ts b/src/modules/ws-proxy/manager.ts new file mode 100644 index 0000000..2edc3b2 --- /dev/null +++ b/src/modules/ws-proxy/manager.ts @@ -0,0 +1,84 @@ +import { nanoid } from 'nanoid'; +import { WebSocket } from 'ws'; +import { logger } from '../logger.ts'; +class WsMessage { + ws: WebSocket; + user?: string; + constructor({ ws, user }: WssMessageOptions) { + this.ws = ws; + this.user = user; + } + async sendData(data: any, opts?: { timeout?: number }) { + if (this.ws.readyState !== WebSocket.OPEN) { + return { code: 500, message: 'WebSocket is not open' }; + } + const timeout = opts?.timeout || 10 * 6 * 1000; // 10 minutes + const id = nanoid(); + const message = JSON.stringify({ + id, + type: 'proxy', + data, + }); + logger.info('ws-proxy sendData', message); + this.ws.send(message); + return new Promise((resolve) => { + const timer = setTimeout(() => { + resolve({ + code: 500, + message: 'timeout', + }); + }, timeout); + this.ws.once('message', (event: Buffer) => { + const eventData = event.toString(); + if (!eventData) { + return; + } + const data = JSON.parse(eventData); + if (data.id === id) { + resolve(data.data); + clearTimeout(timer); + } + }); + }); + } +} +type WssMessageOptions = { + ws: WebSocket; + user?: string; +}; +export class WsProxyManager { + wssMap: Map = new Map(); + constructor() {} + getId(id: string, user?: string) { + return id + '/' + user; + } + register(id: string, opts?: { ws: WebSocket; user: string }) { + const _id = this.getId(id, opts?.user || ''); + if (this.wssMap.has(_id)) { + const value = this.wssMap.get(_id); + if (value) { + value.ws.close(); + } + } + const value = new WsMessage({ ws: opts?.ws, user: opts?.user }); + this.wssMap.set(_id, value); + } + unregister(id: string, user?: string) { + const _id = this.getId(id, user || ''); + const value = this.wssMap.get(_id); + if (value) { + value.ws.close(); + } + this.wssMap.delete(_id); + } + getIds() { + return Array.from(this.wssMap.keys()); + } + get(id: string, user?: string) { + if (user) { + const _id = this.getId(id, user); + return this.wssMap.get(_id); + } + return this.wssMap.get(id); + } +} diff --git a/src/modules/ws-proxy/proxy.ts b/src/modules/ws-proxy/proxy.ts new file mode 100644 index 0000000..70dd08b --- /dev/null +++ b/src/modules/ws-proxy/proxy.ts @@ -0,0 +1,44 @@ +import { IncomingMessage, ServerResponse } from 'http'; +import { wsProxyManager } from './index.ts'; + +import { App } from '@kevisual/router'; +import { logger } from '../logger.ts'; +import { getLoginUser } from '@/modules/auth.ts'; + +type ProxyOptions = { + createNotFoundPage: (msg?: string) => any; +}; +export const UserV1Proxy = async (req: IncomingMessage, res: ServerResponse, opts?: ProxyOptions) => { + const { url } = req; + const { pathname } = new URL(url || '', `http://localhost`); + const [user, app, userAppKey] = pathname.split('/').slice(1); + if (!user || !app || !userAppKey) { + opts?.createNotFoundPage?.('应用未找到'); + return false; + } + const data = await App.handleRequest(req, res); + const loginUser = await getLoginUser(req); + if (!loginUser) { + opts?.createNotFoundPage?.('没有登录'); + return false; + } + if (loginUser.tokenUser?.username !== user) { + opts?.createNotFoundPage?.('没有访问应用权限'); + return false; + } + logger.debug('data', data); + const client = wsProxyManager.get(userAppKey, user); + const ids = wsProxyManager.getIds(); + if (!client) { + opts?.createNotFoundPage?.(`未找到应用, ${userAppKey}, ${ids.join(',')}`); + return false; + } + const value = await client.sendData(data); + if (value) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(value)); + return true; + } + opts?.createNotFoundPage?.('应用未启动'); + return true; +}; diff --git a/src/program.ts b/src/program.ts index bce5205..b042d74 100644 --- a/src/program.ts +++ b/src/program.ts @@ -5,8 +5,6 @@ import { program, Command } from 'commander'; // import * as minioLib from './modules/minio.ts'; // export const redis = useContextKey('redis', () => redisLib.redis); -// export const redisPublisher = useContextKey('redisPublisher', () => redisLib.redisPublisher); -// export const redisSubscriber = useContextKey('redisSubscriber', () => redisLib.redisSubscriber); // export const minioClient = useContextKey('minioClient', () => minioLib.minioClient); // export const sequelize = useContextKey('sequelize', () => sequelizeLib.sequelize); diff --git a/src/routes-simple/upload.ts b/src/routes-simple/handle-request.ts similarity index 84% rename from src/routes-simple/upload.ts rename to src/routes-simple/handle-request.ts index 1235f3b..a79f37c 100644 --- a/src/routes-simple/upload.ts +++ b/src/routes-simple/handle-request.ts @@ -1,7 +1,6 @@ import { useFileStore } from '@kevisual/use-config/file-store'; -import http from 'http'; -import fs, { rm } from 'fs'; -import path from 'path'; +import http from 'node:http'; +import fs from 'fs'; import { IncomingForm } from 'formidable'; import { app, minioClient } from '@/app.ts'; @@ -9,16 +8,10 @@ import { bucketName } from '@/modules/minio.ts'; import { getContentType } from '@/utils/get-content-type.ts'; import { User } from '@/models/user.ts'; import { getContainerById } from '@/routes/container/module/get-container-file.ts'; -import { router, error, checkAuth, clients, writeEvents } from './router.ts'; +import { router, error, checkAuth, writeEvents } from './router.ts'; import './index.ts'; - +import { handleRequest as PageProxy } from './page-proxy.ts'; const cacheFilePath = useFileStore('cache-file', { needExists: true }); -// curl -X POST http://localhost:4000/api/upload -F "file=@readme.md" -// curl -X POST http://localhost:4000/api/upload \ -// -F "file=@readme.md" \ -// -F "file=@types/index.d.ts" \ -// -F "description=This is a test upload" \ -// -F "username=testuser" router.get('/api/app/upload', async (req, res) => { res.writeHead(200, { 'Content-Type': 'text/plain' }); @@ -177,21 +170,28 @@ router.get('/api/container/file/:id', async (req, res) => { res.end(JSON.stringify(container)); }); -// router.get('/api/code/version', async (req, res) => { -// const version = VERSION; -// res.writeHead(200, { -// 'Content-Type': 'application/json', -// }); -// res.end(JSON.stringify({ code: 200, data: { version } })); -// }); +const simpleAppsPrefixs = [ + "/api/app/", + "/api/micro-app/", + "/api/events", + "/api/s1/", + "/api/container/", + "/api/resource/", +]; -export const uploadMiddleware = async (req: http.IncomingMessage, res: http.ServerResponse) => { +export const handleRequest = async (req: http.IncomingMessage, res: http.ServerResponse) => { if (req.url?.startsWith('/api/router')) { + // router自己管理 return; } - // 设置跨域 - res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Methods', 'GET, POST'); - res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization'); - return router.parse(req, res); + if (req.url && simpleAppsPrefixs.some(prefix => req.url!.startsWith(prefix))) { + // 简单应用路由处理 + // 设置跨域 + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST'); + res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization'); + return router.parse(req, res); + } + // 其他请求交给页面代理处理 + return PageProxy(req, res); }; diff --git a/src/routes-simple/page-proxy.ts b/src/routes-simple/page-proxy.ts new file mode 100644 index 0000000..a36b5f1 --- /dev/null +++ b/src/routes-simple/page-proxy.ts @@ -0,0 +1,386 @@ +import { getDNS, isIpv4OrIpv6, isLocalhost } from '../modules/fm-manager/index.ts'; +import http from 'node:http'; +import https from 'node:https'; +import { UserApp } from '../modules/user-app/index.ts'; +import { config, fileStore } from '../modules/config.ts'; +import path from 'node:path'; +import fs from 'node:fs'; +import { getContentType } from '../modules/fm-manager/index.ts'; +import { createRefreshHtml } from '../modules/html/create-refresh-html.ts'; +import { fileProxy, getTextFromStreamAndAddStat, httpProxy, aiProxy } from '../modules/fm-manager/index.ts'; +import { UserPermission } from '@kevisual/permission'; +import { getLoginUser } from '../modules/auth.ts'; +import { rediretHome } from '../modules/user-app/index.ts'; +import { logger } from '../modules/logger.ts'; +import { UserV1Proxy } from '../modules/ws-proxy/proxy.ts'; +const domain = config?.proxy?.domain; +const allowedOrigins = config?.proxy?.allowedOrigin || []; + +const noProxyUrl = ['/', '/favicon.ico']; +const notAuthPathList = [ + { + user: 'root', + paths: ['center'], + }, + { + user: 'admin', + paths: ['center'], + }, + { + user: 'user', + paths: ['login'], + }, + { + user: 'public', + paths: ['center'], + all: true, + }, + { + user: 'test', + paths: ['center'], + all: true, + }, +]; +const checkNotAuthPath = (user, app) => { + const notAuthPath = notAuthPathList.find((item) => { + if (item.user === user) { + if (item.all) { + return true; + } + return item.paths?.includes?.(app); + } + return false; + }); + return notAuthPath; +}; +export const handleRequest = async (req: http.IncomingMessage, res: http.ServerResponse) => { + const querySearch = new URL(req.url, `http://${req.headers.host}`).searchParams; + const password = querySearch.get('p'); + if (req.url === '/favicon.ico') { + res.writeHead(200, { 'Content-Type': 'image/x-icon' }); + res.end('proxy no favicon.ico\n'); + return; + } + const proxyApiList = config?.apiList || []; + const proxyApi = proxyApiList.find((item) => req.url.startsWith(item.path)); + if (proxyApi && proxyApi?.type === 'static') { + return fileProxy(req, res, proxyApi); + } + if (proxyApi) { + const _u = new URL(req.url, `${proxyApi.target}`); + // 设置代理请求的目标 URL 和请求头 + let header: any = {}; + if (req.headers?.['Authorization']) { + header.authorization = req.headers['Authorization']; + } else if (req.headers?.['authorization']) { + header.authorization = req.headers['authorization']; + } + // 提取req的headers中的非HOST的header + const headers = Object.keys(req.headers).filter((item) => item && item.toLowerCase() !== 'host'); + const host = req.headers['host']; + logger.info('proxy host', host); + logger.info('headers', headers); + + headers.forEach((item) => { + header[item] = req.headers[item]; + }); + const options = { + host: _u.hostname, + path: req.url, + method: req.method, + headers: { + ...header, + }, + }; + if (_u.port) { + // @ts-ignore + options.port = _u.port; + } + logger.info('proxy options', options); + const isHttps = _u.protocol === 'https:'; + const protocol = isHttps ? https : http; + if (isHttps) { + // 不验证https + // @ts-ignore + options.rejectUnauthorized = false; + } + // 创建代理请求 + const proxyReq = protocol.request(options, (proxyRes) => { + // 将代理服务器的响应头和状态码返回给客户端 + res.writeHead(proxyRes.statusCode, proxyRes.headers); + // 将代理响应流写入客户端响应 + proxyRes.pipe(res, { end: true }); + }); + // 处理代理请求的错误事件 + proxyReq.on('error', (err) => { + logger.error(`Proxy request error: ${err.message}`); + res.writeHead(500, { 'Content-Type': 'text/plain' }); + res.write(`Proxy request error: ${err.message}`); + }); + // 处理 POST 请求的请求体（传递数据到目标服务器） + req.pipe(proxyReq, { end: true }); + return; + } + if (req.url.startsWith('/api') || req.url.startsWith('/v1')) { + res.end('not catch api'); + return; + } + + const dns = getDNS(req); + // 配置可以跨域 + // 配置可以访问的域名 localhost, xiongxiao.me + const _orings = allowedOrigins || []; + const host = dns.hostName; + if ( + _orings.some((item) => { + return host.includes(item); + }) + ) { + res.setHeader('Access-Control-Allow-Origin', '*'); + } + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, DELETE, PUT'); + + let user, app; + let domainApp = false; + if (isLocalhost(dns.hostName)) { + // 本地开发环境 测试 + // user = 'root'; + // app = 'codeflow'; + // domainApp = true; + } else { + if (isIpv4OrIpv6(dns.hostName)) { + // 打印出 req.url 和错误信息 + console.error('Invalid domain: ', req.url, dns.hostName); + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Invalid domain\n'); + return res.end(); + } + // 验证域名 + if (domain && dns.hostName !== domain) { + // redis获取域名对应的用户和应用 + domainApp = true; + const data = await UserApp.getDomainApp(dns.hostName); + if (!data) { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Invalid domain\n'); + return; + } + if (!data.user || !data.app) { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Invalid domain config\n'); + return; + } + user = data.user; + app = data.app; + } + } + // const url = req.url; + const pathname = new URL(req.url, `http://${dns.hostName}`).pathname; + + /** + * url是pathname的路径 + */ + const url = pathname; + if (!domainApp && noProxyUrl.includes(url)) { + if (url === '/') { + // TODO: 获取一下登陆用户，如果没有登陆用户，重定向到ai-chat页面 + // 重定向到 + // res.writeHead(302, { Location: home }); + // return res.end(); + rediretHome(req, res); + return; + } + // 不是域名代理，且是在不代理的url当中 + res.write('No proxy for this URL\n'); + return res.end(); + } + if (!domainApp) { + // 原始url地址 + const urls = url.split('/'); + if (urls.length < 3) { + console.log('urls errpr', urls); + res.writeHead(404, { 'Content-Type': 'text/html' }); + res.write('Invalid Proxy URL\n'); + return res.end(); + } + const [_, _user, _app] = urls; + if (_app && urls.length === 3) { + // 重定向到 + res.writeHead(302, { Location: `${url}/` }); + return res.end(); + } + if (!_user || !_app) { + res.write('Invalid URL\n'); + return res.end(); + } + user = _user; + app = _app; + } + const createRefreshPage = (user, app) => { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(createRefreshHtml(user, app)); + }; + const createErrorPage = () => { + res.writeHead(500, { 'Content-Type': 'text/html; charset=utf-8' }); + res.write('Server Error\n'); + res.end(); + }; + const createNotFoundPage = async (msg?: string, code = 404) => { + res.writeHead(code, { 'Content-Type': 'text/html; charset=utf-8' }); + res.write(msg || 'Not Found App\n'); + res.end(); + }; + if (app === 'ai' || app === 'resources' || app === 'r') { + return aiProxy(req, res, { + createNotFoundPage, + }); + } + if (user !== 'api' && app === 'v1') { + return UserV1Proxy(req, res, { + createNotFoundPage, + }); + } + + const userApp = new UserApp({ user, app }); + let isExist = await userApp.getExist(); + logger.debug('userApp', userApp, isExist); + if (!isExist) { + try { + const { code, loading, message } = await userApp.setCacheData(); + if (loading || code === 20000) { + return createRefreshPage(user, app); + } else if (code === 500) { + return createNotFoundPage(message || 'Not Found App\n'); + } else if (code !== 200) { + return createErrorPage(); + } + isExist = await userApp.getExist(); + } catch (error) { + console.error('setCacheData error', error); + createErrorPage(); + userApp.setLoaded('error', 'setCacheData error'); + return; + } + } + if (!isExist) { + return createNotFoundPage(); + } + + if (!checkNotAuthPath(user, app)) { + const { permission } = isExist; + const permissionInstance = new UserPermission({ permission, owner: user }); + const loginUser = await getLoginUser(req); + const checkPermission = permissionInstance.checkPermissionSuccess({ + username: loginUser?.tokenUser?.username || '', + password: password, + }); + if (!checkPermission.success) { + return createNotFoundPage('no permission'); + } + } + const indexFile = isExist.indexFilePath; // 已经必定存在了 + try { + let appFileUrl: string; + if (domainApp) { + appFileUrl = (url + '').replace(`/`, ''); + } else { + appFileUrl = (url + '').replace(`/${user}/${app}/`, ''); + } + appFileUrl = decodeURIComponent(appFileUrl); // Decode URL components + let appFile = await userApp.getFile(appFileUrl); + if (!appFile && url.endsWith('/')) { + appFile = await userApp.getFile(appFileUrl + 'index.html'); + } + if (isExist.proxy) { + let proxyUrl = appFile || isExist.indexFilePath; + if (!proxyUrl.startsWith('http')) { + return createNotFoundPage('Invalid proxy url'); + } + console.log('proxyUrl', appFileUrl, proxyUrl); + httpProxy(req, res, { + proxyUrl, + userApp, + createNotFoundPage, + }); + return; + } + console.log('appFile', appFile, appFileUrl, isExist); + // console.log('isExist', isExist); + if (!appFile) { + const [indexFilePath, etag] = indexFile.split('||'); + // console.log('indexFilePath', indexFile, path.join(fileStore, indexFilePath)); + const contentType = getContentType(indexFilePath); + const isHTML = contentType.includes('html'); + const filePath = path.join(fileStore, indexFilePath); + if (!userApp.fileCheck(filePath)) { + res.writeHead(500, { 'Content-Type': 'text/html; charset=utf-8', tips: 'App Cache expired, Please refresh' }); + res.write(createRefreshHtml(user, app)); + res.end(); + await userApp.clearCacheData(); + return; + } + // 如果 content是 'application/octet-stream' 会下载文件, 添加文件后缀 + if (contentType === 'application/octet-stream') { + // 提取文件名，只保留文件名而不是整个路径 + const fileName = path.basename(indexFilePath); + res.setHeader('Content-Disposition', `attachment; filename=${fileName}`); + } + // 不存在的文件,返回indexFile的文件 + res.writeHead(200, { 'Content-Type': contentType, 'Cache-Control': isHTML ? 'no-cache' : 'public, max-age=3600' }); + if (isHTML) { + const newHtml = await getTextFromStreamAndAddStat(fs.createReadStream(filePath)); + res.end(newHtml.html); + } else { + const readStream = fs.createReadStream(filePath); + readStream.pipe(res); + } + + return; + } else { + const [appFilePath, eTag] = appFile.split('||'); + // 检查 If-None-Match 头判断缓存是否有效 + if (req.headers['if-none-match'] === eTag) { + res.statusCode = 304; // 内容未修改 + res.end(); + return; + } + const filePath = path.join(fileStore, appFilePath); + let contentType = getContentType(filePath); + const isHTML = contentType.includes('html'); + // 如果 content是 'application/octet-stream' 会下载文件, 添加文件后缀 + if (contentType === 'application/octet-stream') { + // 提取文件名，只保留文件名而不是整个路径 + const fileName = path.basename(appFilePath); + res.setHeader('Content-Disposition', `attachment; filename=${fileName}`); + } + + if (!userApp.fileCheck(filePath)) { + console.error('File expired', filePath); + res.writeHead(500, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end('File expired\n'); + await userApp.clearCacheData(); + return; + } + let resContent = ''; + const headers = new Map(); + headers.set('Content-Type', contentType); + headers.set('Cache-Control', isHTML ? 'no-cache' : 'public, max-age=3600'); // 设置缓存时间为 1 小时 + headers.set('ETag', eTag); + res?.setHeaders?.(headers); + if (isHTML) { + const newHtml = await getTextFromStreamAndAddStat(fs.createReadStream(filePath)); + resContent = newHtml.html; + headers.set('Content-Length', newHtml.contentLength.toString()); + res.writeHead(200); + res.end(resContent); + } else { + res.writeHead(200); + const readStream = fs.createReadStream(filePath); + readStream.pipe(res); + } + return; + } + } catch (error) { + console.error('getFile error', error); + } +}; diff --git a/src/routes-simple/resources/upload.ts b/src/routes-simple/resources/upload.ts index 792ca18..3adcb87 100644 --- a/src/routes-simple/resources/upload.ts +++ b/src/routes-simple/resources/upload.ts @@ -11,7 +11,7 @@ import { ConfigModel } from '@/routes/config/models/model.ts'; import { validateDirectory } from './util.ts'; import { pick } from 'lodash-es'; import { getFileStat } from '@/routes/file/index.ts'; -import { logger } from '@/logger/index.ts'; +import { logger } from '@/modules/logger.ts'; const cacheFilePath = useFileStore('cache-file', { needExists: true }); diff --git a/src/routes/app-manager/proxy/index.ts b/src/routes/app-manager/proxy/index.ts index f7f45b7..0d12935 100644 --- a/src/routes/app-manager/proxy/index.ts +++ b/src/routes/app-manager/proxy/index.ts @@ -1 +1,2 @@ import './page-proxy.ts'; +import './list.ts' \ No newline at end of file diff --git a/src/routes/app-manager/proxy/list.ts b/src/routes/app-manager/proxy/list.ts new file mode 100644 index 0000000..94ed347 --- /dev/null +++ b/src/routes/app-manager/proxy/list.ts @@ -0,0 +1,189 @@ +import { deleteUserAppFiles } from '@/modules/user-app/index.ts'; +import { app, redis } from '@/app.ts'; +import fs from 'fs'; +import { fileStore } from '@/modules/config.ts'; +import { getAppLoadStatus } from '@/modules/user-app/index.ts'; +import { getLoginUser } from '@/modules/auth.ts'; + +export class CenterUserApp { + user: string; + app: string; + constructor({ user, app }: { user: string; app: string }) { + this.user = user; + this.app = app; + } + async clearCache() { + const keys = await redis.keys('user:app:*'); + return keys; + } + async getCache() { + const app = this.app; + const user = this.user; + const key = 'user:app:' + app + ':' + user; + const value = await redis.get(key); + if (!value) { + return null; + } + return JSON.parse(value); + } + async getLoaded() { + const app = this.app; + const user = this.user; + const value = await getAppLoadStatus(user, app); + return value; + } + async clearCacheData() { + const app = this.app; + const user = this.user; + const key = 'user:app:' + app + ':' + user; + await redis.del(key); + await redis.del('user:app:exist:' + app + ':' + user); + await redis.del('user:app:set:' + app + ':' + user); + await redis.del('user:app:status:' + app + ':' + user); + await redis.del('user:app:permission:' + app + ':' + user); + const userDomainApp = 'user:domain:app:' + user + ':' + app; + const domainKeys = await redis.get(userDomainApp); + if (domainKeys) { + const domainKeysList = JSON.parse(domainKeys); + domainKeysList.forEach(async (domain: string) => { + await redis.del('domain:' + domain); + }); + } + await redis.del(userDomainApp); + + // 删除所有文件 + deleteUserAppFiles(user, app); + } +} +app + .route({ + path: 'page-proxy-app', + key: 'auth-admin', + id: 'auth-admin', + }) + .define(async (ctx) => { + const { user } = ctx.query; + const loginUser = await getLoginUser(ctx.req); + if (loginUser) { + const root = ['admin', 'root']; + if (root.includes(loginUser.tokenUser?.username)) { + return; + } + ctx.throw(401, 'No Proxy App Permission'); + } + ctx.throw(401, 'No Login And No Proxy App Permission'); + }) + .addTo(app); + +app + .route({ + path: 'page-proxy-app', + key: 'list', + middleware: ['auth-admin'], + description: '获取应用列表', + isDebug: true, + }) + .define(async (ctx) => { + const keys = await redis.keys('user:app:*'); + // const keys = await redis.keys('user:app:exist:*'); + // const data = await redis.mget(...keys); + const domainList = await redis.keys('domain:*'); + ctx.body = { + // data: data, + keys, + domainList, + }; + }) + .addTo(app); + +app + .route({ + path: 'page-proxy-app', + key: 'delete', + middleware: ['auth-admin'], + }) + .define(async (ctx) => { + const { user, app } = ctx.query; + try { + const userApp = new CenterUserApp({ user, app }); + await userApp.clearCacheData(); + } catch (error) { + console.error(error); + ctx.throw('删除失败'); + } + ctx.body = 'successfully'; + }) + .addTo(app); + +app + .route({ + path: 'page-proxy-app', + key: 'deleteAll', + }) + .define(async (ctx) => { + const keys = await redis.keys('user:app:*'); + for (const key of keys) { + await redis.set(key, '', 'EX', 1); + } + ctx.body = { + keys, + }; + }) + .addTo(app); + +app + .route({ + path: 'page-proxy-app', + key: 'clear', + }) + .define(async (ctx) => { + const keys = await redis.keys('user:app:*'); + if (keys.length > 0) { + await redis.del(...keys); + } + fs.rmSync(fileStore, { recursive: true }); + + ctx.body = { + keys, + }; + }) + .addTo(app); + +app + .route({ + path: 'page-proxy-app', + key: 'get', + middleware: ['auth-admin'], + }) + .define(async (ctx) => { + const { user, app } = ctx.query; + if (!user || !app) { + if (!user) { + ctx.throw('user is required'); + } + if (!app) { + ctx.throw('app is required'); + } + } + const userApp = new CenterUserApp({ user, app }); + const cache = await userApp.getCache(); + if (!cache) { + ctx.throw('Not Found App'); + } + ctx.body = cache; + }) + .addTo(app); + +app + .route({ + path: 'page-proxy-app', + key: 'status', + middleware: [], + }) + .define(async (ctx) => { + const { user, app } = ctx.query; + const userApp = new CenterUserApp({ user, app }); + const status = await userApp.getLoaded(); + ctx.body = status; + }) + .addTo(app);