kevisual/code-center
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

update new way

Browse Source
This commit is contained in:
xion 2024-11-17 01:35:05 +08:00
parent 7ec8a001de
commit dc69b95f39
13 changed files with 463 additions and 1030 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
package.json
View File

@ -26,21 +26,20 @@
"files": [
"types"
],
"license": "ISC",
"license": "UNLICENSED",
"dependencies": {
"@abearxiong/auth": "1.0.2",
"@abearxiong/router": "0.0.1-alpha.43",
"@abearxiong/use-config": "^0.0.2",
"@babel/core": "^7.26.0",
"@babel/preset-env": "^7.26.0",
"@babel/preset-typescript": "^7.26.0",
"@kevisual/ai-graph": "workspace:^",
"@kevisual/ai-lang": "workspace:^",
"@kevisual/router": "0.0.4-alpha-8",
"@kevisual/router": "0.0.5-alpha-1",
"@supabase/supabase-js": "^2.46.1",
"@types/semver": "^7.5.8",
"archiver": "^7.0.1",
"bullmq": "^5.25.6",
"bullmq": "^5.26.2",
"dayjs": "^1.11.13",
"dts-bundle-generator": "^9.5.1",
"formidable": "^3.5.2",
@ -84,14 +83,12 @@
"cross-env": "^7.0.3",
"glob": "^11.0.0",
"nodemon": "^3.1.7",
"patch-package": "^8.0.0",
"pm2": "^5.4.3",
"rimraf": "^6.0.1",
"rollup": "^4.26.0",
"rollup": "^4.27.2",
"rollup-plugin-copy": "^3.5.0",
"rollup-plugin-dts": "^6.1.1",
"tape": "^5.9.0",
"ts-loader": "^9.5.1",
"tsx": "^4.19.2",
"typescript": "^5.6.3"
},

955
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

9
src/app.ts
View File

@ -23,15 +23,6 @@ export const app = new App<{ import: any; emit: typeof emit; sequelize: typeof s
emit,
sequelize,
},
// routerHandle(res) {
// console.log('routerHandle', res.query);
// const { code, data, message } = res;
// return {
// code,
// data,
// message,
// };
// },
});
const clients = [];

4
src/index.ts
View File

@ -2,10 +2,10 @@ import { useConfig } from '@abearxiong/use-config';
import { app } from './app.ts';
import './route.ts';
const config = useConfig();
import { app as aiApp } from '@kevisual/ai-lang/src/index.ts';
// import { app as aiApp } from '@kevisual/ai-lang/src/index.ts';
import { uploadMiddleware } from './lib/upload.ts';
import { loadApps } from './load-apps.ts';
export { aiApp };
// export { aiApp };
export { app };
loadApps(app);
app.listen(config.port, () => {

400
src/lib/upload.ts
View File

@ -4,9 +4,11 @@ import fs, { rm } from 'fs';
import path from 'path';
import { IncomingForm } from 'formidable';
import { app, minioClient } from '@/app.ts';
import { SimpleRouter } from '@kevisual/router/simple';
import { bucketName } from '@/modules/minio.ts';
import { getContentType } from '@/utils/get-content-type.ts';
import { User } from '@/models/user.ts';
import { getContainerById } from '@/routes/container/module/get-container-file.ts';
const filePath = useFileStore('upload', { needExists: true });
const cacheFilePath = useFileStore('cache-file', { needExists: true });
// curl -X POST http://localhost:4000/api/upload -F "file=@readme.md"
@ -17,193 +19,194 @@ const cacheFilePath = useFileStore('cache-file', { needExists: true });
// -F "username=testuser"
let clients = [];
export const uploadMiddleware = async (req: http.IncomingMessage, res: http.ServerResponse) => {
if (req.method === 'GET' && req.url === '/api/app/upload') {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('Upload API is ready');
return;
}
const error = (msg: string) => {
return JSON.stringify({ code: 500, message: msg });
};
const checkAuth = async () => {
const authroization = req.headers?.['authorization'] as string;
if (!authroization) {
res.statusCode = 401;
res.end(error('Invalid authorization'));
return { tokenUser: null, token: null };
}
const token = authroization.split(' ')[1];
let tokenUser;
try {
tokenUser = await User.verifyToken(token);
} catch (e) {
res.statusCode = 401;
res.end(error('Invalid token'));
return { tokenUser: null, token: null };
}
return { tokenUser, token };
};
if (req.method === 'POST' && req.url === '/api/upload') {
if (res.headersSent) return; // 如果响应已发送，不再处理
res.writeHead(200, { 'Content-Type': 'application/json' });
const { tokenUser } = await checkAuth();
if (!tokenUser) return;
// 使用 formidable 解析 multipart/form-data
const form = new IncomingForm({
multiples: true, // 支持多文件上传
uploadDir: filePath, // 上传文件存储目录
allowEmptyFiles: true, // 允许空文件
});
// 解析上传的文件
form.parse(req, async (err, fields, files) => {
if (err) {
res.end(error(`Upload error: ${err.message}`));
// 删除临时文件
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
uploadedFiles.forEach((file) => {
fs.unlinkSync(file.filepath);
});
return;
}
// 逐个处理每个上传的文件
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
const uploadResults = [];
for (let i = 0; i < uploadedFiles.length; i++) {
const file = uploadedFiles[i];
// @ts-ignore
const tempPath = file.filepath; // 文件上传时的临时路径
const relativePath = file.originalFilename; // 保留表单中上传的文件名 (包含文件夹结构)
// 比如 child2/b.txt
const minioPath = `${tokenUser.username}/${relativePath}`;
// 上传到 MinIO 并保留文件夹结构
const isHTML = relativePath.endsWith('.html');
await minioClient.fPutObject(bucketName, minioPath, tempPath, {
'Content-Type': getContentType(relativePath),
'app-source': 'user-files',
'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年
});
uploadResults.push({
name: relativePath,
path: minioPath,
});
fs.unlinkSync(tempPath); // 删除临时文件
}
res.end(JSON.stringify({ code: 200, data: uploadResults }));
});
}
if (req.method === 'POST' && req.url === '/api/app/upload') {
if (res.headersSent) return; // 如果响应已发送，不再处理
res.writeHead(200, { 'Content-Type': 'application/json' });
const { tokenUser, token } = await checkAuth();
if (!tokenUser) return;
//
// 使用 formidable 解析 multipart/form-data
const form = new IncomingForm({
multiples: true, // 支持多文件上传
uploadDir: cacheFilePath, // 上传文件存储目录
allowEmptyFiles: true, // 允许空
minFileSize: 0, // 最小文件大小
createDirsFromUploads: false, // 根据上传的文件夹结构创建目录
keepExtensions: true, // 保留文件
hashAlgorithm: 'md5', // 文件哈希算法
});
form.on('progress', (bytesReceived, bytesExpected) => {
const progress = (bytesReceived / bytesExpected) * 100;
console.log(`Upload progress: ${progress.toFixed(2)}%`);
const data = {
progress: progress.toFixed(2),
message: `Upload progress: ${progress.toFixed(2)}%`,
};
// 向所有连接的客户端推送进度信息
clients.forEach((client) => client.write(`${JSON.stringify(data)}\n`));
});
// 解析上传的文件
form.parse(req, async (err, fields, files) => {
if (err) {
res.end(error(`Upload error: ${err.message}`));
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
uploadedFiles.forEach((file) => {
fs.unlinkSync(file.filepath);
});
return;
}
const clearFiles = () => {
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
uploadedFiles.forEach((file) => {
fs.unlinkSync(file.filepath);
});
};
let appKey, version;
const { appKey: _appKey, version: _version } = fields;
if (Array.isArray(_appKey)) {
appKey = _appKey?.[0];
} else {
appKey = _appKey;
}
if (Array.isArray(_version)) {
version = _version?.[0];
} else {
version = _version;
}
if (!appKey) {
res.end(error('appKey is required'));
clearFiles();
return;
}
if (!version) {
res.end(error('version is required'));
clearFiles();
return;
}
console.log('Appkey', appKey, version);
// 逐个处理每个上传的文件
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
const uploadResults = [];
for (let i = 0; i < uploadedFiles.length; i++) {
const file = uploadedFiles[i];
// @ts-ignore
const tempPath = file.filepath; // 文件上传时的临时路径
const relativePath = file.originalFilename; // 保留表单中上传的文件名 (包含文件夹结构)
// 比如 child2/b.txt
const minioPath = `${tokenUser.username}/${appKey}/${version}/${relativePath}`;
// 上传到 MinIO 并保留文件夹结构
const isHTML = relativePath.endsWith('.html');
await minioClient.fPutObject(bucketName, minioPath, tempPath, {
'Content-Type': getContentType(relativePath),
'app-source': 'user-app',
'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年
});
uploadResults.push({
name: relativePath,
path: minioPath,
});
fs.unlinkSync(tempPath); // 删除临时文件
}
// 受控
const r = await app.call({
path: 'app',
key: 'uploadFiles',
payload: {
token: token,
data: {
appKey,
version,
files: uploadResults,
},
},
});
const data: any = {
code: r.code,
data: r.body,
};
if (r.message) {
data.message = r.message;
}
res.end(JSON.stringify(data));
});
const router = new SimpleRouter();
const error = (msg: string, code = 500) => {
return JSON.stringify({ code, message: msg });
};
const checkAuth = async (req: http.IncomingMessage, res: http.ServerResponse) => {
const authroization = req.headers?.['authorization'] as string;
if (!authroization) {
res.statusCode = 401;
res.end(error('Invalid authorization'));
return { tokenUser: null, token: null };
}
const token = authroization.split(' ')[1];
let tokenUser;
try {
tokenUser = await User.verifyToken(token);
} catch (e) {
res.statusCode = 401;
res.end(error('Invalid token'));
return { tokenUser: null, token: null };
}
return { tokenUser, token };
};
router.get('/api/app/upload', async (req, res) => {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('Upload API is ready');
});
router.post('/api/upload', async (req, res) => {
if (res.headersSent) return; // 如果响应已发送，不再处理
res.writeHead(200, { 'Content-Type': 'application/json' });
const { tokenUser } = await checkAuth(req, res);
if (!tokenUser) return;
// 使用 formidable 解析 multipart/form-data
const form = new IncomingForm({
multiples: true, // 支持多文件上传
uploadDir: filePath, // 上传文件存储目录
allowEmptyFiles: true, // 允许空文件
});
// 解析上传的文件
form.parse(req, async (err, fields, files) => {
if (err) {
res.end(error(`Upload error: ${err.message}`));
// 删除临时文件
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
uploadedFiles.forEach((file) => {
fs.unlinkSync(file.filepath);
});
return;
}
// 逐个处理每个上传的文件
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
const uploadResults = [];
for (let i = 0; i < uploadedFiles.length; i++) {
const file = uploadedFiles[i];
// @ts-ignore
const tempPath = file.filepath; // 文件上传时的临时路径
const relativePath = file.originalFilename; // 保留表单中上传的文件名 (包含文件夹结构)
// 比如 child2/b.txt
const minioPath = `${tokenUser.username}/${relativePath}`;
// 上传到 MinIO 并保留文件夹结构
const isHTML = relativePath.endsWith('.html');
await minioClient.fPutObject(bucketName, minioPath, tempPath, {
'Content-Type': getContentType(relativePath),
'app-source': 'user-files',
'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年
});
uploadResults.push({
name: relativePath,
path: minioPath,
});
fs.unlinkSync(tempPath); // 删除临时文件
}
res.end(JSON.stringify({ code: 200, data: uploadResults }));
});
});
router.post('/api/app/upload', async (req, res) => {
if (res.headersSent) return; // 如果响应已发送，不再处理
res.writeHead(200, { 'Content-Type': 'application/json' });
const { tokenUser, token } = await checkAuth(req, res);
if (!tokenUser) return;
//
// 使用 formidable 解析 multipart/form-data
const form = new IncomingForm({
multiples: true, // 支持多文件上传
uploadDir: cacheFilePath, // 上传文件存储目录
allowEmptyFiles: true, // 允许空
minFileSize: 0, // 最小文件大小
createDirsFromUploads: false, // 根据上传的文件夹结构创建目录
keepExtensions: true, // 保留文件
hashAlgorithm: 'md5', // 文件哈希算法
});
form.on('progress', (bytesReceived, bytesExpected) => {
const progress = (bytesReceived / bytesExpected) * 100;
console.log(`Upload progress: ${progress.toFixed(2)}%`);
const data = {
progress: progress.toFixed(2),
message: `Upload progress: ${progress.toFixed(2)}%`,
};
// 向所有连接的客户端推送进度信息
clients.forEach((client) => client.write(`${JSON.stringify(data)}\n`));
});
// 解析上传的文件
form.parse(req, async (err, fields, files) => {
if (err) {
res.end(error(`Upload error: ${err.message}`));
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
uploadedFiles.forEach((file) => {
fs.unlinkSync(file.filepath);
});
return;
}
const clearFiles = () => {
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
uploadedFiles.forEach((file) => {
fs.unlinkSync(file.filepath);
});
};
let appKey, version;
const { appKey: _appKey, version: _version } = fields;
if (Array.isArray(_appKey)) {
appKey = _appKey?.[0];
} else {
appKey = _appKey;
}
if (Array.isArray(_version)) {
version = _version?.[0];
} else {
version = _version;
}
if (!appKey) {
res.end(error('appKey is required'));
clearFiles();
return;
}
if (!version) {
res.end(error('version is required'));
clearFiles();
return;
}
console.log('Appkey', appKey, version);
// 逐个处理每个上传的文件
const uploadedFiles = Array.isArray(files.file) ? files.file : [files.file];
const uploadResults = [];
for (let i = 0; i < uploadedFiles.length; i++) {
const file = uploadedFiles[i];
// @ts-ignore
const tempPath = file.filepath; // 文件上传时的临时路径
const relativePath = file.originalFilename; // 保留表单中上传的文件名 (包含文件夹结构)
// 比如 child2/b.txt
const minioPath = `${tokenUser.username}/${appKey}/${version}/${relativePath}`;
// 上传到 MinIO 并保留文件夹结构
const isHTML = relativePath.endsWith('.html');
await minioClient.fPutObject(bucketName, minioPath, tempPath, {
'Content-Type': getContentType(relativePath),
'app-source': 'user-app',
'Cache-Control': isHTML ? 'no-cache' : 'max-age=31536000, immutable', // 缓存一年
});
uploadResults.push({
name: relativePath,
path: minioPath,
});
fs.unlinkSync(tempPath); // 删除临时文件
}
// 受控
const r = await app.call({
path: 'app',
key: 'uploadFiles',
payload: {
token: token,
data: {
appKey,
version,
files: uploadResults,
},
},
});
const data: any = {
code: r.code,
data: r.body,
};
if (r.message) {
data.message = r.message;
}
res.end(JSON.stringify(data));
});
});
router.get('/api/events', async (req, res) => {
if (req.url === '/api/events') {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
@ -217,4 +220,31 @@ export const uploadMiddleware = async (req: http.IncomingMessage, res: http.Serv
clients = clients.filter((client) => client !== res);
});
}
});
router.get('/api/container/file/:id', async (req, res) => {
const id = req.params.id;
if (!id) {
res.end(error('id is required'));
return;
}
const container = await getContainerById(id);
if (container.id) {
const code = container.code;
res.writeHead(200, {
'Content-Type': 'application/javascript; charset=utf-8',
'container-id': container.id,
});
res.end(code);
} else {
res.end(error('Container not found'));
}
res.writeHead(200, {
'Content-Type': 'application/json',
});
res.end(JSON.stringify(container));
});
export const uploadMiddleware = async (req: http.IncomingMessage, res: http.ServerResponse) => {
return router.parse(req, res);
};

1
src/modules/redis.ts
View File

@ -18,6 +18,7 @@ export const redis = new Redis({
maxRetriesPerRequest: null, // 允许请求重试的次数 (如果需要无限次重试)
...config.redis,
});
console.log('redis', config.redis);
// 监听连接事件
redis.on('connect', () => {

6
src/route.ts
View File

@ -1,5 +1,5 @@
import './demo/index.ts';
import { app as adminApp, appendTo } from './admin/index.ts';
// import './demo/index.ts';
// import { app as adminApp, appendTo } from './admin/index.ts';
import './routes/index.ts';
import { app } from './app.ts';
import { useConfig } from '@abearxiong/use-config';
@ -13,4 +13,4 @@ createAuthRoute({
// app.importApp(adminApp);
appendTo(app);
// appendTo(app);

48
src/routes/agent/list.ts
View File

@ -1,7 +1,7 @@
import { app } from '@/app.ts';
import { AiAgent, AiProperties } from '@/models/agent.ts';
import { CustomError } from '@kevisual/router';
import { agentManger } from '@kevisual/ai-lang';
// import { agentManger } from '@kevisual/ai-lang';
import { v4 } from 'uuid';
app
.route({
@ -66,25 +66,25 @@ app
})
.addTo(app);
app
.route('agent', 'test')
.define(async (ctx) => {
const { message } = ctx.query;
const data: AiProperties = {
type: 'ollama',
id: 'test',
model: 'qwen2.5:14b',
baseUrl: 'http://mz.zxj.im:11434',
cache: 'memory',
};
const agent = agentManger.createAgent(data as any);
const res = await agent.sendHumanMessage(message);
// agent.close();
agentManger.removeAgent(agent.id);
ctx.body = res;
return ctx;
})
.addTo(app);
// app
// .route('agent', 'test')
// .define(async (ctx) => {
// const { message } = ctx.query;
// const data: AiProperties = {
// type: 'ollama',
// id: 'test',
// model: 'qwen2.5:14b',
// baseUrl: 'http://mz.zxj.im:11434',
// cache: 'memory',
// };
// const agent = agentManger.createAgent(data as any);
// const res = await agent.sendHumanMessage(message);
// // agent.close();
// agentManger.removeAgent(agent.id);
// ctx.body = res;
// return ctx;
// })
// .addTo(app);
export const agentModelList = ['qwen2.5:14b', 'qwen2.5-coder:7b', 'llama3.1:8b', 'bakllava:latest'] as const;
export const openAiModels = ['gpt-4o'];
@ -130,8 +130,8 @@ const initManager = async () => {
cacheName: item.cacheName,
};
});
agentManger.createAgentList(data);
// agentManger.createAgentList(data);
};
setTimeout(() => {
initManager();
}, 1000);
// setTimeout(() => {
// initManager();
// }, 1000);

5
src/routes/container/list.ts
View File

@ -1,7 +1,6 @@
import { CustomError } from '@kevisual/router';
import { app } from '../../app.ts';
import { ContainerModel, ContainerData, Container } from './models/index.ts';
import semver from 'semver';
import { uploadMinioContainer } from '../page/module/cache-file.ts';
const list = app.route({
path: 'container',
@ -49,7 +48,7 @@ add.run = async (ctx) => {
const container = {
...data,
};
let containerModel: any = null;
let containerModel: ContainerModel | null = null;
if (container.id) {
containerModel = await ContainerModel.findByPk(container.id);
if (containerModel) {
@ -134,7 +133,7 @@ app
version: version,
code: container.code,
filePath: fileName,
saveHTML
saveHTML,
});
await ctx.call({
path: 'app',

15
src/routes/container/models/index.ts
View File

@ -1,6 +1,6 @@
import { sequelize } from '../../../modules/sequelize.ts';
import { DataTypes, Model } from 'sequelize';
import crypto from 'crypto';
export interface ContainerData {}
export type ContainerPublish = {
key: string;
@ -21,11 +21,20 @@ export class ContainerModel extends Model {
declare type: string;
declare tags: string[];
declare code: string;
declare hash: string;
declare source: string;
declare sourceType: string;
declare data: ContainerData;
declare publish: ContainerPublish;
declare uid: string;
declare updatedAt: Date;
declare createdAt: Date;
createHash() {
const { code } = this;
const hash = crypto.createHash('md5');
hash.update(code);
this.hash = hash.digest('hex');
}
}
ContainerModel.init(
{
@ -55,6 +64,10 @@ ContainerModel.init(
type: DataTypes.TEXT,
defaultValue: '',
},
hash: {
type: DataTypes.TEXT,
defaultValue: '',
},
source: {
type: DataTypes.STRING,
defaultValue: '',

11
src/routes/container/module/get-container-file.ts Normal file
View File

@ -0,0 +1,11 @@
import { ContainerModel } from '../models/index.ts';
export const getContainerById = async (id: string) => {
const container = await ContainerModel.findByPk(id);
const code = container?.code;
return {
code,
id: container?.id,
updatedAt: new Date(container?.updatedAt).getTime(),
};
};

4
src/routes/index.ts
View File

@ -10,9 +10,9 @@ import './agent/index.ts';
import './user/index.ts';
import './chat-prompt/index.ts';
// import './chat-prompt/index.ts';
import './chat-history/index.ts';
// import './chat-history/index.ts';
import './github/index.ts';

22
src/routes/user/me.ts
View File

@ -16,7 +16,7 @@ app
logging: false,
});
if (!user) {
throw new CustomError(500, 'user not found');
ctx.throw(500, 'user not found');
}
user.setTokenUser(tokenUser);
ctx.body = await user.getInfo();
@ -30,7 +30,7 @@ app
.define(async (ctx) => {
const { username, email, password } = ctx.query;
if (!username && !email) {
throw new CustomError(400, 'username or email is required');
ctx.throw(400, 'username or email is required');
}
let user: User | null = null;
if (username) {
@ -40,10 +40,10 @@ app
user = await User.findOne({ where: { email } });
}
if (!user) {
throw new CustomError(500, 'Login Failed');
ctx.throw(500, 'Login Failed');
}
if (!user.checkPassword(password)) {
throw new CustomError(500, 'Password error');
ctx.throw(500, 'Password error');
}
const token = await user.createToken();
ctx.body = token;
@ -58,7 +58,7 @@ app
const result = await User.verifyToken(token);
ctx.body = result || {};
} catch (e) {
throw new CustomError(401, 'Token InValid ');
ctx.throw(401, 'Token InValid ');
}
})
.addTo(app);
@ -73,7 +73,7 @@ app
const { id } = tokenUser;
const user = await User.findByPk(id);
if (!user) {
throw new CustomError(500, 'user not found');
ctx.throw(500, 'user not found');
}
user.setTokenUser(tokenUser);
if (username) {
@ -105,13 +105,13 @@ app
const tokenUser = ctx.state.tokenUser;
const { username, type = 'org' } = ctx.query.data || {};
if (!username && type === 'org') {
throw new CustomError('username is required');
ctx.throw('username is required');
}
if (tokenUser.username === username) {
// 自己刷新自己的token
const user = await User.findByPk(tokenUser.id);
if (!user) {
throw new CustomError('user not found');
ctx.throw('user not found');
}
if (user.type === 'user') {
const token = await user.createToken();
@ -131,7 +131,7 @@ app
}
if (!me || me.type === 'org') {
console.log('switch Error ', me.username, me.type);
throw new CustomError('Permission denied');
ctx.throw('Permission denied');
}
if (type === 'user') {
const token = await me.createToken();
@ -140,13 +140,13 @@ app
}
const orgUser = await User.findOne({ where: { username } });
if (!orgUser) {
throw new CustomError('org user not found');
ctx.throw('org user not found');
}
const user = await Org.findOne({ where: { username } });
const users = user.users;
const index = users.findIndex((u) => u.uid === me.id);
if (index === -1) {
throw new CustomError('Permission denied');
ctx.throw('Permission denied');
}
const token = await orgUser.createToken(me.id);
ctx.body = token;