feat: 更新资源迁移逻辑,优化用户 ID 处理,删除冗余代码
This commit is contained in:
@@ -1,83 +1,9 @@
|
|||||||
import { oss } from '@/modules/s3.ts';
|
import { oss } from '@/modules/s3.ts';
|
||||||
import { db, schema } from '../src/app.ts';
|
|
||||||
import { eq } from 'drizzle-orm';
|
|
||||||
import { UserId } from '@/routes/user/modules/user-id.ts';
|
import { UserId } from '@/routes/user/modules/user-id.ts';
|
||||||
import { mvUserAToUserB } from '@/routes/file/index.ts';
|
import { mvUserAToUserB } from '@/routes/file/index.ts';
|
||||||
|
|
||||||
|
|
||||||
// 迁移资源,原本的是 ${username}/appKey 改为 ${userId}/appKey
|
// 迁移资源,原本的是 ${username}/appKey 改为 ${userId}/appKey
|
||||||
|
|
||||||
// 第一步,迁移 表kv_app 和 kv_app_list, 对应的 data 中,把data 对应的 files 中path 去掉第一个前缀,比如
|
|
||||||
// {
|
|
||||||
// "files": [
|
|
||||||
// {
|
|
||||||
// "name": "README.md",
|
|
||||||
// "path": "root/code-center/0.0.6/README.md"
|
|
||||||
// },
|
|
||||||
// 把 path 中的 root 去掉,变成 code-center/0.0.6/README.md
|
|
||||||
// ]
|
|
||||||
// }
|
|
||||||
|
|
||||||
type Data = {
|
|
||||||
files: {
|
|
||||||
name: string;
|
|
||||||
path: string;
|
|
||||||
}[];
|
|
||||||
}
|
|
||||||
const BATCH_SIZE = 1000;
|
|
||||||
|
|
||||||
// 迁移 kv_app;
|
|
||||||
const firstMigration = async () => {
|
|
||||||
let offset = 0;
|
|
||||||
while (true) {
|
|
||||||
const kvAppList = await db.select().from(schema.kvApp).limit(BATCH_SIZE).offset(offset);
|
|
||||||
if (kvAppList.length === 0) break;
|
|
||||||
|
|
||||||
for (const kvApp of kvAppList) {
|
|
||||||
const data = kvApp.data as Data;
|
|
||||||
const uid = kvApp.uid;
|
|
||||||
const username = await UserId.getUserNameById(uid);
|
|
||||||
if (!data.files) continue;
|
|
||||||
for (const file of data.files) {
|
|
||||||
// const pathParts = file.path.split('/');
|
|
||||||
// pathParts.shift();
|
|
||||||
// file.path = pathParts.join('/');
|
|
||||||
file.path = username + '/' + file.path;
|
|
||||||
}
|
|
||||||
await db.update(schema.kvApp).set({ data: { ...data } }).where(eq(schema.kvApp.id, kvApp.id));
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Processed ${offset + kvAppList.length} records`);
|
|
||||||
offset += BATCH_SIZE;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 迁移 kv_app_list
|
|
||||||
const secondMigration = async () => {
|
|
||||||
let offset = 0;
|
|
||||||
while (true) {
|
|
||||||
const kvAppList = await db.select().from(schema.kvAppList).limit(BATCH_SIZE).offset(offset);
|
|
||||||
if (kvAppList.length === 0) break;
|
|
||||||
|
|
||||||
for (const kvApp of kvAppList) {
|
|
||||||
const data = kvApp.data as Data;
|
|
||||||
const uid = kvApp.uid;
|
|
||||||
const username = await UserId.getUserNameById(uid);
|
|
||||||
if (!data.files) continue;
|
|
||||||
for (const file of data.files) {
|
|
||||||
// const pathParts = file.path.split('/');
|
|
||||||
// pathParts.shift();
|
|
||||||
// file.path = pathParts.join('/');
|
|
||||||
file.path = username + '/' + file.path;
|
|
||||||
}
|
|
||||||
await db.update(schema.kvAppList).set({ data: { ...data } }).where(eq(schema.kvAppList.id, kvApp.id));
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Processed ${offset + kvAppList.length} records`);
|
|
||||||
offset += BATCH_SIZE;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 迁移对象存储
|
* 迁移对象存储
|
||||||
*/
|
*/
|
||||||
@@ -95,7 +21,9 @@ const migrateOss = async () => {
|
|||||||
const id = await UserId.getUserIdByName(username);
|
const id = await UserId.getUserIdByName(username);
|
||||||
if (id) {
|
if (id) {
|
||||||
name.id = id;
|
name.id = id;
|
||||||
mvUserAToUserB(username, `data/${id}`, true);
|
console.log(`migrating ${name.prefix} to data/${id}`);
|
||||||
|
await mvUserAToUserB(username, `data/${id}`, true);
|
||||||
|
console.log(`migrated ${name.prefix} to data/${id}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,15 +4,14 @@ import { filterKeys } from './http-proxy.ts';
|
|||||||
import { getUserFromRequest } from '../utils.ts';
|
import { getUserFromRequest } from '../utils.ts';
|
||||||
import { UserPermission, Permission } from '@kevisual/permission';
|
import { UserPermission, Permission } from '@kevisual/permission';
|
||||||
import { getLoginUser } from '@/modules/auth.ts';
|
import { getLoginUser } from '@/modules/auth.ts';
|
||||||
import busboy from 'busboy';
|
|
||||||
import { getContentType, getTextContentType } from '../get-content-type.ts';
|
import { getContentType, getTextContentType } from '../get-content-type.ts';
|
||||||
import { OssBase } from '@kevisual/oss';
|
import { OssBase } from '@kevisual/oss';
|
||||||
import { parseSearchValue } from '@kevisual/router/src/server/parse-body.ts';
|
import { parseSearchValue } from '@kevisual/router/src/server/parse-body.ts';
|
||||||
import { logger } from '@/modules/logger.ts';
|
import { logger } from '@/modules/logger.ts';
|
||||||
import { pipeBusboy } from '../pipe-busboy.ts';
|
|
||||||
import { pipeMinioStream } from '../pipe.ts';
|
import { pipeMinioStream } from '../pipe.ts';
|
||||||
import { Readable } from 'stream';
|
import { Readable } from 'stream';
|
||||||
import { postChunkProxy, postProxy } from './ai-proxy-chunk/post-proxy.ts'
|
import { postChunkProxy, postProxy } from './ai-proxy-chunk/post-proxy.ts'
|
||||||
|
import { UserId } from '@/routes/user/modules/user-id.ts';
|
||||||
type FileList = {
|
type FileList = {
|
||||||
name: string;
|
name: string;
|
||||||
prefix?: string;
|
prefix?: string;
|
||||||
@@ -24,28 +23,18 @@ type FileList = {
|
|||||||
url?: string;
|
url?: string;
|
||||||
pathname?: string;
|
pathname?: string;
|
||||||
};
|
};
|
||||||
export const getFileList = async (list: any, opts?: { objectName: string; app: string; host?: string }) => {
|
export const getFileList = async (list: any, opts?: { objectName: string; app: string; host?: string; uid?: string, username?: string }) => {
|
||||||
const { app, host } = opts || {};
|
const { app, host, uid, username } = opts || {};
|
||||||
const objectName = opts?.objectName || '';
|
const objectName = opts?.objectName || '';
|
||||||
const [user] = objectName.split('/');
|
let beforePath = `data/${uid}`;
|
||||||
let replaceUser = user + '/';
|
let replaceUser = `${username}/resources/`;
|
||||||
if (app === 'resources') {
|
|
||||||
replaceUser = `${user}/resources/`;
|
|
||||||
}
|
|
||||||
return list.map((item: FileList) => {
|
return list.map((item: FileList) => {
|
||||||
if (item.name) {
|
if (item.name) {
|
||||||
item.path = item.name?.replace?.(objectName, '');
|
item.path = item.name?.replace?.(objectName, '');
|
||||||
item.pathname = '/' + item.name.replace(`${user}/`, replaceUser);
|
item.pathname = '/' + item.name.replace(`${beforePath}/`, replaceUser);
|
||||||
} else {
|
} else {
|
||||||
item.path = item.prefix?.replace?.(objectName, '');
|
item.path = item.prefix?.replace?.(objectName, '');
|
||||||
item.pathname = '/' + item.prefix.replace(`${user}/`, replaceUser);
|
item.pathname = '/' + item.prefix.replace(`${beforePath}/`, replaceUser);
|
||||||
}
|
|
||||||
if (item.name && app === 'ai') {
|
|
||||||
const [_user, _app, _version, ...rest] = item.name.split('/');
|
|
||||||
item.pathname = item.pathname.replace(`/${_user}/${_app}/${_version}/`, `/${_user}/${_app}/`);
|
|
||||||
} else if (app === 'ai') {
|
|
||||||
const [_user, _app, _version, ...rest] = item.prefix?.split('/');
|
|
||||||
item.pathname = item.pathname.replace(`/${_user}/${_app}/${_version}/`, `/${_user}/${_app}/`);
|
|
||||||
}
|
}
|
||||||
item.url = new URL(item.pathname, `https://${host}`).toString();
|
item.url = new URL(item.pathname, `https://${host}`).toString();
|
||||||
return item;
|
return item;
|
||||||
@@ -71,7 +60,7 @@ const getAiProxy = async (req: IncomingMessage, res: ServerResponse, opts: Proxy
|
|||||||
const edit = !!params.get('edit');
|
const edit = !!params.get('edit');
|
||||||
const recursive = !!params.get('recursive');
|
const recursive = !!params.get('recursive');
|
||||||
const showStat = !!params.get('stat');
|
const showStat = !!params.get('stat');
|
||||||
const { objectName, app, owner, loginUser, isOwner } = await getObjectName(req);
|
const { objectName, app, owner, loginUser, isOwner, uid, user } = await getObjectName(req);
|
||||||
if (!dir && _u.pathname.endsWith('/')) {
|
if (!dir && _u.pathname.endsWith('/')) {
|
||||||
dir = true; // 如果是目录请求,强制设置为true
|
dir = true; // 如果是目录请求,强制设置为true
|
||||||
}
|
}
|
||||||
@@ -98,6 +87,8 @@ const getAiProxy = async (req: IncomingMessage, res: ServerResponse, opts: Proxy
|
|||||||
objectName: objectName,
|
objectName: objectName,
|
||||||
app: app,
|
app: app,
|
||||||
host,
|
host,
|
||||||
|
uid,
|
||||||
|
username: user,
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
@@ -198,33 +189,24 @@ export const getObjectByPathname = (opts: {
|
|||||||
const [_, user, app] = opts.pathname.split('/');
|
const [_, user, app] = opts.pathname.split('/');
|
||||||
let prefix = '';
|
let prefix = '';
|
||||||
let replaceKey = '';
|
let replaceKey = '';
|
||||||
if (app === 'ai') {
|
replaceKey = `/${user}/${app}/`;
|
||||||
const version = opts?.version || '1.0.0';
|
prefix = `${user}/`; // root/resources
|
||||||
replaceKey = `/${user}/${app}/`;
|
|
||||||
prefix = `${user}/${app}/${version}/`;
|
|
||||||
} else {
|
|
||||||
replaceKey = `/${user}/${app}/`;
|
|
||||||
prefix = `${user}/`; // root/resources
|
|
||||||
}
|
|
||||||
let objectName = opts.pathname.replace(replaceKey, prefix);
|
let objectName = opts.pathname.replace(replaceKey, prefix);
|
||||||
// 解码decodeURIComponent编码的路径
|
// 解码decodeURIComponent编码的路径
|
||||||
objectName = decodeURIComponent(objectName);
|
objectName = decodeURIComponent(objectName);
|
||||||
return { prefix, replaceKey, objectName, user, app };
|
return { prefix, replaceKey, objectName, user, app };
|
||||||
}
|
}
|
||||||
export const getObjectName = async (req: IncomingMessage, opts?: { checkOwner?: boolean }) => {
|
export const getObjectName = async (req: IncomingMessage, opts?: { checkOwner?: boolean, uid?: string }) => {
|
||||||
const _u = new URL(req.url, 'http://localhost');
|
const _u = new URL(req.url, 'http://localhost');
|
||||||
const pathname = decodeURIComponent(_u.pathname);
|
const pathname = decodeURIComponent(_u.pathname);
|
||||||
const params = _u.searchParams;
|
const params = _u.searchParams;
|
||||||
const { user, app } = getUserFromRequest(req);
|
const { user, app } = getUserFromRequest(req);
|
||||||
const checkOwner = opts?.checkOwner ?? true;
|
const checkOwner = opts?.checkOwner ?? true;
|
||||||
|
const uid = opts?.uid || await UserId.getUserIdByName(user);
|
||||||
let objectName = '';
|
let objectName = '';
|
||||||
let owner = '';
|
let owner = '';
|
||||||
if (app === 'ai') {
|
|
||||||
const version = params.get('version') || '1.0.0'; // root/ai
|
objectName = pathname.replace(`/${user}/${app}/`, `data/${uid}/`); // root/resources
|
||||||
objectName = pathname.replace(`/${user}/${app}/`, `${user}/${app}/${version}/`);
|
|
||||||
} else {
|
|
||||||
objectName = pathname.replace(`/${user}/${app}/`, `${user}/`); // root/resources
|
|
||||||
}
|
|
||||||
// 解码decodeURIComponent编码的路径
|
// 解码decodeURIComponent编码的路径
|
||||||
objectName = decodeURIComponent(objectName);
|
objectName = decodeURIComponent(objectName);
|
||||||
owner = user;
|
owner = user;
|
||||||
@@ -242,6 +224,7 @@ export const getObjectName = async (req: IncomingMessage, opts?: { checkOwner?:
|
|||||||
isOwner,
|
isOwner,
|
||||||
app,
|
app,
|
||||||
user,
|
user,
|
||||||
|
uid,
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
export const deleteProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => {
|
export const deleteProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { Logger } from '@kevisual/logger';
|
|||||||
const config = useConfig();
|
const config = useConfig();
|
||||||
|
|
||||||
export const logger = new Logger({
|
export const logger = new Logger({
|
||||||
|
// @ts-ignore
|
||||||
level: config.LOG_LEVEL || 'info',
|
level: config.LOG_LEVEL || 'info',
|
||||||
showTime: true,
|
showTime: true,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ const wrapperResources = (resources: string, urlpath: string) => {
|
|||||||
if (urlpath.startsWith('http')) {
|
if (urlpath.startsWith('http')) {
|
||||||
return urlpath;
|
return urlpath;
|
||||||
}
|
}
|
||||||
return `${resources}/data/${urlpath}`;
|
return `${resources}/${urlpath}`;
|
||||||
};
|
};
|
||||||
const demoData = {
|
const demoData = {
|
||||||
user: 'root',
|
user: 'root',
|
||||||
@@ -218,6 +218,7 @@ export class UserApp {
|
|||||||
// console.log('fetchData', JSON.stringify(fetchData.data.files, null, 2));
|
// console.log('fetchData', JSON.stringify(fetchData.data.files, null, 2));
|
||||||
// const getFileSize
|
// const getFileSize
|
||||||
this.setLoaded('loading', 'loading');
|
this.setLoaded('loading', 'loading');
|
||||||
|
|
||||||
const loadProxy = async () => {
|
const loadProxy = async () => {
|
||||||
const value = fetchData;
|
const value = fetchData;
|
||||||
await redis.set(key, JSON.stringify(value));
|
await redis.set(key, JSON.stringify(value));
|
||||||
@@ -227,14 +228,12 @@ export class UserApp {
|
|||||||
const files = value?.data?.files || [];
|
const files = value?.data?.files || [];
|
||||||
const permission = value?.data?.permission || { share: 'private' };
|
const permission = value?.data?.permission || { share: 'private' };
|
||||||
const data = {};
|
const data = {};
|
||||||
|
const realPath = await getRealPath(files);
|
||||||
|
indexHtml = realPath.indexHtml || indexHtml;
|
||||||
|
|
||||||
// 将文件名和路径添加到 `data` 对象中
|
realPath.files.forEach((file) => {
|
||||||
files.forEach((file) => {
|
data[file.name] = file.path;
|
||||||
const noUserPath = file.path.startsWith(`${user}/`) ? file.path.replace(`${user}/`, '') : file.path;
|
console.log('proxy realPath file', file.name, file.path);
|
||||||
if (file.name === 'index.html') {
|
|
||||||
indexHtml = wrapperResources(resources, noUserPath);
|
|
||||||
}
|
|
||||||
data[file.name] = wrapperResources(resources, noUserPath);
|
|
||||||
});
|
});
|
||||||
await redis.set('user:app:exist:' + app + ':' + user, indexHtml + '||etag||true', 'EX', 60 * 60 * 24 * 7); // 7天
|
await redis.set('user:app:exist:' + app + ':' + user, indexHtml + '||etag||true', 'EX', 60 * 60 * 24 * 7); // 7天
|
||||||
await redis.set('user:app:permission:' + app + ':' + user, JSON.stringify(permission), 'EX', 60 * 60 * 24 * 7); // 7天
|
await redis.set('user:app:permission:' + app + ':' + user, JSON.stringify(permission), 'EX', 60 * 60 * 24 * 7); // 7天
|
||||||
@@ -338,13 +337,13 @@ export const downloadUserAppFiles = async (user: string, app: string, data: type
|
|||||||
fs.mkdirSync(uploadFiles, { recursive: true });
|
fs.mkdirSync(uploadFiles, { recursive: true });
|
||||||
}
|
}
|
||||||
const newFiles = [];
|
const newFiles = [];
|
||||||
|
const realPath = await getRealPath(files);
|
||||||
if (data.type === 'oss') {
|
if (data.type === 'oss') {
|
||||||
let serverPath = new URL(resources).href;
|
let serverPath = new URL(resources).href;
|
||||||
let hasIndexHtml = false;
|
let hasIndexHtml = false;
|
||||||
// server download file
|
// server download file
|
||||||
for (let i = 0; i < files.length; i++) {
|
for (let i = 0; i < realPath.files.length; i++) {
|
||||||
const file = files[i];
|
const file = realPath.files[i];
|
||||||
const destFile = path.join(uploadFiles, file.name);
|
const destFile = path.join(uploadFiles, file.name);
|
||||||
const destDir = path.dirname(destFile); // 获取目标文件所在的目录路径
|
const destDir = path.dirname(destFile); // 获取目标文件所在的目录路径
|
||||||
if (file.name === 'index.html') {
|
if (file.name === 'index.html') {
|
||||||
@@ -368,7 +367,7 @@ export const downloadUserAppFiles = async (user: string, app: string, data: type
|
|||||||
name: 'index.html',
|
name: 'index.html',
|
||||||
path: path.join(uploadFiles, 'index.html'),
|
path: path.join(uploadFiles, 'index.html'),
|
||||||
});
|
});
|
||||||
fs.writeFileSync(path.join(uploadFiles, 'index.html'), JSON.stringify(files), {
|
fs.writeFileSync(path.join(uploadFiles, 'index.html'), JSON.stringify(realPath.files), {
|
||||||
encoding: 'utf-8',
|
encoding: 'utf-8',
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -465,3 +464,41 @@ subscriber.on('message', (channel, message) => {
|
|||||||
userApp.clearCacheData();
|
userApp.clearCacheData();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
const getRealPath = async (files: { name: string; path: string }[] = []) => {
|
||||||
|
let _files: { name: string; path: string }[] = [];
|
||||||
|
let indexHtml = '';
|
||||||
|
const userset = new Map<string, string>();
|
||||||
|
for (const file of files) {
|
||||||
|
let noUserPath = file.path;
|
||||||
|
if (file.path.startsWith(`http`)) {
|
||||||
|
noUserPath = file.path;
|
||||||
|
} else {
|
||||||
|
const user = file.path.split('/')[0];
|
||||||
|
let uid = '';
|
||||||
|
if (!userset.has(user)) {
|
||||||
|
const _uid = await UserId.getUserIdByName(user);
|
||||||
|
if (_uid) {
|
||||||
|
userset.set(user, _uid);
|
||||||
|
}
|
||||||
|
uid = _uid;
|
||||||
|
} else {
|
||||||
|
uid = userset.get(user)!;
|
||||||
|
}
|
||||||
|
if (uid) {
|
||||||
|
noUserPath = file.path.replace(`${user}/`, `data/${uid}/`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const resourcePath = wrapperResources(resources, noUserPath);
|
||||||
|
if (file.name === 'index.html') {
|
||||||
|
indexHtml = resourcePath;
|
||||||
|
}
|
||||||
|
_files.push({
|
||||||
|
name: file.name,
|
||||||
|
path: resourcePath,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
indexHtml,
|
||||||
|
files: _files,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,7 +16,6 @@ import { UserV1Proxy } from '../modules/v1-ws-proxy/proxy.ts';
|
|||||||
import { UserV3Proxy } from '@/modules/v3/index.ts';
|
import { UserV3Proxy } from '@/modules/v3/index.ts';
|
||||||
import { hasBadUser, userIsBanned, appIsBanned, userPathIsBanned } from '@/modules/off/index.ts';
|
import { hasBadUser, userIsBanned, appIsBanned, userPathIsBanned } from '@/modules/off/index.ts';
|
||||||
import { robotsTxt } from '@/modules/html/index.ts';
|
import { robotsTxt } from '@/modules/html/index.ts';
|
||||||
import { isBun } from '@/utils/get-engine.ts';
|
|
||||||
import { N5Proxy } from '@/modules/n5/index.ts';
|
import { N5Proxy } from '@/modules/n5/index.ts';
|
||||||
const domain = config?.proxy?.domain;
|
const domain = config?.proxy?.domain;
|
||||||
const allowedOrigins = config?.proxy?.allowedOrigin || [];
|
const allowedOrigins = config?.proxy?.allowedOrigin || [];
|
||||||
@@ -251,7 +250,7 @@ export const handleRequest = async (req: http.IncomingMessage, res: http.ServerR
|
|||||||
res.write(msg || 'Not Found App\n');
|
res.write(msg || 'Not Found App\n');
|
||||||
res.end();
|
res.end();
|
||||||
};
|
};
|
||||||
if (app === 'ai' || app === 'resources' || app === 'r') {
|
if (app === 'resources') {
|
||||||
return aiProxy(req, res, {
|
return aiProxy(req, res, {
|
||||||
createNotFoundPage,
|
createNotFoundPage,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { App as AppType, AppList, AppData } from './module/app-drizzle.ts';
|
|||||||
import { app, db, oss, schema } from '@/app.ts';
|
import { app, db, oss, schema } from '@/app.ts';
|
||||||
import { uniqBy } from 'es-toolkit';
|
import { uniqBy } from 'es-toolkit';
|
||||||
import { getUidByUsername, prefixFix } from './util.ts';
|
import { getUidByUsername, prefixFix } from './util.ts';
|
||||||
import { deleteFiles, getMinioList, getMinioListAndSetToAppList } from '../file/index.ts';
|
import { deleteFileByPrefix, getMinioList, getMinioListAndSetToAppList } from '../file/index.ts';
|
||||||
import { setExpire } from './revoke.ts';
|
import { setExpire } from './revoke.ts';
|
||||||
import { User } from '@/models/user.ts';
|
import { User } from '@/models/user.ts';
|
||||||
import { callDetectAppVersion } from './export.ts';
|
import { callDetectAppVersion } from './export.ts';
|
||||||
@@ -186,12 +186,10 @@ app
|
|||||||
ctx.throw('app not found');
|
ctx.throw('app not found');
|
||||||
}
|
}
|
||||||
if (am.version === app.version) {
|
if (am.version === app.version) {
|
||||||
ctx.throw('app is published');
|
ctx.throw('app处于于发布状态,无法删除');
|
||||||
}
|
}
|
||||||
const appData = app.data as AppData;
|
if (deleteFile) {
|
||||||
const files = appData.files || [];
|
await deleteFileByPrefix(`data/${app.uid}/${app.key}/${app.version}`);
|
||||||
if (deleteFile && files.length > 0) {
|
|
||||||
await deleteFiles(files.map((item) => item.path));
|
|
||||||
}
|
}
|
||||||
await db.delete(schema.kvAppList).where(eq(schema.kvAppList.id, id));
|
await db.delete(schema.kvAppList).where(eq(schema.kvAppList.id, id));
|
||||||
ctx.body = 'success';
|
ctx.body = 'success';
|
||||||
|
|||||||
@@ -190,8 +190,8 @@ app
|
|||||||
await db.delete(schema.kvApp).where(eq(schema.kvApp.id, id));
|
await db.delete(schema.kvApp).where(eq(schema.kvApp.id, id));
|
||||||
await Promise.all(list.map((item) => db.delete(schema.kvAppList).where(eq(schema.kvAppList.id, item.id))));
|
await Promise.all(list.map((item) => db.delete(schema.kvAppList).where(eq(schema.kvAppList.id, item.id))));
|
||||||
if (deleteFile) {
|
if (deleteFile) {
|
||||||
const username = tokenUser.username;
|
const id = tokenUser.id;
|
||||||
await deleteFileByPrefix(`${username}/${am.key}`);
|
await deleteFileByPrefix(`data/${id}/${am.key}`);
|
||||||
}
|
}
|
||||||
ctx.body = 'success';
|
ctx.body = 'success';
|
||||||
return ctx;
|
return ctx;
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
import { app } from '@/app.ts';
|
import { app } from '@/app.ts';
|
||||||
import { getFileStat, getMinioList, deleteFile, updateFileStat, deleteFiles } from './module/get-minio-list.ts';
|
import { getFileStat, getMinioList, deleteFile, updateFileStat, deleteFileByPrefix } from './module/get-minio-list.ts';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { CustomError } from '@kevisual/router';
|
import { CustomError } from '@kevisual/router';
|
||||||
import { callDetectAppVersion } from '../app-manager/export.ts';
|
import { callDetectAppVersion } from '../app-manager/export.ts';
|
||||||
|
import { UserId } from '../user/modules/user-id.ts';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 清理prefix中的'..'
|
* 清理prefix中的'..'
|
||||||
@@ -23,8 +24,9 @@ const handlePrefix = (prefix: string) => {
|
|||||||
* @param tokenUser
|
* @param tokenUser
|
||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
const getPrefixByUser = (data: { prefix: string }, tokenUser: { username: string }) => {
|
const getPrefixByUser = async (data: { prefix: string }, tokenUser: { username: string }) => {
|
||||||
const prefixBase = '/' + tokenUser.username;
|
const uid = await UserId.getUserIdByName(tokenUser.username);
|
||||||
|
const prefixBase = '/data/' + uid;
|
||||||
const _prefix = handlePrefix(data.prefix);
|
const _prefix = handlePrefix(data.prefix);
|
||||||
return {
|
return {
|
||||||
len: prefixBase.length,
|
len: prefixBase.length,
|
||||||
@@ -40,7 +42,7 @@ app
|
|||||||
.define(async (ctx) => {
|
.define(async (ctx) => {
|
||||||
const tokenUser = ctx.state.tokenUser;
|
const tokenUser = ctx.state.tokenUser;
|
||||||
const data = ctx.query.data || {};
|
const data = ctx.query.data || {};
|
||||||
const { len, prefix } = getPrefixByUser(data, tokenUser);
|
const { len, prefix } = await getPrefixByUser(data, tokenUser);
|
||||||
const recursive = data.recursive;
|
const recursive = data.recursive;
|
||||||
const list = await getMinioList({ prefix: prefix.slice(1), recursive: recursive });
|
const list = await getMinioList({ prefix: prefix.slice(1), recursive: recursive });
|
||||||
|
|
||||||
@@ -67,7 +69,7 @@ app
|
|||||||
.define(async (ctx) => {
|
.define(async (ctx) => {
|
||||||
const tokenUser = ctx.state.tokenUser;
|
const tokenUser = ctx.state.tokenUser;
|
||||||
const data = ctx.query.data || {};
|
const data = ctx.query.data || {};
|
||||||
const { prefix } = getPrefixByUser(data, tokenUser);
|
const { prefix } = await getPrefixByUser(data, tokenUser);
|
||||||
console.log('prefix', prefix);
|
console.log('prefix', prefix);
|
||||||
const stat = await getFileStat(prefix.slice(1));
|
const stat = await getFileStat(prefix.slice(1));
|
||||||
ctx.body = stat;
|
ctx.body = stat;
|
||||||
@@ -83,7 +85,8 @@ app
|
|||||||
})
|
})
|
||||||
.define(async (ctx) => {
|
.define(async (ctx) => {
|
||||||
const tokenUser = ctx.state.tokenUser;
|
const tokenUser = ctx.state.tokenUser;
|
||||||
const list = await getMinioList({ prefix: '' + tokenUser.username, recursive: true });
|
const uid = await UserId.getUserIdByName(tokenUser.username);
|
||||||
|
const list = await getMinioList({ prefix: `data/${uid}/`, recursive: true });
|
||||||
const size = list.reduce((acc, item) => {
|
const size = list.reduce((acc, item) => {
|
||||||
if ('size' in item) {
|
if ('size' in item) {
|
||||||
return acc + item.size;
|
return acc + item.size;
|
||||||
@@ -109,7 +112,7 @@ app
|
|||||||
.define(async (ctx) => {
|
.define(async (ctx) => {
|
||||||
const tokenUser = ctx.state.tokenUser;
|
const tokenUser = ctx.state.tokenUser;
|
||||||
const data = ctx.query.data || {};
|
const data = ctx.query.data || {};
|
||||||
const { prefix } = getPrefixByUser(data, tokenUser);
|
const { prefix } = await getPrefixByUser(data, tokenUser);
|
||||||
const [username, appKey, version] = prefix.slice(1).split('/');
|
const [username, appKey, version] = prefix.slice(1).split('/');
|
||||||
const res = await deleteFile(prefix.slice(1));
|
const res = await deleteFile(prefix.slice(1));
|
||||||
if (res.code === 200) {
|
if (res.code === 200) {
|
||||||
@@ -136,7 +139,7 @@ app
|
|||||||
if (!data.metadata || JSON.stringify(data.metadata) === '{}') {
|
if (!data.metadata || JSON.stringify(data.metadata) === '{}') {
|
||||||
ctx.throw(400, 'metadata is required');
|
ctx.throw(400, 'metadata is required');
|
||||||
}
|
}
|
||||||
const { prefix } = getPrefixByUser(data, tokenUser);
|
const { prefix } = await getPrefixByUser(data, tokenUser);
|
||||||
const res = await updateFileStat(prefix.slice(1), data.metadata);
|
const res = await updateFileStat(prefix.slice(1), data.metadata);
|
||||||
if (res.code === 200) {
|
if (res.code === 200) {
|
||||||
ctx.body = 'update metadata success';
|
ctx.body = 'update metadata success';
|
||||||
@@ -155,6 +158,7 @@ app
|
|||||||
})
|
})
|
||||||
.define(async (ctx) => {
|
.define(async (ctx) => {
|
||||||
const tokenUser = ctx.state.tokenUser;
|
const tokenUser = ctx.state.tokenUser;
|
||||||
|
const uid = await UserId.getUserIdByName(tokenUser.username);
|
||||||
let directory = ctx.query.data?.directory as string;
|
let directory = ctx.query.data?.directory as string;
|
||||||
if (!directory) {
|
if (!directory) {
|
||||||
ctx.throw(400, 'directory is required');
|
ctx.throw(400, 'directory is required');
|
||||||
@@ -165,12 +169,15 @@ app
|
|||||||
if (directory.endsWith('/')) {
|
if (directory.endsWith('/')) {
|
||||||
ctx.throw(400, 'directory is invalid, cannot end with /');
|
ctx.throw(400, 'directory is invalid, cannot end with /');
|
||||||
}
|
}
|
||||||
const prefix = tokenUser.username + '/' + directory + '/';
|
const prefix = `data/${uid}/${directory}/`;
|
||||||
const list = await getMinioList<true>({ prefix, recursive: true });
|
const list = await getMinioList<true>({ prefix, recursive: true });
|
||||||
if (list.length === 0) {
|
if (list.length === 0) {
|
||||||
ctx.throw(400, 'directory is empty');
|
ctx.throw(400, 'directory is empty');
|
||||||
}
|
}
|
||||||
const res = await deleteFiles(list.map((item) => item.name));
|
if(list.length > 5000) {
|
||||||
|
ctx.throw(400, '删除文件数量过多,请分批删除');
|
||||||
|
}
|
||||||
|
const res = await deleteFileByPrefix(prefix);
|
||||||
if (!res) {
|
if (!res) {
|
||||||
ctx.throw(500, 'delete all failed');
|
ctx.throw(500, 'delete all failed');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import dayjs from 'dayjs';
|
import dayjs from 'dayjs';
|
||||||
import { oss } from '@/modules/s3.ts';
|
import { oss } from '@/modules/s3.ts';
|
||||||
import { StatObjectResult } from '@kevisual/oss';
|
import { StatObjectResult } from '@kevisual/oss';
|
||||||
|
import { UserId } from '@/routes/user/modules/user-id.ts';
|
||||||
type MinioListOpt = {
|
type MinioListOpt = {
|
||||||
prefix: string;
|
prefix: string;
|
||||||
recursive?: boolean;
|
recursive?: boolean;
|
||||||
@@ -62,23 +63,14 @@ export const deleteFile = async (prefix: string): Promise<{ code: number; messag
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// 批量删除文件
|
|
||||||
export const deleteFiles = async (prefixs: string[]): Promise<any> => {
|
|
||||||
try {
|
|
||||||
for (const prefix of prefixs) {
|
|
||||||
await oss.deleteObject(prefix);
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
} catch (e) {
|
|
||||||
console.error('delete Files Error not handle', e);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
export const deleteFileByPrefix = async (prefix: string): Promise<any> => {
|
export const deleteFileByPrefix = async (prefix: string): Promise<any> => {
|
||||||
try {
|
try {
|
||||||
const allFiles = await getMinioList<true>({ prefix, recursive: true });
|
const objects = await oss.listObjects<true>(prefix, { recursive: true });
|
||||||
const files = allFiles.filter((item) => item.name.startsWith(prefix));
|
if (objects.length > 0) {
|
||||||
await deleteFiles(files.map((item) => item.name));
|
for (const obj of objects) {
|
||||||
|
await oss.deleteObject(obj.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error('delete File Error not handle', e);
|
console.error('delete File Error not handle', e);
|
||||||
@@ -93,7 +85,8 @@ type GetMinioListAndSetToAppListOpts = {
|
|||||||
// 批量列出文件,并设置到appList的files中
|
// 批量列出文件,并设置到appList的files中
|
||||||
export const getMinioListAndSetToAppList = async (opts: GetMinioListAndSetToAppListOpts) => {
|
export const getMinioListAndSetToAppList = async (opts: GetMinioListAndSetToAppListOpts) => {
|
||||||
const { username, appKey, version } = opts;
|
const { username, appKey, version } = opts;
|
||||||
const minioList = await getMinioList({ prefix: `${username}/${appKey}/${version}`, recursive: true });
|
const uid = await UserId.getUserIdByName(username);
|
||||||
|
const minioList = await getMinioList({ prefix: `data/${uid}/${appKey}/${version}`, recursive: true });
|
||||||
const files = minioList;
|
const files = minioList;
|
||||||
return files as MinioFile[];
|
return files as MinioFile[];
|
||||||
};
|
};
|
||||||
@@ -178,7 +171,8 @@ export const backupUserA = async (usernameA: string, id: string, backName?: stri
|
|||||||
* @param username
|
* @param username
|
||||||
*/
|
*/
|
||||||
export const deleteUser = async (username: string) => {
|
export const deleteUser = async (username: string) => {
|
||||||
const list = await getMinioList<true>({ prefix: `${username}/`, recursive: true });
|
const uid = await UserId.getUserIdByName(username);
|
||||||
|
const list = await getMinioList<true>({ prefix: `data/${uid}/`, recursive: true });
|
||||||
for (const item of list) {
|
for (const item of list) {
|
||||||
await oss.deleteObject(item.name);
|
await oss.deleteObject(item.name);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,10 +2,8 @@ import { app, db, schema } from '@/app.ts';
|
|||||||
import { User } from '@/models/user.ts';
|
import { User } from '@/models/user.ts';
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import { CustomError } from '@kevisual/router';
|
import { CustomError } from '@kevisual/router';
|
||||||
import { backupUserA, deleteUser, mvUserAToUserB } from '@/routes/file/index.ts';
|
|
||||||
import { AppHelper } from '@/routes/app-manager/module/index.ts';
|
import { AppHelper } from '@/routes/app-manager/module/index.ts';
|
||||||
import { eq } from 'drizzle-orm';
|
import { eq } from 'drizzle-orm';
|
||||||
// import { mvAppFromUserAToUserB } from '@/routes/app-manager/admin/mv-user-app.ts';
|
|
||||||
|
|
||||||
export const checkUsername = (username: string) => {
|
export const checkUsername = (username: string) => {
|
||||||
if (username.length > 30) {
|
if (username.length > 30) {
|
||||||
@@ -45,10 +43,6 @@ export const toChangeName = async (opts: { id: string; newName: string; admin?:
|
|||||||
user.data = data;
|
user.data = data;
|
||||||
try {
|
try {
|
||||||
await user.save();
|
await user.save();
|
||||||
// 迁移文件数据
|
|
||||||
await backupUserA(oldName, user.id); // 备份文件数据
|
|
||||||
await mvUserAToUserB(oldName, newName, true); // 迁移文件数据
|
|
||||||
// await mvAppFromUserAToUserB(oldName, newName); // 迁移应用数据
|
|
||||||
|
|
||||||
if (['org', 'user'].includes(user.type)) {
|
if (['org', 'user'].includes(user.type)) {
|
||||||
const type = user.type === 'org' ? 'org' : 'user';
|
const type = user.type === 'org' ? 'org' : 'user';
|
||||||
@@ -175,8 +169,8 @@ app
|
|||||||
ctx.throw(404, { message: 'User not found' });
|
ctx.throw(404, { message: 'User not found' });
|
||||||
}
|
}
|
||||||
await db.delete(schema.cfUser).where(eq(schema.cfUser.id, user.id));
|
await db.delete(schema.cfUser).where(eq(schema.cfUser.id, user.id));
|
||||||
backupUserA(user.username, user.id);
|
// backupUserA(user.username, user.id);
|
||||||
deleteUser(user.username);
|
// deleteUser(user.username);
|
||||||
// TODO: EXPIRE 删除token
|
// TODO: EXPIRE 删除token
|
||||||
ctx.body = {
|
ctx.body = {
|
||||||
id: user.id,
|
id: user.id,
|
||||||
|
|||||||
@@ -1,15 +0,0 @@
|
|||||||
process.env.NODE_ENV = 'development';
|
|
||||||
import { mvUserAToUserB, backupUserA } from '../routes/file/module/get-minio-list.ts';
|
|
||||||
|
|
||||||
|
|
||||||
// mvUserAToUserB('demo', 'demo2');
|
|
||||||
|
|
||||||
// backupUserA('demo', '123', '2026-01-31-16-00');
|
|
||||||
// backupUserA('demo', '123', '2025-04-02-16-01');
|
|
||||||
// backupUserA('demo', '123', '2025-04-02-16-02');
|
|
||||||
// backupUserA('demo', '123', '2025-04-02-16-03');
|
|
||||||
// backupUserA('demo', '123', '2025-04-02-16-04');
|
|
||||||
// backupUserA('demo', '123', '2025-04-02-16-05');
|
|
||||||
// backupUserA('demo', '123', '2025-04-02-16-06');
|
|
||||||
// backupUserA('demo', '123', '2025-04-02-16-07');
|
|
||||||
// backupUserA('demo', '123', '2025-04-02-16-08');
|
|
||||||
Reference in New Issue
Block a user