feat: 更新资源迁移逻辑,优化用户 ID 处理,删除冗余代码

This commit is contained in:
xiongxiao
2026-03-25 02:36:27 +08:00
committed by cnb
parent 55378f4c94
commit b39f7d6028
11 changed files with 107 additions and 181 deletions

View File

@@ -1,83 +1,9 @@
import { oss } from '@/modules/s3.ts'; import { oss } from '@/modules/s3.ts';
import { db, schema } from '../src/app.ts';
import { eq } from 'drizzle-orm';
import { UserId } from '@/routes/user/modules/user-id.ts'; import { UserId } from '@/routes/user/modules/user-id.ts';
import { mvUserAToUserB } from '@/routes/file/index.ts'; import { mvUserAToUserB } from '@/routes/file/index.ts';
// 迁移资源,原本的是 ${username}/appKey 改为 ${userId}/appKey // 迁移资源,原本的是 ${username}/appKey 改为 ${userId}/appKey
// 第一步,迁移 表kv_app 和 kv_app_list, 对应的 data 中把data 对应的 files 中path 去掉第一个前缀,比如
// {
// "files": [
// {
// "name": "README.md",
// "path": "root/code-center/0.0.6/README.md"
// },
// 把 path 中的 root 去掉,变成 code-center/0.0.6/README.md
// ]
// }
type Data = {
files: {
name: string;
path: string;
}[];
}
const BATCH_SIZE = 1000;
// 迁移 kv_app;
const firstMigration = async () => {
let offset = 0;
while (true) {
const kvAppList = await db.select().from(schema.kvApp).limit(BATCH_SIZE).offset(offset);
if (kvAppList.length === 0) break;
for (const kvApp of kvAppList) {
const data = kvApp.data as Data;
const uid = kvApp.uid;
const username = await UserId.getUserNameById(uid);
if (!data.files) continue;
for (const file of data.files) {
// const pathParts = file.path.split('/');
// pathParts.shift();
// file.path = pathParts.join('/');
file.path = username + '/' + file.path;
}
await db.update(schema.kvApp).set({ data: { ...data } }).where(eq(schema.kvApp.id, kvApp.id));
}
console.log(`Processed ${offset + kvAppList.length} records`);
offset += BATCH_SIZE;
}
}
// 迁移 kv_app_list
const secondMigration = async () => {
let offset = 0;
while (true) {
const kvAppList = await db.select().from(schema.kvAppList).limit(BATCH_SIZE).offset(offset);
if (kvAppList.length === 0) break;
for (const kvApp of kvAppList) {
const data = kvApp.data as Data;
const uid = kvApp.uid;
const username = await UserId.getUserNameById(uid);
if (!data.files) continue;
for (const file of data.files) {
// const pathParts = file.path.split('/');
// pathParts.shift();
// file.path = pathParts.join('/');
file.path = username + '/' + file.path;
}
await db.update(schema.kvAppList).set({ data: { ...data } }).where(eq(schema.kvAppList.id, kvApp.id));
}
console.log(`Processed ${offset + kvAppList.length} records`);
offset += BATCH_SIZE;
}
}
/** /**
* 迁移对象存储 * 迁移对象存储
*/ */
@@ -95,7 +21,9 @@ const migrateOss = async () => {
const id = await UserId.getUserIdByName(username); const id = await UserId.getUserIdByName(username);
if (id) { if (id) {
name.id = id; name.id = id;
mvUserAToUserB(username, `data/${id}`, true); console.log(`migrating ${name.prefix} to data/${id}`);
await mvUserAToUserB(username, `data/${id}`, true);
console.log(`migrated ${name.prefix} to data/${id}`);
} }
} }
} }

View File

@@ -4,15 +4,14 @@ import { filterKeys } from './http-proxy.ts';
import { getUserFromRequest } from '../utils.ts'; import { getUserFromRequest } from '../utils.ts';
import { UserPermission, Permission } from '@kevisual/permission'; import { UserPermission, Permission } from '@kevisual/permission';
import { getLoginUser } from '@/modules/auth.ts'; import { getLoginUser } from '@/modules/auth.ts';
import busboy from 'busboy';
import { getContentType, getTextContentType } from '../get-content-type.ts'; import { getContentType, getTextContentType } from '../get-content-type.ts';
import { OssBase } from '@kevisual/oss'; import { OssBase } from '@kevisual/oss';
import { parseSearchValue } from '@kevisual/router/src/server/parse-body.ts'; import { parseSearchValue } from '@kevisual/router/src/server/parse-body.ts';
import { logger } from '@/modules/logger.ts'; import { logger } from '@/modules/logger.ts';
import { pipeBusboy } from '../pipe-busboy.ts';
import { pipeMinioStream } from '../pipe.ts'; import { pipeMinioStream } from '../pipe.ts';
import { Readable } from 'stream'; import { Readable } from 'stream';
import { postChunkProxy, postProxy } from './ai-proxy-chunk/post-proxy.ts' import { postChunkProxy, postProxy } from './ai-proxy-chunk/post-proxy.ts'
import { UserId } from '@/routes/user/modules/user-id.ts';
type FileList = { type FileList = {
name: string; name: string;
prefix?: string; prefix?: string;
@@ -24,28 +23,18 @@ type FileList = {
url?: string; url?: string;
pathname?: string; pathname?: string;
}; };
export const getFileList = async (list: any, opts?: { objectName: string; app: string; host?: string }) => { export const getFileList = async (list: any, opts?: { objectName: string; app: string; host?: string; uid?: string, username?: string }) => {
const { app, host } = opts || {}; const { app, host, uid, username } = opts || {};
const objectName = opts?.objectName || ''; const objectName = opts?.objectName || '';
const [user] = objectName.split('/'); let beforePath = `data/${uid}`;
let replaceUser = user + '/'; let replaceUser = `${username}/resources/`;
if (app === 'resources') {
replaceUser = `${user}/resources/`;
}
return list.map((item: FileList) => { return list.map((item: FileList) => {
if (item.name) { if (item.name) {
item.path = item.name?.replace?.(objectName, ''); item.path = item.name?.replace?.(objectName, '');
item.pathname = '/' + item.name.replace(`${user}/`, replaceUser); item.pathname = '/' + item.name.replace(`${beforePath}/`, replaceUser);
} else { } else {
item.path = item.prefix?.replace?.(objectName, ''); item.path = item.prefix?.replace?.(objectName, '');
item.pathname = '/' + item.prefix.replace(`${user}/`, replaceUser); item.pathname = '/' + item.prefix.replace(`${beforePath}/`, replaceUser);
}
if (item.name && app === 'ai') {
const [_user, _app, _version, ...rest] = item.name.split('/');
item.pathname = item.pathname.replace(`/${_user}/${_app}/${_version}/`, `/${_user}/${_app}/`);
} else if (app === 'ai') {
const [_user, _app, _version, ...rest] = item.prefix?.split('/');
item.pathname = item.pathname.replace(`/${_user}/${_app}/${_version}/`, `/${_user}/${_app}/`);
} }
item.url = new URL(item.pathname, `https://${host}`).toString(); item.url = new URL(item.pathname, `https://${host}`).toString();
return item; return item;
@@ -71,7 +60,7 @@ const getAiProxy = async (req: IncomingMessage, res: ServerResponse, opts: Proxy
const edit = !!params.get('edit'); const edit = !!params.get('edit');
const recursive = !!params.get('recursive'); const recursive = !!params.get('recursive');
const showStat = !!params.get('stat'); const showStat = !!params.get('stat');
const { objectName, app, owner, loginUser, isOwner } = await getObjectName(req); const { objectName, app, owner, loginUser, isOwner, uid, user } = await getObjectName(req);
if (!dir && _u.pathname.endsWith('/')) { if (!dir && _u.pathname.endsWith('/')) {
dir = true; // 如果是目录请求强制设置为true dir = true; // 如果是目录请求强制设置为true
} }
@@ -98,6 +87,8 @@ const getAiProxy = async (req: IncomingMessage, res: ServerResponse, opts: Proxy
objectName: objectName, objectName: objectName,
app: app, app: app,
host, host,
uid,
username: user,
}), }),
}), }),
); );
@@ -198,33 +189,24 @@ export const getObjectByPathname = (opts: {
const [_, user, app] = opts.pathname.split('/'); const [_, user, app] = opts.pathname.split('/');
let prefix = ''; let prefix = '';
let replaceKey = ''; let replaceKey = '';
if (app === 'ai') {
const version = opts?.version || '1.0.0';
replaceKey = `/${user}/${app}/`;
prefix = `${user}/${app}/${version}/`;
} else {
replaceKey = `/${user}/${app}/`; replaceKey = `/${user}/${app}/`;
prefix = `${user}/`; // root/resources prefix = `${user}/`; // root/resources
}
let objectName = opts.pathname.replace(replaceKey, prefix); let objectName = opts.pathname.replace(replaceKey, prefix);
// 解码decodeURIComponent编码的路径 // 解码decodeURIComponent编码的路径
objectName = decodeURIComponent(objectName); objectName = decodeURIComponent(objectName);
return { prefix, replaceKey, objectName, user, app }; return { prefix, replaceKey, objectName, user, app };
} }
export const getObjectName = async (req: IncomingMessage, opts?: { checkOwner?: boolean }) => { export const getObjectName = async (req: IncomingMessage, opts?: { checkOwner?: boolean, uid?: string }) => {
const _u = new URL(req.url, 'http://localhost'); const _u = new URL(req.url, 'http://localhost');
const pathname = decodeURIComponent(_u.pathname); const pathname = decodeURIComponent(_u.pathname);
const params = _u.searchParams; const params = _u.searchParams;
const { user, app } = getUserFromRequest(req); const { user, app } = getUserFromRequest(req);
const checkOwner = opts?.checkOwner ?? true; const checkOwner = opts?.checkOwner ?? true;
const uid = opts?.uid || await UserId.getUserIdByName(user);
let objectName = ''; let objectName = '';
let owner = ''; let owner = '';
if (app === 'ai') {
const version = params.get('version') || '1.0.0'; // root/ai objectName = pathname.replace(`/${user}/${app}/`, `data/${uid}/`); // root/resources
objectName = pathname.replace(`/${user}/${app}/`, `${user}/${app}/${version}/`);
} else {
objectName = pathname.replace(`/${user}/${app}/`, `${user}/`); // root/resources
}
// 解码decodeURIComponent编码的路径 // 解码decodeURIComponent编码的路径
objectName = decodeURIComponent(objectName); objectName = decodeURIComponent(objectName);
owner = user; owner = user;
@@ -242,6 +224,7 @@ export const getObjectName = async (req: IncomingMessage, opts?: { checkOwner?:
isOwner, isOwner,
app, app,
user, user,
uid,
}; };
}; };
export const deleteProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => { export const deleteProxy = async (req: IncomingMessage, res: ServerResponse, opts: ProxyOptions) => {

View File

@@ -3,6 +3,7 @@ import { Logger } from '@kevisual/logger';
const config = useConfig(); const config = useConfig();
export const logger = new Logger({ export const logger = new Logger({
// @ts-ignore
level: config.LOG_LEVEL || 'info', level: config.LOG_LEVEL || 'info',
showTime: true, showTime: true,
}); });

View File

@@ -21,7 +21,7 @@ const wrapperResources = (resources: string, urlpath: string) => {
if (urlpath.startsWith('http')) { if (urlpath.startsWith('http')) {
return urlpath; return urlpath;
} }
return `${resources}/data/${urlpath}`; return `${resources}/${urlpath}`;
}; };
const demoData = { const demoData = {
user: 'root', user: 'root',
@@ -218,6 +218,7 @@ export class UserApp {
// console.log('fetchData', JSON.stringify(fetchData.data.files, null, 2)); // console.log('fetchData', JSON.stringify(fetchData.data.files, null, 2));
// const getFileSize // const getFileSize
this.setLoaded('loading', 'loading'); this.setLoaded('loading', 'loading');
const loadProxy = async () => { const loadProxy = async () => {
const value = fetchData; const value = fetchData;
await redis.set(key, JSON.stringify(value)); await redis.set(key, JSON.stringify(value));
@@ -227,14 +228,12 @@ export class UserApp {
const files = value?.data?.files || []; const files = value?.data?.files || [];
const permission = value?.data?.permission || { share: 'private' }; const permission = value?.data?.permission || { share: 'private' };
const data = {}; const data = {};
const realPath = await getRealPath(files);
indexHtml = realPath.indexHtml || indexHtml;
// 将文件名和路径添加到 `data` 对象中 realPath.files.forEach((file) => {
files.forEach((file) => { data[file.name] = file.path;
const noUserPath = file.path.startsWith(`${user}/`) ? file.path.replace(`${user}/`, '') : file.path; console.log('proxy realPath file', file.name, file.path);
if (file.name === 'index.html') {
indexHtml = wrapperResources(resources, noUserPath);
}
data[file.name] = wrapperResources(resources, noUserPath);
}); });
await redis.set('user:app:exist:' + app + ':' + user, indexHtml + '||etag||true', 'EX', 60 * 60 * 24 * 7); // 7天 await redis.set('user:app:exist:' + app + ':' + user, indexHtml + '||etag||true', 'EX', 60 * 60 * 24 * 7); // 7天
await redis.set('user:app:permission:' + app + ':' + user, JSON.stringify(permission), 'EX', 60 * 60 * 24 * 7); // 7天 await redis.set('user:app:permission:' + app + ':' + user, JSON.stringify(permission), 'EX', 60 * 60 * 24 * 7); // 7天
@@ -338,13 +337,13 @@ export const downloadUserAppFiles = async (user: string, app: string, data: type
fs.mkdirSync(uploadFiles, { recursive: true }); fs.mkdirSync(uploadFiles, { recursive: true });
} }
const newFiles = []; const newFiles = [];
const realPath = await getRealPath(files);
if (data.type === 'oss') { if (data.type === 'oss') {
let serverPath = new URL(resources).href; let serverPath = new URL(resources).href;
let hasIndexHtml = false; let hasIndexHtml = false;
// server download file // server download file
for (let i = 0; i < files.length; i++) { for (let i = 0; i < realPath.files.length; i++) {
const file = files[i]; const file = realPath.files[i];
const destFile = path.join(uploadFiles, file.name); const destFile = path.join(uploadFiles, file.name);
const destDir = path.dirname(destFile); // 获取目标文件所在的目录路径 const destDir = path.dirname(destFile); // 获取目标文件所在的目录路径
if (file.name === 'index.html') { if (file.name === 'index.html') {
@@ -368,7 +367,7 @@ export const downloadUserAppFiles = async (user: string, app: string, data: type
name: 'index.html', name: 'index.html',
path: path.join(uploadFiles, 'index.html'), path: path.join(uploadFiles, 'index.html'),
}); });
fs.writeFileSync(path.join(uploadFiles, 'index.html'), JSON.stringify(files), { fs.writeFileSync(path.join(uploadFiles, 'index.html'), JSON.stringify(realPath.files), {
encoding: 'utf-8', encoding: 'utf-8',
}); });
} }
@@ -465,3 +464,41 @@ subscriber.on('message', (channel, message) => {
userApp.clearCacheData(); userApp.clearCacheData();
} }
}); });
const getRealPath = async (files: { name: string; path: string }[] = []) => {
let _files: { name: string; path: string }[] = [];
let indexHtml = '';
const userset = new Map<string, string>();
for (const file of files) {
let noUserPath = file.path;
if (file.path.startsWith(`http`)) {
noUserPath = file.path;
} else {
const user = file.path.split('/')[0];
let uid = '';
if (!userset.has(user)) {
const _uid = await UserId.getUserIdByName(user);
if (_uid) {
userset.set(user, _uid);
}
uid = _uid;
} else {
uid = userset.get(user)!;
}
if (uid) {
noUserPath = file.path.replace(`${user}/`, `data/${uid}/`);
}
}
const resourcePath = wrapperResources(resources, noUserPath);
if (file.name === 'index.html') {
indexHtml = resourcePath;
}
_files.push({
name: file.name,
path: resourcePath,
});
}
return {
indexHtml,
files: _files,
}
}

View File

@@ -16,7 +16,6 @@ import { UserV1Proxy } from '../modules/v1-ws-proxy/proxy.ts';
import { UserV3Proxy } from '@/modules/v3/index.ts'; import { UserV3Proxy } from '@/modules/v3/index.ts';
import { hasBadUser, userIsBanned, appIsBanned, userPathIsBanned } from '@/modules/off/index.ts'; import { hasBadUser, userIsBanned, appIsBanned, userPathIsBanned } from '@/modules/off/index.ts';
import { robotsTxt } from '@/modules/html/index.ts'; import { robotsTxt } from '@/modules/html/index.ts';
import { isBun } from '@/utils/get-engine.ts';
import { N5Proxy } from '@/modules/n5/index.ts'; import { N5Proxy } from '@/modules/n5/index.ts';
const domain = config?.proxy?.domain; const domain = config?.proxy?.domain;
const allowedOrigins = config?.proxy?.allowedOrigin || []; const allowedOrigins = config?.proxy?.allowedOrigin || [];
@@ -251,7 +250,7 @@ export const handleRequest = async (req: http.IncomingMessage, res: http.ServerR
res.write(msg || 'Not Found App\n'); res.write(msg || 'Not Found App\n');
res.end(); res.end();
}; };
if (app === 'ai' || app === 'resources' || app === 'r') { if (app === 'resources') {
return aiProxy(req, res, { return aiProxy(req, res, {
createNotFoundPage, createNotFoundPage,
}); });

View File

@@ -2,7 +2,7 @@ import { App as AppType, AppList, AppData } from './module/app-drizzle.ts';
import { app, db, oss, schema } from '@/app.ts'; import { app, db, oss, schema } from '@/app.ts';
import { uniqBy } from 'es-toolkit'; import { uniqBy } from 'es-toolkit';
import { getUidByUsername, prefixFix } from './util.ts'; import { getUidByUsername, prefixFix } from './util.ts';
import { deleteFiles, getMinioList, getMinioListAndSetToAppList } from '../file/index.ts'; import { deleteFileByPrefix, getMinioList, getMinioListAndSetToAppList } from '../file/index.ts';
import { setExpire } from './revoke.ts'; import { setExpire } from './revoke.ts';
import { User } from '@/models/user.ts'; import { User } from '@/models/user.ts';
import { callDetectAppVersion } from './export.ts'; import { callDetectAppVersion } from './export.ts';
@@ -186,12 +186,10 @@ app
ctx.throw('app not found'); ctx.throw('app not found');
} }
if (am.version === app.version) { if (am.version === app.version) {
ctx.throw('app is published'); ctx.throw('app处于于发布状态,无法删除');
} }
const appData = app.data as AppData; if (deleteFile) {
const files = appData.files || []; await deleteFileByPrefix(`data/${app.uid}/${app.key}/${app.version}`);
if (deleteFile && files.length > 0) {
await deleteFiles(files.map((item) => item.path));
} }
await db.delete(schema.kvAppList).where(eq(schema.kvAppList.id, id)); await db.delete(schema.kvAppList).where(eq(schema.kvAppList.id, id));
ctx.body = 'success'; ctx.body = 'success';

View File

@@ -190,8 +190,8 @@ app
await db.delete(schema.kvApp).where(eq(schema.kvApp.id, id)); await db.delete(schema.kvApp).where(eq(schema.kvApp.id, id));
await Promise.all(list.map((item) => db.delete(schema.kvAppList).where(eq(schema.kvAppList.id, item.id)))); await Promise.all(list.map((item) => db.delete(schema.kvAppList).where(eq(schema.kvAppList.id, item.id))));
if (deleteFile) { if (deleteFile) {
const username = tokenUser.username; const id = tokenUser.id;
await deleteFileByPrefix(`${username}/${am.key}`); await deleteFileByPrefix(`data/${id}/${am.key}`);
} }
ctx.body = 'success'; ctx.body = 'success';
return ctx; return ctx;

View File

@@ -1,8 +1,9 @@
import { app } from '@/app.ts'; import { app } from '@/app.ts';
import { getFileStat, getMinioList, deleteFile, updateFileStat, deleteFiles } from './module/get-minio-list.ts'; import { getFileStat, getMinioList, deleteFile, updateFileStat, deleteFileByPrefix } from './module/get-minio-list.ts';
import path from 'path'; import path from 'path';
import { CustomError } from '@kevisual/router'; import { CustomError } from '@kevisual/router';
import { callDetectAppVersion } from '../app-manager/export.ts'; import { callDetectAppVersion } from '../app-manager/export.ts';
import { UserId } from '../user/modules/user-id.ts';
/** /**
* 清理prefix中的'..' * 清理prefix中的'..'
@@ -23,8 +24,9 @@ const handlePrefix = (prefix: string) => {
* @param tokenUser * @param tokenUser
* @returns * @returns
*/ */
const getPrefixByUser = (data: { prefix: string }, tokenUser: { username: string }) => { const getPrefixByUser = async (data: { prefix: string }, tokenUser: { username: string }) => {
const prefixBase = '/' + tokenUser.username; const uid = await UserId.getUserIdByName(tokenUser.username);
const prefixBase = '/data/' + uid;
const _prefix = handlePrefix(data.prefix); const _prefix = handlePrefix(data.prefix);
return { return {
len: prefixBase.length, len: prefixBase.length,
@@ -40,7 +42,7 @@ app
.define(async (ctx) => { .define(async (ctx) => {
const tokenUser = ctx.state.tokenUser; const tokenUser = ctx.state.tokenUser;
const data = ctx.query.data || {}; const data = ctx.query.data || {};
const { len, prefix } = getPrefixByUser(data, tokenUser); const { len, prefix } = await getPrefixByUser(data, tokenUser);
const recursive = data.recursive; const recursive = data.recursive;
const list = await getMinioList({ prefix: prefix.slice(1), recursive: recursive }); const list = await getMinioList({ prefix: prefix.slice(1), recursive: recursive });
@@ -67,7 +69,7 @@ app
.define(async (ctx) => { .define(async (ctx) => {
const tokenUser = ctx.state.tokenUser; const tokenUser = ctx.state.tokenUser;
const data = ctx.query.data || {}; const data = ctx.query.data || {};
const { prefix } = getPrefixByUser(data, tokenUser); const { prefix } = await getPrefixByUser(data, tokenUser);
console.log('prefix', prefix); console.log('prefix', prefix);
const stat = await getFileStat(prefix.slice(1)); const stat = await getFileStat(prefix.slice(1));
ctx.body = stat; ctx.body = stat;
@@ -83,7 +85,8 @@ app
}) })
.define(async (ctx) => { .define(async (ctx) => {
const tokenUser = ctx.state.tokenUser; const tokenUser = ctx.state.tokenUser;
const list = await getMinioList({ prefix: '' + tokenUser.username, recursive: true }); const uid = await UserId.getUserIdByName(tokenUser.username);
const list = await getMinioList({ prefix: `data/${uid}/`, recursive: true });
const size = list.reduce((acc, item) => { const size = list.reduce((acc, item) => {
if ('size' in item) { if ('size' in item) {
return acc + item.size; return acc + item.size;
@@ -109,7 +112,7 @@ app
.define(async (ctx) => { .define(async (ctx) => {
const tokenUser = ctx.state.tokenUser; const tokenUser = ctx.state.tokenUser;
const data = ctx.query.data || {}; const data = ctx.query.data || {};
const { prefix } = getPrefixByUser(data, tokenUser); const { prefix } = await getPrefixByUser(data, tokenUser);
const [username, appKey, version] = prefix.slice(1).split('/'); const [username, appKey, version] = prefix.slice(1).split('/');
const res = await deleteFile(prefix.slice(1)); const res = await deleteFile(prefix.slice(1));
if (res.code === 200) { if (res.code === 200) {
@@ -136,7 +139,7 @@ app
if (!data.metadata || JSON.stringify(data.metadata) === '{}') { if (!data.metadata || JSON.stringify(data.metadata) === '{}') {
ctx.throw(400, 'metadata is required'); ctx.throw(400, 'metadata is required');
} }
const { prefix } = getPrefixByUser(data, tokenUser); const { prefix } = await getPrefixByUser(data, tokenUser);
const res = await updateFileStat(prefix.slice(1), data.metadata); const res = await updateFileStat(prefix.slice(1), data.metadata);
if (res.code === 200) { if (res.code === 200) {
ctx.body = 'update metadata success'; ctx.body = 'update metadata success';
@@ -155,6 +158,7 @@ app
}) })
.define(async (ctx) => { .define(async (ctx) => {
const tokenUser = ctx.state.tokenUser; const tokenUser = ctx.state.tokenUser;
const uid = await UserId.getUserIdByName(tokenUser.username);
let directory = ctx.query.data?.directory as string; let directory = ctx.query.data?.directory as string;
if (!directory) { if (!directory) {
ctx.throw(400, 'directory is required'); ctx.throw(400, 'directory is required');
@@ -165,12 +169,15 @@ app
if (directory.endsWith('/')) { if (directory.endsWith('/')) {
ctx.throw(400, 'directory is invalid, cannot end with /'); ctx.throw(400, 'directory is invalid, cannot end with /');
} }
const prefix = tokenUser.username + '/' + directory + '/'; const prefix = `data/${uid}/${directory}/`;
const list = await getMinioList<true>({ prefix, recursive: true }); const list = await getMinioList<true>({ prefix, recursive: true });
if (list.length === 0) { if (list.length === 0) {
ctx.throw(400, 'directory is empty'); ctx.throw(400, 'directory is empty');
} }
const res = await deleteFiles(list.map((item) => item.name)); if(list.length > 5000) {
ctx.throw(400, '删除文件数量过多,请分批删除');
}
const res = await deleteFileByPrefix(prefix);
if (!res) { if (!res) {
ctx.throw(500, 'delete all failed'); ctx.throw(500, 'delete all failed');
} }

View File

@@ -1,6 +1,7 @@
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import { oss } from '@/modules/s3.ts'; import { oss } from '@/modules/s3.ts';
import { StatObjectResult } from '@kevisual/oss'; import { StatObjectResult } from '@kevisual/oss';
import { UserId } from '@/routes/user/modules/user-id.ts';
type MinioListOpt = { type MinioListOpt = {
prefix: string; prefix: string;
recursive?: boolean; recursive?: boolean;
@@ -62,23 +63,14 @@ export const deleteFile = async (prefix: string): Promise<{ code: number; messag
} }
}; };
// 批量删除文件
export const deleteFiles = async (prefixs: string[]): Promise<any> => {
try {
for (const prefix of prefixs) {
await oss.deleteObject(prefix);
}
return true;
} catch (e) {
console.error('delete Files Error not handle', e);
return false;
}
};
export const deleteFileByPrefix = async (prefix: string): Promise<any> => { export const deleteFileByPrefix = async (prefix: string): Promise<any> => {
try { try {
const allFiles = await getMinioList<true>({ prefix, recursive: true }); const objects = await oss.listObjects<true>(prefix, { recursive: true });
const files = allFiles.filter((item) => item.name.startsWith(prefix)); if (objects.length > 0) {
await deleteFiles(files.map((item) => item.name)); for (const obj of objects) {
await oss.deleteObject(obj.name);
}
}
return true; return true;
} catch (e) { } catch (e) {
console.error('delete File Error not handle', e); console.error('delete File Error not handle', e);
@@ -93,7 +85,8 @@ type GetMinioListAndSetToAppListOpts = {
// 批量列出文件并设置到appList的files中 // 批量列出文件并设置到appList的files中
export const getMinioListAndSetToAppList = async (opts: GetMinioListAndSetToAppListOpts) => { export const getMinioListAndSetToAppList = async (opts: GetMinioListAndSetToAppListOpts) => {
const { username, appKey, version } = opts; const { username, appKey, version } = opts;
const minioList = await getMinioList({ prefix: `${username}/${appKey}/${version}`, recursive: true }); const uid = await UserId.getUserIdByName(username);
const minioList = await getMinioList({ prefix: `data/${uid}/${appKey}/${version}`, recursive: true });
const files = minioList; const files = minioList;
return files as MinioFile[]; return files as MinioFile[];
}; };
@@ -178,7 +171,8 @@ export const backupUserA = async (usernameA: string, id: string, backName?: stri
* @param username * @param username
*/ */
export const deleteUser = async (username: string) => { export const deleteUser = async (username: string) => {
const list = await getMinioList<true>({ prefix: `${username}/`, recursive: true }); const uid = await UserId.getUserIdByName(username);
const list = await getMinioList<true>({ prefix: `data/${uid}/`, recursive: true });
for (const item of list) { for (const item of list) {
await oss.deleteObject(item.name); await oss.deleteObject(item.name);
} }

View File

@@ -2,10 +2,8 @@ import { app, db, schema } from '@/app.ts';
import { User } from '@/models/user.ts'; import { User } from '@/models/user.ts';
import { nanoid } from 'nanoid'; import { nanoid } from 'nanoid';
import { CustomError } from '@kevisual/router'; import { CustomError } from '@kevisual/router';
import { backupUserA, deleteUser, mvUserAToUserB } from '@/routes/file/index.ts';
import { AppHelper } from '@/routes/app-manager/module/index.ts'; import { AppHelper } from '@/routes/app-manager/module/index.ts';
import { eq } from 'drizzle-orm'; import { eq } from 'drizzle-orm';
// import { mvAppFromUserAToUserB } from '@/routes/app-manager/admin/mv-user-app.ts';
export const checkUsername = (username: string) => { export const checkUsername = (username: string) => {
if (username.length > 30) { if (username.length > 30) {
@@ -45,10 +43,6 @@ export const toChangeName = async (opts: { id: string; newName: string; admin?:
user.data = data; user.data = data;
try { try {
await user.save(); await user.save();
// 迁移文件数据
await backupUserA(oldName, user.id); // 备份文件数据
await mvUserAToUserB(oldName, newName, true); // 迁移文件数据
// await mvAppFromUserAToUserB(oldName, newName); // 迁移应用数据
if (['org', 'user'].includes(user.type)) { if (['org', 'user'].includes(user.type)) {
const type = user.type === 'org' ? 'org' : 'user'; const type = user.type === 'org' ? 'org' : 'user';
@@ -175,8 +169,8 @@ app
ctx.throw(404, { message: 'User not found' }); ctx.throw(404, { message: 'User not found' });
} }
await db.delete(schema.cfUser).where(eq(schema.cfUser.id, user.id)); await db.delete(schema.cfUser).where(eq(schema.cfUser.id, user.id));
backupUserA(user.username, user.id); // backupUserA(user.username, user.id);
deleteUser(user.username); // deleteUser(user.username);
// TODO: EXPIRE 删除token // TODO: EXPIRE 删除token
ctx.body = { ctx.body = {
id: user.id, id: user.id,

View File

@@ -1,15 +0,0 @@
process.env.NODE_ENV = 'development';
import { mvUserAToUserB, backupUserA } from '../routes/file/module/get-minio-list.ts';
// mvUserAToUserB('demo', 'demo2');
// backupUserA('demo', '123', '2026-01-31-16-00');
// backupUserA('demo', '123', '2025-04-02-16-01');
// backupUserA('demo', '123', '2025-04-02-16-02');
// backupUserA('demo', '123', '2025-04-02-16-03');
// backupUserA('demo', '123', '2025-04-02-16-04');
// backupUserA('demo', '123', '2025-04-02-16-05');
// backupUserA('demo', '123', '2025-04-02-16-06');
// backupUserA('demo', '123', '2025-04-02-16-07');
// backupUserA('demo', '123', '2025-04-02-16-08');