feat: remove old data import scripts and JSON files
- Deleted `import-data.ts` and `import-life.ts` scripts for importing short-link and life JSON data into the database. - Removed `ncode-list.json` containing short-link data. - Added new script `mv-resources.ts` for migrating resources from username-based paths to userId-based paths in the database and object storage. - Introduced `UserId` module for fetching user IDs and usernames from the database with caching. - Updated `UserApp` class to use user IDs instead of usernames for resource paths. - Modified routes to include a new endpoint for retrieving user IDs based on usernames.
This commit is contained in:
xiongxiao
@@ -1,78 +0,0 @@
|
||||
/**
|
||||
* 导入 short-link JSON 数据到数据库
|
||||
* 运行: bun run scripts/import-data.ts
|
||||
*/
|
||||
|
||||
import { drizzle } from 'drizzle-orm/node-postgres';
|
||||
import { shortLink } from '@/db/schemas/n-code-schema.ts';
|
||||
import { useConfig } from '@kevisual/use-config';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const config = useConfig() as any;
|
||||
const DATABASE_URL = config.DATABASE_URL || process.env.DATABASE_URL || '';
|
||||
if (!DATABASE_URL) {
|
||||
console.error('缺少 DATABASE_URL 配置');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const db = drizzle(DATABASE_URL);
|
||||
|
||||
// 读取 JSON 数据
|
||||
const jsonPath = resolve(import.meta.dir, 'ncode-list.json');
|
||||
const rawData = JSON.parse(readFileSync(jsonPath, 'utf-8')) as Array<{
|
||||
code: string;
|
||||
data: Record<string, any>;
|
||||
description: string;
|
||||
slug: string;
|
||||
tags: string[];
|
||||
title: string;
|
||||
type: string;
|
||||
userId: string;
|
||||
version: string;
|
||||
}>;
|
||||
|
||||
async function importData() {
|
||||
console.log(`准备导入 ${rawData.length} 条 short-link 数据...`);
|
||||
|
||||
let inserted = 0;
|
||||
let skipped = 0;
|
||||
|
||||
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
for (const item of rawData) {
|
||||
const userId = item.userId && uuidRegex.test(item.userId) ? item.userId : null;
|
||||
|
||||
try {
|
||||
await db
|
||||
.insert(shortLink)
|
||||
.values({
|
||||
slug: item.slug,
|
||||
code: item.code,
|
||||
type: item.type || 'link',
|
||||
version: item.version || '1.0',
|
||||
title: item.title || '',
|
||||
description: item.description || '',
|
||||
tags: item.tags ?? [],
|
||||
data: item.data ?? {},
|
||||
userId: userId as any,
|
||||
})
|
||||
.onConflictDoNothing();
|
||||
|
||||
console.log(` ✓ 导入: slug=${item.slug}, code=${item.code}, title=${item.title}`);
|
||||
inserted++;
|
||||
} catch (err: any) {
|
||||
const cause = err.cause || err;
|
||||
console.warn(` ✗ 跳过: slug=${item.slug}, code=${item.code} — ${cause.message || err.message}`);
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n完成: 成功 ${inserted} 条，跳过 ${skipped} 条`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
importData().catch((err) => {
|
||||
console.error('导入失败:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,83 +0,0 @@
|
||||
/**
|
||||
* 导入 life JSON 数据到数据库
|
||||
* 运行: bun run scripts/import-life.ts
|
||||
*/
|
||||
|
||||
import { drizzle } from 'drizzle-orm/node-postgres';
|
||||
import { life } from '@/db/schemas/life-schema.ts';
|
||||
import { useConfig } from '@kevisual/use-config';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const config = useConfig() as any;
|
||||
const DATABASE_URL = config.DATABASE_URL || process.env.DATABASE_URL || '';
|
||||
if (!DATABASE_URL) {
|
||||
console.error('缺少 DATABASE_URL 配置');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const db = drizzle(DATABASE_URL);
|
||||
|
||||
// 读取 JSON 数据
|
||||
const jsonPath = resolve(import.meta.dir, 'life-list.json');
|
||||
const rawData = JSON.parse(readFileSync(jsonPath, 'utf-8')) as Array<{
|
||||
data: Record<string, any>;
|
||||
description: string;
|
||||
effectiveAt: string;
|
||||
link: string;
|
||||
prompt: string;
|
||||
summary: string;
|
||||
tags: string[];
|
||||
taskResult: Record<string, any>;
|
||||
taskType: string;
|
||||
title: string;
|
||||
type: string;
|
||||
updatedAt: string;
|
||||
userId: string;
|
||||
}>;
|
||||
|
||||
async function importData() {
|
||||
console.log(`准备导入 ${rawData.length} 条 flowme-life 数据...`);
|
||||
|
||||
let inserted = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const item of rawData) {
|
||||
const uid = item.userId;
|
||||
|
||||
try {
|
||||
await db
|
||||
.insert(life)
|
||||
.values({
|
||||
title: item.title || '',
|
||||
summary: item.summary || '',
|
||||
description: item.description || '',
|
||||
tags: item.tags ?? [],
|
||||
link: item.link || '',
|
||||
data: item.data ?? {},
|
||||
effectiveAt: item.effectiveAt || '',
|
||||
type: item.type || '',
|
||||
prompt: item.prompt || '',
|
||||
taskType: item.taskType || '',
|
||||
taskResult: item.taskResult ?? {},
|
||||
uid: uid as any,
|
||||
})
|
||||
.onConflictDoNothing();
|
||||
|
||||
console.log(` ✓ 导入: title=${item.title}, type=${item.type}`);
|
||||
inserted++;
|
||||
} catch (err: any) {
|
||||
const cause = err.cause || err;
|
||||
console.warn(` ✗ 跳过: title=${item.title} — ${cause.message || err.message}`);
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n完成: 成功 ${inserted} 条，跳过 ${skipped} 条`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
importData().catch((err) => {
|
||||
console.error('导入失败:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
106
scripts/mv-resources.ts
Normal file
106
scripts/mv-resources.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import { oss } from '@/modules/s3.ts';
|
||||
import { db, schema } from '../src/app.ts';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { UserId } from '@/routes/user/modules/user-id.ts';
|
||||
import { mvUserAToUserB } from '@/routes/file/index.ts';
|
||||
|
||||
|
||||
// 迁移资源，原本的是 ${username}/appKey 改为 ${userId}/appKey
|
||||
|
||||
// 第一步，迁移 表kv_app 和 kv_app_list, 对应的 data 中，把data 对应的 files 中path 去掉第一个前缀，比如
|
||||
// {
|
||||
// "files": [
|
||||
// {
|
||||
// "name": "README.md",
|
||||
// "path": "root/code-center/0.0.6/README.md"
|
||||
// },
|
||||
// 把 path 中的 root 去掉，变成 code-center/0.0.6/README.md
|
||||
// ]
|
||||
// }
|
||||
|
||||
type Data = {
|
||||
files: {
|
||||
name: string;
|
||||
path: string;
|
||||
}[];
|
||||
}
|
||||
const BATCH_SIZE = 1000;
|
||||
|
||||
// 迁移 kv_app;
|
||||
const firstMigration = async () => {
|
||||
let offset = 0;
|
||||
while (true) {
|
||||
const kvAppList = await db.select().from(schema.kvApp).limit(BATCH_SIZE).offset(offset);
|
||||
if (kvAppList.length === 0) break;
|
||||
|
||||
for (const kvApp of kvAppList) {
|
||||
const data = kvApp.data as Data;
|
||||
const uid = kvApp.uid;
|
||||
const username = await UserId.getUserNameById(uid);
|
||||
if (!data.files) continue;
|
||||
for (const file of data.files) {
|
||||
// const pathParts = file.path.split('/');
|
||||
// pathParts.shift();
|
||||
// file.path = pathParts.join('/');
|
||||
file.path = username + '/' + file.path;
|
||||
}
|
||||
await db.update(schema.kvApp).set({ data: { ...data } }).where(eq(schema.kvApp.id, kvApp.id));
|
||||
}
|
||||
|
||||
console.log(`Processed ${offset + kvAppList.length} records`);
|
||||
offset += BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
// 迁移 kv_app_list
|
||||
const secondMigration = async () => {
|
||||
let offset = 0;
|
||||
while (true) {
|
||||
const kvAppList = await db.select().from(schema.kvAppList).limit(BATCH_SIZE).offset(offset);
|
||||
if (kvAppList.length === 0) break;
|
||||
|
||||
for (const kvApp of kvAppList) {
|
||||
const data = kvApp.data as Data;
|
||||
const uid = kvApp.uid;
|
||||
const username = await UserId.getUserNameById(uid);
|
||||
if (!data.files) continue;
|
||||
for (const file of data.files) {
|
||||
// const pathParts = file.path.split('/');
|
||||
// pathParts.shift();
|
||||
// file.path = pathParts.join('/');
|
||||
file.path = username + '/' + file.path;
|
||||
}
|
||||
await db.update(schema.kvAppList).set({ data: { ...data } }).where(eq(schema.kvAppList.id, kvApp.id));
|
||||
}
|
||||
|
||||
console.log(`Processed ${offset + kvAppList.length} records`);
|
||||
offset += BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 迁移对象存储
|
||||
*/
|
||||
const migrateOss = async () => {
|
||||
const list = await oss.listObjects('')
|
||||
const _names = list.filter(item => item.size === 0)
|
||||
type Name = {
|
||||
prefix: string;
|
||||
size: 0;
|
||||
id?: string;
|
||||
}
|
||||
const names: Name[] = _names as any;
|
||||
for (const name of names) {
|
||||
const username = name.prefix.split('/')[0];
|
||||
const id = await UserId.getUserIdByName(username);
|
||||
if (id) {
|
||||
name.id = id;
|
||||
mvUserAToUserB(username, `data/${id}`, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
// await firstMigration();
|
||||
// await secondMigration();
|
||||
await migrateOss();
|
||||
|
||||
console.log('Migration completed');
|
||||
@@ -1,104 +0,0 @@
|
||||
[
|
||||
{
|
||||
"code": "anwjgg",
|
||||
"data": {
|
||||
"link": "https://kevisual.cn/root/name-card/"
|
||||
},
|
||||
"description": "这是一个测试码",
|
||||
"slug": "pmzsq4gp4g",
|
||||
"tags": [],
|
||||
"title": "测试码",
|
||||
"type": "link",
|
||||
"userId": "0e700dc8-90dd-41b7-91dd-336ea51de3d2",
|
||||
"version": "1.0"
|
||||
},
|
||||
{
|
||||
"code": "0gmn12",
|
||||
"data": {
|
||||
"link": "https://kevisual.cn/root/v1/ha-api?path=ha&key=open-balcony-light",
|
||||
"permission": {
|
||||
"share": "public"
|
||||
},
|
||||
"useOwnerToken": true
|
||||
},
|
||||
"description": "test 阳台灯",
|
||||
"slug": "3z1nbdogew",
|
||||
"tags": [],
|
||||
"title": "阳台灯",
|
||||
"type": "link",
|
||||
"userId": "0e700dc8-90dd-41b7-91dd-336ea51de3d2",
|
||||
"version": "1.0"
|
||||
},
|
||||
{
|
||||
"code": "abc111",
|
||||
"data": {
|
||||
"link": "https://kevisual.cn/root/nfc/",
|
||||
"permission": {
|
||||
"share": "public"
|
||||
}
|
||||
},
|
||||
"description": "nfc link",
|
||||
"slug": "0000000001",
|
||||
"tags": [],
|
||||
"title": "nfc link",
|
||||
"type": "link",
|
||||
"userId": "",
|
||||
"version": "1.0"
|
||||
},
|
||||
{
|
||||
"code": "ej73jm",
|
||||
"data": {
|
||||
"link": "https://kevisual.cn/root/nfc/",
|
||||
"permission": {
|
||||
"share": "public"
|
||||
}
|
||||
},
|
||||
"description": "nfc link",
|
||||
"slug": "001",
|
||||
"tags": [],
|
||||
"title": "nfc link",
|
||||
"type": "link",
|
||||
"userId": "",
|
||||
"version": "1.0"
|
||||
},
|
||||
{
|
||||
"code": "09dd42",
|
||||
"data": {
|
||||
"details": [
|
||||
{
|
||||
"description": "算法基于七卡瓦，自建",
|
||||
"title": "beads（kevisual.cn）",
|
||||
"url": "https://kevisual.cn/root/beads/"
|
||||
},
|
||||
{
|
||||
"description": "算法很不错，图片转图纸很好",
|
||||
"title": "拼豆七卡瓦(zippland.com)",
|
||||
"url": "https://perlerbeads.zippland.com/"
|
||||
},
|
||||
{
|
||||
"description": "功能偏向PS，画板类，像素类",
|
||||
"title": "拼豆像素格子（zwpyyds.com）",
|
||||
"url": "https://www.zwpyyds.com/"
|
||||
},
|
||||
{
|
||||
"description": "编辑不错,拼豆社区",
|
||||
"title": "我嘞个豆(ohmybead.cn)",
|
||||
"url": "https://ohmybead.cn/"
|
||||
}
|
||||
],
|
||||
"link": "https://kevisual.cn/root/nfc/way/",
|
||||
"permission": {
|
||||
"share": "public"
|
||||
},
|
||||
"title": "拼豆图纸自取方案",
|
||||
"type": "html-render"
|
||||
},
|
||||
"description": "Pindou",
|
||||
"slug": "pindou",
|
||||
"tags": [],
|
||||
"title": "Pindou",
|
||||
"type": "link",
|
||||
"userId": "",
|
||||
"version": "1.0"
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user