good/browser-helper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重构数据库模式，增加用户信息和笔记字段；更新配置文件路径，优化浏览器启动参数；添加用户和笔记索引；更新初始化脚本和快照文件

Browse Source
This commit is contained in:
熊潇
2026-01-02 18:30:52 +08:00
parent 2c3bc79e6e
commit 2472cb0059
8 changed files with 547 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
drizzle.config.ts
View File

@@ -11,9 +11,10 @@ if (!fs.existsSync(dir)) {
export default {
schema: './src/db/schema.ts',
out: './storage/browser-helper/drizzle',
out: './src/db/drizzle',
dialect: 'sqlite',
dbCredentials: {
url: process.env.DATABASE_URL || 'storage/browser-helper/data.sqlite3',
},
strict: false,
} satisfies Config;

2
package.json
View File

@@ -18,10 +18,12 @@
"init:browser": "npx playwright install",
"build": "bun run bun.config.ts",
"browser": "pm2 start start-browser.js --name browser ",
"dev:browser": "node start-browser.js ",
"cmd": "tsx src/test/cmd.ts ",
"init": "pnpm run init:pnpm && pnpm run init:db && pnpm run init:browser",
"init:pnpm": "pnpm approve-builds",
"init:db": "npx drizzle-kit push",
"push": "npx drizzle-kit push",
"studio": "npx drizzle-kit studio",
"drizzle:migrate": "npx drizzle-kit migrate",
"drizzle:push": "npx drizzle-kit push"

58
src/db/drizzle/0000_rapid_genesis.sql Normal file
View File

@@ -0,0 +1,58 @@
CREATE TABLE `cache` (
`key` text PRIMARY KEY NOT NULL,
`value` text NOT NULL,
`expire_at` integer NOT NULL,
`created_at` integer NOT NULL
);
--> statement-breakpoint
CREATE TABLE `xhs_note` (
`id` text PRIMARY KEY NOT NULL,
`title` text,
`summary` text,
`description` text,
`link` text,
`data` text,
`tags` text,
`status` text,
`author_url` text,
`cover` text,
`sync_status` integer NOT NULL,
`sync_at` integer NOT NULL,
`star` integer,
`user_id` text,
`pushed_at` integer,
`created_at` integer NOT NULL,
`updated_at` integer NOT NULL,
`deleted_at` integer
);
--> statement-breakpoint
CREATE INDEX `idx_xhs_note_user_id` ON `xhs_note` (`user_id`);--> statement-breakpoint
CREATE INDEX `idx_xhs_note_tags` ON `xhs_note` (`tags`);--> statement-breakpoint
CREATE TABLE `xhs_user` (
`id` text PRIMARY KEY NOT NULL,
`user_id` text NOT NULL,
`xsec_token` text,
`username` text,
`nickname` text,
`avatar` text,
`title` text,
`summary` text,
`description` text,
`link` text,
`data` text,
`tags` text,
`bun_tags` text,
`followers_count` integer,
`following_count` integer,
`status` text,
`sync_status` integer DEFAULT 0 NOT NULL,
`sync_at` integer DEFAULT 0 NOT NULL,
`star` integer,
`created_at` integer DEFAULT 1767349555883 NOT NULL,
`updated_at` integer DEFAULT 1767349555883 NOT NULL,
`deleted_at` integer
);
--> statement-breakpoint
CREATE INDEX `idx_xhs_user_user_id` ON `xhs_user` (`user_id`);--> statement-breakpoint
CREATE INDEX `idx_xhs_user_tags` ON `xhs_user` (`tags`);--> statement-breakpoint
CREATE INDEX `idx_xhs_user_bun_tags` ON `xhs_user` (`bun_tags`);

397
src/db/drizzle/meta/0000_snapshot.json Normal file
View File

@@ -0,0 +1,397 @@
{
"version": "6",
"dialect": "sqlite",
"id": "6e34d9c0-5f26-4fcf-8f85-9de7832cd139",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"cache": {
"name": "cache",
"columns": {
"key": {
"name": "key",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"value": {
"name": "value",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"expire_at": {
"name": "expire_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"xhs_note": {
"name": "xhs_note",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"title": {
"name": "title",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"summary": {
"name": "summary",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"description": {
"name": "description",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"link": {
"name": "link",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"data": {
"name": "data",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"tags": {
"name": "tags",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"author_url": {
"name": "author_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"cover": {
"name": "cover",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"sync_status": {
"name": "sync_status",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"sync_at": {
"name": "sync_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"star": {
"name": "star",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"pushed_at": {
"name": "pushed_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"deleted_at": {
"name": "deleted_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"idx_xhs_note_user_id": {
"name": "idx_xhs_note_user_id",
"columns": [
"user_id"
],
"isUnique": false
},
"idx_xhs_note_tags": {
"name": "idx_xhs_note_tags",
"columns": [
"tags"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"xhs_user": {
"name": "xhs_user",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"xsec_token": {
"name": "xsec_token",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"username": {
"name": "username",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"nickname": {
"name": "nickname",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"avatar": {
"name": "avatar",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"title": {
"name": "title",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"summary": {
"name": "summary",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"description": {
"name": "description",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"link": {
"name": "link",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"data": {
"name": "data",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"tags": {
"name": "tags",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"bun_tags": {
"name": "bun_tags",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"followers_count": {
"name": "followers_count",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"following_count": {
"name": "following_count",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"sync_status": {
"name": "sync_status",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"sync_at": {
"name": "sync_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"star": {
"name": "star",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 1767349555883
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 1767349555883
},
"deleted_at": {
"name": "deleted_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"idx_xhs_user_user_id": {
"name": "idx_xhs_user_user_id",
"columns": [
"user_id"
],
"isUnique": false
},
"idx_xhs_user_tags": {
"name": "idx_xhs_user_tags",
"columns": [
"tags"
],
"isUnique": false
},
"idx_xhs_user_bun_tags": {
"name": "idx_xhs_user_bun_tags",
"columns": [
"bun_tags"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}

13
src/db/drizzle/meta/_journal.json Normal file
View File

@@ -0,0 +1,13 @@
{
"version": "7",
"dialect": "sqlite",
"entries": [
{
"idx": 0,
"version": "6",
"when": 1767349555897,
"tag": "0000_rapid_genesis",
"breakpoints": true
}
]
}

34
src/db/schema.ts
View File

@@ -1,4 +1,4 @@
import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core';
import { sqliteTable, text, integer, index } from 'drizzle-orm/sqlite-core';
import { randomUUID } from 'node:crypto';
export const cache = sqliteTable('cache', {
key: text('key').primaryKey(),
@@ -17,7 +17,7 @@ export const xhsNote = sqliteTable('xhs_note', {
data: text('data'),
tags: text('tags'),
status: text('status'),
status: text('status'), // 正常笔记，归档，禁止用户，已删除
authorUrl: text('author_url'),
cover: text('cover'),
@@ -25,16 +25,20 @@ export const xhsNote = sqliteTable('xhs_note', {
syncAt: integer('sync_at').notNull(),
star: integer('star'),
userId: text('user_id'),
pushedAt: integer('pushed_at'),
createdAt: integer('created_at').notNull(),
updatedAt: integer('updated_at').notNull(),
deletedAt: integer('deleted_at'),
});
}, (table) => ([
index('idx_xhs_note_user_id').on(table.userId),
index('idx_xhs_note_tags').on(table.tags),
]));
export const xhsUser = sqliteTable('xhs_user', {
id: text('id').primaryKey().$defaultFn(() => randomUUID()),
user_id: text('user_id').notNull(),
user_id: text('user_id').primaryKey(),
xsec_token: text('xsec_token'),
username: text('username'),
nickname: text('nickname'),
@@ -47,17 +51,23 @@ export const xhsUser = sqliteTable('xhs_user', {
data: text('data'),
tags: text('tags'),
bunTags: text('bun_tags'),
followersCount: integer('followers_count'),
followingCount: integer('following_count'),
status: text('status'),
status: text('status'), // 笔记用户（从笔记中添加，没有获取具体详情） 正常用户，封禁，已删除
syncStatus: integer('sync_status').notNull(),
syncAt: integer('sync_at').notNull(),
syncStatus: integer('sync_status').default(0).notNull(),
syncAt: integer('sync_at').default(0).notNull(),
star: integer('star'),
star: integer('star'), // 标记
createdAt: integer('created_at').notNull(),
updatedAt: integer('updated_at').notNull(),
createdAt: integer('created_at').default(Date.now()).notNull(),
updatedAt: integer('updated_at').default(Date.now()).notNull(),
deletedAt: integer('deleted_at'),
});
}, (table) => ([
index('idx_xhs_user_user_id').on(table.user_id),
index('idx_xhs_user_tags').on(table.tags),
index('idx_xhs_user_bun_tags').on(table.bunTags),
]));

3
src/playwright/browser.ts
View File

@@ -57,6 +57,9 @@ export const main = async (opts?: {
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--no-first-run',
'--disable-session-crashed-bubble',
'--disable-infobars',
'--disable-default-apps',
`--user-agent=${userAgent}`,
];

57
src/routes/xhs/search-notes.ts
View File

@@ -1,4 +1,4 @@
import { xhsNote } from '@/db/schema.ts';
import { xhsNote, xhsUser } from '@/db/schema.ts';
import { app, core, db } from '../../app.ts';
import { sql } from 'drizzle-orm';
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -173,34 +173,41 @@ app.route({
const secToken = note.xsec_token;
return `https://www.xiaohongshu.com/explore/${id}?xsec_token=${secToken}`
}
const getUserUrl = (note: XHS.SearchNote) => {
const getUser = (note: XHS.SearchNote) => {
const user = note.note_card?.user;
const id = user?.user_id;
const secToken = user?.xsec_token;
if (user) {
return `https://www.xiaohongshu.com/user/profile/${id}?xsec_token=${secToken}`
return {
user: user,
link: `https://www.xiaohongshu.com/user/profile/${id}?xsec_token=${secToken}`
}
}
return ``
return { user: null, link: '' }
}
const getCover = (note: XHS.SearchNote) => {
const cover = note.note_card?.cover
return cover?.url_default || ''
}
const keyword = sessionCache.get('xhs-search-keyword');
const notes = data.filter(note => note.model_type === 'note').map(note => {
const dataNotes = data.filter(note => note.model_type === 'note');
let notes = dataNotes.map(note => {
const cornnerTag = note.note_card?.corner_tag_info;
const pushTime = cornnerTag?.find(tag => tag.type === 'publish_time')?.text || '';
// 一天前 pushTime 包含 "前"
const user = getUser(note);
return {
id: note.id,
title: note.note_card?.display_title || '',
tags: '',
summary: '',
status: '正常笔记',
description: keyword || '',
link: getNoteUrl(note),
data: JSON.stringify(note),
data: JSON.stringify({ note }),
cover: getCover(note),
authorUrl: getUserUrl(note),
authorUrl: user.link,
user_id: user.user?.user_id || '',
syncStatus: 0,
// pushedAt: 0,
syncAt: 0,
@@ -208,6 +215,31 @@ app.route({
updatedAt: Date.now(),
}
});
let notesUser = dataNotes.map(note => {
const userData = getUser(note);
const user = userData.user;
if (!user) return null;
return {
user_id: user?.user_id || '',
nickname: user?.nickname || '',
avatar: user?.avatar || '',
status: '笔记用户',
xsec_token: user?.xsec_token || '',
data: JSON.stringify({ user }),
}
})
const userIds = notes.map(note => note.user_id).filter(id => id);
const userList = await db.select().from(xhsUser).where(sql`user_id IN (${userIds.join(',')})`);
// 如果用户表有bun的tags，对关键字进行屏蔽，对应的笔记默认打上禁止标签
for (const note of notes) {
const user = userList.find(u => u.user_id === note.user_id);
if (user) {
const bunTags = user.bunTags || '-';
if (bunTags.includes(keyword || '')) {
note.status = '禁止用户'; // 直接修改 notes 数组中的对象
}
}
}
await db.insert(xhsNote).values(notes).onConflictDoUpdate({
target: xhsNote.id,
set: {
@@ -215,7 +247,18 @@ app.route({
updatedAt: Date.now(),
},
}).execute();
console.log(`已保存 ${data.length} 条搜索笔记结果`);
// 保存用户信息，去重
const uniqueUsers = Array.from(new Map(notesUser.filter(u => u !== null).map(u => [u!.user_id, u!])).values());
await db.insert(xhsUser).values(uniqueUsers).onConflictDoUpdate({
target: xhsUser.user_id,
set: {
nickname: sql`excluded.nickname`,
avatar: sql`excluded.avatar`,
},
}).execute();
console.log(`已保存 ${uniqueUsers.length} 条用户信息`);
} catch (error) {
console.error('保存搜索笔记结果时出错:', error);
}