From 2472cb0059707648ea76e14f1e1580b94fa6203a Mon Sep 17 00:00:00 2001 From: xiongxiao Date: Fri, 2 Jan 2026 18:30:52 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=EF=BC=8C=E5=A2=9E=E5=8A=A0=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=92=8C=E7=AC=94=E8=AE=B0=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=EF=BC=9B=E6=9B=B4=E6=96=B0=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=EF=BC=8C=E4=BC=98=E5=8C=96=E6=B5=8F=E8=A7=88?= =?UTF-8?q?=E5=99=A8=E5=90=AF=E5=8A=A8=E5=8F=82=E6=95=B0=EF=BC=9B=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E7=94=A8=E6=88=B7=E5=92=8C=E7=AC=94=E8=AE=B0=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=EF=BC=9B=E6=9B=B4=E6=96=B0=E5=88=9D=E5=A7=8B=E5=8C=96?= =?UTF-8?q?=E8=84=9A=E6=9C=AC=E5=92=8C=E5=BF=AB=E7=85=A7=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drizzle.config.ts | 3 +- package.json | 2 + src/db/drizzle/0000_rapid_genesis.sql | 58 ++++ src/db/drizzle/meta/0000_snapshot.json | 397 +++++++++++++++++++++++++ src/db/drizzle/meta/_journal.json | 13 + src/db/schema.ts | 34 ++- src/playwright/browser.ts | 3 + src/routes/xhs/search-notes.ts | 57 +++- 8 files changed, 547 insertions(+), 20 deletions(-) create mode 100644 src/db/drizzle/0000_rapid_genesis.sql create mode 100644 src/db/drizzle/meta/0000_snapshot.json create mode 100644 src/db/drizzle/meta/_journal.json diff --git a/drizzle.config.ts b/drizzle.config.ts index 6664fbd..82af8ea 100644 --- a/drizzle.config.ts +++ b/drizzle.config.ts @@ -11,9 +11,10 @@ if (!fs.existsSync(dir)) { export default { schema: './src/db/schema.ts', - out: './storage/browser-helper/drizzle', + out: './src/db/drizzle', dialect: 'sqlite', dbCredentials: { url: process.env.DATABASE_URL || 'storage/browser-helper/data.sqlite3', }, + strict: false, } satisfies Config; diff --git a/package.json b/package.json index 2f0b374..48f48a3 100644 --- a/package.json +++ b/package.json @@ -18,10 +18,12 @@ "init:browser": "npx playwright install", "build": "bun run bun.config.ts", "browser": "pm2 start start-browser.js --name browser ", + "dev:browser": "node start-browser.js ", "cmd": "tsx src/test/cmd.ts ", "init": "pnpm run init:pnpm && pnpm run init:db && pnpm run init:browser", "init:pnpm": "pnpm approve-builds", "init:db": "npx drizzle-kit push", + "push": "npx drizzle-kit push", "studio": "npx drizzle-kit studio", "drizzle:migrate": "npx drizzle-kit migrate", "drizzle:push": "npx drizzle-kit push" diff --git a/src/db/drizzle/0000_rapid_genesis.sql b/src/db/drizzle/0000_rapid_genesis.sql new file mode 100644 index 0000000..78fe38d --- /dev/null +++ b/src/db/drizzle/0000_rapid_genesis.sql @@ -0,0 +1,58 @@ +CREATE TABLE `cache` ( + `key` text PRIMARY KEY NOT NULL, + `value` text NOT NULL, + `expire_at` integer NOT NULL, + `created_at` integer NOT NULL +); +--> statement-breakpoint +CREATE TABLE `xhs_note` ( + `id` text PRIMARY KEY NOT NULL, + `title` text, + `summary` text, + `description` text, + `link` text, + `data` text, + `tags` text, + `status` text, + `author_url` text, + `cover` text, + `sync_status` integer NOT NULL, + `sync_at` integer NOT NULL, + `star` integer, + `user_id` text, + `pushed_at` integer, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL, + `deleted_at` integer +); +--> statement-breakpoint +CREATE INDEX `idx_xhs_note_user_id` ON `xhs_note` (`user_id`);--> statement-breakpoint +CREATE INDEX `idx_xhs_note_tags` ON `xhs_note` (`tags`);--> statement-breakpoint +CREATE TABLE `xhs_user` ( + `id` text PRIMARY KEY NOT NULL, + `user_id` text NOT NULL, + `xsec_token` text, + `username` text, + `nickname` text, + `avatar` text, + `title` text, + `summary` text, + `description` text, + `link` text, + `data` text, + `tags` text, + `bun_tags` text, + `followers_count` integer, + `following_count` integer, + `status` text, + `sync_status` integer DEFAULT 0 NOT NULL, + `sync_at` integer DEFAULT 0 NOT NULL, + `star` integer, + `created_at` integer DEFAULT 1767349555883 NOT NULL, + `updated_at` integer DEFAULT 1767349555883 NOT NULL, + `deleted_at` integer +); +--> statement-breakpoint +CREATE INDEX `idx_xhs_user_user_id` ON `xhs_user` (`user_id`);--> statement-breakpoint +CREATE INDEX `idx_xhs_user_tags` ON `xhs_user` (`tags`);--> statement-breakpoint +CREATE INDEX `idx_xhs_user_bun_tags` ON `xhs_user` (`bun_tags`); \ No newline at end of file diff --git a/src/db/drizzle/meta/0000_snapshot.json b/src/db/drizzle/meta/0000_snapshot.json new file mode 100644 index 0000000..305459c --- /dev/null +++ b/src/db/drizzle/meta/0000_snapshot.json @@ -0,0 +1,397 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "6e34d9c0-5f26-4fcf-8f85-9de7832cd139", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "cache": { + "name": "cache", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expire_at": { + "name": "expire_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "xhs_note": { + "name": "xhs_note", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "link": { + "name": "link", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author_url": { + "name": "author_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cover": { + "name": "cover", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sync_status": { + "name": "sync_status", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sync_at": { + "name": "sync_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "star": { + "name": "star", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_at": { + "name": "pushed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_xhs_note_user_id": { + "name": "idx_xhs_note_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_xhs_note_tags": { + "name": "idx_xhs_note_tags", + "columns": [ + "tags" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "xhs_user": { + "name": "xhs_user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "xsec_token": { + "name": "xsec_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "nickname": { + "name": "nickname", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar": { + "name": "avatar", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "link": { + "name": "link", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bun_tags": { + "name": "bun_tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "followers_count": { + "name": "followers_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "following_count": { + "name": "following_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sync_status": { + "name": "sync_status", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "sync_at": { + "name": "sync_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "star": { + "name": "star", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1767349555883 + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1767349555883 + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_xhs_user_user_id": { + "name": "idx_xhs_user_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_xhs_user_tags": { + "name": "idx_xhs_user_tags", + "columns": [ + "tags" + ], + "isUnique": false + }, + "idx_xhs_user_bun_tags": { + "name": "idx_xhs_user_bun_tags", + "columns": [ + "bun_tags" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/db/drizzle/meta/_journal.json b/src/db/drizzle/meta/_journal.json new file mode 100644 index 0000000..f368d28 --- /dev/null +++ b/src/db/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1767349555897, + "tag": "0000_rapid_genesis", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts index 4649a80..8b99b59 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core'; +import { sqliteTable, text, integer, index } from 'drizzle-orm/sqlite-core'; import { randomUUID } from 'node:crypto'; export const cache = sqliteTable('cache', { key: text('key').primaryKey(), @@ -17,7 +17,7 @@ export const xhsNote = sqliteTable('xhs_note', { data: text('data'), tags: text('tags'), - status: text('status'), + status: text('status'), // 正常笔记，归档，禁止用户，已删除 authorUrl: text('author_url'), cover: text('cover'), @@ -25,16 +25,20 @@ export const xhsNote = sqliteTable('xhs_note', { syncAt: integer('sync_at').notNull(), star: integer('star'), + userId: text('user_id'), pushedAt: integer('pushed_at'), createdAt: integer('created_at').notNull(), updatedAt: integer('updated_at').notNull(), deletedAt: integer('deleted_at'), -}); +}, (table) => ([ + index('idx_xhs_note_user_id').on(table.userId), + index('idx_xhs_note_tags').on(table.tags), +])); export const xhsUser = sqliteTable('xhs_user', { - id: text('id').primaryKey().$defaultFn(() => randomUUID()), - user_id: text('user_id').notNull(), + user_id: text('user_id').primaryKey(), + xsec_token: text('xsec_token'), username: text('username'), nickname: text('nickname'), @@ -47,17 +51,23 @@ export const xhsUser = sqliteTable('xhs_user', { data: text('data'), tags: text('tags'), + bunTags: text('bun_tags'), + followersCount: integer('followers_count'), followingCount: integer('following_count'), - status: text('status'), + status: text('status'), // 笔记用户（从笔记中添加，没有获取具体详情） 正常用户，封禁，已删除 - syncStatus: integer('sync_status').notNull(), - syncAt: integer('sync_at').notNull(), + syncStatus: integer('sync_status').default(0).notNull(), + syncAt: integer('sync_at').default(0).notNull(), - star: integer('star'), + star: integer('star'), // 标记 - createdAt: integer('created_at').notNull(), - updatedAt: integer('updated_at').notNull(), + createdAt: integer('created_at').default(Date.now()).notNull(), + updatedAt: integer('updated_at').default(Date.now()).notNull(), deletedAt: integer('deleted_at'), -}); \ No newline at end of file +}, (table) => ([ + index('idx_xhs_user_user_id').on(table.user_id), + index('idx_xhs_user_tags').on(table.tags), + index('idx_xhs_user_bun_tags').on(table.bunTags), +])); diff --git a/src/playwright/browser.ts b/src/playwright/browser.ts index 4b536d1..0a7bb0b 100644 --- a/src/playwright/browser.ts +++ b/src/playwright/browser.ts @@ -57,6 +57,9 @@ export const main = async (opts?: { '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--no-first-run', + '--disable-session-crashed-bubble', + '--disable-infobars', + '--disable-default-apps', `--user-agent=${userAgent}`, ]; diff --git a/src/routes/xhs/search-notes.ts b/src/routes/xhs/search-notes.ts index d078b33..e5e3e93 100644 --- a/src/routes/xhs/search-notes.ts +++ b/src/routes/xhs/search-notes.ts @@ -1,4 +1,4 @@ -import { xhsNote } from '@/db/schema.ts'; +import { xhsNote, xhsUser } from '@/db/schema.ts'; import { app, core, db } from '../../app.ts'; import { sql } from 'drizzle-orm'; const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); @@ -173,34 +173,41 @@ app.route({ const secToken = note.xsec_token; return `https://www.xiaohongshu.com/explore/${id}?xsec_token=${secToken}` } - const getUserUrl = (note: XHS.SearchNote) => { + const getUser = (note: XHS.SearchNote) => { const user = note.note_card?.user; const id = user?.user_id; const secToken = user?.xsec_token; if (user) { - return `https://www.xiaohongshu.com/user/profile/${id}?xsec_token=${secToken}` + return { + user: user, + link: `https://www.xiaohongshu.com/user/profile/${id}?xsec_token=${secToken}` + } } - return `` + return { user: null, link: '' } } const getCover = (note: XHS.SearchNote) => { const cover = note.note_card?.cover return cover?.url_default || '' } const keyword = sessionCache.get('xhs-search-keyword'); - const notes = data.filter(note => note.model_type === 'note').map(note => { + const dataNotes = data.filter(note => note.model_type === 'note'); + let notes = dataNotes.map(note => { const cornnerTag = note.note_card?.corner_tag_info; const pushTime = cornnerTag?.find(tag => tag.type === 'publish_time')?.text || ''; // 一天前 pushTime 包含 "前" + const user = getUser(note); return { id: note.id, title: note.note_card?.display_title || '', tags: '', summary: '', + status: '正常笔记', description: keyword || '', link: getNoteUrl(note), - data: JSON.stringify(note), + data: JSON.stringify({ note }), cover: getCover(note), - authorUrl: getUserUrl(note), + authorUrl: user.link, + user_id: user.user?.user_id || '', syncStatus: 0, // pushedAt: 0, syncAt: 0, @@ -208,6 +215,31 @@ app.route({ updatedAt: Date.now(), } }); + let notesUser = dataNotes.map(note => { + const userData = getUser(note); + const user = userData.user; + if (!user) return null; + return { + user_id: user?.user_id || '', + nickname: user?.nickname || '', + avatar: user?.avatar || '', + status: '笔记用户', + xsec_token: user?.xsec_token || '', + data: JSON.stringify({ user }), + } + }) + const userIds = notes.map(note => note.user_id).filter(id => id); + const userList = await db.select().from(xhsUser).where(sql`user_id IN (${userIds.join(',')})`); + // 如果用户表有bun的tags，对关键字进行屏蔽，对应的笔记默认打上禁止标签 + for (const note of notes) { + const user = userList.find(u => u.user_id === note.user_id); + if (user) { + const bunTags = user.bunTags || '-'; + if (bunTags.includes(keyword || '')) { + note.status = '禁止用户'; // 直接修改 notes 数组中的对象 + } + } + } await db.insert(xhsNote).values(notes).onConflictDoUpdate({ target: xhsNote.id, set: { @@ -215,7 +247,18 @@ app.route({ updatedAt: Date.now(), }, }).execute(); + console.log(`已保存 ${data.length} 条搜索笔记结果`); + // 保存用户信息，去重 + const uniqueUsers = Array.from(new Map(notesUser.filter(u => u !== null).map(u => [u!.user_id, u!])).values()); + await db.insert(xhsUser).values(uniqueUsers).onConflictDoUpdate({ + target: xhsUser.user_id, + set: { + nickname: sql`excluded.nickname`, + avatar: sql`excluded.avatar`, + }, + }).execute(); + console.log(`已保存 ${uniqueUsers.length} 条用户信息`); } catch (error) { console.error('保存搜索笔记结果时出错:', error); }