diff --git a/src/db/schema.ts b/src/db/schema.ts index 2e977f1..7f72b82 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -37,7 +37,7 @@ export const xhsNote = sqliteTable('xhs_note', { ])); export const xhsUser = sqliteTable('xhs_user', { - user_id: text('user_id').primaryKey(), + id: text('id').primaryKey(), xsec_token: text('xsec_token'), username: text('username'), @@ -67,7 +67,7 @@ export const xhsUser = sqliteTable('xhs_user', { updatedAt: integer('updated_at').default(Date.now()).notNull(), deletedAt: integer('deleted_at'), }, (table) => ([ - index('idx_xhs_user_user_id').on(table.user_id), + index('idx_xhs_user_id').on(table.id), index('idx_xhs_user_tags').on(table.tags), index('idx_xhs_user_bun_tags').on(table.bunTags), ])); diff --git a/src/index.ts b/src/index.ts index 660495b..c09f56a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,8 @@ app.route({ description: 'Token 权限验证,临时方案', }).define(async (ctx) => { // token authentication - + console.log('token', ctx.state); + ctx.state.token = 'abc'; }).addTo(app); const isPm2 = !!process.env.PM2_HOME; if (import.meta.main || isPm2) { diff --git a/src/playwright/browser.ts b/src/playwright/browser.ts index 3640670..44ad0ee 100644 --- a/src/playwright/browser.ts +++ b/src/playwright/browser.ts @@ -46,7 +46,7 @@ export const main = async (opts?: { console.log(`端口: ${debugPort}`); console.log(`用户数据目录: ${userDataDir}`); console.log(`无头模式: ${headless}`); - const userAgent = new UserAgent().toString(); + // const userAgent = new UserAgent().toString(); const params = [ `--remote-debugging-port=${debugPort}`, @@ -55,37 +55,7 @@ export const main = async (opts?: { '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--no-first-run', - '--disable-session-crashed-bubble', - '--disable-infobars', - '--disable-default-apps', - '--disable-blink-features=AutomationControlled', - '--exclude-switches=enable-automation', - '--disable-features=IsolateOrigins,site-per-process', - '--disable-web-security', - '--disable-features=VizDisplayCompositor', - `--user-agent=${userAgent}`, - '--disable-sync', - '--no-default-browser-check', - '--no-experiments', - '--disable-popup-blocking', - '--disable-prompt-on-repost', - '--disable-background-networking', - '--disable-component-update', - '--disable-extensions', - '--disable-bundled-ppapi-flash', - // 隐藏automation bar相关特征 - '--disable-renderer-backgrounding', - '--disable-backgrounding-occluded-windows', - '--disable-breakpad', - '--disable-client-side-phishing-detection', - '--disable-component-extensions-with-background-pages', - '--disable-datasaver-prompt', - '--disable-device-discovery-notifications', - '--disable-hang-monitor', - '--disable-ipc-flooding-protection', - '--no-service-autorun', - // 禁用自动化识别 - '--disable-automation', + // `--user-agent=${userAgent}`, ]; // 如果需要无头模式,添加额外参数 @@ -95,9 +65,6 @@ export const main = async (opts?: { '--window-size=1920,1080', ); } - - params.push('about:blank'); - console.log('启动参数:', params); if (opts?.kiosk) { params.push('--kiosk'); // 全屏模式,无修改边框 diff --git a/src/playwright/core.ts b/src/playwright/core.ts index ae485a2..e44b6eb 100644 --- a/src/playwright/core.ts +++ b/src/playwright/core.ts @@ -53,9 +53,7 @@ export class Core { if (opts?.useDebugPort !== undefined) { this.useDebugPort = opts.useDebugPort; } - if (opts?.useCDPConnect !== undefined) { - this.useCDPConnect = opts.useCDPConnect; - } + this.useCDPConnect = opts?.useCDPConnect || true; } async createBrowser() { const chrome = await main({ debugPort: this.debugPort, headless: this.headless }); @@ -63,31 +61,6 @@ export class Core { async init() { const debugPort = this.debugPort; try { - // 如果不使用CDP连接,直接用Playwright启动 - if (!this.useCDPConnect) { - console.log('使用纯Playwright模式启动(无CDP),避免被检测...'); - this.browser = await chromium.launch({ - headless: this.headless, - args: [ - `--user-data-dir=${path.join(process.cwd(), 'browser-context')}`, - '--no-sandbox', - '--disable-blink-features=AutomationControlled', - '--disable-infobars', - '--exclude-switches=enable-automation', - ] - }); - this.browserContext = await this.browser.newContext(); - this.handleRequest(this.browserContext); - this.page = await this.browserContext.newPage(); - - // 应用隐身脚本 - await this.stealthMode(this.page); - - this.emitter.emit('connected'); - return; - } - - // === 以下为CDP连接模式(可选) === const stdout = execSync(`netstat -ano | findstr :${debugPort}`); console.log(`端口 ${debugPort} 已在监听:\n${stdout}`); const debugHost = this.debugHost; @@ -95,50 +68,10 @@ export class Core { console.log('成功连接到 Chrome CDP!'); this.browser = browser; this.browserContext = browser.contexts()[0]; - - // 关闭所有现存的页面,防止复用百度等默认页面 - const existingPages = this.browserContext.pages(); - for (const page of existingPages) { - await page.close(); - } - this.handleRequest(this.browserContext); // 创建全新的空白页面 - this.page = await this.browserContext.newPage(); - - // 在页面创建后立即设置CDP脚本注入(在导航前) - try { - const cdpSession = await this.browserContext.newCDPSession(this.page); - // 禁用webdriver特征 - 在页面加载前注入 - await cdpSession.send('Page.addScriptToEvaluateOnNewDocument', { - source: `Object.defineProperty(navigator, 'webdriver', { get: () => false })` - }); - // 隐藏automation bar相关特征 - await cdpSession.send('Page.addScriptToEvaluateOnNewDocument', { - source: ` - const style = document.createElement('style'); - style.textContent = \` - [class*="automation"], - [id*="automation"], - .infobar, - #infobar-container, - .top-chrome-background, - .automation-bar { - display: none !important; - } - \`; - document.documentElement.appendChild(style); - ` - }); - } catch (e) { - console.log('CDP session设置失败(非致命错误):', (e as Error).message.slice(0, 80)); - } - - // 导航到空白页面,清除任何缓存的导航 - await this.page.goto('about:blank', { waitUntil: 'domcontentloaded' }); - - // 始终启用隐身模式以隐藏debugPort和automation特征 - await this.stealthMode(this.page); + this.page = await this.browserContext.newPage() + // await this.stealthMode(this.page); this.emitter.emit('connected'); return; @@ -203,6 +136,7 @@ export class Core { return this.page!; } throw new Error('无法连接到浏览器实例'); + } async setReady(ready: boolean = true) { if (this.recordReady !== ready) { @@ -250,7 +184,7 @@ export class Core { context.on('response', async response => { const url = response.url(); const recordReady = this.recordReady; - console.log('Response URL:', url); + // console.log('Response URL:', url); for (let listener of this.listeners) { const type = listener.type || 'both'; if (type === 'request') continue; diff --git a/src/routes/good/index.ts b/src/routes/good/index.ts index f3f26f9..cb888ad 100644 --- a/src/routes/good/index.ts +++ b/src/routes/good/index.ts @@ -3,7 +3,7 @@ import { app, core, db } from '../../app.ts'; app.route({ path: 'good', key: 'searchInfo', - description: '搜索小红书今日热门信息差内容。支持自定义关键词,参数keyword(字符串)可选,默认搜索"信息差"', + description: '搜索小红书今日热门信息差内容。参数是keyword,默认搜索"信息差"', middleware: ['auth'], metadata: { tags: ['小红书', '信息差', '热门'], @@ -20,7 +20,7 @@ app.route({ ...rest, token: ctx.query?.token as string, } - }) + }, ctx) ctx.forward(res) }).addTo(app); @@ -28,11 +28,10 @@ app.route({ app.route({ path: 'good', key: 'searchWork', - description: '搜索小红书今日工作机会与招聘信息。支持自定义关键词搜索,默认搜索"工作 杭州"', + description: '搜索小红书今日工作机会与招聘信息。参数是keyword,默认搜索"工作 杭州"', middleware: ['auth'], metadata: { tags: ['小红书', '工作', '招聘'], - icon: 'search', } }).define(async (ctx) => { const { keyword = '工作 杭州', ...rest } = ctx.query; @@ -45,6 +44,53 @@ app.route({ ...rest, token: ctx.query?.token as string, } - }) + }, ctx) + ctx.forward(res) +}).addTo(app); + +app.route({ + path: 'good', + key: 'searchDate', + description: '搜索小红书今日交友信息。参数是keyword,默认搜索"相亲 杭州"', + middleware: ['auth'], + metadata: { + tags: ['小红书', '约会', '交友', '相亲'], + } +}).define(async (ctx) => { + const { keyword = '相亲 杭州', ...rest } = ctx.query; + const res = await app.run({ + path: 'xhs', + key: 'search-notes', + payload: { + keyword: keyword, + scrollTimes: 10, + ...rest, + token: ctx.query?.token as string, + } + }, ctx) + ctx.forward(res) +}).addTo(app); + + +app.route({ + path: 'good', + key: 'searchBean', + description: '搜索小红书的拼豆,参数是keyword,默认搜索"拼豆"', + middleware: ['auth'], + metadata: { + tags: ['小红书', '拼豆'], + } +}).define(async (ctx) => { + const { keyword = '拼豆', ...rest } = ctx.query; + const res = await app.run({ + path: 'xhs', + key: 'search-notes', + payload: { + keyword: keyword, + scrollTimes: 10, + ...rest, + token: ctx.query?.token as string, + } + }, ctx) ctx.forward(res) }).addTo(app); \ No newline at end of file diff --git a/src/routes/xhs/search-notes.ts b/src/routes/xhs/search-notes.ts index 364f019..eb22bfb 100644 --- a/src/routes/xhs/search-notes.ts +++ b/src/routes/xhs/search-notes.ts @@ -79,6 +79,7 @@ const hoverPickerExample = async (page: Page, opts?: HoverPickerOptions) => { } } } + await sleep(2000); // 等待2秒以确保筛选生效 // 将鼠标移到页面外,移除 hover 状态 await page.mouse.move(0, 0); console.log('已移除 hover 状态'); @@ -209,7 +210,7 @@ app.route({ status: '正常笔记', description: keyword || '', link: getNoteUrl(note), - data: JSON.stringify({ note, keyword }), + data: JSON.stringify({ note, keyword, user }), cover: getCover(note), authorUrl: user.link, user_id: user.user?.user_id || '', @@ -225,7 +226,7 @@ app.route({ const user = userData.user; if (!user) return null; return { - user_id: user?.user_id || '', + id: user?.user_id || '', nickname: user?.nickname || '', avatar: user?.avatar || '', status: '笔记用户', @@ -234,11 +235,11 @@ app.route({ data: JSON.stringify({ user }), } }) - const userIds = notes.map(note => note.user_id).filter(id => id); - const userList = await db.select().from(xhsUser).where(sql`user_id IN (${userIds.join(',')})`); + const userIds = notes.map(note => note.id).filter(id => id); + const userList = await db.select().from(xhsUser).where(sql`id IN (${userIds.join(',')})`); // 如果用户表有bun的tags,对关键字进行屏蔽,对应的笔记默认打上禁止标签 for (const note of notes) { - const user = userList.find(u => u.user_id === note.user_id); + const user = userList.find(u => u.id === note.user_id); if (user) { const bunTags = user.bunTags || '-'; if (bunTags.includes(keyword || '')) { @@ -250,15 +251,21 @@ app.route({ target: xhsNote.id, set: { summary: sql`excluded.summary`, + cover: sql`excluded.cover`, + status: sql`excluded.status`, + data: sql`excluded.data`, + link: sql`excluded.link`, + description: sql`excluded.description`, + authorUrl: sql`excluded.author_url`, updatedAt: Date.now(), }, }).execute(); console.log(`已保存 ${data.length} 条搜索笔记结果`); // 保存用户信息,去重 - const uniqueUsers = Array.from(new Map(notesUser.filter(u => u !== null).map(u => [u!.user_id, u!])).values()); + const uniqueUsers = Array.from(new Map(notesUser.filter(u => u !== null).map(u => [u!.id, u!])).values()); await db.insert(xhsUser).values(uniqueUsers).onConflictDoUpdate({ - target: xhsUser.user_id, + target: xhsUser.id, set: { nickname: sql`excluded.nickname`, avatar: sql`excluded.avatar`, diff --git a/src/routes/xhs/xhs-user-list.ts b/src/routes/xhs/xhs-user-list.ts index 0b7d9c7..75e9f71 100644 --- a/src/routes/xhs/xhs-user-list.ts +++ b/src/routes/xhs/xhs-user-list.ts @@ -6,7 +6,12 @@ app.route({ path: 'xhs-users', key: 'list', middleware: ['auth'], - description: '获取小红书用户列表', + description: `获取小红书用户列表, 参数说明: + page: 页码,默认1 + pageSize: 每页数量,默认20 + search: 搜索关键词,模糊匹配昵称、用户名和描述 + sort: 排序方式,ASC或DESC,默认DESC按更新时间降序 +`, metadata: { tags: ['小红书', '用户'], } @@ -66,11 +71,11 @@ app.route({ tags: ['小红书', '用户'], } }).define(async (ctx) => { - const { user_id, createdAt, updatedAt, ...rest } = ctx.query.data || {}; + const { id, createdAt, updatedAt, ...rest } = ctx.query.data || {}; let user; - if (!user_id) { + if (!id) { user = await db.insert(xhsUser).values({ - user_id: rest.user_id || `user_${Date.now()}`, + id: rest.id || `user_${Date.now()}`, nickname: rest.nickname || '', username: rest.username || '', avatar: rest.avatar || '', @@ -85,7 +90,7 @@ app.route({ updatedAt: Date.now(), }).returning(); } else { - const existing = await db.select().from(xhsUser).where(eq(xhsUser.user_id, user_id)).limit(1); + const existing = await db.select().from(xhsUser).where(eq(xhsUser.id, id)).limit(1); if (existing.length === 0) { ctx.throw(404, '没有找到对应的用户'); } @@ -99,7 +104,7 @@ app.route({ link: rest.link, data: rest.data ? JSON.stringify(rest.data) : undefined, updatedAt: Date.now(), - }).where(eq(xhsUser.user_id, user_id)).returning(); + }).where(eq(xhsUser.id, id)).returning(); } ctx.body = user; }).addTo(app); @@ -109,20 +114,20 @@ app.route({ path: 'xhs-users', key: 'delete', middleware: ['auth'], - description: '删除小红书用户, 参数: data.user_id 用户ID', + description: '删除小红书用户, 参数: data.id 用户ID', metadata: { tags: ['小红书', '用户'], } }).define(async (ctx) => { - const { user_id } = ctx.query.data || {}; - if (!user_id) { - ctx.throw(400, 'user_id 参数缺失'); + const { id } = ctx.query.data || {}; + if (!id) { + ctx.throw(400, 'id 参数缺失'); } - const existing = await db.select().from(xhsUser).where(eq(xhsUser.user_id, user_id)).limit(1); + const existing = await db.select().from(xhsUser).where(eq(xhsUser.id, id)).limit(1); if (existing.length === 0) { ctx.throw(404, '没有找到对应的用户'); } - await db.delete(xhsUser).where(eq(xhsUser.user_id, user_id)); + await db.delete(xhsUser).where(eq(xhsUser.id, id)); ctx.body = { success: true }; }).addTo(app); @@ -130,16 +135,16 @@ app.route({ path: 'xhs-users', key: 'get', middleware: ['auth'], - description: '获取单个小红书用户, 参数: data.user_id 用户ID', + description: '获取单个小红书用户, 参数: data.id 用户ID', metadata: { tags: ['小红书', '用户'], } }).define(async (ctx) => { - const { user_id } = ctx.query.data || {}; - if (!user_id) { - ctx.throw(400, 'user_id 参数缺失'); + const { id } = ctx.query.data || {}; + if (!id) { + ctx.throw(400, 'id 参数缺失'); } - const existing = await db.select().from(xhsUser).where(eq(xhsUser.user_id, user_id)).limit(1); + const existing = await db.select().from(xhsUser).where(eq(xhsUser.id, id)).limit(1); if (existing.length === 0) { ctx.throw(404, '没有找到对应的用户'); }