This commit is contained in:
2025-12-26 18:13:15 +08:00
parent 66e6370013
commit 413c147109
32 changed files with 2449 additions and 205 deletions

View File

@@ -0,0 +1,10 @@
{
"permissions": {
"allow": [
"Bash(pnpm drizzle-kit generate:*)",
"Bash(pnpm drizzle-kit push:*)",
"Bash(pnpm rebuild:*)",
"Bash(tsx:*)"
]
}
}

1
.env.example Normal file
View File

@@ -0,0 +1 @@
DATABASE_URL=storage/browser-helper/data.sqlite3

10
.gitignore vendored
View File

@@ -3,3 +3,13 @@ node_modules
browser-context browser-context
cache cache
.env
!.env*example
storage
# storage/browser-helper/data.db
dist
pack-dist

5
bun.config.ts Normal file
View File

@@ -0,0 +1,5 @@
import { buildWithBun } from '@kevisual/code-builder'
buildWithBun({
external: ['playwright', 'better-sqlite3']
})

19
drizzle.config.ts Normal file
View File

@@ -0,0 +1,19 @@
import type { Config } from 'drizzle-kit';
import 'dotenv/config';
const url = process.env.DATABASE_URL || 'storage/browser-helper/data.sqlite3';
// Ensure the directory exists
import fs from "node:fs";
import path from "node:path";
const dir = path.dirname(url);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
export default {
schema: './src/db/schema.ts',
out: './storage/browser-helper/drizzle',
dialect: 'sqlite',
dbCredentials: {
url: process.env.DATABASE_URL || 'storage/browser-helper/data.sqlite3',
},
} satisfies Config;

View File

@@ -1,17 +1,35 @@
{ {
"name": "xhs-helper", "name": "@kevisual/browser-helper",
"version": "0.0.1", "version": "0.0.1",
"description": "", "description": "",
"main": "index.js", "main": "index.js",
"types": "typings/note.ts", "types": "typings/note.d.ts",
"basename": "/root/browser-helper",
"app": {
"type": "pm2-system-app",
"entry": "./app.js",
"runtime": [
"client"
]
},
"scripts": { "scripts": {
"start": "tsx src/playwright/index.ts", "start": "tsx src/playwright/index.ts",
"init:base": "npx playwright install", "dev": "tsx watch src/index.ts",
"browser": "pm2 start start-browser.ts --name xhs-helper-browser --interpreter=tsx" "init:browser": "npx playwright install",
"build": "code-builder build",
"browser": "pm2 start start-browser.js --name browser-helper",
"cmd": "tsx src/test/cmd.ts ",
"init": "pnpm run init:pnpm && pnpm run init:db && pnpm run init:browser",
"init:pnpm": "pnpm approve-builds",
"init:db": "npx drizzle-kit push",
"studio": "npx drizzle-kit studio",
"drizzle:migrate": "npx drizzle-kit migrate",
"drizzle:push": "npx drizzle-kit push"
}, },
"keywords": [], "keywords": [],
"files": [ "files": [
"typings", "typings",
"dist",
"src", "src",
"start-browser.ts" "start-browser.ts"
], ],
@@ -20,9 +38,25 @@
"packageManager": "pnpm@10.26.0", "packageManager": "pnpm@10.26.0",
"type": "module", "type": "module",
"dependencies": { "dependencies": {
"playwright": "^1.57.0" "playwright": "^1.57.0",
"better-sqlite3": "^12.5.0"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^25.0.3" "@kevisual/code-builder": "^0.0.2",
"@kevisual/context": "^0.0.4",
"@kevisual/router": "^0.0.49",
"@kevisual/types": "^0.0.10",
"@kevisual/use-config": "^1.0.21",
"@types/better-sqlite3": "^7.6.13",
"@types/bun": "^1.3.5",
"@types/node": "^25.0.3",
"commander": "^14.0.2",
"dotenv": "^17.2.3",
"drizzle-kit": "^0.31.8",
"drizzle-orm": "^0.45.1",
"es-toolkit": "^1.43.0",
"eventemitter3": "^5.0.1",
"lru-cache": "^11.2.4",
"window-size": "^1.1.1"
} }
} }

1510
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

2
pnpm-workspace.yaml Normal file
View File

@@ -0,0 +1,2 @@
onlyBuiltDependencies:
- better-sqlite3

16
readme.md Normal file
View File

@@ -0,0 +1,16 @@
# 浏览器自动化助手
实现功能,浏览了页面,自动把想要的数据,存储到数据库中,方便后续分析和使用。
## 初始化
```bash
pnpm install
pnpm run init
```
## 启动studio
```bash
pnpm run studio
```

49
src/app.ts Normal file
View File

@@ -0,0 +1,49 @@
import { App } from '@kevisual/router'
import { useConfigKey } from '@kevisual/context'
import { useConfig } from '@kevisual/use-config'
import Database from 'better-sqlite3';
import { drizzle } from 'drizzle-orm/better-sqlite3';
import { Core } from './playwright/core.ts';
import * as schema from './db/schema.ts';
export const config = useConfig()
export const app = useConfigKey<App>('app', () => new App({
}))
app.router.createRouteList();
export const db = useConfigKey('db', () => {
const sqlite = new Database(config.DATABASE_URL || 'storage/browser-helper/data.sqlite3');
sqlite.pragma('journal_mode = WAL');
const db = drizzle({ client: sqlite, schema });
return db;
})
export const core = useConfigKey<Core>('core', () => new Core({
listeners: [
{
path: "search/notes",
response: async (page) => {
console.log('处理搜索笔记响应');
console.log(page.url);
try {
const data = JSON.parse(page.text);
if (data.code === 0) {
const notes = data.data?.items || [];
console.log(`搜索到 ${notes.length} 条笔记`);
app.run({
path: 'xhs',
key: 'save-search-notes',
payload: {
data: notes,
}
})
}
} catch (error) {
console.error('解析搜索笔记响应失败:', error);
}
}
}
]
}));

53
src/db/cache.ts Normal file
View File

@@ -0,0 +1,53 @@
import { eq, and } from 'drizzle-orm';
import { db } from '../app.ts';
import { cache } from './schema.ts';
export const cacheStore = {
set: (key: string, value: unknown, days = 7) => {
const expireAt = Math.floor(Date.now() / 1000) + days * 86400;
return db.insert(cache)
.values({
key,
value: JSON.stringify(value),
expireAt,
createdAt: Math.floor(Date.now() / 1000),
})
.onConflictDoUpdate({
target: cache.key,
set: { value: JSON.stringify(value), expireAt },
});
},
get: <T = unknown>(key: string): T | null => {
const now = Math.floor(Date.now() / 1000);
const row = db.select()
.from(cache)
.where(and(eq(cache.key, key), eq(cache.expireAt, now)))
.get() as { value: string } | undefined;
return row ? JSON.parse(row.value) as T : null;
},
has: (key: string): boolean => {
const now = Math.floor(Date.now() / 1000);
const row = db.select()
.from(cache)
.where(and(eq(cache.key, key), eq(cache.expireAt, now)))
.get();
return !!row;
},
delete: (key: string) => {
return db.delete(cache).where(eq(cache.key, key));
},
// 定期清理过期数据
cleanExpired: () => {
const now = Math.floor(Date.now() / 1000);
return db.delete(cache).where(eq(cache.expireAt, now));
},
};
// 每5小时清理一次过期数据
setInterval(() => {
cacheStore.cleanExpired();
}, 5 * 1000 * 60 * 60);

24
src/db/schema.ts Normal file
View File

@@ -0,0 +1,24 @@
import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core';
export const cache = sqliteTable('cache', {
key: text('key').primaryKey(),
value: text('value').notNull(),
expireAt: integer('expire_at').notNull(),
createdAt: integer('created_at').notNull(),
});
export const xhsNote = sqliteTable('xhs_note', {
id: text('id').primaryKey(),
content: text('content').notNull(),
title: text('title'),
description: text('description'),
tags: text('tags').notNull(),
noteUrl: text('note_url'),
status: text('status'),
authorUrl: text('author_url'),
cover: text('cover'),
syncStatus: integer('sync_status').notNull(),
syncAt: integer('sync_at').notNull(),
createdAt: integer('created_at').notNull(),
updatedAt: integer('updated_at').notNull(),
});

18
src/index.ts Normal file
View File

@@ -0,0 +1,18 @@
import { app } from './app.ts'
export { app, db } from './app.ts';
export { Core } from './playwright/core.ts';
import './routes/index.ts';
// 如果是直接运行,则启动应用
app.route({
id: 'auth'
}).define(async (ctx) => {
// token authentication
}).addTo(app);
if (import.meta.main) {
console.log('Starting application...');
app.listen(52000, () => {
console.log('Application is running on http://localhost:52000');
})
}

6
src/modules/cache.ts Normal file
View File

@@ -0,0 +1,6 @@
import { LRUCache } from 'lru-cache'
export const sessionCache = new LRUCache<string, any>({
max: 500,
ttl: 1000 * 60 * 60, // 1 hour
});

187
src/playwright/core.ts Normal file
View File

@@ -0,0 +1,187 @@
import { chromium, Page, BrowserContext, Browser, CDPSession, Request } from 'playwright';
import { execSync } from 'node:child_process';
import { EventEmitter } from 'eventemitter3'
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
type RequestObject = {
url: string;
path: string;
request: Request
}
type ResponseObject = {
url: string;
path: string;
text: string;
}
export type Listener = {
path: string;
request?: (data: RequestObject) => void;
response?: (data: ResponseObject) => void;
type?: 'request' | 'response' | 'both';
}
export class Core<T = {}> {
browserContext: BrowserContext | null = null;
browser: Browser | null = null;
page: Page | null = null;
debugPort = 9223;
status: 'disconnected' | 'connecting' | 'connected' | 'failed' = 'disconnected';
emitter = new EventEmitter();
listeners: Listener[] = [];
recordReady: boolean = false;
data: T | null = null;
constructor(opts?: { debugPort?: number, listeners?: Listener[] }) {
if (opts?.debugPort) {
this.debugPort = opts.debugPort;
}
if (opts?.listeners) {
this.listeners = opts.listeners;
}
}
async init() {
const debugPort = this.debugPort;
try {
const stdout = execSync(`netstat -ano | findstr :${debugPort}`);
console.log(`端口 ${debugPort} 已在监听:\n${stdout}`);
const browser = await chromium.connectOverCDP(`http://127.0.0.1:${debugPort}`);
console.log('成功连接到 Chrome CDP!');
this.browser = browser;
this.browserContext = browser.contexts()[0];
this.handleRequest(this.browserContext);
this.page = this.browserContext.pages()[0] || await this.browserContext.newPage();
this.emitter.emit('connected');
return;
} catch (error: any) {
throw new Error(`无法连接到 Chrome CDP端口 ${debugPort} 可能未正确启动: ${(error as Error).message.slice(0, 100)}`);
}
}
async connect() {
if (this.status === 'connected') {
return this
}
if (this.status === 'connecting') {
return new Promise<boolean>((resolve) => {
const timer = setTimeout(() => {
this.emitter.removeAllListeners('connected');
resolve(false)
}, 60 * 5 * 1000);
this.emitter.once('connected', () => {
clearInterval(timer)
resolve(true)
});
});
}
if (this.status === 'disconnected' || this.status === 'failed') {
this.status = 'connecting';
for (let i = 0; i < 10; i++) {
try {
await this.init();
this.status = 'connected'
return true;
} catch (e) {
console.log(`尝试 ${i + 1}/10 连接失败: ${(e as Error).message.slice(0, 100)}`);
await sleep(2000);
}
}
}
this.status = 'failed';
return false;
}
async getBrowser() {
if (this.browser) {
return this.browser;
}
const connected = await this.connect();
if (connected) {
return this.browser!;
}
throw new Error('无法连接到浏览器实例');
}
async getPage() {
if (this.page) {
return this.page;
}
const connected = await this.connect();
if (connected) {
return this.page!;
}
throw new Error('无法连接到浏览器实例');
}
async setReady(ready: boolean = true) {
if (this.recordReady !== ready) {
this.recordReady = ready;
}
}
async setData(data?: any) {
if (!data) {
this.data = null;
return
}
this.data = data;
}
async handleRequest(context: BrowserContext) {
context.on('request', request => {
const url = request.url();
for (let listener of this.listeners) {
const type = listener.type || 'both';
if (type === 'response') continue;
const data = { url, path: listener.path, request };
if (url.includes(listener.path)) {
this.emitter.emit(listener.path, data);
}
if (listener.request && typeof listener.request === 'function') {
listener.request(data);
}
}
});
context.on('response', async response => {
const url = response.url();
const recordReady = this.recordReady;
for (let listener of this.listeners) {
const type = listener.type || 'both';
if (type === 'request') continue;
if (url.includes(listener.path)) {
if (!recordReady) {
console.log('记录未就绪,跳过响应处理');
return
}
try {
const status = response.status();
const contentType = response.headers()['content-type'] || '';
if (status >= 400) {
console.log(`请求 ${url} 返回错误状态码: ${status}`);
this.emitter.emit('error', { url, status, error: '错误状态码' });
return;
}
const isText = contentType.includes('application/json') || contentType.includes('text/');
if (!isText) {
console.log(`请求 ${url} 返回非文本内容类型: ${contentType}`);
this.emitter.emit('error', { url, status, error: '非文本内容' });
return;
}
const responseBody = await response.text();
const data: ResponseObject = { url, path: listener.path, text: responseBody };
this.emitter.emit(listener.path, data);
if (listener.response && typeof listener.response === 'function') {
listener.response(data);
}
} catch (e) {
console.log('无法读取响应内容');
}
}
}
});
}
close() {
if (this.browser) {
this.browser.close();
this.browser = null;
this.browserContext = null;
this.page = null;
this.status = 'disconnected';
}
}
}

View File

@@ -1,183 +1 @@
import { chromium, Page, BrowserContext } from 'playwright'; export * from './core.ts'
import path from 'node:path';
import { exec } from 'node:child_process';
import { promisify } from 'node:util';
import fs from 'node:fs';
const execAsync = promisify(exec);
export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
export const main = async () => {
const debugPort = 9223;
// 等待并检查端口是否监听
for (let i = 0; i < 15; i++) {
await new Promise(resolve => setTimeout(resolve, 2000));
try {
// 检查端口是否被监听
const { stdout } = await execAsync(`netstat -ano | findstr :${debugPort}`);
console.log(`端口 ${debugPort} 已在监听:\n${stdout}`);
// 尝试连接
const browser = await chromium.connectOverCDP(`http://127.0.0.1:${debugPort}`);
console.log('成功连接到 Chrome CDP!');
// 获取已有的 context
const context = browser.contexts()[0];
// 获取已有的页面或创建新页面
const page = context.pages()[0] || await context.newPage();
// console.log('Navigating to xiaohongshu.com');
// await page.goto('https://www.xiaohongshu.com/search_result?keyword=%25E5%25A4%259A%25E7%25BB%25B4%25E8%25A1%25A8%25E6%25A0%25BC&type=51');
// console.log('当前页面标题:', await page.title());
// 点击筛选按钮的示例
// await page.click('text=筛选');
// PAIU
await listenFetchRequests(context);
await sleep(2000);
// 关闭浏览器连接,不关闭实际浏览器
await hoverPickerExample(page, { keyword: '多维表格' });
// 等待更长时间让请求有机会发生
console.log('等待 API 请求...');
await sleep(5000);
// 清理所有路由,避免 TargetClosedError
await context.unrouteAll({ behavior: 'ignoreErrors' });
// 获取
await browser.close();
return;
} catch (error: any) {
console.log(`尝试 ${i + 1}/15: ${(error as Error).message.slice(0, 100)}`);
}
}
throw new Error(`无法连接到 Chrome CDP端口 ${debugPort} 可能未正确启动`);
}
main();
type HoverPickerOptions = {
keyword?: string;
pushTime?: '一天内' | '一周内' | '半年内';
sort?: '综合' | '最新' | '最多点赞' | '最多评论';
searchRange?: '不限' | '已看过' | '未看过' | '已关注';
distance?: '不限' | '同城' | '附近';
scrollTimes?: number;
}
const hoverPickerExample = async (page: Page, opts?: HoverPickerOptions) => {
const { pushTime = '一天内', sort = '最新', keyword = '', scrollTimes = 5 } = opts || {};
if (keyword) {
// class为 input-box
const inputBox = await page.$('.input-box');
const input = await inputBox?.$('input');
if (input) {
//先获取当前内容,如果一样就不输入
const currentValue = await input.inputValue();
if (currentValue === keyword) {
console.log('关键词已存在,无需输入');
} else {
await input.fill(keyword);
await input.press('Enter');
console.log(`已输入关键词: ${keyword}`);
await sleep(3000); // 等待搜索结果加载
}
}
}
// 查找筛选按钮并保持 hover 状态
let filterButton = await page.$('text=筛选');
if (filterButton) {
await filterButton.hover();
console.log('鼠标悬停在筛选按钮上');
// 在保持 hover 的情况下,等待筛选面板出现并点击元素
const filterClassPanel = await page.$('.filter-panel');
if (filterClassPanel) {
console.log('筛选面板已打开');
// 点击最新选项
const latestOption = await filterClassPanel.$(`text=${sort}`);
if (latestOption) {
await latestOption.click();
console.log(`已选择${sort}选项`);
}
// 点击一周内选项
const oneWeekOption = await filterClassPanel.$(`text=${pushTime}`);
if (oneWeekOption) {
await oneWeekOption.click();
console.log(`已选择${pushTime}选项`);
}
if (opts?.distance && opts.distance !== '不限') {
const distanceOption = await filterClassPanel.$(`text=${opts.distance}`);
if (distanceOption) {
await distanceOption.click();
console.log(`已选择${opts.distance}选项`);
}
}
if (opts?.searchRange && opts.searchRange !== '不限') {
const rangeOption = await filterClassPanel.$(`text=${opts.searchRange}`);
if (rangeOption) {
await rangeOption.click();
console.log(`已选择${opts.searchRange}选项`);
}
}
// 点击收起按钮
const shouquButton = await filterClassPanel.$('text=收起');
if (shouquButton) {
await shouquButton.click();
console.log('已点击收起按钮');
}
}
}
// 将鼠标移到页面外,移除 hover 状态
await page.mouse.move(0, 0);
console.log('已移除 hover 状态');
// 自动滚动页面5次以触发更多请求
for (let i = 0; i < scrollTimes; i++) {
await page.evaluate(() => {
window.scrollBy({
top: window.innerHeight,
left: 0,
behavior: 'smooth'
});
});
console.log(`已滚动页面 ${i + 1}`);
// 判断是否滚动到底部
const isBottom = await page.evaluate(() => {
return (window.innerHeight + window.scrollY) >= document.body.scrollHeight;
});
if (isBottom) {
console.log('已到达页面底部,停止滚动');
break;
}
await sleep(2000); // 等待2秒以加载新内容
}
}
const listenFetchRequests = async (context: BrowserContext) => {
// 监听访问 https://edith.xiaohongshu.com/api/sns/web/v1/search/notes
// 返回对应的结果 - 使用 context 级别的路由,刷新后不会丢失
await context.route('https://edith.xiaohongshu.com/api/sns/web/v1/search/notes*', async (route) => {
const request = route.request();
console.log('捕获到请求:', request.url());
await route.continue();
});
// 使用 response 事件来获取响应内容
context.on('response', async (response) => {
const url = response.url();
// 打印所有 edith.xiaohongshu.com 的 API 响应
if (url.includes('edith.xiaohongshu.com/api/')) {
console.log('收到 API 响应:', url);
console.log('状态:', response.status());
if (url.includes('search/notes')) {
try {
const responseBody = await response.text();
fs.writeFileSync(path.join(process.cwd(), 'cache', Date.now().toString() + '.json'), responseBody, 'utf-8');
} catch (e) {
console.log('无法读取响应内容');
}
}
}
});
}

View File

@@ -0,0 +1,2 @@
import './pane-manager.ts';
import './page.ts';

View File

@@ -0,0 +1,16 @@
import { app, core } from "../../app.ts";
app.route({
path: 'page',
key: 'go',
middleware: ['auth']
}).define(async (ctx) => {
const url = ctx.query?.url as string;
if (!url) {
ctx.throw!(400, '缺少 url 参数');
}
const page = await core.getPage();
await page.goto(url);
ctx.body = { success: true, message: `已导航到 ${url}` };
}).addTo(app);

View File

@@ -0,0 +1,67 @@
import { app, core } from "../../app.ts";
/**
* 浏览器窗口边界信息
*/
export interface Bounds {
/**
* 窗口距离屏幕左边缘的偏移量(像素)
*/
left?: number;
/**
* 窗口距离屏幕上边缘的偏移量(像素)
*/
top?: number;
/**
* 窗口宽度(像素)
*/
width?: number;
/**
* 窗口高度(像素)
*/
height?: number;
/**
* 窗口状态,默认为 normal
*/
windowState?: "normal" | "minimized" | "maximized" | "fullscreen";
}
const desc = `窗口管理,参数为窗口位置和大小信息
参数示例:
left: 窗口左上角横坐标
top: 窗口左上角纵坐标
width: 窗口宽度
height: 窗口高度
windowState: 窗口状态,可选值有 normal正常, minimized最小化, maximized最大化, fullscreen全屏
`;
app.route({
path: 'window',
key: 'manager',
description: desc,
middleware: ['auth']
}).define(async (ctx) => {
const bounds = ctx.query as Bounds || {};
const { left, top, width, height, windowState } = bounds;
if (!left && !top && !width && !height && !windowState) {
bounds.windowState = 'normal'
}
const page = await core.getPage();
const cdp = await page.context().newCDPSession(page);
try {
const { windowId } = await cdp.send('Browser.getWindowForTarget');
const page2 = { left: -1920, top: 0 }
const full = { windowState: 'fullscreen' }
// 设定传入的窗口参数
await cdp.send('Browser.setWindowBounds', {
windowId,
bounds: bounds
});
ctx.body = { success: true, message: '窗口已调整' };
} catch (error) {
ctx.throw!(500, '调整窗口失败: ' + (error as Error).message);
} finally {
// 关闭session
await cdp.detach();
}
}).addTo(app);

2
src/routes/index.ts Normal file
View File

@@ -0,0 +1,2 @@
import './browser/index.ts';
import './xhs/index.ts';

1
src/routes/xhs/index.ts Normal file
View File

@@ -0,0 +1 @@
import './search-notes.ts';

View File

@@ -0,0 +1,205 @@
import { xhsNote } from '@/db/schema.ts';
import { app, core, db } from '../../app.ts';
import { sql } from 'drizzle-orm';
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
import { Page } from 'playwright';
import { Core } from '@/playwright/core.ts';
import { sessionCache } from '@/modules/cache.ts';
type HoverPickerOptions = {
keyword?: string;
pushTime?: '一天内' | '一周内' | '半年内';
sort?: '综合' | '最新' | '最多点赞' | '最多评论';
searchRange?: '不限' | '已看过' | '未看过' | '已关注';
distance?: '不限' | '同城' | '附近';
scrollTimes?: number;
core?: Core
}
const hoverPickerExample = async (page: Page, opts?: HoverPickerOptions) => {
const { pushTime = '一天内', sort = '最新', keyword = '', scrollTimes = 5 } = opts || {};
core.setReady(false)
if (keyword) {
// class为 input-box
const inputBox = await page.$('.input-box');
const input = await inputBox?.$('input');
if (input) {
//先获取当前内容,如果一样就不输入
const currentValue = await input.inputValue();
if (currentValue === keyword) {
console.log('关键词已存在,无需输入');
} else {
await input.fill(keyword);
await input.press('Enter');
console.log(`已输入关键词: ${keyword}`);
await sleep(3000); // 等待搜索结果加载
}
}
}
// 查找筛选按钮并保持 hover 状态
let filterButton = await page.$('text=筛选');
if (filterButton) {
await filterButton.hover();
console.log('鼠标悬停在筛选按钮上');
// 在保持 hover 的情况下,等待筛选面板出现并点击元素
const filterClassPanel = await page.$('.filter-panel');
if (filterClassPanel) {
console.log('筛选面板已打开');
// 点击最新选项
const latestOption = await filterClassPanel.$(`text=${sort}`);
if (latestOption) {
await latestOption.click();
console.log(`已选择${sort}选项`);
}
// 点击一周内选项
const oneWeekOption = await filterClassPanel.$(`text=${pushTime}`);
if (oneWeekOption) {
await oneWeekOption.click();
console.log(`已选择${pushTime}选项`);
}
core.setReady(true)
if (opts?.distance && opts.distance !== '不限') {
const distanceOption = await filterClassPanel.$(`text=${opts.distance}`);
if (distanceOption) {
await distanceOption.click();
console.log(`已选择${opts.distance}选项`);
}
}
if (opts?.searchRange && opts.searchRange !== '不限') {
const rangeOption = await filterClassPanel.$(`text=${opts.searchRange}`);
if (rangeOption) {
await rangeOption.click();
console.log(`已选择${opts.searchRange}选项`);
}
}
// 点击收起按钮
const shouquButton = await filterClassPanel.$('text=收起');
if (shouquButton) {
await shouquButton.click();
console.log('已点击收起按钮');
}
}
}
// 将鼠标移到页面外,移除 hover 状态
await page.mouse.move(0, 0);
console.log('已移除 hover 状态');
// 自动滚动页面5次以触发更多请求
for (let i = 0; i < scrollTimes; i++) {
await page.evaluate(() => {
window.scrollBy({
top: window.innerHeight,
left: 0,
behavior: 'smooth'
});
});
console.log(`已滚动页面 ${i + 1}`);
// 判断是否滚动到底部
const isBottom = await page.evaluate(() => {
return (window.innerHeight + window.scrollY) >= document.body.scrollHeight;
});
if (isBottom) {
console.log('已到达页面底部,停止滚动');
break;
}
await sleep(2000); // 等待2秒以加载新内容
}
}
const searchNoteDesc = `
根据关键词搜索小红书笔记。
参数说明:
- keyword: 搜索关键词
- pushTime: 笔记发布时间筛选(一天内、一周内、半年内)
- sort: 排序方式(综合、最新、最多点赞、最多评论)
- distance: 距离筛选(不限、同城、附近)
- searchRange: 搜索范围(不限、已看过、未看过、已关注)
- scrollTimes: 页面自动滚动次数默认5次
其中只有keyword是必填项。
`;
app.route({
path: 'xhs',
key: 'search-notes',
description: searchNoteDesc,
middleware: ['auth']
}).define(
async (ctx) => {
const query = ctx.query!;
const page = await core.getPage();
// 获取浏览器的当前URL
const currentURL = page.url();
if (!currentURL.includes('www.xiaohongshu.com/search_result')) {
const url = new URL('https://www.xiaohongshu.com/search_result');
url.searchParams.set('keyword', query.keyword as string || '');
await page.goto(url.toString());
console.log(`导航到搜索页面: ${url.toString()}`);
await sleep(3000); // 等待页面加载
}
const keyword = query.keyword as string;
// 存储关键词到 core 的 data 中,供响应处理使用
sessionCache.set('xhs-search-keyword', keyword);
await hoverPickerExample(page, {
keyword: query.keyword as string,
pushTime: (query.pushTime as '一天内' | '一周内' | '半年内') || '一天内',
sort: (query.sort as '综合' | '最新' | '最多点赞' | '最多评论') || '最新',
distance: (query.distance as '不限' | '同城' | '附近') || '不限',
searchRange: (query.searchRange as '不限' | '已看过' | '未看过' | '已关注') || '不限',
scrollTimes: query.scrollTimes ?? 5,
});
ctx.body = { success: true, message: '操作完成' };
}).addTo(app);
app.route({
path: 'xhs',
key: 'save-search-notes',
description: '保存搜索笔记结果',
middleware: ['auth']
}).define(async (ctx) => {
const data = ctx.query!.data as XHS.SearchNote[];
try {
const getNoteUrl = (note: XHS.SearchNote) => {
const id = note.id;
const secToken = note.xsec_token;
return `https://www.xiaohongshu.com/explore/${id}?xsec_token=${secToken}&xsec_source=pc_feed`
}
const getUserUrl = (note: XHS.SearchNote) => {
const user = note.note_card?.user;
const id = user?.user_id;
const secToken = user?.xsec_token;
if (user) {
return `https://www.xiaohongshu.com/user/profile/${id}?xsec_token=${secToken}&xsec_source=pc_note`
}
return ``
}
const getCover = (note: XHS.SearchNote) => {
const cover = note.note_card?.cover
return cover?.url_default || ''
}
const keyword = sessionCache.get('xhs-search-keyword');
const notes = data.filter(note => note.model_type === 'note').map(note => ({
id: note.id,
content: JSON.stringify(note),
description: keyword || '',
title: note.note_card?.display_title || '',
authorUrl: getUserUrl(note),
tags: '',
syncStatus: 0,
noteUrl: getNoteUrl(note),
cover: getCover(note),
syncAt: 0,
createdAt: Date.now(),
updatedAt: Date.now(),
}));
await db.insert(xhsNote).values(notes).onConflictDoUpdate({
target: xhsNote.id,
set: {
content: sql`excluded.content`,
updatedAt: Date.now(),
},
}).execute();
console.log(`已保存 ${data.length} 条搜索笔记结果`);
} catch (error) {
console.error('保存搜索笔记结果时出错:', error);
}
ctx.body = { success: true, message: '保存搜索笔记结果功能已实现' };
}).addTo(app);

0
src/routes/xhs/user.ts Normal file
View File

13
src/test/browser.ts Normal file
View File

@@ -0,0 +1,13 @@
import { Command } from "commander";
import { core, program, app, showMore } from './common.ts'
program.addCommand(new Command('browser').action(async () => {
const res = await app.run({
path: 'page',
key: 'go',
payload: {
url: 'https://www.xiaohongshu.com/explore'
}
})
console.log(showMore(res));
}));

7
src/test/cmd.ts Normal file
View File

@@ -0,0 +1,7 @@
import { program } from './common.ts'
import './pane.ts'
import './browser.ts'
import './db/add.ts'
import './xhs/index.ts'
program.parse(process.argv);

34
src/test/common.ts Normal file
View File

@@ -0,0 +1,34 @@
import util from 'node:util';
export * from '../index.ts';
import { Core } from '../index.ts';
export const core = new Core();
export const showMore = (data: any) => {
return util.inspect(data, { depth: null, colors: true });
}
export const sleep = async (ms: number) => {
return new Promise(resolve => setTimeout(resolve, ms));
}
export const exit = async (time: number = 0) => {
await sleep(time * 1000);
process.exit(0);
}
// const main = async () => {
// const connected = await core.connect();
// if (!connected) {
// console.log('无法连接到浏览器');
// return;
// }
// const page = core.page!;
// await page.goto('https://www.xiaohongshu.com/explore');
// console.log('已打开小红书首页');
// }
// main()
import { program } from 'commander';
program
.name('xhs-helper')
.description('CLI to manage xhs-helper tasks')
.version('0.1.0');
export { program };

34
src/test/db/add.ts Normal file
View File

@@ -0,0 +1,34 @@
import { cache } from './../../db/schema.ts';
import { Command } from 'commander';
import { program, db } from '../common.ts';
program
.addCommand(new Command('db:add')
.description('Add initial data to the database')
.action(async () => {
const result = await db.select().from(cache).limit(1).all(); // Ensure DB is initialized
if (result.length > 0) {
console.log('Database already has data. No action taken.');
const [a] = result;
console.log(a);
return;
}
await db.insert(cache).values([
{
key: 'example_key_1',
value: JSON.stringify({ data: 'example_value_1' }),
expireAt: Math.floor(Date.now() / 1000) + 7 * 86400,
createdAt: Math.floor(Date.now() / 1000),
},
{
key: 'example_key_2',
value: JSON.stringify({ data: 'example_value_2' }),
expireAt: Math.floor(Date.now() / 1000) + 7 * 86400,
createdAt: Math.floor(Date.now() / 1000),
},
]);
console.log('Initial data added to the database.');
})
);

47
src/test/pane.ts Normal file
View File

@@ -0,0 +1,47 @@
import { Command } from "commander";
import { core, program, exit } from './common.ts'
// import size from 'window-size'
const main = async () => {
const connected = await core.connect();
if (!connected) {
console.log('无法连接到浏览器实例,程序退出');
process.exit(1);
}
const page = core.page!;
// 设置页面视口大小
const browser = core.browser!;
const cdp = await page.context().newCDPSession(page);
const { windowId } = await cdp.send('Browser.getWindowForTarget');
// 第一个窗口(左侧)
// await cdp.send('Browser.setWindowBounds', {
// windowId,
// bounds: { left: -1920, top: 0 }
// });
// 如何获取屏幕宽度
// const { width, height } = size.get();
// console.log(`屏幕宽度: ${width}, 高度: ${height}`);
// await page.goto('https://www.xiaohongshu.com/');
await cdp.send('Browser.setWindowBounds', {
windowId,
bounds: {
windowState: 'maximized'
}
});
// 开启全屏
// const session = await page.context().newCDPSession(page);
// await session.send('Emulation.setDeviceMetricsOverride', {
// width: 1920,
// height: 1080,
// deviceScaleFactor: 1,
// mobile: false,
// });
exit(0)
}
export const paneCommand = new Command('pane').action(async () => {
await main();
});
program.addCommand(paneCommand);

18
src/test/xhs/index.ts Normal file
View File

@@ -0,0 +1,18 @@
import { program, app, showMore } from '../common.ts'
program
.command('xhs:run')
.description('运行小红书浏览器辅助')
.action(async () => {
const res = await app.run({
path: 'xhs',
key: 'search-notes',
payload: {
keyword: '多维表格',
scrollTimes: 1,
}
})
console.log(showMore(res));
});

View File

@@ -1,4 +1,3 @@
import { chromium } from 'playwright';
import { spawn } from 'node:child_process'; import { spawn } from 'node:child_process';
import path from 'node:path'; import path from 'node:path';
@@ -15,21 +14,32 @@ export const main = async () => {
// console.log('注意:需要手动登录账号和安装插件'); // console.log('注意:需要手动登录账号和安装插件');
// 启动 Chrome带远程调试端口 // 启动 Chrome带远程调试端口
const chromeProcess = spawn(executablePath, [ const chromeProcess = spawn(
executablePath,
[
`--remote-debugging-port=${debugPort}`, `--remote-debugging-port=${debugPort}`,
`--user-data-dir=${userDataDir}`, `--user-data-dir=${userDataDir}`,
], { // '--kiosk', // 全屏模式,无修改边框
],
{
windowsHide: true, // 隐藏 CMD 窗口
detached: false, detached: false,
stdio: 'inherit', stdio: ['ignore', 'ignore', 'ignore'],
}); },
);
chromeProcess.on('error', (err) => { chromeProcess.on('error', (err) => {
console.error('Chrome 启动失败:', err); console.error('Chrome 启动失败:', err);
// 需要重新启动
}); });
chromeProcess.on('exit', (code, signal) => { chromeProcess.on('exit', (code, signal) => {
console.log(`Chrome 进程退出,代码: ${code}, 信号: ${signal}`); console.log(`Chrome 进程退出,代码: ${code}, 信号: ${signal}`);
if (code === 0) {
// 重启
main();
}
}); });
} };
main(); main();

26
tsconfig.json Normal file
View File

@@ -0,0 +1,26 @@
{
"extends": "@kevisual/types/json/backend.json",
"compilerOptions": {
"module": "NodeNext",
"target": "esnext",
"baseUrl": ".",
"typeRoots": [
"./node_modules/@types",
"./node_modules/@kevisual/types/index.d.ts",
"./typings"
],
"paths": {
"@/*": [
"src/*"
],
"@agent/*": [
"agent/*"
]
},
},
"include": [
"typings/**/*.d.ts",
"src/**/*",
"agent/**/*",
],
}

View File

@@ -1,4 +1,4 @@
export namespace XHS { declare namespace XHS {
/** 笔记用户信息 */ /** 笔记用户信息 */
export interface NoteUser { export interface NoteUser {
/** 昵称 */ /** 昵称 */
@@ -86,11 +86,11 @@ export namespace XHS {
} }
/** 笔记 */ /** 笔记 */
export interface Note { export interface SearchNote {
/** 笔记ID */ /** 笔记ID */
id: string; id: string;
/** 模型类型如note笔记 */ /** 模型类型如note笔记 */
model_type: string; model_type: 'note' | 'hot_query' | string;
/** 笔记卡片 */ /** 笔记卡片 */
note_card: NoteCard; note_card: NoteCard;
/** 安全令牌 */ /** 安全令牌 */
@@ -98,8 +98,8 @@ export namespace XHS {
} }
} }
export namespace XHS { declare namespace XHS {
export interface ResultList<T = NoteCard> { export interface ResultList<T = SearchNote> {
hasMore: boolean; hasMore: boolean;
items: T[]; items: T[];
} }