更新依赖项,添加 user-agents 库;重构浏览器启动逻辑,支持无头模式和隐身模式
This commit is contained in:
@@ -40,6 +40,7 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"better-sqlite3": "^12.5.0",
|
"better-sqlite3": "^12.5.0",
|
||||||
"playwright": "^1.57.0",
|
"playwright": "^1.57.0",
|
||||||
|
"user-agents": "^1.1.669",
|
||||||
"zod": "^4.2.1",
|
"zod": "^4.2.1",
|
||||||
"zod-to-json-schema": "^3.25.1"
|
"zod-to-json-schema": "^3.25.1"
|
||||||
},
|
},
|
||||||
@@ -52,6 +53,7 @@
|
|||||||
"@types/better-sqlite3": "^7.6.13",
|
"@types/better-sqlite3": "^7.6.13",
|
||||||
"@types/bun": "^1.3.5",
|
"@types/bun": "^1.3.5",
|
||||||
"@types/node": "^25.0.3",
|
"@types/node": "^25.0.3",
|
||||||
|
"@types/user-agents": "^1.0.4",
|
||||||
"commander": "^14.0.2",
|
"commander": "^14.0.2",
|
||||||
"dotenv": "^17.2.3",
|
"dotenv": "^17.2.3",
|
||||||
"drizzle-kit": "^0.31.8",
|
"drizzle-kit": "^0.31.8",
|
||||||
|
|||||||
23
pnpm-lock.yaml
generated
23
pnpm-lock.yaml
generated
@@ -14,6 +14,9 @@ importers:
|
|||||||
playwright:
|
playwright:
|
||||||
specifier: ^1.57.0
|
specifier: ^1.57.0
|
||||||
version: 1.57.0
|
version: 1.57.0
|
||||||
|
user-agents:
|
||||||
|
specifier: ^1.1.669
|
||||||
|
version: 1.1.669
|
||||||
zod:
|
zod:
|
||||||
specifier: ^4.2.1
|
specifier: ^4.2.1
|
||||||
version: 4.2.1
|
version: 4.2.1
|
||||||
@@ -45,6 +48,9 @@ importers:
|
|||||||
'@types/node':
|
'@types/node':
|
||||||
specifier: ^25.0.3
|
specifier: ^25.0.3
|
||||||
version: 25.0.3
|
version: 25.0.3
|
||||||
|
'@types/user-agents':
|
||||||
|
specifier: ^1.0.4
|
||||||
|
version: 1.0.4
|
||||||
commander:
|
commander:
|
||||||
specifier: ^14.0.2
|
specifier: ^14.0.2
|
||||||
version: 14.0.2
|
version: 14.0.2
|
||||||
@@ -436,6 +442,9 @@ packages:
|
|||||||
'@types/node@25.0.3':
|
'@types/node@25.0.3':
|
||||||
resolution: {integrity: sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==}
|
resolution: {integrity: sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==}
|
||||||
|
|
||||||
|
'@types/user-agents@1.0.4':
|
||||||
|
resolution: {integrity: sha512-AjeFc4oX5WPPflgKfRWWJfkEk7Wu82fnj1rROPsiqFt6yElpdGFg8Srtm/4PU4rA9UiDUZlruGPgcwTMQlwq4w==}
|
||||||
|
|
||||||
asn1js@3.0.7:
|
asn1js@3.0.7:
|
||||||
resolution: {integrity: sha512-uLvq6KJu04qoQM6gvBfKFjlh6Gl0vOKQuR5cJMDHQkmwfMOQeN3F3SHCv9SNYSL+CRoHvOGFfllDlVz03GQjvQ==}
|
resolution: {integrity: sha512-uLvq6KJu04qoQM6gvBfKFjlh6Gl0vOKQuR5cJMDHQkmwfMOQeN3F3SHCv9SNYSL+CRoHvOGFfllDlVz03GQjvQ==}
|
||||||
engines: {node: '>=12.0.0'}
|
engines: {node: '>=12.0.0'}
|
||||||
@@ -674,6 +683,9 @@ packages:
|
|||||||
ini@1.3.8:
|
ini@1.3.8:
|
||||||
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
|
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
|
||||||
|
|
||||||
|
lodash.clonedeep@4.5.0:
|
||||||
|
resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==}
|
||||||
|
|
||||||
lru-cache@11.2.4:
|
lru-cache@11.2.4:
|
||||||
resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==}
|
resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==}
|
||||||
engines: {node: 20 || >=22}
|
engines: {node: 20 || >=22}
|
||||||
@@ -838,6 +850,9 @@ packages:
|
|||||||
undici-types@7.16.0:
|
undici-types@7.16.0:
|
||||||
resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==}
|
resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==}
|
||||||
|
|
||||||
|
user-agents@1.1.669:
|
||||||
|
resolution: {integrity: sha512-pbIzG+AOqCaIpySKJ4IAm1l0VyE4jMnK4y1thV8lm8PYxI+7X5uWcppOK7zY79TCKKTAnJH3/4gaVIZHsjrmJA==}
|
||||||
|
|
||||||
util-deprecate@1.0.2:
|
util-deprecate@1.0.2:
|
||||||
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
|
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
|
||||||
|
|
||||||
@@ -1142,6 +1157,8 @@ snapshots:
|
|||||||
dependencies:
|
dependencies:
|
||||||
undici-types: 7.16.0
|
undici-types: 7.16.0
|
||||||
|
|
||||||
|
'@types/user-agents@1.0.4': {}
|
||||||
|
|
||||||
asn1js@3.0.7:
|
asn1js@3.0.7:
|
||||||
dependencies:
|
dependencies:
|
||||||
pvtsutils: 1.3.6
|
pvtsutils: 1.3.6
|
||||||
@@ -1321,6 +1338,8 @@ snapshots:
|
|||||||
|
|
||||||
ini@1.3.8: {}
|
ini@1.3.8: {}
|
||||||
|
|
||||||
|
lodash.clonedeep@4.5.0: {}
|
||||||
|
|
||||||
lru-cache@11.2.4: {}
|
lru-cache@11.2.4: {}
|
||||||
|
|
||||||
mime-db@1.54.0: {}
|
mime-db@1.54.0: {}
|
||||||
@@ -1498,6 +1517,10 @@ snapshots:
|
|||||||
|
|
||||||
undici-types@7.16.0: {}
|
undici-types@7.16.0: {}
|
||||||
|
|
||||||
|
user-agents@1.1.669:
|
||||||
|
dependencies:
|
||||||
|
lodash.clonedeep: 4.5.0
|
||||||
|
|
||||||
util-deprecate@1.0.2: {}
|
util-deprecate@1.0.2: {}
|
||||||
|
|
||||||
wrappy@1.0.2: {}
|
wrappy@1.0.2: {}
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import { spawn } from 'node:child_process';
|
import { spawn } from 'node:child_process';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
|
import UserAgent from 'user-agents';
|
||||||
|
import { chromium } from 'playwright';
|
||||||
|
|
||||||
export const getExecutablePath = () => {
|
export const getExecutablePath = () => {
|
||||||
// 根据不同平台返回 Chrome 的可执行文件路径
|
// 根据不同平台返回 Chrome 的可执行文件路径
|
||||||
@@ -33,22 +35,48 @@ export const main = async (opts?: {
|
|||||||
userDataDir?: string;
|
userDataDir?: string;
|
||||||
debugPort?: number;
|
debugPort?: number;
|
||||||
kiosk?: boolean;
|
kiosk?: boolean;
|
||||||
|
headless?: boolean;
|
||||||
}) => {
|
}) => {
|
||||||
// Chrome 路径和配置
|
// Chrome 路径和配置
|
||||||
const executablePath = opts?.executablePath || getExecutablePath();
|
let executablePath = opts?.executablePath || getExecutablePath();
|
||||||
// 使用独立的用户数据目录,避免与 Chrome 冲突
|
// 使用独立的用户数据目录,避免与 Chrome 冲突
|
||||||
const userDataDir = opts?.userDataDir || path.join(process.cwd(), 'browser-context');
|
const userDataDir = opts?.userDataDir || path.join(process.cwd(), 'browser-context');
|
||||||
const debugPort = opts?.debugPort || 9223;
|
const debugPort = opts?.debugPort || 9223;
|
||||||
|
const headless = opts?.headless || false;
|
||||||
|
|
||||||
console.log('启动 Chrome...', executablePath);
|
console.log('启动 Chrome...', executablePath);
|
||||||
console.log(`端口: ${debugPort}`);
|
console.log(`端口: ${debugPort}`);
|
||||||
console.log(`用户数据目录: ${userDataDir}`);
|
console.log(`用户数据目录: ${userDataDir}`);
|
||||||
// console.log('注意:需要手动登录账号和安装插件');
|
console.log(`无头模式: ${headless}`);
|
||||||
|
const userAgent = new UserAgent().toString();
|
||||||
|
|
||||||
const params = [
|
const params = [
|
||||||
`--remote-debugging-port=${debugPort}`,
|
`--remote-debugging-port=${debugPort}`,
|
||||||
`--user-data-dir=${userDataDir}`,
|
`--user-data-dir=${userDataDir}`,
|
||||||
// '--kiosk', // 全屏模式,无修改边框
|
'--no-sandbox',
|
||||||
|
'--disable-setuid-sandbox',
|
||||||
|
'--disable-dev-shm-usage',
|
||||||
|
'--no-first-run',
|
||||||
|
`--user-agent=${userAgent}`,
|
||||||
];
|
];
|
||||||
|
|
||||||
|
if (headless) {
|
||||||
|
params.push(
|
||||||
|
'--headless',
|
||||||
|
'--disable-blink-features=AutomationControlled',
|
||||||
|
'--disable-infobars',
|
||||||
|
'--disable-features=IsolateOrigins,site-per-process',
|
||||||
|
'--disable-features=VizDisplayCompositor',
|
||||||
|
'--window-size=1920,1080',
|
||||||
|
'--disable-background-networking',
|
||||||
|
'--disable-background-timer-throttling',
|
||||||
|
'--disable-backgrounding-occluded-windows',
|
||||||
|
'--disable-renderer-backgrounding',
|
||||||
|
'--disable-component-extensions-with-background-pages',
|
||||||
|
'--disable-features=TranslateUI',
|
||||||
|
'--disable-ipc-flooding-protection',
|
||||||
|
);
|
||||||
|
}
|
||||||
console.log('启动参数:', params);
|
console.log('启动参数:', params);
|
||||||
if (opts?.kiosk) {
|
if (opts?.kiosk) {
|
||||||
params.push('--kiosk'); // 全屏模式,无修改边框
|
params.push('--kiosk'); // 全屏模式,无修改边框
|
||||||
@@ -62,13 +90,12 @@ export const main = async (opts?: {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 检查 Chrome 可执行文件是否存在
|
// 检查 Chrome 可执行文件是否存在,不存在则使用 Playwright 的浏览器
|
||||||
if (!fs.existsSync(executablePath)) {
|
if (!fs.existsSync(executablePath)) {
|
||||||
console.error('Chrome 可执行文件不存在:', executablePath);
|
console.log('Chrome 可执行文件不存在,使用 Playwright 的浏览器');
|
||||||
return;
|
executablePath = chromium.executablePath();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// 启动 Chrome(带远程调试端口)
|
// 启动 Chrome(带远程调试端口)
|
||||||
const chromeProcess = spawn(
|
const chromeProcess = spawn(
|
||||||
executablePath,
|
executablePath,
|
||||||
|
|||||||
@@ -27,13 +27,14 @@ export class Core<T = {}> {
|
|||||||
page: Page | null = null;
|
page: Page | null = null;
|
||||||
debugPort = 9223;
|
debugPort = 9223;
|
||||||
debugHost = '127.0.0.1';
|
debugHost = '127.0.0.1';
|
||||||
|
headless = false;
|
||||||
status: 'disconnected' | 'connecting' | 'connected' | 'failed' = 'disconnected';
|
status: 'disconnected' | 'connecting' | 'connected' | 'failed' = 'disconnected';
|
||||||
emitter = new EventEmitter();
|
emitter = new EventEmitter();
|
||||||
listeners: Listener[] = [];
|
listeners: Listener[] = [];
|
||||||
recordReady: boolean = false;
|
recordReady: boolean = false;
|
||||||
timer: NodeJS.Timeout | null = null;
|
timer: NodeJS.Timeout | null = null;
|
||||||
data: T | null = null;
|
data: T | null = null;
|
||||||
constructor(opts?: { debugPort?: number, debugHost?: string, listeners?: Listener[] }) {
|
constructor(opts?: { debugPort?: number, debugHost?: string, listeners?: Listener[], headless?: boolean }) {
|
||||||
if (opts?.debugPort) {
|
if (opts?.debugPort) {
|
||||||
this.debugPort = opts.debugPort;
|
this.debugPort = opts.debugPort;
|
||||||
}
|
}
|
||||||
@@ -43,9 +44,12 @@ export class Core<T = {}> {
|
|||||||
if (opts?.listeners) {
|
if (opts?.listeners) {
|
||||||
this.listeners = opts.listeners;
|
this.listeners = opts.listeners;
|
||||||
}
|
}
|
||||||
|
if (opts?.headless !== undefined) {
|
||||||
|
this.headless = opts.headless;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
async createBrowser() {
|
async createBrowser() {
|
||||||
await main({ debugPort: this.debugPort });
|
await main({ debugPort: this.debugPort, headless: this.headless });
|
||||||
}
|
}
|
||||||
async init() {
|
async init() {
|
||||||
const debugPort = this.debugPort;
|
const debugPort = this.debugPort;
|
||||||
@@ -59,6 +63,9 @@ export class Core<T = {}> {
|
|||||||
this.browserContext = browser.contexts()[0];
|
this.browserContext = browser.contexts()[0];
|
||||||
this.handleRequest(this.browserContext);
|
this.handleRequest(this.browserContext);
|
||||||
this.page = this.browserContext.pages()[0] || await this.browserContext.newPage();
|
this.page = this.browserContext.pages()[0] || await this.browserContext.newPage();
|
||||||
|
if (this.headless) {
|
||||||
|
await this.stealthMode(this.page);
|
||||||
|
}
|
||||||
this.emitter.emit('connected');
|
this.emitter.emit('connected');
|
||||||
return;
|
return;
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
@@ -147,6 +154,74 @@ export class Core<T = {}> {
|
|||||||
}
|
}
|
||||||
this.data = data;
|
this.data = data;
|
||||||
}
|
}
|
||||||
|
async stealthMode(page: Page) {
|
||||||
|
const stealthScript = `
|
||||||
|
() => {
|
||||||
|
Object.defineProperty(navigator, 'webdriver', {
|
||||||
|
get: () => undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
window.chrome = {
|
||||||
|
runtime: {},
|
||||||
|
};
|
||||||
|
|
||||||
|
Object.defineProperty(navigator, 'plugins', {
|
||||||
|
get: () => [1, 2, 3, 4, 5],
|
||||||
|
});
|
||||||
|
|
||||||
|
Object.defineProperty(navigator, 'languages', {
|
||||||
|
get: () => ['en-US', 'en'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const originalQuery = window.navigator.permissions.query;
|
||||||
|
window.navigator.permissions.query = (parameters) => (
|
||||||
|
parameters.name === 'notifications' ?
|
||||||
|
Promise.resolve({ state: Notification.permission }) :
|
||||||
|
originalQuery(parameters)
|
||||||
|
);
|
||||||
|
|
||||||
|
Object.defineProperty(navigator, 'hardwareConcurrency', {
|
||||||
|
get: () => 4,
|
||||||
|
});
|
||||||
|
|
||||||
|
Object.defineProperty(navigator, 'deviceMemory', {
|
||||||
|
get: () => 8,
|
||||||
|
});
|
||||||
|
|
||||||
|
const originalGetContext = HTMLCanvasElement.prototype.getContext;
|
||||||
|
HTMLCanvasElement.prototype.getContext = function(type) {
|
||||||
|
const context = originalGetContext.apply(this, arguments);
|
||||||
|
if (type === '2d' && context) {
|
||||||
|
const originalGetImageData = context.getImageData;
|
||||||
|
context.getImageData = function() {
|
||||||
|
const imageData = originalGetImageData.apply(this, arguments);
|
||||||
|
for (let i = 0; i < imageData.data.length; i += 4) {
|
||||||
|
imageData.data[i] = imageData.data[i] + Math.random() * 0.1 - 0.05;
|
||||||
|
}
|
||||||
|
return imageData;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return context;
|
||||||
|
};
|
||||||
|
|
||||||
|
Object.defineProperty(navigator, 'connection', {
|
||||||
|
get: () => ({
|
||||||
|
effectiveType: '4g',
|
||||||
|
rtt: 100,
|
||||||
|
downlink: 10,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
window.navigator.getBattery = () => Promise.resolve({
|
||||||
|
charging: true,
|
||||||
|
chargingTime: 0,
|
||||||
|
dischargingTime: Infinity,
|
||||||
|
level: 1,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
await page.addInitScript(stealthScript);
|
||||||
|
}
|
||||||
async handleRequest(context: BrowserContext) {
|
async handleRequest(context: BrowserContext) {
|
||||||
context.on('request', request => {
|
context.on('request', request => {
|
||||||
const url = request.url();
|
const url = request.url();
|
||||||
|
|||||||
@@ -10,13 +10,14 @@ app.route({
|
|||||||
icon: 'search',
|
icon: 'search',
|
||||||
}
|
}
|
||||||
}).define(async (ctx) => {
|
}).define(async (ctx) => {
|
||||||
const keyword = ctx.query?.keyword as string || '信息差';
|
const { keyword = '信息差', ...rest } = ctx.query;
|
||||||
const res = await app.run({
|
const res = await app.run({
|
||||||
path: 'xhs',
|
path: 'xhs',
|
||||||
key: 'search-notes',
|
key: 'search-notes',
|
||||||
payload: {
|
payload: {
|
||||||
keyword: keyword,
|
keyword: keyword,
|
||||||
scrollTimes: 5,
|
scrollTimes: 5,
|
||||||
|
...rest,
|
||||||
token: ctx.query?.token as string,
|
token: ctx.query?.token as string,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -34,13 +35,14 @@ app.route({
|
|||||||
icon: 'search',
|
icon: 'search',
|
||||||
}
|
}
|
||||||
}).define(async (ctx) => {
|
}).define(async (ctx) => {
|
||||||
const keyword = ctx.query?.keyword as string || '工作 杭州';
|
const { keyword = '工作 杭州', ...rest } = ctx.query;
|
||||||
const res = await app.run({
|
const res = await app.run({
|
||||||
path: 'xhs',
|
path: 'xhs',
|
||||||
key: 'search-notes',
|
key: 'search-notes',
|
||||||
payload: {
|
payload: {
|
||||||
keyword: keyword,
|
keyword: keyword,
|
||||||
scrollTimes: 5,
|
scrollTimes: 5,
|
||||||
|
...rest,
|
||||||
token: ctx.query?.token as string,
|
token: ctx.query?.token as string,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user