generated from tailored/router-template
temp
This commit is contained in:
@@ -1,3 +0,0 @@
|
||||
// https://edith.xiaohongshu.com/api/sns/web/unread_count
|
||||
export const XHS_GET_UNREAD = 'unread_count';
|
||||
export const XHS_QUEUE_NAME = 'XHS_QUEUE';
|
||||
@@ -0,0 +1,4 @@
|
||||
import { queue, taskApp, XHS_QUEUE_NAME } from './task.ts';
|
||||
import './routes/mention.ts';
|
||||
|
||||
export { queue, taskApp, XHS_QUEUE_NAME };
|
||||
|
||||
32
src/task/queue.ts
Normal file
32
src/task/queue.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import { redis } from '@/modules/redis.ts';
|
||||
import { Queue } from 'bullmq';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { addUnreadTask, XHS_QUEUE_NAME } from '@/task/task.ts';
|
||||
|
||||
export const queue = new Queue(XHS_QUEUE_NAME, {
|
||||
connection: redis,
|
||||
});
|
||||
|
||||
// 初始启动
|
||||
async function start() {
|
||||
addUnreadTask();
|
||||
}
|
||||
//
|
||||
start();
|
||||
const getTasks = async () => {
|
||||
const tasks = await queue.getJobs(['waiting', 'active', 'completed', 'failed']);
|
||||
return tasks;
|
||||
};
|
||||
const getTask = async (id: string) => {
|
||||
const task = await queue.getJob(id);
|
||||
return task;
|
||||
};
|
||||
const removeTask = async (id: string) => {
|
||||
const task = await queue.getJob(id);
|
||||
if (task) {
|
||||
await task.remove();
|
||||
}
|
||||
};
|
||||
|
||||
// const task = await getTask('4');
|
||||
// console.log('task', task);
|
||||
100
src/task/routes/mention.ts
Normal file
100
src/task/routes/mention.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
import { taskApp, queue, xhsApp } from '../task.ts';
|
||||
import { random, omit } from 'lodash-es';
|
||||
import util from 'node:util';
|
||||
|
||||
export const sleep = (ms: number) => {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
};
|
||||
|
||||
taskApp
|
||||
.route({
|
||||
path: 'task',
|
||||
key: 'getUnread',
|
||||
})
|
||||
.define(async (ctx) => {
|
||||
const res = await xhsApp.call({
|
||||
path: 'mention',
|
||||
key: 'getUnread',
|
||||
});
|
||||
if (res.code === 200) {
|
||||
const data = res.body;
|
||||
const unread_count = data.unread_count;
|
||||
console.log('unread_count====', data);
|
||||
if (unread_count > 0) {
|
||||
queue.add(
|
||||
'mention',
|
||||
{
|
||||
path: 'task',
|
||||
key: 'getMention',
|
||||
payload: {
|
||||
unread_count,
|
||||
},
|
||||
},
|
||||
{
|
||||
attempts: 3,
|
||||
delay: 0,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: {
|
||||
age: 24 * 3600, // keep up to 24 hours
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
ctx.body = {
|
||||
job: unread_count,
|
||||
};
|
||||
}
|
||||
})
|
||||
.addTo(taskApp);
|
||||
|
||||
taskApp
|
||||
.route({
|
||||
path: 'task',
|
||||
key: 'getMention',
|
||||
})
|
||||
.define(async (ctx) => {
|
||||
const { unread_count } = ctx.query;
|
||||
if (unread_count > 0) {
|
||||
const mentionRes = await xhsApp.call({
|
||||
path: 'mention',
|
||||
key: 'getMention',
|
||||
payload: {
|
||||
num: unread_count,
|
||||
},
|
||||
});
|
||||
console.log('mentionRes', mentionRes.body);
|
||||
if (mentionRes.code === 200) {
|
||||
let data = mentionRes.body || [];
|
||||
// data = data.map((item) => omit(item, 'mention'));
|
||||
console.log('queryMention', util.inspect(data, { depth: 10 }));
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const item = data[i];
|
||||
const note_id = item.note_id;
|
||||
const xsec_token = item.xsec_token;
|
||||
const comment_id = item.comment.comment_id;
|
||||
const content = item.comment?.content || 'test';
|
||||
const postData = {
|
||||
note_id,
|
||||
content,
|
||||
comment_id,
|
||||
};
|
||||
const res = await xhsApp.call({
|
||||
path: 'mention',
|
||||
key: 'addComment',
|
||||
payload: postData,
|
||||
});
|
||||
console.log('addComment', postData, 'res', res.body);
|
||||
}
|
||||
}
|
||||
const postRead = await xhsApp.call({
|
||||
path: 'mention',
|
||||
key: 'postRead',
|
||||
});
|
||||
console.log('postRead', postRead.body);
|
||||
}
|
||||
await sleep(1000);
|
||||
ctx.body = {
|
||||
job: unread_count,
|
||||
};
|
||||
})
|
||||
.addTo(taskApp);
|
||||
@@ -1,44 +1,38 @@
|
||||
// https://edith.xiaohongshu.com/api/sns/web/unread_count
|
||||
|
||||
import { QueryRouterServer } from '@kevisual/router';
|
||||
import { redis } from '@/modules/redis.ts';
|
||||
import { Queue } from 'bullmq';
|
||||
import { app as xhsApp } from '@kevisual/xhs/index';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { XHS_QUEUE_NAME } from '@/task/common.ts';
|
||||
export const XHS_GET_UNREAD = 'unread_count';
|
||||
export const XHS_QUEUE_NAME = 'XHS_QUEUE';
|
||||
|
||||
export const taskApp = new QueryRouterServer();
|
||||
export { xhsApp };
|
||||
export const queue = new Queue(XHS_QUEUE_NAME, {
|
||||
connection: redis,
|
||||
});
|
||||
|
||||
// 初始启动
|
||||
async function start() {
|
||||
const res = await queue.add(
|
||||
'start-job',
|
||||
export const addUnreadTask = async (nextTime = 0) => {
|
||||
const id = 'unread';
|
||||
const job = await queue.add(
|
||||
'unread',
|
||||
{
|
||||
name: 'initialJob',
|
||||
path: 'task',
|
||||
key: 'getUnread',
|
||||
},
|
||||
{
|
||||
delay: 0, // 立即执行
|
||||
delay: nextTime,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
jobId: nanoid(), // 使用 nanoid 生成唯一 ID
|
||||
removeOnFail: {
|
||||
age: 24 * 3600, // keep up to 24 hours
|
||||
},
|
||||
jobId: id,
|
||||
},
|
||||
);
|
||||
console.log('Queue started:', res.id);
|
||||
}
|
||||
//
|
||||
start();
|
||||
const getTasks = async () => {
|
||||
const tasks = await queue.getJobs(['waiting', 'active', 'completed', 'failed']);
|
||||
return tasks;
|
||||
return {
|
||||
id,
|
||||
job,
|
||||
};
|
||||
};
|
||||
const getTask = async (id: string) => {
|
||||
const task = await queue.getJob(id);
|
||||
return task;
|
||||
};
|
||||
const removeTask = async (id: string) => {
|
||||
const task = await queue.getJob(id);
|
||||
if (task) {
|
||||
await task.remove();
|
||||
}
|
||||
};
|
||||
|
||||
// const task = await getTask('4');
|
||||
// console.log('task', task);
|
||||
|
||||
@@ -1,52 +1,112 @@
|
||||
import { redis } from '@/modules/redis.ts';
|
||||
import { Queue, Worker } from 'bullmq';
|
||||
import { clamp } from 'lodash-es';
|
||||
import { Worker } from 'bullmq';
|
||||
import { add, clamp } from 'lodash-es';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
const XHS_QUEUE_NAME = 'XHS_QUEUE';
|
||||
export const queue = new Queue(XHS_QUEUE_NAME);
|
||||
import { queue, XHS_QUEUE_NAME, taskApp } from './index.ts';
|
||||
import { addUnreadTask } from './task.ts';
|
||||
import dayjs from 'dayjs';
|
||||
export const sleep = (ms: number) => {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
};
|
||||
class TimeRecorder {
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
duration: number;
|
||||
updateTime: number;
|
||||
maxDuration: number = 60 * 1000; // 20s;
|
||||
constructor() {
|
||||
const now = Date.now();
|
||||
this.startTime = now;
|
||||
this.endTime = now;
|
||||
this.updateTime = now;
|
||||
this.duration = 0;
|
||||
}
|
||||
start() {
|
||||
this.startTime = Date.now();
|
||||
return this.startTime;
|
||||
}
|
||||
end() {
|
||||
this.endTime = Date.now();
|
||||
this.duration = this.endTime - this.startTime;
|
||||
return this.endTime;
|
||||
}
|
||||
update() {
|
||||
this.updateTime = Date.now();
|
||||
return this.updateTime;
|
||||
}
|
||||
getClampDuration() {
|
||||
const duration = Date.now() - this.updateTime;
|
||||
return {
|
||||
duration: duration,
|
||||
maxDuration: this.maxDuration,
|
||||
updateTime: this.updateTime,
|
||||
nextTime: clamp(this.maxDuration - duration, 0, this.maxDuration),
|
||||
};
|
||||
}
|
||||
time() {
|
||||
return {
|
||||
startTime: this.startTime,
|
||||
endTime: this.endTime,
|
||||
duration: this.duration,
|
||||
updateTime: this.updateTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
const timeRecorder = new TimeRecorder();
|
||||
let errorCount = 0;
|
||||
export const worker = new Worker(
|
||||
XHS_QUEUE_NAME,
|
||||
async (job) => {
|
||||
const startTime = Date.now();
|
||||
console.log('job', job.name, job.data);
|
||||
await sleep(1000);
|
||||
const endTime = Date.now();
|
||||
const duration = endTime - startTime;
|
||||
const timer = new TimeRecorder();
|
||||
const data = job.data;
|
||||
if (data.path === 'task' && data.key === 'getUnread') {
|
||||
console.log('====run time', dayjs().format('YYYY-MM-DD HH:mm:ss'));
|
||||
timeRecorder.update();
|
||||
}
|
||||
const res = await taskApp.call(data);
|
||||
if (res.code !== 200) {
|
||||
console.log('job error', job.name, job.id, res);
|
||||
|
||||
return {
|
||||
startTime: startTime,
|
||||
endTime: endTime,
|
||||
duration: duration,
|
||||
};
|
||||
errorCount++;
|
||||
if (errorCount > 3) {
|
||||
queue.pause();
|
||||
console.log('error count', errorCount);
|
||||
}
|
||||
throw new Error('job error' + job.name + ' ' + job.id);
|
||||
}
|
||||
errorCount = 0;
|
||||
timer.end();
|
||||
return timer.time();
|
||||
},
|
||||
{ connection: redis, concurrency: 1 },
|
||||
{ connection: redis, concurrency: 1, limiter: { max: 1, duration: 2000 } },
|
||||
);
|
||||
worker.on('completed', async (job) => {
|
||||
console.log('job completed', job.name, job.id, job.returnvalue);
|
||||
const duration = job.returnvalue.duration || 0;
|
||||
const maxNextTime = 20 * 1000; // 5 minutes
|
||||
const nextTime = clamp(maxNextTime - duration, 0, maxNextTime);
|
||||
const hasJobs = await queue.getJobCounts('waiting', 'wait', 'delayed');
|
||||
console.log('hasJobs', nextTime, 'joblenght', hasJobs);
|
||||
if (hasJobs.delayed + hasJobs.wait > 0) {
|
||||
console.log('======has jobs, no need to add new job');
|
||||
const jobCounts = await queue.getJobCounts('waiting', 'wait', 'delayed');
|
||||
if (job.name !== 'unread') {
|
||||
console.log('job completed', job.name, job.id, job.returnvalue, jobCounts.delayed, jobCounts.wait);
|
||||
}
|
||||
if (jobCounts.delayed + jobCounts.wait > 0) {
|
||||
// console.log('======has jobs, no need to add new job');
|
||||
} else {
|
||||
const id = nanoid();
|
||||
queue.add(
|
||||
'repeact-call-job' + id,
|
||||
{},
|
||||
{
|
||||
delay: nextTime,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: {
|
||||
age: 24 * 3600, // keep up to 24 hours
|
||||
},
|
||||
jobId: id,
|
||||
},
|
||||
);
|
||||
const up = timeRecorder.getClampDuration();
|
||||
const nextTime = up.nextTime;
|
||||
const unread = await queue.getJob('unread');
|
||||
if (!unread) {
|
||||
addUnreadTask(nextTime);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const init = async () => {
|
||||
const jobCounts = await queue.getJobCounts('waiting', 'wait', 'delayed');
|
||||
if (jobCounts.delayed + jobCounts.wait > 0) {
|
||||
// console.log('======has jobs, no need to add new job');
|
||||
} else {
|
||||
const unread = await queue.getJob('unread');
|
||||
if (!unread) {
|
||||
addUnreadTask(0);
|
||||
timeRecorder.update();
|
||||
}
|
||||
}
|
||||
};
|
||||
init();
|
||||
|
||||
Reference in New Issue
Block a user