feat: initialize project structure with essential files and configurations

- Add .gitignore to exclude unnecessary files and directories
- Create .npmrc for npm authentication
- Add AGENTS.md for project documentation
- Initialize package.json with project metadata and dependencies
- Implement app.ts to set up the application and project manager
- Create file-search module for searching files in a directory
- Set up project manager and listener for managing project files
- Implement project search functionality with MeiliSearch integration
- Add routes for authentication and project management
- Create scheduler for task management
- Add tests for file searching and project management functionalities
This commit is contained in:
xiongxiao
2026-03-13 17:22:14 +08:00
committed by cnb
commit 1b131b3961
27 changed files with 1336 additions and 0 deletions

14
.gitignore vendored Normal file
View File

@@ -0,0 +1,14 @@
node_modules
.DS_Store
dist
pack-dist
.env
!.env*example
libs
cache-file

2
.npmrc Normal file
View File

@@ -0,0 +1,2 @@
//npm.xiongxiao.me/:_authToken=${ME_NPM_TOKEN}
//registry.npmjs.org/:_authToken=${NPM_TOKEN}

1
AGENTS.md Normal file
View File

@@ -0,0 +1 @@
# AGENTS.md

4
bun.config.ts Normal file
View File

@@ -0,0 +1,4 @@
import { buildWithBun } from '@kevisual/code-builder';
await buildWithBun({ naming: 'app', entry: 'src/index.ts', dts: true, clean: true });

31
package.json Normal file
View File

@@ -0,0 +1,31 @@
{
"name": "@kevisual/project-search",
"version": "0.0.1",
"description": "",
"main": "index.js",
"scripts": {
"dev": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "abearxiong <xiongxiao@xiongxiao.me> (https://www.xiongxiao.me)",
"license": "MIT",
"packageManager": "pnpm@10.32.1",
"type": "module",
"devDependencies": {
"@kevisual/code-builder": "^0.0.6",
"@kevisual/context": "^0.0.8",
"@kevisual/remote-app": "^0.0.7",
"@kevisual/router": "^0.1.1",
"fast-glob": "^3.3.3",
"meilisearch": "^0.55.0",
"zod": "^4.3.6"
},
"dependencies": {
"@parcel/watcher": "^2.5.6",
"es-toolkit": "^1.45.1",
"eventemitter3": "^5.0.4"
},
"exports": {
".": "./dist/index.js"
}
}

10
readme.md Normal file
View File

@@ -0,0 +1,10 @@
# 对文件项目快速搜索展示
## 介绍
比如文件夹 `/workspace/projects/project-search` 有很多文件监听修改后同步到meilisearh中。增删改查同步。
1. 获取文件夹下的.gitignore文件解析出需要忽略的文件列表。
2. 监听文件夹下的文件修改事件过滤掉需要忽略的文件将修改的文件同步到meilisearch中。
3. 需要提供一个scheduler任务调度同步到meilisearch的任务中如果存在同一个任务还没有被执行新的任务就不需要被添加到队列中。
4. 任务调度是一个队列任务列,先进先出,单线程执行。每次执行一个任务,执行完后再执行下一个任务。

12
src/app.ts Normal file
View File

@@ -0,0 +1,12 @@
import { App } from '@kevisual/router';
import { ProjectManager } from './project/manager';
import { useKey, useContextKey } from '@kevisual/context';
export const app = useContextKey<App>('app', new App());
export const manager = useContextKey<ProjectManager>('project-manager', new ProjectManager({
meiliSearchOptions: {
apiKey: useKey('CNB_TOKEN')
}
}));

30
src/file-search/index.ts Normal file
View File

@@ -0,0 +1,30 @@
import fg from 'fast-glob';
import path from 'node:path';
import fs from 'node:fs';
const defaultIgnore = ['node_modules', 'dist', 'pack-dist', 'build', 'out', 'coverage', '.git', '.svn', '.hg',
'.next',
'.astro',
".swc"
];
export class FileSearch {
async searchFiles(options: { cwd?: string; ignore?: string[] } = {}) {
const { cwd = process.cwd(), ignore = [] } = options;
const ignoreFile = path.join(cwd, '.gitignore');
const gitIgnorePatterns = fs.existsSync(ignoreFile)
? fs.readFileSync(ignoreFile, 'utf-8')
.split('\n')
.map(line => line.trim())
.filter(line => line && !line.startsWith('#'))
: [];
const allIgnore = [...defaultIgnore, ...ignore, ...gitIgnorePatterns];
const files = await fg('**', {
cwd,
ignore: allIgnore,
dot: true,
onlyFiles: true,
});
return files;
}
}

15
src/index.ts Normal file
View File

@@ -0,0 +1,15 @@
export { app, manager } from './app';
export { ProjectManager } from './project/manager';
export { ProjectSearch } from './project/project-search/index';
export { ProjectListener } from './project/project-listener/listener';
export { FileSearch } from './file-search/index';
export { Scheduler } from './scheduler/index';
import './routes/auth';
import './routes/project';
import './routes/search';
import './routes/file';
if (import.meta.main) {
//
}

176
src/project/manager.ts Normal file
View File

@@ -0,0 +1,176 @@
import { ProjectManagerInterface } from "./user-interface";
import { ProjectSearch } from "./project-search/index";
import { ProjectStore } from "./project-store";
import { ProjectListener } from "./project-listener/listener";
import { EventEmitter } from "eventemitter3";
import fs from 'node:fs';
import { CustomError } from "@kevisual/router";
type Project = {
name?: string;
path: string;
/**
* 用 git remote url 唯一标识项目,方便搜索结果展示和过滤
*/
repo: string;
listener: ProjectListener;
};
export type ProjectInput = Omit<Project, "listener">;
export type ProjectInfo = Omit<Project, "listener"> & { status: "active" | "inactive" };
type ProjectManagerOpt = {
meiliSearchOptions?: {
apiHost?: string;
apiKey?: string;
}
}
export class ProjectManager implements ProjectManagerInterface {
projects = new Map<string, Project>();
projectSearch: ProjectSearch;
projectStore: ProjectStore;
emitter: EventEmitter;
constructor(options: ProjectManagerOpt = {}) {
const projectSearch = new ProjectSearch(options.meiliSearchOptions);
const projectStore = new ProjectStore(options.meiliSearchOptions);
const emitter = new EventEmitter();
this.projectSearch = projectSearch;
this.projectStore = projectStore;
this.emitter = emitter;
const that = this;
emitter.on('file-change', ({ type, path: filePath, projectPath }) => {
console.log('[ProjectManager] file-change:', type, filePath);
const project = that.projects.get(projectPath);
if (!project) return;
projectSearch.syncFileChange({ type, path: filePath, projectPath, repo: project.repo });
});
}
/**
* 初始化:从 store 加载已持久化的项目,检查本地目录存在后启动监听
*/
async init(): Promise<void> {
await this.projectStore.init();
const docs = await this.projectStore.listProjects();
for (const doc of docs) {
if (!fs.existsSync(doc.path)) {
console.log(`[ProjectManager] init: skip missing path ${doc.path}`);
continue;
}
try {
await this.addProject({ name: doc.name, path: doc.path, repo: doc.repo });
} catch (err) {
console.error(`[ProjectManager] init: failed to add project ${doc.path}`, err);
}
}
console.log(`[ProjectManager] init done, loaded ${this.projects.size} project(s)`);
}
/**
* 添加项目:创建监听器、启动监听、触发全量初始同步
*/
async addProject(input: ProjectInput): Promise<ProjectInfo> {
// 检查本地目录是否存在
if (!fs.existsSync(input.path)) {
throw new CustomError(`[ProjectManager] addProject: path does not exist: ${input.path}`);
}
// 若已存在则先停止旧监听
if (this.projects.has(input.path)) {
await this.stopProject(input.path);
}
const listener = new ProjectListener({ projectPath: input.path, emitter: this.emitter });
const project: Project = { ...input, listener };
this.projects.set(input.path, project);
// 持久化存储项目信息
this.projectStore.ensureIndex().then(() =>
this.projectStore.addProject({ name: input.name, path: input.path, repo: input.repo })
).catch(err => {
console.error('[ProjectManager] projectStore.addProject failed:', err);
});
// 启动文件监听
await listener.startListening();
console.log(`[ProjectManager] watching: ${input.path}`);
// 触发全量初始同步(不阻塞返回)
this.projectSearch.initialSync(input.path, input.repo).catch(err => {
console.error('[ProjectManager] initialSync failed:', err);
});
return this.getProjectInfo(project);
}
async removeProject(projectPath: string): Promise<void> {
const project = this.projects.get(projectPath);
if (!project) return;
await project.listener.stopListening();
this.projects.delete(projectPath);
// 从持久化存储中删除项目及其所有索引文件
this.projectStore.removeProject(projectPath).catch(err => {
console.error('[ProjectManager] projectStore.removeProject failed:', err);
});
this.projectSearch.removeProjectFiles(projectPath).catch(err => {
console.error('[ProjectManager] projectSearch.removeProjectFiles failed:', err);
});
console.log(`[ProjectManager] removed: ${projectPath}`);
}
getProject(projectPath: string) {
return this.projects.get(projectPath);
}
/** 停止项目文件监听,但不删除索引数据 */
async stopProject(projectPath: string): Promise<void> {
const project = this.projects.get(projectPath);
if (!project) return;
await project.listener.stopListening();
console.log(`[ProjectManager] stopped: ${projectPath}`);
}
getProjectInfo(project: Project): ProjectInfo {
const { listener, ...info } = project;
return {
...info,
status: listener.isWatching ? "active" : "inactive",
};
}
listProjects(): ProjectInfo[] {
return Array.from(this.projects.values()).map(p => this.getProjectInfo(p));
}
async getFile(filepath: string): Promise<{ filepath: string, content: string; type: string } | null> {
if (!fileIsExist(filepath)) {
return null;
}
const read = fs.readFileSync(filepath);
const base64 = read.toString('base64');
return {
filepath,
content: base64,
type: 'base64',
}
}
async deleteFile(filepath: string): Promise<boolean> {
if (!fileIsExist(filepath)) {
return false;
}
fs.unlinkSync(filepath);
return true;
}
}
const fileIsExist = (filePath: string): boolean => {
try {
fs.accessSync(filePath, fs.constants.F_OK);
return true;
} catch (err) {
return false;
}
};

View File

@@ -0,0 +1,90 @@
import ParcelWatcher, { subscribe } from '@parcel/watcher';
import path from 'node:path';
import fs from 'node:fs';
import { EventEmitter } from "eventemitter3";
const defaultIgnore = ['node_modules', 'dist', 'pack-dist', 'build', 'out', 'coverage', '.git', '.svn', '.hg',
'.next',
'.astro',
".swc"
];
export class ProjectListener {
projectPath: string;
subscribe: ParcelWatcher.AsyncSubscription | null = null;
isWatching = false;
emitter: EventEmitter;
private updateDebounceMap: Map<string, ReturnType<typeof setTimeout>> = new Map();
constructor(options: { projectPath: string, emitter: EventEmitter }) {
this.projectPath = options.projectPath;
this.emitter = options.emitter;
}
async startListening() {
if (this.isWatching) return;
const projectPath = this.projectPath;
const ignoreFile = path.join(projectPath, '.gitignore');
let ignorePatterns: string[] = [];
if (fs.existsSync(ignoreFile)) {
ignorePatterns = fs.readFileSync(ignoreFile, 'utf-8')
.split('\n')
.map(line => line.trim())
.filter(line => line && !line.startsWith('#'));
}
const allIgnore = [...defaultIgnore, ...ignorePatterns];
const sub = await subscribe(
this.projectPath, // 监听路径(支持数组)
async (err, events) => {
if (err) throw err;
for (const event of events) {
// 跳过非文件事件delete 时文件不存在,直接放行)
if (event.type !== 'delete') {
try {
if (!fs.statSync(event.path).isFile()) continue;
} catch {
continue;
}
}
// event: { type: 'create' | 'update' | 'delete', path: '/path/to/file.ts' }
if (event.type === 'update') {
const existing = this.updateDebounceMap.get(event.path);
if (existing) clearTimeout(existing);
const timer = setTimeout(() => {
this.updateDebounceMap.delete(event.path);
this.emitter.emit('file-change', {
type: event.type,
path: event.path,
projectPath: this.projectPath
});
}, 100);
this.updateDebounceMap.set(event.path, timer);
} else {
this.emitter.emit('file-change', {
// 事件类型:'create', 'update', 'delete'...
type: event.type,
// 具体文件路径
path: event.path,
projectPath: this.projectPath
});
}
}
},
{
ignore: allIgnore.map(pattern => `**/${pattern}/**`),
backend: 'watchman', // 可选:'fs-events'(默认)、'inotify'、'windows'、'watchman'
}
);
this.subscribe = sub;
this.isWatching = true;
}
async stopListening() {
if (this.subscribe) {
await this.subscribe.unsubscribe();
this.subscribe = null;
}
for (const timer of this.updateDebounceMap.values()) {
clearTimeout(timer);
}
this.updateDebounceMap.clear();
this.isWatching = false;
}
}

View File

@@ -0,0 +1,43 @@
// 后缀是可以展示为纯文本的文件
import path from 'node:path';
// 常见的文本文件扩展名
const TEXT_FILE_EXTENSIONS = [
// 前端/脚本
'.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx', '.vue', '.svelte', '.astro',
// node 相关
'.npmrc', '.yarnrc', '.babelrc', '.eslintrc', '.prettierrc', '.stylelintrc',
// 样式
'.css', '.scss', '.sass', '.less', '.styl', '.postcss',
// 后端/后端语言
'.java', '.py', '.rb', '.go', '.rs', '.php', '.cs', '.c', '.cpp', '.h', '.hpp', '.kt', '.scala', '.swift', '.m', '.mm', '.dart', '.lua', '.r',
// 数据/配置文件
'.json', '.yaml', '.yml', '.xml', '.toml', '.ini', '.env', '.properties', '.conf', '.config',
// 文档
'.md', '.txt', '.rst', '.adoc', '.tex', '.org',
// Shell/脚本
'.sh', '.bash', '.zsh', '.fish', '.ps1', '.bat', '.cmd', '.psm1',
// 其他
'.sql', '.graphql', '.gql', '.proto', '.dockerfile', '.gitignore', '.gitattributes', '.editorconfig', '.eslintrc', '.prettierrc', '.prettierignore', '.npmrc', '.yarnrc', '.babelrc', '.webpackrc',
];
// 没有扩展名的配置文件
const CONFIG_FILE_NAMES = [
'.gitignore', '.gitattributes', '.dockerignore', '.editorconfig', '.env', '.env.local', '.env.development', '.env.production',
'Dockerfile', 'Makefile', 'CMakeLists.txt', 'Vagrantfile', 'Gemfile', 'Podfile', 'Cartfile',
'.bashrc', '.zshrc', '.vimrc', '.inputrc',
];
export const isText = (filePath: string): boolean => {
const fileName = path.basename(filePath);
const ext = path.extname(filePath).toLowerCase();
// 检查扩展名
if (TEXT_FILE_EXTENSIONS.includes(ext)) {
return true;
}
// 检查没有扩展名的配置文件
if (CONFIG_FILE_NAMES.includes(fileName)) {
return true;
}
return false;
}

View File

@@ -0,0 +1,311 @@
import { MeiliSearch } from 'meilisearch';
import { isText } from "./file-list-content";
import crypto from 'node:crypto';
import fs from 'node:fs';
import fg from 'fast-glob';
import path from 'node:path';
import { Scheduler } from '../../scheduler/index';
export type FileProjectData = {
/**
* meilisearch 文档 ID文件绝对路径的 base64url 编码,无碰撞且唯一)
*/
id: string;
/**
* 文件路径的 md5 hash用于快速索引/查找
*/
hash: string;
/**
* 文件对应相关信息
*
* 具体文件路径
*/
filepath: string;
content?: string;
lastModified: number;
size: number;
/**
* 项目路径,文件所在的项目路径,方便搜索结果展示和过滤
*/
projectPath: string;
repo: string;
/**
* 人工编辑的字段,方便搜索结果展示和过滤
*/
title?: string;
tags?: string[];
summary?: string;
description?: string;
link?: string;
};
/** 人工编辑的字段update 时不覆盖 */
const MANUAL_FIELDS: (keyof FileProjectData)[] = ['title', 'tags', 'summary', 'description', 'link'];
const INDEX_NAME = 'project-files';
/** 根据文件绝对路径生成稳定的文档 IDbase64url 编码,无碰撞) */
function fileId(filePath: string): string {
return Buffer.from(filePath).toString('base64url');
}
/** 根据文件绝对路径生成 md5 hash */
function fileHash(filePath: string): string {
return crypto.createHash('md5').update(filePath).digest('hex');
}
export class ProjectSearch {
meiliSearch: MeiliSearch;
scheduler: Scheduler;
constructor({ apiHost, apiKey }: { apiHost?: string; apiKey?: string } = {}) {
this.meiliSearch = new MeiliSearch({
host: apiHost || 'http://localhost:7700',
apiKey: apiKey || 'masterKey',
});
this.scheduler = new Scheduler();
}
private get index() {
return this.meiliSearch.index(INDEX_NAME);
}
/** 全文搜索 */
async searchFiles(query: string = '', options?: {
projectPath?: string;
repo?: string;
title?: string;
tags?: string | string[];
summary?: string;
description?: string;
link?: string;
/**
* ['projectPath:asc']
*/
sort?: string[],
limit?: number,
getContent?: boolean;
}) {
const filter: string[] = [];
if (options?.projectPath) filter.push(`projectPath = "${options.projectPath}"`);
if (options?.repo) filter.push(`repo = "${options.repo}"`);
if (options?.title) filter.push(`title = "${options.title}"`);
if (options?.tags) {
const tags = Array.isArray(options.tags) ? options.tags : [options.tags];
tags.forEach(tag => filter.push(`tags = "${tag}"`));
}
if (options?.summary) filter.push(`summary = "${options.summary}"`);
if (options?.description) filter.push(`description = "${options.description}"`);
if (options?.link) filter.push(`link = "${options.link}"`);
const limit = options?.limit ?? 1000;
const search = {
filter: filter.length ? filter.join(' AND ') : undefined,
sort: options?.sort ?? ['projectPath:asc'],
}
let hits: FileProjectData[] = [];
if (limit > 1000) {
// meilisearch 单次查询限制 1000 条,超过则使用游标分页查询并合并结果
let allHits: FileProjectData[] = [];
let offset = 0;
while (true) {
const searchResults = await this.index.search(query, {
...search,
limit: Math.min(limit - allHits.length, 1000),
offset,
});
allHits = allHits.concat(searchResults.hits as FileProjectData[]);
if (allHits.length >= limit || searchResults.hits.length < 1000) {
break;
}
offset += 1000;
}
hits = allHits;
} else {
const searchResults = await this.index.search(query, {
...search,
limit: options?.limit ?? 1000,
});
hits = searchResults.hits as FileProjectData[];
}
if (!options?.getContent) {
hits = hits.map(({ content, ...rest }) => rest);
}
return hits;
}
/** 初始化索引(设置可过滤字段) */
async ensureIndex() {
try {
await this.meiliSearch.createIndex(INDEX_NAME, { primaryKey: 'id' });
} catch (_) {
// index 已存在则忽略
}
const t1 = await this.index.updateFilterableAttributes(['projectPath', 'repo', 'filepath', 'hash', 'title', 'tags', 'summary', 'description', 'link']);
const t2 = await this.index.updateSearchableAttributes(['content', 'filepath', 'title', 'summary', 'description', 'tags']);
const t3 = await this.index.updateSortableAttributes(['projectPath', 'filepath', 'lastModified', 'size']);
// 等待任务完成,确保过滤字段生效后才能使用
await this.meiliSearch.tasks.waitForTasks([t1.taskUid, t2.taskUid, t3.taskUid]);
}
/** 添加或完整覆盖一个文件文档 */
private async _upsertFile(filePath: string, projectPath: string, repo: string, existingManual?: Partial<FileProjectData>) {
let stat: fs.Stats;
try {
stat = fs.statSync(filePath);
} catch {
// 文件可能刚被删除
return;
}
// 文本文件才读取内容
let content: string | undefined;
if (isText(filePath)) {
try {
content = fs.readFileSync(filePath, 'utf-8');
} catch {
// 读取失败忽略content 保持 undefined
}
}
const newHash = fileHash(filePath);
// hash 未变化且已有文档,且无人工编辑字段,则跳过更新
if (existingManual?.hash === newHash && !MANUAL_FIELDS.some(field => existingManual[field] !== undefined)) {
return;
}
console.log(`[ProjectSearch] updated: ${filePath}`);
const doc: FileProjectData = {
id: fileId(filePath),
hash: newHash,
// 保留人工编辑字段
...existingManual,
filepath: filePath,
...(content !== undefined ? { content } : {}),
lastModified: stat.mtimeMs,
size: stat.size,
projectPath,
repo,
};
await this.index.addDocuments([doc]);
}
/** 更新文档的人工编辑字段title/tags/summary/description/link不影响文件内容等其他字段 */
async updateDocument(filePath: string, opts: {
title?: string;
tags?: string[];
summary?: string;
description?: string;
link?: string;
}) {
const partial: Record<string, any> = { id: fileId(filePath) };
if (opts.title !== undefined) partial.title = opts.title;
if (opts.tags !== undefined) partial.tags = opts.tags;
if (opts.summary !== undefined) partial.summary = opts.summary;
if (opts.description !== undefined) partial.description = opts.description;
if (opts.link !== undefined) partial.link = opts.link;
await this.index.updateDocuments([partial]);
}
/** 删除某个项目下所有已索引的文件文档 */
async removeProjectFiles(projectPath: string): Promise<void> {
await this.index.deleteDocuments({ filter: `projectPath = "${projectPath}"` });
}
/** 将文件变更通过 scheduler 安全入队 */
syncFileChange({ type, path: filePath, projectPath, repo }: { type: 'create' | 'update' | 'delete'; path: string; projectPath: string; repo: string }) {
if (!filePath) {
console.warn('[ProjectSearch] syncFileChange: filePath is undefined, skipping');
return;
}
this.scheduler.add(filePath, async () => {
if (type === 'delete') {
try {
await this.index.deleteDocument(fileId(filePath));
console.log(`[ProjectSearch] deleted: ${filePath}`);
} catch (err) {
console.error(`[ProjectSearch] delete failed: ${filePath}`, err);
}
return;
}
if (type === 'update') {
// 获取已有文档,保留人工编辑字段
let existingManual: Partial<FileProjectData> = {};
try {
const existing = await this.index.getDocument(fileId(filePath)) as FileProjectData;
for (const field of MANUAL_FIELDS) {
if (existing[field] !== undefined) {
(existingManual as any)[field] = existing[field];
}
}
existingManual.hash = existing.hash; // 保留 hash 字段,避免重复更新
} catch {
// 文档不存在,忽略
}
await this._upsertFile(filePath, projectPath, repo, existingManual);
return;
}
// create
await this._upsertFile(filePath, projectPath, repo);
console.log(`[ProjectSearch] created: ${filePath}`);
});
}
/**
* 全量同步整个项目目录到 meilisearch
* 1. 扫描本地文件,逐一 upsert
* 2. 对比 meilisearch 中已有文档,删除本地已不存在的文件记录
*/
async initialSync(projectPath: string, repo: string) {
await this.ensureIndex();
const ignoreFile = path.join(projectPath, '.gitignore');
const gitIgnore = fs.existsSync(ignoreFile)
? fs.readFileSync(ignoreFile, 'utf-8')
.split('\n')
.map(l => l.trim())
.filter(l => l && !l.startsWith('#'))
: [];
const defaultIgnore = ['node_modules', 'dist', 'pack-dist', 'build', 'out', 'coverage', '.git', '.svn', '.hg', '.next', '.astro', '.swc'];
const allIgnore = [...defaultIgnore, ...gitIgnore];
const files = await fg('**', {
cwd: projectPath,
ignore: allIgnore,
dot: true,
onlyFiles: true,
absolute: true,
});
const localFileSet = new Set(files);
console.log(`[ProjectSearch] initialSync: ${files.length} files in ${projectPath}`);
// 1. 从 meilisearch 拉取该项目所有已有文档,找出本地不存在的,加入 scheduler 删除
try {
const PAGE_SIZE = 1000;
let offset = 0;
while (true) {
const result = await this.index.getDocuments<FileProjectData>({
filter: `projectPath = "${projectPath}"`,
fields: ['id', 'filepath'],
limit: PAGE_SIZE,
offset,
});
for (const doc of result.results) {
if (!doc.filepath) continue; // 跳过 filepath 缺失的损坏文档
if (!localFileSet.has(doc.filepath)) {
this.syncFileChange({ type: 'delete', path: doc.filepath, projectPath, repo });
}
}
offset += PAGE_SIZE;
if (result.results.length < PAGE_SIZE) break;
}
} catch (err) {
console.error('[ProjectSearch] initialSync: failed to clean stale documents', err);
}
// 2. upsert 所有本地文件
for (const filePath of files) {
this.syncFileChange({ type: 'update', path: filePath, projectPath, repo });
}
}
}

View File

@@ -0,0 +1,124 @@
import { MeiliSearch } from 'meilisearch';
const PROJECT_INDEX_NAME = 'projects';
export type ProjectDoc = {
/**
* 文档 ID使用 path 的 base64url 编码
*/
id: string;
name?: string;
path: string;
/**
* 用 git remote url 唯一标识项目
*/
repo: string;
/**
* 人工编辑的字段,方便搜索结果展示和过滤
*/
title?: string;
tags?: string[];
summary?: string;
description?: string;
link?: string;
/**
* 项目状态active 活动中 | inactive 非活动
*/
status?: 'active' | 'inactive';
};
function projectId(projectPath: string): string {
return Buffer.from(projectPath).toString('base64url');
}
export class ProjectStore {
meiliSearch: MeiliSearch;
constructor({ apiHost, apiKey }: { apiHost?: string; apiKey?: string } = {}) {
this.meiliSearch = new MeiliSearch({
host: apiHost || 'http://localhost:7700',
apiKey: apiKey || 'masterKey',
});
}
private get index() {
return this.meiliSearch.index(PROJECT_INDEX_NAME);
}
async ensureIndex() {
try {
await this.meiliSearch.createIndex(PROJECT_INDEX_NAME, { primaryKey: 'id' });
} catch (_) {
// index 已存在则忽略
}
const t1 = await this.index.updateFilterableAttributes(['path', 'repo', 'name', 'status']);
const t2 = await this.index.updateSortableAttributes(['path', 'name']);
await this.meiliSearch.tasks.waitForTasks([t1.taskUid, t2.taskUid]);
}
async init() {
await this.ensureIndex();
}
async addProject(input: { name?: string; path: string; repo: string; status?: 'active' | 'inactive' }): Promise<void> {
const id = projectId(input.path);
// 先查询是否已存在
let existingDoc: ProjectDoc | null = null;
try {
existingDoc = await this.index.getDocument<ProjectDoc>(id);
} catch {
// 不存在则忽略
}
// 如果存在,合并文档:只更新 name, path, repo保留其他字段
const doc: ProjectDoc = {
id,
name: input.name,
path: input.path,
repo: input.repo,
// 保留原有字段
title: existingDoc?.title,
tags: existingDoc?.tags,
summary: existingDoc?.summary,
description: existingDoc?.description,
link: existingDoc?.link,
status: input.status ?? existingDoc?.status ?? 'active',
};
// MeiliSearch 的 addDocuments 本身就是 upsert 行为
await this.index.addDocuments([doc]);
}
async removeProject(projectPath: string): Promise<void> {
await this.index.deleteDocument(projectId(projectPath));
}
async listProjects(): Promise<ProjectDoc[]> {
const result = await this.index.getDocuments<ProjectDoc>({
limit: 1000,
sort: ['path:asc'],
});
return result.results;
}
async getProject(projectPath: string): Promise<ProjectDoc | null> {
try {
return await this.index.getDocument<ProjectDoc>(projectId(projectPath));
} catch {
return null;
}
}
async updateProject(projectPath: string, opts: { name?: string; repo?: string; title?: string; tags?: string[]; summary?: string; description?: string; link?: string; status?: 'active' | 'inactive' }): Promise<void> {
const partial: Record<string, any> = { id: projectId(projectPath) };
if (opts.name !== undefined) partial.name = opts.name;
if (opts.repo !== undefined) partial.repo = opts.repo;
if (opts.title !== undefined) partial.title = opts.title;
if (opts.tags !== undefined) partial.tags = opts.tags;
if (opts.summary !== undefined) partial.summary = opts.summary;
if (opts.description !== undefined) partial.description = opts.description;
if (opts.link !== undefined) partial.link = opts.link;
if (opts.status !== undefined) partial.status = opts.status;
await this.index.updateDocuments([partial]);
}
}

View File

@@ -0,0 +1,6 @@
export interface ProjectManagerInterface {
removeProject(path: string): void;
getProject(path: string): any;
listProjects(): any[];
addProject(project: any): void;
}

16
src/project/util/git.ts Normal file
View File

@@ -0,0 +1,16 @@
import { execSync } from 'node:child_process'
export const getGitPathname = (repoPath: string): string | null => {
try {
const url = execSync('git config --get remote.origin.url', { cwd: repoPath, encoding: 'utf-8' }).trim();
if (url) {
const _url = new URL(url.replace(/\.git$/, ''));
const pathname = _url.pathname.replace(/^\/+/, '').replace(/\/+$/, ''); // 去除开头和结尾的斜杠
return pathname;
}
return null;
} catch (err) {
console.error(`Failed to get remote URL for repo at ${repoPath}:`, err);
return null;
}
};

17
src/routes/auth.ts Normal file
View File

@@ -0,0 +1,17 @@
import { app } from '../app.ts';
app.route({
path: 'auth',
key: 'auth',
id: 'auth',
description: '用户身份验证中间件,校验请求令牌的合法性,验证通过后方可访问受保护接口',
}).define(async (ctx) => { }).addTo(app, { overwrite: false });
app.route({
path: 'auth-admin',
key: 'auth-admin',
id: 'auth-admin',
description: '管理员权限验证中间件,在 auth 基础上进一步校验请求方是否具有管理员权限',
}).define(async (ctx) => {
}).addTo(app, { overwrite: false });

81
src/routes/file.ts Normal file
View File

@@ -0,0 +1,81 @@
import { app, manager } from '../app.ts';
import { z } from 'zod'
app.route({
path: 'project-file',
key: 'get',
middleware: ['auth-admin'],
description: '读取指定路径的项目文件内容,以 base64 格式返回,适用于前端预览或下载场景',
metadata: {
args: {
filepath: z.string().nonempty().describe('要读取的文件绝对路径,必填'),
}
}
}).define(async (ctx) => {
const { filepath } = ctx.query as { filepath: string };
if (!filepath) ctx.throw(400, 'filepath 不能为空');
try {
const file = await manager.getFile(filepath);
ctx.body = { file };
} catch (error) {
ctx.throw(500, '读取文件失败');
}
}).addTo(app);
/**
* 更新文件自定义信息title/tags/summary/description/link
*/
app.route({
path: 'project-file',
key: 'update',
middleware: ['auth-admin'],
description: '更新指定文件的自定义描述信息title / tags / summary / description / link用于人工标注和补充文件元数据',
metadata: {
args: {
filepath: z.string().nonempty().describe('要更新的文件绝对路径,必填'),
title: z.string().optional().describe('文件标题,用于展示和搜索,选填'),
tags: z.array(z.string()).optional().describe('文件标签列表,用于分类过滤,选填'),
summary: z.string().optional().describe('文件摘要,简短描述文件内容,选填'),
description: z.string().optional().describe('文件详细描述,选填'),
link: z.string().optional().describe('与文件关联的外部链接如文档、Issue 等),选填'),
}
}
}).define(async (ctx) => {
const { filepath, title, tags, summary, description, link } = ctx.query as {
filepath: string;
title?: string;
tags?: string[];
summary?: string;
description?: string;
link?: string;
};
if (!filepath) ctx.throw(400, 'filepath 不能为空');
await manager.projectSearch.updateDocument(filepath, { title, tags, summary, description, link });
ctx.body = { success: true };
}).addTo(app);
app.route({
path: 'project-file',
key: 'delete',
middleware: ['auth-admin'],
description: '从搜索索引中删除指定路径的文件记录,文件本身不受影响',
metadata: {
args: {
filepath: z.string().nonempty().describe('要删除的文件绝对路径,必填'),
}
}
}).define(async (ctx) => {
const { filepath } = ctx.query as { filepath: string };
if (!filepath) ctx.throw(400, 'filepath 不能为空');
try {
const success = await manager.deleteFile(filepath);
if (!success) {
ctx.throw(404, '文件不存在');
}
ctx.body = { success: true };
} catch (error) {
ctx.throw(500, '删除文件失败');
}
}).addTo(app);

162
src/routes/project.ts Normal file
View File

@@ -0,0 +1,162 @@
import { z } from 'zod'
import { getGitPathname } from '../project/util/git';
import { app, manager } from '../app.ts';
/**
* 添加项目:启动监听 + 全量初始同步
* query: { path, repo, name? }
*/
app
.route({
path: 'project',
key: 'add',
description: '注册并启动项目监控:开始监听项目目录的文件变更,并将所有文件全量同步到 Meilisearch 搜索索引',
middleware: ['auth-admin'],
metadata: {
args: {
filepath: z.string().describe('项目根目录的绝对路径,必填'),
repo: z.string().optional().describe('代码仓库标识,用于搜索结果展示和过滤,格式如 owner/repo例如 kevisual/cnb选填默认自动从 git 配置读取)'),
name: z.string().optional().describe('项目显示名称,用于搜索结果展示,选填(默认使用目录名)'),
}
}
})
.define(async (ctx) => {
let { filepath, repo, name } = ctx.query as { filepath: string; repo?: string; name?: string };
if (!filepath) ctx.throw(400, 'filepath 不能为空');
if (!repo) {
const gitPathname = getGitPathname(filepath);
if (gitPathname) {
repo = gitPathname;
} else {
const pathParts = filepath.split('/');
repo = pathParts[pathParts.length - 1];
}
}
const info = await manager.addProject({ path: filepath, repo, name });
ctx.body = { success: true, data: info };
})
.addTo(app);
app
.route({
path: 'project',
key: 'remove',
description: '移除项目:停止文件变更监听,并清除该项目在 Meilisearch 索引中的所有数据',
middleware: ['auth-admin'],
metadata: {
args: {
filepath: z.string().nonempty().describe('要移除的项目根目录绝对路径,必填'),
}
}
})
.define(async (ctx) => {
const { filepath } = ctx.query as { filepath: string };
if (!filepath) ctx.throw(400, 'filepath 不能为空');
await manager.removeProject(filepath);
ctx.body = { success: true };
})
.addTo(app);
/**
* 停止项目文件监听
* query: { filepath }
*/
app
.route({
path: 'project',
key: 'stop',
description: '暂停项目的文件变更监听,暂停后文件修改将不再实时同步;可通过 add 接口重新启动监听',
middleware: ['auth-admin'],
metadata: {
args: {
filepath: z.string().nonempty().describe('要暂停监听的项目根目录绝对路径,必填'),
}
}
})
.define(async (ctx) => {
const { filepath } = ctx.query as { filepath: string };
if (!filepath) ctx.throw(400, 'filepath 不能为空');
await manager.stopProject(filepath);
ctx.body = { success: true };
})
.addTo(app);
/**
* 获取单个项目信息
* query: { filepath }
*/
app
.route({
path: 'project',
key: 'get',
description: '获取指定项目的详细信息,包括路径、仓库名称、项目名称及当前监听状态等',
middleware: ['auth-admin'],
metadata: {
args: {
filepath: z.string().nonempty().describe('要查询的项目根目录绝对路径,必填'),
}
}
})
.define(async (ctx) => {
const { filepath } = ctx.query as { filepath: string };
if (!filepath) ctx.throw(400, 'filepath 不能为空');
const project = manager.getProject(filepath);
if (!project) {
ctx.throw(404, '项目不存在');
return;
}
ctx.body = { success: true, data: manager.getProjectInfo(project) };
})
.addTo(app);
/**
* 列出所有项目
*/
app
.route({
path: 'project',
key: 'list',
description: '列出所有已注册的项目及其当前运行状态(路径、仓库名称、监听是否活跃等)',
middleware: ['auth-admin'],
})
.define(async (ctx) => {
ctx.body = { list: manager.listProjects() };
})
.addTo(app);
/**
* 更新项目自定义信息name/title/tags/summary/description/link
* query: { filepath, name?, title?, tags?, summary?, description?, link? }
*/
app
.route({
path: 'project',
key: 'update',
description: '更新项目的自定义描述信息name / title / tags / summary / description / link用于丰富项目元数据以便更好地展示和检索',
middleware: ['auth-admin'],
metadata: {
args: {
filepath: z.string().nonempty().describe('要更新的项目根目录绝对路径,必填'),
name: z.string().optional().describe('项目显示名称,选填'),
title: z.string().optional().describe('项目标题,用于展示,选填'),
tags: z.array(z.string()).optional().describe('项目标签列表,用于分类过滤,选填'),
summary: z.string().optional().describe('项目摘要,简短描述项目用途,选填'),
description: z.string().optional().describe('项目详细描述,选填'),
link: z.string().optional().describe('项目关联的外部链接(如官网、文档等),选填'),
}
}
})
.define(async (ctx) => {
const { filepath, name, title, tags, summary, description, link } = ctx.query as {
filepath: string;
name?: string;
title?: string;
tags?: string[];
summary?: string;
description?: string;
link?: string;
};
if (!filepath) ctx.throw(400, 'filepath 不能为空');
await manager.projectStore.updateProject(filepath, { name, title, tags, summary, description, link });
ctx.body = { success: true };
})
.addTo(app);

39
src/routes/search.ts Normal file
View File

@@ -0,0 +1,39 @@
import { app, manager } from '../app.ts';
import { z } from 'zod'
/**
* 搜索文件
* query: { q, projectPath?, repo? }
*/
app
.route({
path: 'project-search',
key: 'files',
description: '在已索引的项目文件中执行全文搜索,支持按仓库、目录、标签等字段过滤,以及自定义排序和数量限制',
middleware: ['auth-admin'],
metadata: {
args: {
q: z.string().optional().describe('搜索关键词,选填;留空或不传则返回全部文件'),
projectPath: z.string().optional().describe('按项目根目录路径过滤,仅返回该项目下的文件,选填'),
repo: z.string().optional().describe('按代码仓库标识过滤(如 owner/repo选填'),
title: z.string().optional().describe('按人工标注的标题字段过滤,选填'),
tags: z.array(z.string()).optional().describe('按人工标注的标签列表过滤,选填'),
summary: z.string().optional().describe('按人工标注的摘要字段过滤,选填'),
description: z.string().optional().describe('按人工标注的描述字段过滤,选填'),
link: z.string().optional().describe('按人工标注的外部链接字段过滤,选填'),
sort: z.array(z.string()).optional().describe('排序规则数组,格式为 ["字段:asc"] 或 ["字段:desc"],选填,当 q 为空时默认为 ["projectPath:asc"]'),
limit: z.number().optional().describe('返回结果数量上限,选填,当 q 为空时默认为 1000'),
getContent: z.boolean().optional().describe('是否返回文件内容,默认为 false如果为 true则在结果中包含 content 字段,内容以 base64 编码返回,适用于前端预览或下载场景'),
}
}
})
.define(async (ctx) => {
let { q, projectPath, repo, title, tags, summary, description, link, sort, limit, getContent = false } = ctx.query as { q?: string; projectPath?: string; repo?: string; title?: string; tags?: string[]; summary?: string; description?: string; link?: string; sort?: string[]; limit?: number; getContent?: boolean };
if (!q) {
sort = sort ?? ['projectPath:asc'];
limit = limit ?? 1000;
}
const projectSearch = manager.projectSearch;
const hits = await projectSearch.searchFiles(q, { projectPath, repo, title, tags, summary, description, link, sort, limit, getContent });
ctx.body = { list: hits };
})
.addTo(app);

56
src/scheduler/index.ts Normal file
View File

@@ -0,0 +1,56 @@
/**
* 任务调度器
* - FIFO 先进先出队列
* - 单线程执行(一次只执行一个任务)
* - 去重:同一个 key 的任务如果已在队列中等待(未执行),则不重复添加
*/
type Task = {
key: string;
execute: () => Promise<void>;
};
export class Scheduler {
private queue: Task[] = [];
private running = false;
/** 当前在队列中等待执行的任务 key 集合(不含正在执行的) */
private pendingKeys = new Set<string>();
/**
* 添加任务
* @param key 任务唯一标识(如文件路径)
* @param execute 任务执行函数
* @returns true 表示已加入队列false 表示该 key 已存在,跳过
*/
add(key: string, execute: () => Promise<void>): boolean {
if (this.pendingKeys.has(key)) {
// 同一任务已在队列中,跳过
return false;
}
this.pendingKeys.add(key);
this.queue.push({ key, execute });
// 触发执行(如果没有在跑的话)
this._run();
return true;
}
private async _run() {
if (this.running) return;
this.running = true;
while (this.queue.length > 0) {
const task = this.queue.shift()!;
// 任务开始执行前,从 pendingKeys 移除,允许后续同 key 任务重新入队
this.pendingKeys.delete(task.key);
try {
await task.execute();
} catch (err) {
console.error(`[Scheduler] Task "${task.key}" failed:`, err);
}
}
this.running = false;
}
/** 队列中等待执行的任务数量(不含正在执行的) */
get pendingCount(): number {
return this.queue.length;
}
}

11
test/common.ts Normal file
View File

@@ -0,0 +1,11 @@
import path from 'node:path';
export const projectPath = path.join('/workspace/projects/project-search');
import { ProjectManager } from '../src/index';
import { useKey } from '@kevisual/context';
export const manager = new ProjectManager({
meiliSearchOptions: {
apiKey: useKey('CNB_TOKEN')
}
});

11
test/file.ts Normal file
View File

@@ -0,0 +1,11 @@
import path from "node:path";
import { FileSearch } from "../src/file-search";
import { projectPath } from "./common";
const fileSearch = new FileSearch();
async function testSearchFiles() {
const files = await fileSearch.searchFiles({ cwd: projectPath });
console.log('Found files:', files);
}
testSearchFiles().catch(console.error);

19
test/remote.ts Normal file
View File

@@ -0,0 +1,19 @@
import { app, manager } from '../src/index'
import { RemoteApp } from '@kevisual/remote-app'
app.createRouteList()
manager.init().then(() => {
console.log('ProjectManager initialized successfully');
}).catch((error) => {
console.error('Failed to initialize ProjectManager:', error);
});
const remote = new RemoteApp({
app,
id: 'project-search',
username: 'root'
});
const isConnect = await remote.isConnect();
if (isConnect) {
console.log('Remote app connected successfully', isConnect);
remote.listenProxy();
}

7
test/search.ts Normal file
View File

@@ -0,0 +1,7 @@
import { manager, projectPath } from "./common";
manager.projectSearch.searchFiles('routes', { projectPath }).then(result => {
console.log('Search results:', result);
}).catch(err => {
console.error('Search error:', err);
});

7
test/start.ts Normal file
View File

@@ -0,0 +1,7 @@
import { manager, projectPath } from "./common";
manager.addProject({
name: 'Test Project',
path: projectPath,
repo: 'kevisual/test-repo',
})

41
test/wather.ts Normal file
View File

@@ -0,0 +1,41 @@
import { subscribe } from '@parcel/watcher';
import { debounce } from 'es-toolkit';
import fs from 'node:fs';
import { projectPath } from "./common";
// 每个路径独立的防抖处理函数
const debouncedHandlers = new Map<string, ReturnType<typeof debounce>>();
const subscription = await subscribe(
projectPath, // 监听路径(支持数组)
async (err, events) => {
if (err) throw err;
for (const event of events) {
// 如果已有防抖函数,先取消
if (debouncedHandlers.has(event.path)) {
debouncedHandlers.get(event.path)?.cancel();
}
// 为每个路径创建独立的防抖处理
const handler = debounce(() => {
console.log(event.type, event.path, new Date()); // 'create', 'update', 'delete'...
if (!fs.statSync(event.path).isFile()) return;
// 跳过非文件事件
// event: { type: 'update', path: '/path/to/file.ts', file: true }
debouncedHandlers.delete(event.path);
}, 300);
debouncedHandlers.set(event.path, handler);
handler();
}
},
{
ignore: ['**/node_modules/**', '**/.git/**'],
backend: 'watchman', // 可选:'fs-events'(默认)、'inotify'、'windows'、'watchman'
}
);
// 后续可取消监听
// await subscription.unsubscribe();