diff --git a/kevisual.json b/kevisual.json index cf4d258..630907d 100644 --- a/kevisual.json +++ b/kevisual.json @@ -3,6 +3,12 @@ "name": "kevisual", "share": "public" }, + "checkDir": { + "./build/tools/kevisual-sync": { + "url": "https://kevisual.xiongxiao.me/root/ai/kevisual/tools/kevisual-sync/", + "enabled": true + } + }, "syncDirectory": [ { "files": [ diff --git a/package.json b/package.json index e57aabf..f5e8256 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@kevisual/envision-cli", - "version": "0.0.48", + "version": "0.0.49", "description": "envision command tools", "main": "dist/app.mjs", "type": "module", @@ -38,6 +38,7 @@ ], "author": "abearxiong", "dependencies": { + "micromatch": "^4.0.8", "pm2": "^6.0.5" }, "devDependencies": { @@ -48,6 +49,7 @@ "@types/bun": "^1.2.13", "@types/crypto-js": "^4.2.2", "@types/jsonwebtoken": "^9.0.9", + "@types/micromatch": "^4.0.9", "@types/node": "^22.15.17", "chalk": "^5.4.1", "commander": "^13.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f35971a..4f7512b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + micromatch: + specifier: ^4.0.8 + version: 4.0.8 pm2: specifier: ^6.0.5 version: 6.0.5(supports-color@10.0.0) @@ -33,6 +36,9 @@ importers: '@types/jsonwebtoken': specifier: ^9.0.9 version: 9.0.9 + '@types/micromatch': + specifier: ^4.0.9 + version: 4.0.9 '@types/node': specifier: ^22.15.17 version: 22.15.17 @@ -814,6 +820,9 @@ packages: '@tootallnate/quickjs-emscripten@0.23.0': resolution: {integrity: sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==} + '@types/braces@3.0.5': + resolution: {integrity: sha512-SQFof9H+LXeWNz8wDe7oN5zu7ket0qwMu5vZubW4GCJ8Kkeh6nBWUz87+KTz/G3Kqsrp0j/W253XJb3KMEeg3w==} + '@types/bun@1.2.10': resolution: {integrity: sha512-eilv6WFM3M0c9ztJt7/g80BDusK98z/FrFwseZgT4bXCq2vPhXD4z8R3oddmAn+R/Nmz9vBn4kweJKmGTZj+lg==} @@ -838,6 +847,9 @@ packages: '@types/lodash@4.17.16': resolution: {integrity: sha512-HX7Em5NYQAXKW+1T+FiuG27NGwzJfCX3s1GjOa7ujxZa52kjJLOr4FUxT+giF6Tgxv1e+/czV/iTtBw27WTU9g==} + '@types/micromatch@4.0.9': + resolution: {integrity: sha512-7V+8ncr22h4UoYRLnLXSpTxjQrNUXtWHGeMPRJt1nULXI57G9bIcpyrHlmrQ7QK24EyyuXvYcSSWAM8GA9nqCg==} + '@types/mime@1.3.5': resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} @@ -2739,6 +2751,8 @@ snapshots: '@tootallnate/quickjs-emscripten@0.23.0': {} + '@types/braces@3.0.5': {} + '@types/bun@1.2.10': dependencies: bun-types: 1.2.10 @@ -2764,6 +2778,10 @@ snapshots: '@types/lodash@4.17.16': {} + '@types/micromatch@4.0.9': + dependencies: + '@types/braces': 3.0.5 + '@types/mime@1.3.5': {} '@types/ms@0.7.34': {} diff --git a/src/command/sync/modules/base.ts b/src/command/sync/modules/base.ts index e45c58e..7bfaffd 100644 --- a/src/command/sync/modules/base.ts +++ b/src/command/sync/modules/base.ts @@ -4,6 +4,7 @@ import { Config, SyncList, SyncConfigType, SyncConfig } from './type.ts'; import { fileIsExist } from '@/uitls/file.ts'; import { getHash } from '@/uitls/hash.ts'; import glob from 'fast-glob'; +import { isMatch } from 'micromatch'; import { logger } from '@/module/logger.ts'; export type SyncOptions = { @@ -11,11 +12,18 @@ export type SyncOptions = { configFilename?: string; baseURL?: string; }; +const checkAuth = (value: string = '', baseURL: string = '') => { + if (value.startsWith(baseURL)) { + return true; + } + return false; +}; export class SyncBase { config: Config; #filename: string; #dir: string; baseURL: string; + defaultIgnore: string[] = ['node_modules/**', '.git/**']; constructor(opts?: SyncOptions) { const filename = opts?.configFilename || 'kevisual.json'; const dir = opts?.dir || process.cwd(); @@ -47,7 +55,7 @@ export class SyncBase { return {} as Config; } } - getRelativeFile(filename?: string) { + getRelativePath(filename?: string) { if (!filename) return false; const dir = this.#dir; const file = path.join(dir, filename); @@ -57,6 +65,18 @@ export class SyncBase { if (syncType === 'sync') return true; return syncType === type; } + getIngore(ignore: string[] = []) { + const defaultIgnore = [...this.defaultIgnore, ...ignore]; + const set = new Set(defaultIgnore); + return new Array(...set); + } + getMatchList(opts?: { matchList?: string[]; ignore: string[]; matchObjectList?: { path: string; [key: string]: any }[] }) { + const { matchList = [], ignore = [], matchObjectList = [] } = opts || {}; + const _ignore = this.getIngore(ignore); + const _matchList = matchList.filter((file) => !isMatch(file, _ignore)); + const _matchObjectList = matchObjectList.filter((item) => !isMatch(item.path, _ignore)); + return { matchList: _matchList, matchObjectList: _matchObjectList }; + } /** * * @param opts @@ -73,13 +93,6 @@ export class SyncBase { const syncList = syncKeys.map((key) => { const value = sync[key]; const filepath = path.join(this.#dir, key); // 文件的路径 - - const checkAuth = (value: string = '', baseURL: string = '') => { - if (value.startsWith(baseURL)) { - return true; - } - return false; - }; if (typeof value === 'string') { const auth = checkAuth(value, baseURL); const type = auth ? 'sync' : 'none'; @@ -106,6 +119,25 @@ export class SyncBase { } return syncList; } + async getCheckList() { + const checkDir = this.config?.checkDir || {}; + const dirKeys = Object.keys(checkDir); + const files = dirKeys.map((key) => { + return { key, ...this.getRelativePath(key) }; + }); + return files + .map((item) => { + if (!item) return; + let auth = checkAuth(checkDir[item.key]?.url, this.baseURL); + return { + key: item.key, + ...checkDir[item.key], + filepath: item?.absolute, + auth, + }; + }) + .filter((item) => item); + } getMergeSync(sync: Config['sync'] = {}, fileSync: Config['sync'] = {}) { const syncFileSyncKeys = Object.keys(fileSync); const syncKeys = Object.keys(sync); @@ -126,7 +158,7 @@ export class SyncBase { const { registry, ignore = [], files = [], replace = {} } = item; const cwd = this.#dir; const glob_files = await glob(files, { - ignore, + ignore: this.getIngore(ignore), onlyFiles: true, cwd, dot: true, diff --git a/src/command/sync/modules/type.ts b/src/command/sync/modules/type.ts index c08b8a5..5bf18b4 100644 --- a/src/command/sync/modules/type.ts +++ b/src/command/sync/modules/type.ts @@ -20,12 +20,19 @@ export interface Config { name?: string; // 项目名称 version?: string; // 项目版本号 registry?: string; // 项目仓库地址 - user?: string; // 同步用户，否则会自动 query 一次 metadata?: Record; // 元数据, 统一的配置 - syncDirectory: SyncDirectory[]; - sync: { + syncDirectory?: SyncDirectory[]; + sync?: { [key: string]: SyncConfig | string; }; + checkDir?: { + [key: string]: { + url: string; // 需要检查的 url + replace?: Record; // 替换的路径 + ignore?: string[]; // 忽略的目录 + enabled?: boolean; // 是否启用 + }; + }; } export type SyncList = { diff --git a/src/command/sync/sync.ts b/src/command/sync/sync.ts index 7cc1394..9acf587 100644 --- a/src/command/sync/sync.ts +++ b/src/command/sync/sync.ts @@ -1,11 +1,13 @@ import { program as app, Command } from '@/program.ts'; import { SyncBase } from './modules/base.ts'; import { baseURL, storage } from '@/module/query.ts'; -import { fetchLink } from '@/module/download/install.ts'; +import { fetchLink, fetchAiList } from '@/module/download/install.ts'; import fs from 'node:fs'; import { upload } from '@/module/download/upload.ts'; import { logger } from '@/module/logger.ts'; import { chalk } from '@/module/chalk.ts'; +import path, { relative } from 'node:path'; +import { fileIsExist } from '@/uitls/file.ts'; const command = new Command('sync') .option('-d --dir ') @@ -33,7 +35,7 @@ const syncUpload = new Command('upload') if (opts.share) { meta.share = opts.share; } - const filepath = sync.getRelativeFile(opts.file); + const filepath = sync.getRelativePath(opts.file); for (const item of syncList) { if (!item.auth || !item.exist) { nodonwArr.push(item); @@ -79,7 +81,7 @@ const syncDownload = new Command('download') const syncList = await sync.getSyncList(); logger.debug(syncList); const nodonwArr: (typeof syncList)[number][] = []; - const filepath = sync.getRelativeFile(opts.file); + const filepath = sync.getRelativePath(opts.file); for (const item of syncList) { if (!sync.canDone(item.type, 'download')) { nodonwArr.push(item); @@ -134,7 +136,7 @@ const syncCreateList = new Command('create') }); const newJson = { ...sync.config }; newJson.sync = newSync; - const filepath = sync.getRelativeFile(opts.output); + const filepath = sync.getRelativePath(opts.output); if (filepath) { logger.debug('输出文件', filepath); fs.writeFileSync(filepath.absolute, JSON.stringify(newJson, null, 2)); @@ -144,9 +146,78 @@ const syncCreateList = new Command('create') } }); +const checkDir = new Command('check') + .option('-d --dir ', '配置目录') + .option('-c --config ', '配置文件的名字', 'kevisual.json') + .description('检查目录') + .action(async (opts) => { + const sync = new SyncBase({ dir: opts.dir, baseURL: baseURL, configFilename: opts.config }); + const syncList = await sync.getSyncList(); + logger.debug(syncList); + logger.info('检查目录\n'); + const checkList = await sync.getCheckList(); + for (const item of checkList) { + if (!item.auth) { + continue; + } + if (!item.enabled) { + logger.info('提示:', item.key, chalk.yellow('未启用')); + continue; + } + const res = await fetchAiList(item.url, { recursive: true }); + if (res.code === 200) { + const data = res?.data || []; + let matchObjectList = data.filter((dataItem) => { + // 把 pathname 和 path 合并成一个路径 + dataItem.pathname = path.join(item.key || '', dataItem.path); + return dataItem; + }); + matchObjectList = sync.getMatchList({ ignore: item.ignore, matchObjectList }).matchObjectList; + const matchList = matchObjectList + .map((item2) => { + const rp = sync.getRelativePath(item2.pathname); + if (!rp) return false; + return { ...item2, relative: rp.relative, absolute: rp.absolute }; + }) + .filter((i) => i); + for (const matchItem of matchList) { + if (!matchItem) continue; + let needDownload = true; + let hash = ''; + await sync.getDir(matchItem.absolute, true); + logger.debug('文件路径', matchItem.absolute); + if (fileIsExist(matchItem.absolute)) { + hash = sync.getHash(matchItem.absolute); + if (hash !== matchItem.etag) { + logger.error('文件不一致', matchItem.pathname, chalk.red(matchItem.url), chalk.red('文件不一致')); + } else { + needDownload = false; + logger.info('文件一致', matchItem.pathname, chalk.green(matchItem.url), chalk.green('文件一致')); + } + } + if (needDownload) { + const { content, status } = await fetchLink(matchItem.url, { setToken: item.auth, returnContent: true, hash }); + if (status === 200) { + fs.writeFileSync(matchItem.absolute, content); + logger.info('下载成功', matchItem.pathname, chalk.green(matchItem.url)); + } else if (status === 304) { + logger.info('文件未修改', matchItem.pathname, chalk.green(matchItem.url)); + } else { + logger.error('下载失败', matchItem.pathname, chalk.red(matchItem.url)); + } + } + } + } else { + logger.error('检查失败', item.url, res.code); + break; + } + } + }); + command.addCommand(syncUpload); command.addCommand(syncDownload); command.addCommand(syncList); command.addCommand(syncCreateList); +command.addCommand(checkDir); app.addCommand(command); diff --git a/src/module/download/install.ts b/src/module/download/install.ts index 1c0400a..95d4051 100644 --- a/src/module/download/install.ts +++ b/src/module/download/install.ts @@ -2,7 +2,7 @@ import path from 'path'; import fs from 'fs'; import { storage, baseURL } from '../query.ts'; import { chalk } from '../chalk.ts'; - +import { Result } from '@kevisual/query'; type DownloadTask = { downloadPath: string; downloadUrl: string; @@ -194,3 +194,35 @@ export const uninstallApp = async (app: Partial, opts: UninstallAppOpts }; } }; + +export type AiList = { + name?: string; + lastModified?: string; + etag?: string; + size?: number; + path: string; + pathname?: string; + url?: string; +}; +export const fetchAiList = async (url: string, opts?: { recursive: boolean }): Promise> => { + const token = process.env.KEVISUAL_TOKEN || storage.getItem('token'); + const _url = new URL(url); + const dir = _url.searchParams.get('dir'); + if (!dir) { + _url.searchParams.set('dir', 'true'); + } + if (opts?.recursive) { + _url.searchParams.set('recursive', 'true'); + } + if (!_url.pathname.endsWith('/')) { + _url.pathname += '/'; + } + const res = await fetch(_url.toString(), { + method: 'GET', + headers: { + Authorization: 'Bearer ' + token, + }, + }); + const data = await res.json(); + return data; +};