add check sync

kevisual.json

@@ -3,6 +3,12 @@
     "name": "kevisual",
     "share": "public"
   },
+  "checkDir": {
+    "./build/tools/kevisual-sync": {
+      "url": "https://kevisual.xiongxiao.me/root/ai/kevisual/tools/kevisual-sync/",
+      "enabled": true
+    }
+  },
   "syncDirectory": [
     {
       "files": [

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kevisual/envision-cli",
-  "version": "0.0.48",
+  "version": "0.0.49",
   "description": "envision command tools",
   "main": "dist/app.mjs",
   "type": "module",
@@ -38,6 +38,7 @@
   ],
   "author": "abearxiong",
   "dependencies": {
+    "micromatch": "^4.0.8",
     "pm2": "^6.0.5"
   },
   "devDependencies": {
@@ -48,6 +49,7 @@
     "@types/bun": "^1.2.13",
     "@types/crypto-js": "^4.2.2",
     "@types/jsonwebtoken": "^9.0.9",
+    "@types/micromatch": "^4.0.9",
     "@types/node": "^22.15.17",
     "chalk": "^5.4.1",
     "commander": "^13.1.0",

pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     dependencies:
+      micromatch:
+        specifier: ^4.0.8
+        version: 4.0.8
       pm2:
         specifier: ^6.0.5
         version: 6.0.5(supports-color@10.0.0)
@@ -33,6 +36,9 @@ importers:
       '@types/jsonwebtoken':
         specifier: ^9.0.9
         version: 9.0.9
+      '@types/micromatch':
+        specifier: ^4.0.9
+        version: 4.0.9
       '@types/node':
         specifier: ^22.15.17
         version: 22.15.17
@@ -814,6 +820,9 @@ packages:
   '@tootallnate/quickjs-emscripten@0.23.0':
     resolution: {integrity: sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==}
 
+  '@types/braces@3.0.5':
+    resolution: {integrity: sha512-SQFof9H+LXeWNz8wDe7oN5zu7ket0qwMu5vZubW4GCJ8Kkeh6nBWUz87+KTz/G3Kqsrp0j/W253XJb3KMEeg3w==}
+
   '@types/bun@1.2.10':
     resolution: {integrity: sha512-eilv6WFM3M0c9ztJt7/g80BDusK98z/FrFwseZgT4bXCq2vPhXD4z8R3oddmAn+R/Nmz9vBn4kweJKmGTZj+lg==}
 
@@ -838,6 +847,9 @@ packages:
   '@types/lodash@4.17.16':
     resolution: {integrity: sha512-HX7Em5NYQAXKW+1T+FiuG27NGwzJfCX3s1GjOa7ujxZa52kjJLOr4FUxT+giF6Tgxv1e+/czV/iTtBw27WTU9g==}
 
+  '@types/micromatch@4.0.9':
+    resolution: {integrity: sha512-7V+8ncr22h4UoYRLnLXSpTxjQrNUXtWHGeMPRJt1nULXI57G9bIcpyrHlmrQ7QK24EyyuXvYcSSWAM8GA9nqCg==}
+
   '@types/mime@1.3.5':
     resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==}
 
@@ -2739,6 +2751,8 @@ snapshots:
 
   '@tootallnate/quickjs-emscripten@0.23.0': {}
 
+  '@types/braces@3.0.5': {}
+
   '@types/bun@1.2.10':
     dependencies:
       bun-types: 1.2.10
@@ -2764,6 +2778,10 @@ snapshots:
 
   '@types/lodash@4.17.16': {}
 
+  '@types/micromatch@4.0.9':
+    dependencies:
+      '@types/braces': 3.0.5
+
   '@types/mime@1.3.5': {}
 
   '@types/ms@0.7.34': {}

src/command/sync/modules/base.ts

@@ -4,6 +4,7 @@ import { Config, SyncList, SyncConfigType, SyncConfig } from './type.ts';
 import { fileIsExist } from '@/uitls/file.ts';
 import { getHash } from '@/uitls/hash.ts';
 import glob from 'fast-glob';
+import { isMatch } from 'micromatch';
 import { logger } from '@/module/logger.ts';
 
 export type SyncOptions = {
@@ -11,11 +12,18 @@ export type SyncOptions = {
   configFilename?: string;
   baseURL?: string;
 };
+const checkAuth = (value: string = '', baseURL: string = '') => {
+  if (value.startsWith(baseURL)) {
+    return true;
+  }
+  return false;
+};
 export class SyncBase {
   config: Config;
   #filename: string;
   #dir: string;
   baseURL: string;
+  defaultIgnore: string[] = ['node_modules/**', '.git/**'];
   constructor(opts?: SyncOptions) {
     const filename = opts?.configFilename || 'kevisual.json';
     const dir = opts?.dir || process.cwd();
@@ -47,7 +55,7 @@ export class SyncBase {
       return {} as Config;
     }
   }
-  getRelativeFile(filename?: string) {
+  getRelativePath(filename?: string) {
     if (!filename) return false;
     const dir = this.#dir;
     const file = path.join(dir, filename);
@@ -57,6 +65,18 @@ export class SyncBase {
     if (syncType === 'sync') return true;
     return syncType === type;
   }
+  getIngore(ignore: string[] = []) {
+    const defaultIgnore = [...this.defaultIgnore, ...ignore];
+    const set = new Set(defaultIgnore);
+    return new Array(...set);
+  }
+  getMatchList(opts?: { matchList?: string[]; ignore: string[]; matchObjectList?: { path: string; [key: string]: any }[] }) {
+    const { matchList = [], ignore = [], matchObjectList = [] } = opts || {};
+    const _ignore = this.getIngore(ignore);
+    const _matchList = matchList.filter((file) => !isMatch(file, _ignore));
+    const _matchObjectList = matchObjectList.filter((item) => !isMatch(item.path, _ignore));
+    return { matchList: _matchList, matchObjectList: _matchObjectList };
+  }
   /**
    *
    * @param opts
@@ -73,13 +93,6 @@ export class SyncBase {
     const syncList = syncKeys.map((key) => {
       const value = sync[key];
       const filepath = path.join(this.#dir, key); // 文件的路径
-
-      const checkAuth = (value: string = '', baseURL: string = '') => {
-        if (value.startsWith(baseURL)) {
-          return true;
-        }
-        return false;
-      };
       if (typeof value === 'string') {
         const auth = checkAuth(value, baseURL);
         const type = auth ? 'sync' : 'none';
@@ -106,6 +119,25 @@ export class SyncBase {
     }
     return syncList;
   }
+  async getCheckList() {
+    const checkDir = this.config?.checkDir || {};
+    const dirKeys = Object.keys(checkDir);
+    const files = dirKeys.map((key) => {
+      return { key, ...this.getRelativePath(key) };
+    });
+    return files
+      .map((item) => {
+        if (!item) return;
+        let auth = checkAuth(checkDir[item.key]?.url, this.baseURL);
+        return {
+          key: item.key,
+          ...checkDir[item.key],
+          filepath: item?.absolute,
+          auth,
+        };
+      })
+      .filter((item) => item);
+  }
   getMergeSync(sync: Config['sync'] = {}, fileSync: Config['sync'] = {}) {
     const syncFileSyncKeys = Object.keys(fileSync);
     const syncKeys = Object.keys(sync);
@@ -126,7 +158,7 @@ export class SyncBase {
       const { registry, ignore = [], files = [], replace = {} } = item;
       const cwd = this.#dir;
       const glob_files = await glob(files, {
-        ignore,
+        ignore: this.getIngore(ignore),
         onlyFiles: true,
         cwd,
         dot: true,

src/command/sync/modules/type.ts

@@ -20,12 +20,19 @@ export interface Config {
   name?: string; // 项目名称
   version?: string; // 项目版本号
   registry?: string; // 项目仓库地址
-  user?: string; // 同步用户，否则会自动 query 一次
   metadata?: Record<string, any>; // 元数据, 统一的配置
-  syncDirectory: SyncDirectory[];
+  syncDirectory?: SyncDirectory[];
-  sync: {
+  sync?: {
     [key: string]: SyncConfig | string;
   };
+  checkDir?: {
+    [key: string]: {
+      url: string; // 需要检查的 url
+      replace?: Record<string, string>; // 替换的路径
+      ignore?: string[]; // 忽略的目录
+      enabled?: boolean; // 是否启用
+    };
+  };
 }
 
 export type SyncList = {

src/command/sync/sync.ts

@@ -1,11 +1,13 @@
 import { program as app, Command } from '@/program.ts';
 import { SyncBase } from './modules/base.ts';
 import { baseURL, storage } from '@/module/query.ts';
-import { fetchLink } from '@/module/download/install.ts';
+import { fetchLink, fetchAiList } from '@/module/download/install.ts';
 import fs from 'node:fs';
 import { upload } from '@/module/download/upload.ts';
 import { logger } from '@/module/logger.ts';
 import { chalk } from '@/module/chalk.ts';
+import path, { relative } from 'node:path';
+import { fileIsExist } from '@/uitls/file.ts';
 
 const command = new Command('sync')
   .option('-d --dir <dir>')
@@ -33,7 +35,7 @@ const syncUpload = new Command('upload')
     if (opts.share) {
       meta.share = opts.share;
     }
-    const filepath = sync.getRelativeFile(opts.file);
+    const filepath = sync.getRelativePath(opts.file);
     for (const item of syncList) {
       if (!item.auth || !item.exist) {
         nodonwArr.push(item);
@@ -79,7 +81,7 @@ const syncDownload = new Command('download')
     const syncList = await sync.getSyncList();
     logger.debug(syncList);
     const nodonwArr: (typeof syncList)[number][] = [];
-    const filepath = sync.getRelativeFile(opts.file);
+    const filepath = sync.getRelativePath(opts.file);
     for (const item of syncList) {
       if (!sync.canDone(item.type, 'download')) {
         nodonwArr.push(item);
@@ -134,7 +136,7 @@ const syncCreateList = new Command('create')
     });
     const newJson = { ...sync.config };
     newJson.sync = newSync;
-    const filepath = sync.getRelativeFile(opts.output);
+    const filepath = sync.getRelativePath(opts.output);
     if (filepath) {
       logger.debug('输出文件', filepath);
       fs.writeFileSync(filepath.absolute, JSON.stringify(newJson, null, 2));
@@ -144,9 +146,78 @@ const syncCreateList = new Command('create')
     }
   });
 
+const checkDir = new Command('check')
+  .option('-d --dir <dir>', '配置目录')
+  .option('-c --config <config>', '配置文件的名字', 'kevisual.json')
+  .description('检查目录')
+  .action(async (opts) => {
+    const sync = new SyncBase({ dir: opts.dir, baseURL: baseURL, configFilename: opts.config });
+    const syncList = await sync.getSyncList();
+    logger.debug(syncList);
+    logger.info('检查目录\n');
+    const checkList = await sync.getCheckList();
+    for (const item of checkList) {
+      if (!item.auth) {
+        continue;
+      }
+      if (!item.enabled) {
+        logger.info('提示:', item.key, chalk.yellow('未启用'));
+        continue;
+      }
+      const res = await fetchAiList(item.url, { recursive: true });
+      if (res.code === 200) {
+        const data = res?.data || [];
+        let matchObjectList = data.filter((dataItem) => {
+          // 把 pathname 和 path 合并成一个路径
+          dataItem.pathname = path.join(item.key || '', dataItem.path);
+          return dataItem;
+        });
+        matchObjectList = sync.getMatchList({ ignore: item.ignore, matchObjectList }).matchObjectList;
+        const matchList = matchObjectList
+          .map((item2) => {
+            const rp = sync.getRelativePath(item2.pathname);
+            if (!rp) return false;
+            return { ...item2, relative: rp.relative, absolute: rp.absolute };
+          })
+          .filter((i) => i);
+        for (const matchItem of matchList) {
+          if (!matchItem) continue;
+          let needDownload = true;
+          let hash = '';
+          await sync.getDir(matchItem.absolute, true);
+          logger.debug('文件路径', matchItem.absolute);
+          if (fileIsExist(matchItem.absolute)) {
+            hash = sync.getHash(matchItem.absolute);
+            if (hash !== matchItem.etag) {
+              logger.error('文件不一致', matchItem.pathname, chalk.red(matchItem.url), chalk.red('文件不一致'));
+            } else {
+              needDownload = false;
+              logger.info('文件一致', matchItem.pathname, chalk.green(matchItem.url), chalk.green('文件一致'));
+            }
+          }
+          if (needDownload) {
+            const { content, status } = await fetchLink(matchItem.url, { setToken: item.auth, returnContent: true, hash });
+            if (status === 200) {
+              fs.writeFileSync(matchItem.absolute, content);
+              logger.info('下载成功', matchItem.pathname, chalk.green(matchItem.url));
+            } else if (status === 304) {
+              logger.info('文件未修改', matchItem.pathname, chalk.green(matchItem.url));
+            } else {
+              logger.error('下载失败', matchItem.pathname, chalk.red(matchItem.url));
+            }
+          }
+        }
+      } else {
+        logger.error('检查失败', item.url, res.code);
+        break;
+      }
+    }
+  });
+
 command.addCommand(syncUpload);
 command.addCommand(syncDownload);
 command.addCommand(syncList);
 command.addCommand(syncCreateList);
+command.addCommand(checkDir);
 
 app.addCommand(command);

src/module/download/install.ts

@@ -2,7 +2,7 @@ import path from 'path';
 import fs from 'fs';
 import { storage, baseURL } from '../query.ts';
 import { chalk } from '../chalk.ts';
-
+import { Result } from '@kevisual/query';
 type DownloadTask = {
   downloadPath: string;
   downloadUrl: string;
@@ -194,3 +194,35 @@ export const uninstallApp = async (app: Partial<Package>, opts: UninstallAppOpts
     };
   }
 };
+
+export type AiList = {
+  name?: string;
+  lastModified?: string;
+  etag?: string;
+  size?: number;
+  path: string;
+  pathname?: string;
+  url?: string;
+};
+export const fetchAiList = async (url: string, opts?: { recursive: boolean }): Promise<Result<AiList[]>> => {
+  const token = process.env.KEVISUAL_TOKEN || storage.getItem('token');
+  const _url = new URL(url);
+  const dir = _url.searchParams.get('dir');
+  if (!dir) {
+    _url.searchParams.set('dir', 'true');
+  }
+  if (opts?.recursive) {
+    _url.searchParams.set('recursive', 'true');
+  }
+  if (!_url.pathname.endsWith('/')) {
+    _url.pathname += '/';
+  }
+  const res = await fetch(_url.toString(), {
+    method: 'GET',
+    headers: {
+      Authorization: 'Bearer ' + token,
+    },
+  });
+  const data = await res.json();
+  return data;
+};