"feat: 同步功能增强与配置优化，支持多类型同步及日志分级"

.gitignore

@@ -5,4 +5,6 @@ dist
 
 pack-dist
 apps
-assistant-app
+assistant-app
+
+build

kevisual.json

@@ -1,5 +1,23 @@
 {
+  "metadata": {
+    "name": "kevisual",
+    "share": "public"
+  },
+  "syncDirectory": [
+    {
+      "files": [
+        "build/**/*"
+      ],
+      "ignore": [
+        "build/ignore.md"
+      ],
+      "registry": "https://kevisual.xiongxiao.me/root/ai/kevisual",
+      "replace": {
+        "build/": ""
+      }
+    }
+  ],
   "sync": {
-    "build/01-summary.md": "https://kevisual.xiongxiao.me/root/ai/kevisual/01-summary.md"
+    "./build/01-summary.md": "https://kevisual.xiongxiao.me/root/ai/kevisual/01-summary.md"
   }
 }

package.json

@@ -42,7 +42,7 @@
   },
   "devDependencies": {
     "@kevisual/load": "^0.0.6",
-    "@kevisual/logger": "^0.0.2",
+    "@kevisual/logger": "^0.0.3",
     "@kevisual/query": "0.0.17",
     "@kevisual/query-login": "0.0.5",
     "@types/bun": "^1.2.13",

pnpm-lock.yaml

@@ -16,8 +16,8 @@ importers:
         specifier: ^0.0.6
         version: 0.0.6
       '@kevisual/logger':
-        specifier: ^0.0.2
+        specifier: ^0.0.3
-        version: 0.0.2
+        version: 0.0.3
       '@kevisual/query':
         specifier: 0.0.17
         version: 0.0.17(encoding@0.1.13)(ws@8.18.0)
@@ -612,8 +612,8 @@ packages:
       '@kevisual/use-config': ^1.0.11
       pm2: ^5.4.3
 
-  '@kevisual/logger@0.0.2':
+  '@kevisual/logger@0.0.3':
-    resolution: {integrity: sha512-4NVdNsOHmMRg+OuZPoNNdI3p7jRII7lMJHRar1IoBck7fFIV7YGMNQirrrjk07MHv+Eh+U+uUljjgEWbse92RA==}
+    resolution: {integrity: sha512-8emqxg+ab62WAK6VY4FQqetXPSSVKFAjGctD1NDbdnxt7YWuI/PyuDltCpsVz+uvWpV1dO5OKZOoHU7ow59Omw==}
 
   '@kevisual/query-login@0.0.5':
     resolution: {integrity: sha512-389cMMWAisjQoafxX+cUEa2z41S5koDjiyHkucfCkhRoP4M6g0iqbBMavLKmLOWSKx3R8e3ZmXT6RfsYGBb8Ww==}
@@ -2519,7 +2519,7 @@ snapshots:
       '@kevisual/use-config': 1.0.11(dotenv@16.5.0)
       pm2: 6.0.5(supports-color@10.0.0)
 
-  '@kevisual/logger@0.0.2': {}
+  '@kevisual/logger@0.0.3': {}
 
   '@kevisual/query-login@0.0.5(@kevisual/query@0.0.17(@kevisual/ws@8.0.0)(encoding@0.1.13))(rollup@4.40.2)(typescript@5.8.2)':
     dependencies:

src/command/sync/modules/base.ts

@@ -1,7 +1,9 @@
 import path from 'node:path';
 import fs from 'node:fs';
-import { Config, SyncList } from './type.ts';
+import { Config, SyncList, SyncConfigType, SyncConfig } from './type.ts';
 import { fileIsExist } from '@/uitls/file.ts';
+import { getHash } from '@/uitls/hash.ts';
+import glob from 'fast-glob';
 
 export type SyncOptions = {
   dir?: string;
@@ -28,6 +30,15 @@ export class SyncBase {
       const filepath = path.join(dir, filename);
       if (!fileIsExist(filepath)) throw new Error('config file not found');
       const config = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
+      const sync = config.sync || {};
+      const keys = Object.keys(sync);
+      const newConfigSync: any = {};
+      for (let key of keys) {
+        const keyPath = path.join(dir, key);
+        const newKey = path.relative(dir, keyPath);
+        newConfigSync[newKey] = sync[key];
+      }
+      config.sync = newConfigSync;
       this.config = config;
       return config;
     } catch (err) {
@@ -35,10 +46,15 @@ export class SyncBase {
       return {} as Config;
     }
   }
-
+  async canDone(syncType: SyncConfigType, type?: SyncConfigType) {
-  async getSyncList(): Promise<SyncList[]> {
+    if (syncType === 'sync') return true;
+    return syncType === type;
+  }
+  async getSyncList(opts?: { getFile?: boolean }): Promise<SyncList[]> {
     const config = this.config!;
-    const sync = config?.sync || {};
+    let sync = config?.sync || {};
+    const syncDirectory = await this.getSyncDirectoryList();
+    sync = this.getMergeSync(sync, syncDirectory.sync);
     const syncKeys = Object.keys(sync);
     const baseURL = this.baseURL;
     const syncList = syncKeys.map((key) => {
@@ -52,21 +68,92 @@ export class SyncBase {
         return false;
       };
       if (typeof value === 'string') {
+        const auth = checkAuth(value, baseURL);
+        const type = auth ? 'sync' : 'none';
         return {
+          key,
+          type: type as any,
           filepath,
           url: value,
-          auth: checkAuth(value, baseURL),
+          auth,
         };
       }
+      const auth = checkAuth(value.url, baseURL);
+      const type = auth ? 'sync' : 'none';
       return {
+        key,
         filepath,
         ...value,
+        type: value?.type ?? type,
         auth: checkAuth(value.url, baseURL),
       };
     });
-
+    if (opts?.getFile) {
+      return this.getSyncListFile(syncList);
+    }
     return syncList;
   }
+  getMergeSync(sync: Config['sync'] = {}, fileSync: Config['sync'] = {}) {
+    const syncFileSyncKeys = Object.keys(fileSync);
+    const syncKeys = Object.keys(sync);
+    const keys = [...syncKeys, ...syncFileSyncKeys];
+    const obj: Config['sync'] = {};
+    for (let key of keys) {
+      const value = sync[key] ?? fileSync[key];
+      obj[key] = value;
+    }
+    return obj;
+  }
+  async getSyncDirectoryList() {
+    const config = this.config;
+    const syncDirectory = config?.syncDirectory || [];
+    let obj: Record<string, string> = {};
+    const keys: string[] = [];
+    for (let item of syncDirectory) {
+      const { registry, ignore = [], files, replace = {} } = item;
+      const cwd = this.#dir;
+      const glob_files = await glob(files, {
+        ignore,
+        onlyFiles: true,
+        cwd,
+        dot: true,
+        absolute: true,
+      });
+      for (let file of glob_files) {
+        const key = path.relative(cwd, file);
+        const _registryURL = new URL(registry);
+        const replaceKeys = Object.keys(replace);
+        let newKey = key;
+        for (let replaceKey of replaceKeys) {
+          if (newKey.startsWith(replaceKey)) {
+            newKey = key.replace(replaceKey, replace[replaceKey]);
+          }
+        }
+        const pathname = path.join(_registryURL.pathname, newKey);
+        _registryURL.pathname = pathname;
+        keys.push(key);
+        obj[key] = _registryURL.toString();
+      }
+    }
+    return { sync: obj, keys };
+  }
+  async getSyncListFile(syncList: SyncList[]) {
+    let syncListFile: SyncList[] = [];
+    for (let item of syncList) {
+      const { filepath, auth } = item;
+      if (filepath && fileIsExist(filepath) && auth) {
+        syncListFile.push({
+          ...item,
+          exist: true,
+          hash: getHash(filepath),
+        });
+      } else {
+        syncListFile.push({ ...item, exist: false });
+      }
+    }
+    return syncListFile;
+  }
+  getHash = getHash;
   async getDir(filepath: string, check = false) {
     const dir = path.dirname(filepath);
     if (check) {

src/command/sync/modules/type.ts

@@ -1,19 +1,38 @@
+export type SyncConfigType = 'sync' | 'download' | 'upload' | 'none';
 export type SyncConfig = {
-  type?: 'sync'; // 是否可以同步
+  type?: SyncConfigType; // 是否可以同步
   url: string; // 文件具体的 url 的地址
 };
+export type SyncDirectory = {
+  /**
+   * 忽略的目录或则文件，默认忽略 node_modules 使用 fast-glob 去匹配,
+   * 当同步文件夹的时候生效
+   **/
+  ignore?: string[];
+  /**
+   * 合并路径的源地址，https://kevisual.xiongxiao.me/root/ai/kevisual
+   */
+  registry?: string;
+  files?: string[];
+  replace?: Record<string, string>;
+};
 export interface Config {
   name?: string; // 项目名称
   version?: string; // 项目版本号
-  ignore?: string[]; // 忽略的目录或则文件，默认忽略 node_modules 使用 fast-glob 去匹配
+  registry?: string; // 项目仓库地址
-
+  user?: string; // 同步用户，否则会自动 query 一次
+  metadata?: Record<string, any>; // 元数据, 统一的配置
+  syncDirectory: SyncDirectory[];
   sync: {
     [key: string]: SyncConfig | string;
   };
 }
 
 export type SyncList = {
+  key?: string;
   filepath: string;
+  exist?: boolean;
+  hash?: string;
   /**
    * 是否需要鉴权, baseURL 为 kevisual 服务时，需要鉴权
    */

src/command/sync/sync.ts

@@ -1,8 +1,11 @@
 import { program as app, Command } from '@/program.ts';
 import { SyncBase } from './modules/base.ts';
-import { baseURL } from '@/module/query.ts';
+import { baseURL, storage } from '@/module/query.ts';
 import { fetchLink } from '@/module/download/install.ts';
 import fs from 'node:fs';
+import { upload } from '@/module/download/upload.ts';
+import { logger } from '@/module/logger.ts';
+import { chalk } from '@/module/chalk.ts';
 
 const command = new Command('sync')
   .option('-d --dir <dir>')
@@ -10,21 +13,76 @@ const command = new Command('sync')
   .action(() => {
     console.log('同步项目');
   });
-const syncUpload = new Command('upload').description('上传项目').action(() => {
+const syncUpload = new Command('upload')
-  console.log('上传项目');
+  .option('-d --dir <dir>', '配置目录')
-});
+  .option('-s --share <share>', '共享设置')
+  .description('上传项目')
+  .action(async (opts) => {
+    console.log('上传项目');
+    const sync = new SyncBase({ baseURL: baseURL });
+    const syncList = await sync.getSyncList({ getFile: true });
+    logger.debug(syncList);
+    const nodonwArr: (typeof syncList)[number][] = [];
+    const token = storage.getItem('token');
+    const meta: Record<string, string> = {
+      ...sync.config.metadata,
+    };
+    if (opts.share) {
+      meta.share = opts.share;
+    }
+    for (const item of syncList) {
+      if (!item.auth || !item.exist) {
+        nodonwArr.push(item);
+        continue;
+      }
+      if (!sync.canDone(item.type, 'upload')) {
+        nodonwArr.push(item);
+        continue;
+      }
+      const res = await upload({
+        token,
+        file: fs.readFileSync(item.filepath),
+        url: item.url,
+        needHash: true,
+        hash: item.hash,
+        meta,
+      });
+      if (res.code === 200) {
+        logger.info('上传成功', item.key, chalk.green(item.url));
+      }
+      logger.debug(res);
+    }
+    if (nodonwArr.length) {
+      logger.warn('以下文件未上传\n', nodonwArr.map((item) => item.key).join(','));
+    }
+  });
 const syncDownload = new Command('download')
   .option('-d --dir <dir>', '配置目录')
   .description('下载项目')
   .action(async () => {
-    console.log('下载项目');
     const sync = new SyncBase({ baseURL: baseURL });
     const syncList = await sync.getSyncList();
-    console.log(syncList);
+    logger.debug(syncList);
+    const nodonwArr: (typeof syncList)[number][] = [];
     for (const item of syncList) {
-      const { content } = await fetchLink(item.url, { setToken: item.auth, returnContent: true });
+      if (!sync.canDone(item.type, 'download')) {
-      await sync.getDir(item.filepath, true);
+        nodonwArr.push(item);
-      fs.writeFileSync(item.filepath, content);
+        continue;
+      }
+      const hash = sync.getHash(item.filepath);
+      const { content, status } = await fetchLink(item.url, { setToken: item.auth, returnContent: true, hash });
+      if (status === 200) {
+        await sync.getDir(item.filepath, true);
+        fs.writeFileSync(item.filepath, content);
+        logger.info('下载成功', item.key, chalk.green(item.url));
+      } else if (status === 304) {
+        logger.info('文件未修改', item.key, chalk.green(item.url));
+      } else {
+        logger.error('下载失败', item.key, chalk.red(item.url));
+      }
+    }
+    if (nodonwArr.length) {
+      logger.warn('以下文件未下载', nodonwArr.map((item) => item.key).join(','));
     }
   });
 

src/module/download/install.ts

@@ -24,6 +24,7 @@ type Options = {
   check?: boolean;
   returnContent?: boolean;
   setToken?: boolean;
+  hash?: string;
   [key: string]: any;
 };
 export const fetchLink = async (url: string, opts?: Options) => {
@@ -39,8 +40,10 @@ export const fetchLink = async (url: string, opts?: Options) => {
   if (token && setToken) {
     fetchURL.searchParams.set('token', token);
   }
+  if (opts?.hash) {
+    fetchURL.searchParams.set('hash', opts.hash);
+  }
   fetchURL.searchParams.set('download', 'true');
-  console.log('fetchURL', fetchURL.toString());
 
   const res = await fetch(fetchURL.toString());
   const blob = await res.blob();
@@ -52,6 +55,7 @@ export const fetchLink = async (url: string, opts?: Options) => {
   const pathname = fetchURL.pathname;
   const filename = pathname.split('/').pop();
   return {
+    status: res.status,
     filename,
     blob,
     type,

src/module/download/upload.ts

@@ -1,5 +1,6 @@
 import { getBufferHash, getHash } from '@/uitls/hash.ts';
 import FormData from 'form-data';
+import { logger } from '../logger.ts';
 export const handleResponse = async (err: any, res: any) => {
   return new Promise((resolve) => {
     if (err) {
@@ -27,6 +28,9 @@ export const getFormParams = (opts: UploadOptions, headers: any): FormData.Submi
   if (opts.token) {
     // url.searchParams.append('token', opts.token);
   }
+  if (opts.meta) {
+    url.searchParams.append('meta', encodeURIComponent(JSON.stringify(opts.meta)));
+  }
   const value: FormData.SubmitOptions = {
     path: url.pathname + url.search,
     host: url.hostname,
@@ -38,7 +42,7 @@ export const getFormParams = (opts: UploadOptions, headers: any): FormData.Submi
       ...headers,
     },
   };
-  console.log('getFormParams', value);
+  logger.debug('getFormParams', value);
   return value;
 };
 type UploadOptions = {
@@ -47,7 +51,21 @@ type UploadOptions = {
   token?: string;
   form?: FormData;
   needHash?: boolean;
+  hash?: string;
+  meta?: Record<string, any>;
 };
+/**
+ * 单个文件上传
+ * @param opts
+ * @param opts.url 上传地址
+ * @param opts.file 文件路径或Buffer
+ * @param opts.token token
+ * @param opts.form form对象
+ * @param opts.needHash 是否需要hash
+ * @param opts.hash hash
+ * @param opts.meta meta
+ * @returns
+ */
 export const upload = (opts: UploadOptions): Promise<{ code?: number; message?: string; [key: string]: any }> => {
   const form = opts?.form || new FormData();
   if (!opts.form) {
@@ -62,7 +80,7 @@ export const upload = (opts: UploadOptions): Promise<{ code?: number; message?:
     }
     form.append('file', value);
     if (opts.needHash) {
-      hash = getBufferHash(value);
+      hash = opts?.hash || getBufferHash(value);
       opts.url = new URL(opts.url.toString());
       opts.url.searchParams.append('hash', hash);
     }

src/module/logger.ts

@@ -1,5 +1,6 @@
 import { Logger } from '@kevisual/logger/node';
 
+const level = process.env.LOG_LEVEL || 'info';
 export const logger = new Logger({
-  level: 'info',
+  level: level as any,
 });

src/scripts/glob.ts

@@ -0,0 +1,20 @@
+import { logger } from '@/module/logger.ts';
+import glob from 'fast-glob';
+import path from 'node:path';
+const root = process.cwd();
+
+export const globFiles = async (pattern: string) => {
+  const res = await glob(pattern, {
+    cwd: root,
+    onlyFiles: true,
+    dot: true,
+    // absolute: true,
+    // ignore: ['build/01-summary.md'],
+  });
+  logger.info(`globFiles:,`, res);
+
+  const key = path.relative(root, res[0]);
+  logger.info(`key: `, key);
+};
+
+globFiles('./build/**/*');

src/uitls/file.ts

@@ -1,11 +1,26 @@
-import fs from 'fs';
+import fs from 'node:fs';
-export const fileIsExist = (filePath: string, isFile = false) => {
+
+export const fileIsExist = (filePath: string) => {
   try {
     // 检查文件或者目录是否存在
     fs.accessSync(filePath, fs.constants.F_OK);
-    if (isFile) {
+    return true;
-      fs.accessSync(filePath, fs.constants.R_OK);
+  } catch (error) {
-    }
+    return false;
+  }
+};
+export const pathExists = (path: string, type?: 'file' | 'directory') => {
+  try {
+    // 检查路径是否存在
+    fs.accessSync(path, fs.constants.F_OK);
+
+    // 如果需要检查类型
+    if (type) {
+      const stats = fs.statSync(path);
+      if (type === 'file' && !stats.isFile()) return false;
+      if (type === 'directory' && !stats.isDirectory()) return false;
+    }
+
     return true;
   } catch (error) {
     return false;

src/uitls/hash.ts

@@ -2,6 +2,7 @@ import MD5 from 'crypto-js/md5.js';
 import fs from 'node:fs';
 
 export const getHash = (file: string) => {
+  if (!fs.existsSync(file)) return '';
   const content = fs.readFileSync(file, 'utf-8');
   return MD5(content).toString();
 };