"feat: 同步功能增强与配置优化，支持多类型同步及日志分级"

2025-05-12 23:53:45 +08:00 · 2025-05-12 23:53:45 +08:00 · 785bd7b004
commit 785bd7b004
parent eaccbf5ada
13 changed files with 278 additions and 35 deletions
--- a/.gitignore
+++ b/.gitignore
@ -5,4 +5,6 @@ dist

 pack-dist
 apps
-assistant-app
+assistant-app
+
+build
--- a/kevisual.json
+++ b/kevisual.json
@ -1,5 +1,23 @@
 {
+  "metadata": {
+    "name": "kevisual",
+    "share": "public"
+  },
+  "syncDirectory": [
+    {
+      "files": [
+        "build/**/*"
+      ],
+      "ignore": [
+        "build/ignore.md"
+      ],
+      "registry": "https://kevisual.xiongxiao.me/root/ai/kevisual",
+      "replace": {
+        "build/": ""
+      }
+    }
+  ],
  "sync": {
-    "build/01-summary.md": "https://kevisual.xiongxiao.me/root/ai/kevisual/01-summary.md"
+    "./build/01-summary.md": "https://kevisual.xiongxiao.me/root/ai/kevisual/01-summary.md"
  }
 }
--- a/package.json
+++ b/package.json
@ -42,7 +42,7 @@
  },
  "devDependencies": {
    "@kevisual/load": "^0.0.6",
-    "@kevisual/logger": "^0.0.2",
+    "@kevisual/logger": "^0.0.3",
    "@kevisual/query": "0.0.17",
    "@kevisual/query-login": "0.0.5",
    "@types/bun": "^1.2.13",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -16,8 +16,8 @@ importers:
        specifier: ^0.0.6
        version: 0.0.6
      '@kevisual/logger':
-        specifier: ^0.0.2
-        version: 0.0.2
+        specifier: ^0.0.3
+        version: 0.0.3
      '@kevisual/query':
        specifier: 0.0.17
        version: 0.0.17(encoding@0.1.13)(ws@8.18.0)
@ -612,8 +612,8 @@ packages:
      '@kevisual/use-config': ^1.0.11
      pm2: ^5.4.3

-  '@kevisual/logger@0.0.2':
-    resolution: {integrity: sha512-4NVdNsOHmMRg+OuZPoNNdI3p7jRII7lMJHRar1IoBck7fFIV7YGMNQirrrjk07MHv+Eh+U+uUljjgEWbse92RA==}
+  '@kevisual/logger@0.0.3':
+    resolution: {integrity: sha512-8emqxg+ab62WAK6VY4FQqetXPSSVKFAjGctD1NDbdnxt7YWuI/PyuDltCpsVz+uvWpV1dO5OKZOoHU7ow59Omw==}

  '@kevisual/query-login@0.0.5':
    resolution: {integrity: sha512-389cMMWAisjQoafxX+cUEa2z41S5koDjiyHkucfCkhRoP4M6g0iqbBMavLKmLOWSKx3R8e3ZmXT6RfsYGBb8Ww==}
@ -2519,7 +2519,7 @@ snapshots:
      '@kevisual/use-config': 1.0.11(dotenv@16.5.0)
      pm2: 6.0.5(supports-color@10.0.0)

-  '@kevisual/logger@0.0.2': {}
+  '@kevisual/logger@0.0.3': {}

  '@kevisual/query-login@0.0.5(@kevisual/query@0.0.17(@kevisual/ws@8.0.0)(encoding@0.1.13))(rollup@4.40.2)(typescript@5.8.2)':
    dependencies:
--- a/src/command/sync/modules/base.ts
+++ b/src/command/sync/modules/base.ts
@ -1,7 +1,9 @@
 import path from 'node:path';
 import fs from 'node:fs';
-import { Config, SyncList } from './type.ts';
+import { Config, SyncList, SyncConfigType, SyncConfig } from './type.ts';
 import { fileIsExist } from '@/uitls/file.ts';
+import { getHash } from '@/uitls/hash.ts';
+import glob from 'fast-glob';

 export type SyncOptions = {
  dir?: string;
@ -28,6 +30,15 @@ export class SyncBase {
      const filepath = path.join(dir, filename);
      if (!fileIsExist(filepath)) throw new Error('config file not found');
      const config = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
+      const sync = config.sync || {};
+      const keys = Object.keys(sync);
+      const newConfigSync: any = {};
+      for (let key of keys) {
+        const keyPath = path.join(dir, key);
+        const newKey = path.relative(dir, keyPath);
+        newConfigSync[newKey] = sync[key];
+      }
+      config.sync = newConfigSync;
      this.config = config;
      return config;
    } catch (err) {
@ -35,10 +46,15 @@ export class SyncBase {
      return {} as Config;
    }
  }
-
-  async getSyncList(): Promise<SyncList[]> {
+  async canDone(syncType: SyncConfigType, type?: SyncConfigType) {
+    if (syncType === 'sync') return true;
+    return syncType === type;
+  }
+  async getSyncList(opts?: { getFile?: boolean }): Promise<SyncList[]> {
    const config = this.config!;
-    const sync = config?.sync || {};
+    let sync = config?.sync || {};
+    const syncDirectory = await this.getSyncDirectoryList();
+    sync = this.getMergeSync(sync, syncDirectory.sync);
    const syncKeys = Object.keys(sync);
    const baseURL = this.baseURL;
    const syncList = syncKeys.map((key) => {
@ -52,21 +68,92 @@ export class SyncBase {
        return false;
      };
      if (typeof value === 'string') {
+        const auth = checkAuth(value, baseURL);
+        const type = auth ? 'sync' : 'none';
        return {
+          key,
+          type: type as any,
          filepath,
          url: value,
-          auth: checkAuth(value, baseURL),
+          auth,
        };
      }
+      const auth = checkAuth(value.url, baseURL);
+      const type = auth ? 'sync' : 'none';
      return {
+        key,
        filepath,
        ...value,
+        type: value?.type ?? type,
        auth: checkAuth(value.url, baseURL),
      };
    });
-
+    if (opts?.getFile) {
+      return this.getSyncListFile(syncList);
+    }
    return syncList;
  }
+  getMergeSync(sync: Config['sync'] = {}, fileSync: Config['sync'] = {}) {
+    const syncFileSyncKeys = Object.keys(fileSync);
+    const syncKeys = Object.keys(sync);
+    const keys = [...syncKeys, ...syncFileSyncKeys];
+    const obj: Config['sync'] = {};
+    for (let key of keys) {
+      const value = sync[key] ?? fileSync[key];
+      obj[key] = value;
+    }
+    return obj;
+  }
+  async getSyncDirectoryList() {
+    const config = this.config;
+    const syncDirectory = config?.syncDirectory || [];
+    let obj: Record<string, string> = {};
+    const keys: string[] = [];
+    for (let item of syncDirectory) {
+      const { registry, ignore = [], files, replace = {} } = item;
+      const cwd = this.#dir;
+      const glob_files = await glob(files, {
+        ignore,
+        onlyFiles: true,
+        cwd,
+        dot: true,
+        absolute: true,
+      });
+      for (let file of glob_files) {
+        const key = path.relative(cwd, file);
+        const _registryURL = new URL(registry);
+        const replaceKeys = Object.keys(replace);
+        let newKey = key;
+        for (let replaceKey of replaceKeys) {
+          if (newKey.startsWith(replaceKey)) {
+            newKey = key.replace(replaceKey, replace[replaceKey]);
+          }
+        }
+        const pathname = path.join(_registryURL.pathname, newKey);
+        _registryURL.pathname = pathname;
+        keys.push(key);
+        obj[key] = _registryURL.toString();
+      }
+    }
+    return { sync: obj, keys };
+  }
+  async getSyncListFile(syncList: SyncList[]) {
+    let syncListFile: SyncList[] = [];
+    for (let item of syncList) {
+      const { filepath, auth } = item;
+      if (filepath && fileIsExist(filepath) && auth) {
+        syncListFile.push({
+          ...item,
+          exist: true,
+          hash: getHash(filepath),
+        });
+      } else {
+        syncListFile.push({ ...item, exist: false });
+      }
+    }
+    return syncListFile;
+  }
+  getHash = getHash;
  async getDir(filepath: string, check = false) {
    const dir = path.dirname(filepath);
    if (check) {
--- a/src/command/sync/modules/type.ts
+++ b/src/command/sync/modules/type.ts
@ -1,19 +1,38 @@
+export type SyncConfigType = 'sync' | 'download' | 'upload' | 'none';
 export type SyncConfig = {
-  type?: 'sync'; // 是否可以同步
+  type?: SyncConfigType; // 是否可以同步
  url: string; // 文件具体的 url 的地址
 };
+export type SyncDirectory = {
+  /**
+   * 忽略的目录或则文件，默认忽略 node_modules 使用 fast-glob 去匹配,
+   * 当同步文件夹的时候生效
+   **/
+  ignore?: string[];
+  /**
+   * 合并路径的源地址，https://kevisual.xiongxiao.me/root/ai/kevisual
+   */
+  registry?: string;
+  files?: string[];
+  replace?: Record<string, string>;
+};
 export interface Config {
  name?: string; // 项目名称
  version?: string; // 项目版本号
-  ignore?: string[]; // 忽略的目录或则文件，默认忽略 node_modules 使用 fast-glob 去匹配
-
+  registry?: string; // 项目仓库地址
+  user?: string; // 同步用户，否则会自动 query 一次
+  metadata?: Record<string, any>; // 元数据, 统一的配置
+  syncDirectory: SyncDirectory[];
  sync: {
    [key: string]: SyncConfig | string;
  };
 }

 export type SyncList = {
+  key?: string;
  filepath: string;
+  exist?: boolean;
+  hash?: string;
  /**
   * 是否需要鉴权, baseURL 为 kevisual 服务时，需要鉴权
   */
--- a/src/command/sync/sync.ts
+++ b/src/command/sync/sync.ts
@ -1,8 +1,11 @@
 import { program as app, Command } from '@/program.ts';
 import { SyncBase } from './modules/base.ts';
-import { baseURL } from '@/module/query.ts';
+import { baseURL, storage } from '@/module/query.ts';
 import { fetchLink } from '@/module/download/install.ts';
 import fs from 'node:fs';
+import { upload } from '@/module/download/upload.ts';
+import { logger } from '@/module/logger.ts';
+import { chalk } from '@/module/chalk.ts';

 const command = new Command('sync')
  .option('-d --dir <dir>')
@ -10,21 +13,76 @@ const command = new Command('sync')
  .action(() => {
    console.log('同步项目');
  });
-const syncUpload = new Command('upload').description('上传项目').action(() => {
-  console.log('上传项目');
-});
+const syncUpload = new Command('upload')
+  .option('-d --dir <dir>', '配置目录')
+  .option('-s --share <share>', '共享设置')
+  .description('上传项目')
+  .action(async (opts) => {
+    console.log('上传项目');
+    const sync = new SyncBase({ baseURL: baseURL });
+    const syncList = await sync.getSyncList({ getFile: true });
+    logger.debug(syncList);
+    const nodonwArr: (typeof syncList)[number][] = [];
+    const token = storage.getItem('token');
+    const meta: Record<string, string> = {
+      ...sync.config.metadata,
+    };
+    if (opts.share) {
+      meta.share = opts.share;
+    }
+    for (const item of syncList) {
+      if (!item.auth || !item.exist) {
+        nodonwArr.push(item);
+        continue;
+      }
+      if (!sync.canDone(item.type, 'upload')) {
+        nodonwArr.push(item);
+        continue;
+      }
+      const res = await upload({
+        token,
+        file: fs.readFileSync(item.filepath),
+        url: item.url,
+        needHash: true,
+        hash: item.hash,
+        meta,
+      });
+      if (res.code === 200) {
+        logger.info('上传成功', item.key, chalk.green(item.url));
+      }
+      logger.debug(res);
+    }
+    if (nodonwArr.length) {
+      logger.warn('以下文件未上传\n', nodonwArr.map((item) => item.key).join(','));
+    }
+  });
 const syncDownload = new Command('download')
  .option('-d --dir <dir>', '配置目录')
  .description('下载项目')
  .action(async () => {
-    console.log('下载项目');
    const sync = new SyncBase({ baseURL: baseURL });
    const syncList = await sync.getSyncList();
-    console.log(syncList);
+    logger.debug(syncList);
+    const nodonwArr: (typeof syncList)[number][] = [];
    for (const item of syncList) {
-      const { content } = await fetchLink(item.url, { setToken: item.auth, returnContent: true });
-      await sync.getDir(item.filepath, true);
-      fs.writeFileSync(item.filepath, content);
+      if (!sync.canDone(item.type, 'download')) {
+        nodonwArr.push(item);
+        continue;
+      }
+      const hash = sync.getHash(item.filepath);
+      const { content, status } = await fetchLink(item.url, { setToken: item.auth, returnContent: true, hash });
+      if (status === 200) {
+        await sync.getDir(item.filepath, true);
+        fs.writeFileSync(item.filepath, content);
+        logger.info('下载成功', item.key, chalk.green(item.url));
+      } else if (status === 304) {
+        logger.info('文件未修改', item.key, chalk.green(item.url));
+      } else {
+        logger.error('下载失败', item.key, chalk.red(item.url));
+      }
+    }
+    if (nodonwArr.length) {
+      logger.warn('以下文件未下载', nodonwArr.map((item) => item.key).join(','));
    }
  });

--- a/src/module/download/install.ts
+++ b/src/module/download/install.ts
@ -24,6 +24,7 @@ type Options = {
  check?: boolean;
  returnContent?: boolean;
  setToken?: boolean;
+  hash?: string;
  [key: string]: any;
 };
 export const fetchLink = async (url: string, opts?: Options) => {
@ -39,8 +40,10 @@ export const fetchLink = async (url: string, opts?: Options) => {
  if (token && setToken) {
    fetchURL.searchParams.set('token', token);
  }
+  if (opts?.hash) {
+    fetchURL.searchParams.set('hash', opts.hash);
+  }
  fetchURL.searchParams.set('download', 'true');
-  console.log('fetchURL', fetchURL.toString());

  const res = await fetch(fetchURL.toString());
  const blob = await res.blob();
@ -52,6 +55,7 @@ export const fetchLink = async (url: string, opts?: Options) => {
  const pathname = fetchURL.pathname;
  const filename = pathname.split('/').pop();
  return {
+    status: res.status,
    filename,
    blob,
    type,
--- a/src/module/download/upload.ts
+++ b/src/module/download/upload.ts
@ -1,5 +1,6 @@
 import { getBufferHash, getHash } from '@/uitls/hash.ts';
 import FormData from 'form-data';
+import { logger } from '../logger.ts';
 export const handleResponse = async (err: any, res: any) => {
  return new Promise((resolve) => {
    if (err) {
@ -27,6 +28,9 @@ export const getFormParams = (opts: UploadOptions, headers: any): FormData.Submi
  if (opts.token) {
    // url.searchParams.append('token', opts.token);
  }
+  if (opts.meta) {
+    url.searchParams.append('meta', encodeURIComponent(JSON.stringify(opts.meta)));
+  }
  const value: FormData.SubmitOptions = {
    path: url.pathname + url.search,
    host: url.hostname,
@ -38,7 +42,7 @@ export const getFormParams = (opts: UploadOptions, headers: any): FormData.Submi
      ...headers,
    },
  };
-  console.log('getFormParams', value);
+  logger.debug('getFormParams', value);
  return value;
 };
 type UploadOptions = {
@ -47,7 +51,21 @@ type UploadOptions = {
  token?: string;
  form?: FormData;
  needHash?: boolean;
+  hash?: string;
+  meta?: Record<string, any>;
 };
+/**
+ * 单个文件上传
+ * @param opts
+ * @param opts.url 上传地址
+ * @param opts.file 文件路径或Buffer
+ * @param opts.token token
+ * @param opts.form form对象
+ * @param opts.needHash 是否需要hash
+ * @param opts.hash hash
+ * @param opts.meta meta
+ * @returns
+ */
 export const upload = (opts: UploadOptions): Promise<{ code?: number; message?: string; [key: string]: any }> => {
  const form = opts?.form || new FormData();
  if (!opts.form) {
@ -62,7 +80,7 @@ export const upload = (opts: UploadOptions): Promise<{ code?: number; message?:
    }
    form.append('file', value);
    if (opts.needHash) {
-      hash = getBufferHash(value);
+      hash = opts?.hash || getBufferHash(value);
      opts.url = new URL(opts.url.toString());
      opts.url.searchParams.append('hash', hash);
    }
--- a/src/module/logger.ts
+++ b/src/module/logger.ts
@ -1,5 +1,6 @@
 import { Logger } from '@kevisual/logger/node';

+const level = process.env.LOG_LEVEL || 'info';
 export const logger = new Logger({
-  level: 'info',
+  level: level as any,
 });
--- a/src/scripts/glob.ts
+++ b/src/scripts/glob.ts
@ -0,0 +1,20 @@
+import { logger } from '@/module/logger.ts';
+import glob from 'fast-glob';
+import path from 'node:path';
+const root = process.cwd();
+
+export const globFiles = async (pattern: string) => {
+  const res = await glob(pattern, {
+    cwd: root,
+    onlyFiles: true,
+    dot: true,
+    // absolute: true,
+    // ignore: ['build/01-summary.md'],
+  });
+  logger.info(`globFiles:,`, res);
+
+  const key = path.relative(root, res[0]);
+  logger.info(`key: `, key);
+};
+
+globFiles('./build/**/*');
--- a/src/uitls/file.ts
+++ b/src/uitls/file.ts
@ -1,11 +1,26 @@
-import fs from 'fs';
-export const fileIsExist = (filePath: string, isFile = false) => {
+import fs from 'node:fs';
+
+export const fileIsExist = (filePath: string) => {
  try {
    // 检查文件或者目录是否存在
    fs.accessSync(filePath, fs.constants.F_OK);
-    if (isFile) {
-      fs.accessSync(filePath, fs.constants.R_OK);
-    }
+    return true;
+  } catch (error) {
+    return false;
+  }
+};
+export const pathExists = (path: string, type?: 'file' | 'directory') => {
+  try {
+    // 检查路径是否存在
+    fs.accessSync(path, fs.constants.F_OK);
+
+    // 如果需要检查类型
+    if (type) {
+      const stats = fs.statSync(path);
+      if (type === 'file' && !stats.isFile()) return false;
+      if (type === 'directory' && !stats.isDirectory()) return false;
+    }
+
    return true;
  } catch (error) {
    return false;
--- a/src/uitls/hash.ts
+++ b/src/uitls/hash.ts
@ -2,6 +2,7 @@ import MD5 from 'crypto-js/md5.js';
 import fs from 'node:fs';

 export const getHash = (file: string) => {
+  if (!fs.existsSync(file)) return '';
  const content = fs.readFileSync(file, 'utf-8');
  return MD5(content).toString();
 };