diff --git a/.gitignore b/.gitignore index 79cc375..416cd97 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ dist pack-dist apps -assistant-app \ No newline at end of file +assistant-app + +build diff --git a/kevisual.json b/kevisual.json index 8b998a6..cf4d258 100644 --- a/kevisual.json +++ b/kevisual.json @@ -1,5 +1,23 @@ { + "metadata": { + "name": "kevisual", + "share": "public" + }, + "syncDirectory": [ + { + "files": [ + "build/**/*" + ], + "ignore": [ + "build/ignore.md" + ], + "registry": "https://kevisual.xiongxiao.me/root/ai/kevisual", + "replace": { + "build/": "" + } + } + ], "sync": { - "build/01-summary.md": "https://kevisual.xiongxiao.me/root/ai/kevisual/01-summary.md" + "./build/01-summary.md": "https://kevisual.xiongxiao.me/root/ai/kevisual/01-summary.md" } } \ No newline at end of file diff --git a/package.json b/package.json index d15aa7b..0219b53 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ }, "devDependencies": { "@kevisual/load": "^0.0.6", - "@kevisual/logger": "^0.0.2", + "@kevisual/logger": "^0.0.3", "@kevisual/query": "0.0.17", "@kevisual/query-login": "0.0.5", "@types/bun": "^1.2.13", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a5a9b0..f35971a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,8 +16,8 @@ importers: specifier: ^0.0.6 version: 0.0.6 '@kevisual/logger': - specifier: ^0.0.2 - version: 0.0.2 + specifier: ^0.0.3 + version: 0.0.3 '@kevisual/query': specifier: 0.0.17 version: 0.0.17(encoding@0.1.13)(ws@8.18.0) @@ -612,8 +612,8 @@ packages: '@kevisual/use-config': ^1.0.11 pm2: ^5.4.3 - '@kevisual/logger@0.0.2': - resolution: {integrity: sha512-4NVdNsOHmMRg+OuZPoNNdI3p7jRII7lMJHRar1IoBck7fFIV7YGMNQirrrjk07MHv+Eh+U+uUljjgEWbse92RA==} + '@kevisual/logger@0.0.3': + resolution: {integrity: sha512-8emqxg+ab62WAK6VY4FQqetXPSSVKFAjGctD1NDbdnxt7YWuI/PyuDltCpsVz+uvWpV1dO5OKZOoHU7ow59Omw==} '@kevisual/query-login@0.0.5': resolution: {integrity: sha512-389cMMWAisjQoafxX+cUEa2z41S5koDjiyHkucfCkhRoP4M6g0iqbBMavLKmLOWSKx3R8e3ZmXT6RfsYGBb8Ww==} @@ -2519,7 +2519,7 @@ snapshots: '@kevisual/use-config': 1.0.11(dotenv@16.5.0) pm2: 6.0.5(supports-color@10.0.0) - '@kevisual/logger@0.0.2': {} + '@kevisual/logger@0.0.3': {} '@kevisual/query-login@0.0.5(@kevisual/query@0.0.17(@kevisual/ws@8.0.0)(encoding@0.1.13))(rollup@4.40.2)(typescript@5.8.2)': dependencies: diff --git a/src/command/sync/modules/base.ts b/src/command/sync/modules/base.ts index 8551cfb..7232c87 100644 --- a/src/command/sync/modules/base.ts +++ b/src/command/sync/modules/base.ts @@ -1,7 +1,9 @@ import path from 'node:path'; import fs from 'node:fs'; -import { Config, SyncList } from './type.ts'; +import { Config, SyncList, SyncConfigType, SyncConfig } from './type.ts'; import { fileIsExist } from '@/uitls/file.ts'; +import { getHash } from '@/uitls/hash.ts'; +import glob from 'fast-glob'; export type SyncOptions = { dir?: string; @@ -28,6 +30,15 @@ export class SyncBase { const filepath = path.join(dir, filename); if (!fileIsExist(filepath)) throw new Error('config file not found'); const config = JSON.parse(fs.readFileSync(filepath, 'utf-8')); + const sync = config.sync || {}; + const keys = Object.keys(sync); + const newConfigSync: any = {}; + for (let key of keys) { + const keyPath = path.join(dir, key); + const newKey = path.relative(dir, keyPath); + newConfigSync[newKey] = sync[key]; + } + config.sync = newConfigSync; this.config = config; return config; } catch (err) { @@ -35,10 +46,15 @@ export class SyncBase { return {} as Config; } } - - async getSyncList(): Promise { + async canDone(syncType: SyncConfigType, type?: SyncConfigType) { + if (syncType === 'sync') return true; + return syncType === type; + } + async getSyncList(opts?: { getFile?: boolean }): Promise { const config = this.config!; - const sync = config?.sync || {}; + let sync = config?.sync || {}; + const syncDirectory = await this.getSyncDirectoryList(); + sync = this.getMergeSync(sync, syncDirectory.sync); const syncKeys = Object.keys(sync); const baseURL = this.baseURL; const syncList = syncKeys.map((key) => { @@ -52,21 +68,92 @@ export class SyncBase { return false; }; if (typeof value === 'string') { + const auth = checkAuth(value, baseURL); + const type = auth ? 'sync' : 'none'; return { + key, + type: type as any, filepath, url: value, - auth: checkAuth(value, baseURL), + auth, }; } + const auth = checkAuth(value.url, baseURL); + const type = auth ? 'sync' : 'none'; return { + key, filepath, ...value, + type: value?.type ?? type, auth: checkAuth(value.url, baseURL), }; }); - + if (opts?.getFile) { + return this.getSyncListFile(syncList); + } return syncList; } + getMergeSync(sync: Config['sync'] = {}, fileSync: Config['sync'] = {}) { + const syncFileSyncKeys = Object.keys(fileSync); + const syncKeys = Object.keys(sync); + const keys = [...syncKeys, ...syncFileSyncKeys]; + const obj: Config['sync'] = {}; + for (let key of keys) { + const value = sync[key] ?? fileSync[key]; + obj[key] = value; + } + return obj; + } + async getSyncDirectoryList() { + const config = this.config; + const syncDirectory = config?.syncDirectory || []; + let obj: Record = {}; + const keys: string[] = []; + for (let item of syncDirectory) { + const { registry, ignore = [], files, replace = {} } = item; + const cwd = this.#dir; + const glob_files = await glob(files, { + ignore, + onlyFiles: true, + cwd, + dot: true, + absolute: true, + }); + for (let file of glob_files) { + const key = path.relative(cwd, file); + const _registryURL = new URL(registry); + const replaceKeys = Object.keys(replace); + let newKey = key; + for (let replaceKey of replaceKeys) { + if (newKey.startsWith(replaceKey)) { + newKey = key.replace(replaceKey, replace[replaceKey]); + } + } + const pathname = path.join(_registryURL.pathname, newKey); + _registryURL.pathname = pathname; + keys.push(key); + obj[key] = _registryURL.toString(); + } + } + return { sync: obj, keys }; + } + async getSyncListFile(syncList: SyncList[]) { + let syncListFile: SyncList[] = []; + for (let item of syncList) { + const { filepath, auth } = item; + if (filepath && fileIsExist(filepath) && auth) { + syncListFile.push({ + ...item, + exist: true, + hash: getHash(filepath), + }); + } else { + syncListFile.push({ ...item, exist: false }); + } + } + return syncListFile; + } + getHash = getHash; async getDir(filepath: string, check = false) { const dir = path.dirname(filepath); if (check) { diff --git a/src/command/sync/modules/type.ts b/src/command/sync/modules/type.ts index 9f7646b..c08b8a5 100644 --- a/src/command/sync/modules/type.ts +++ b/src/command/sync/modules/type.ts @@ -1,19 +1,38 @@ +export type SyncConfigType = 'sync' | 'download' | 'upload' | 'none'; export type SyncConfig = { - type?: 'sync'; // 是否可以同步 + type?: SyncConfigType; // 是否可以同步 url: string; // 文件具体的 url 的地址 }; +export type SyncDirectory = { + /** + * 忽略的目录或则文件,默认忽略 node_modules 使用 fast-glob 去匹配, + * 当同步文件夹的时候生效 + **/ + ignore?: string[]; + /** + * 合并路径的源地址,https://kevisual.xiongxiao.me/root/ai/kevisual + */ + registry?: string; + files?: string[]; + replace?: Record; +}; export interface Config { name?: string; // 项目名称 version?: string; // 项目版本号 - ignore?: string[]; // 忽略的目录或则文件,默认忽略 node_modules 使用 fast-glob 去匹配 - + registry?: string; // 项目仓库地址 + user?: string; // 同步用户,否则会自动 query 一次 + metadata?: Record; // 元数据, 统一的配置 + syncDirectory: SyncDirectory[]; sync: { [key: string]: SyncConfig | string; }; } export type SyncList = { + key?: string; filepath: string; + exist?: boolean; + hash?: string; /** * 是否需要鉴权, baseURL 为 kevisual 服务时,需要鉴权 */ diff --git a/src/command/sync/sync.ts b/src/command/sync/sync.ts index a0a485a..434d664 100644 --- a/src/command/sync/sync.ts +++ b/src/command/sync/sync.ts @@ -1,8 +1,11 @@ import { program as app, Command } from '@/program.ts'; import { SyncBase } from './modules/base.ts'; -import { baseURL } from '@/module/query.ts'; +import { baseURL, storage } from '@/module/query.ts'; import { fetchLink } from '@/module/download/install.ts'; import fs from 'node:fs'; +import { upload } from '@/module/download/upload.ts'; +import { logger } from '@/module/logger.ts'; +import { chalk } from '@/module/chalk.ts'; const command = new Command('sync') .option('-d --dir ') @@ -10,21 +13,76 @@ const command = new Command('sync') .action(() => { console.log('同步项目'); }); -const syncUpload = new Command('upload').description('上传项目').action(() => { - console.log('上传项目'); -}); +const syncUpload = new Command('upload') + .option('-d --dir ', '配置目录') + .option('-s --share ', '共享设置') + .description('上传项目') + .action(async (opts) => { + console.log('上传项目'); + const sync = new SyncBase({ baseURL: baseURL }); + const syncList = await sync.getSyncList({ getFile: true }); + logger.debug(syncList); + const nodonwArr: (typeof syncList)[number][] = []; + const token = storage.getItem('token'); + const meta: Record = { + ...sync.config.metadata, + }; + if (opts.share) { + meta.share = opts.share; + } + for (const item of syncList) { + if (!item.auth || !item.exist) { + nodonwArr.push(item); + continue; + } + if (!sync.canDone(item.type, 'upload')) { + nodonwArr.push(item); + continue; + } + const res = await upload({ + token, + file: fs.readFileSync(item.filepath), + url: item.url, + needHash: true, + hash: item.hash, + meta, + }); + if (res.code === 200) { + logger.info('上传成功', item.key, chalk.green(item.url)); + } + logger.debug(res); + } + if (nodonwArr.length) { + logger.warn('以下文件未上传\n', nodonwArr.map((item) => item.key).join(',')); + } + }); const syncDownload = new Command('download') .option('-d --dir ', '配置目录') .description('下载项目') .action(async () => { - console.log('下载项目'); const sync = new SyncBase({ baseURL: baseURL }); const syncList = await sync.getSyncList(); - console.log(syncList); + logger.debug(syncList); + const nodonwArr: (typeof syncList)[number][] = []; for (const item of syncList) { - const { content } = await fetchLink(item.url, { setToken: item.auth, returnContent: true }); - await sync.getDir(item.filepath, true); - fs.writeFileSync(item.filepath, content); + if (!sync.canDone(item.type, 'download')) { + nodonwArr.push(item); + continue; + } + const hash = sync.getHash(item.filepath); + const { content, status } = await fetchLink(item.url, { setToken: item.auth, returnContent: true, hash }); + if (status === 200) { + await sync.getDir(item.filepath, true); + fs.writeFileSync(item.filepath, content); + logger.info('下载成功', item.key, chalk.green(item.url)); + } else if (status === 304) { + logger.info('文件未修改', item.key, chalk.green(item.url)); + } else { + logger.error('下载失败', item.key, chalk.red(item.url)); + } + } + if (nodonwArr.length) { + logger.warn('以下文件未下载', nodonwArr.map((item) => item.key).join(',')); } }); diff --git a/src/module/download/install.ts b/src/module/download/install.ts index 87e8811..1c0400a 100644 --- a/src/module/download/install.ts +++ b/src/module/download/install.ts @@ -24,6 +24,7 @@ type Options = { check?: boolean; returnContent?: boolean; setToken?: boolean; + hash?: string; [key: string]: any; }; export const fetchLink = async (url: string, opts?: Options) => { @@ -39,8 +40,10 @@ export const fetchLink = async (url: string, opts?: Options) => { if (token && setToken) { fetchURL.searchParams.set('token', token); } + if (opts?.hash) { + fetchURL.searchParams.set('hash', opts.hash); + } fetchURL.searchParams.set('download', 'true'); - console.log('fetchURL', fetchURL.toString()); const res = await fetch(fetchURL.toString()); const blob = await res.blob(); @@ -52,6 +55,7 @@ export const fetchLink = async (url: string, opts?: Options) => { const pathname = fetchURL.pathname; const filename = pathname.split('/').pop(); return { + status: res.status, filename, blob, type, diff --git a/src/module/download/upload.ts b/src/module/download/upload.ts index 05ab46b..9761446 100644 --- a/src/module/download/upload.ts +++ b/src/module/download/upload.ts @@ -1,5 +1,6 @@ import { getBufferHash, getHash } from '@/uitls/hash.ts'; import FormData from 'form-data'; +import { logger } from '../logger.ts'; export const handleResponse = async (err: any, res: any) => { return new Promise((resolve) => { if (err) { @@ -27,6 +28,9 @@ export const getFormParams = (opts: UploadOptions, headers: any): FormData.Submi if (opts.token) { // url.searchParams.append('token', opts.token); } + if (opts.meta) { + url.searchParams.append('meta', encodeURIComponent(JSON.stringify(opts.meta))); + } const value: FormData.SubmitOptions = { path: url.pathname + url.search, host: url.hostname, @@ -38,7 +42,7 @@ export const getFormParams = (opts: UploadOptions, headers: any): FormData.Submi ...headers, }, }; - console.log('getFormParams', value); + logger.debug('getFormParams', value); return value; }; type UploadOptions = { @@ -47,7 +51,21 @@ type UploadOptions = { token?: string; form?: FormData; needHash?: boolean; + hash?: string; + meta?: Record; }; +/** + * 单个文件上传 + * @param opts + * @param opts.url 上传地址 + * @param opts.file 文件路径或Buffer + * @param opts.token token + * @param opts.form form对象 + * @param opts.needHash 是否需要hash + * @param opts.hash hash + * @param opts.meta meta + * @returns + */ export const upload = (opts: UploadOptions): Promise<{ code?: number; message?: string; [key: string]: any }> => { const form = opts?.form || new FormData(); if (!opts.form) { @@ -62,7 +80,7 @@ export const upload = (opts: UploadOptions): Promise<{ code?: number; message?: } form.append('file', value); if (opts.needHash) { - hash = getBufferHash(value); + hash = opts?.hash || getBufferHash(value); opts.url = new URL(opts.url.toString()); opts.url.searchParams.append('hash', hash); } diff --git a/src/module/logger.ts b/src/module/logger.ts index 41c028e..00b486a 100644 --- a/src/module/logger.ts +++ b/src/module/logger.ts @@ -1,5 +1,6 @@ import { Logger } from '@kevisual/logger/node'; +const level = process.env.LOG_LEVEL || 'info'; export const logger = new Logger({ - level: 'info', + level: level as any, }); diff --git a/src/scripts/glob.ts b/src/scripts/glob.ts new file mode 100644 index 0000000..44c054b --- /dev/null +++ b/src/scripts/glob.ts @@ -0,0 +1,20 @@ +import { logger } from '@/module/logger.ts'; +import glob from 'fast-glob'; +import path from 'node:path'; +const root = process.cwd(); + +export const globFiles = async (pattern: string) => { + const res = await glob(pattern, { + cwd: root, + onlyFiles: true, + dot: true, + // absolute: true, + // ignore: ['build/01-summary.md'], + }); + logger.info(`globFiles:,`, res); + + const key = path.relative(root, res[0]); + logger.info(`key: `, key); +}; + +globFiles('./build/**/*'); diff --git a/src/uitls/file.ts b/src/uitls/file.ts index 26e6b33..ac3c8f5 100644 --- a/src/uitls/file.ts +++ b/src/uitls/file.ts @@ -1,11 +1,26 @@ -import fs from 'fs'; -export const fileIsExist = (filePath: string, isFile = false) => { +import fs from 'node:fs'; + +export const fileIsExist = (filePath: string) => { try { // 检查文件或者目录是否存在 fs.accessSync(filePath, fs.constants.F_OK); - if (isFile) { - fs.accessSync(filePath, fs.constants.R_OK); - } + return true; + } catch (error) { + return false; + } +}; +export const pathExists = (path: string, type?: 'file' | 'directory') => { + try { + // 检查路径是否存在 + fs.accessSync(path, fs.constants.F_OK); + + // 如果需要检查类型 + if (type) { + const stats = fs.statSync(path); + if (type === 'file' && !stats.isFile()) return false; + if (type === 'directory' && !stats.isDirectory()) return false; + } + return true; } catch (error) { return false; diff --git a/src/uitls/hash.ts b/src/uitls/hash.ts index e9c03d5..c435b7c 100644 --- a/src/uitls/hash.ts +++ b/src/uitls/hash.ts @@ -2,6 +2,7 @@ import MD5 from 'crypto-js/md5.js'; import fs from 'node:fs'; export const getHash = (file: string) => { + if (!fs.existsSync(file)) return ''; const content = fs.readFileSync(file, 'utf-8'); return MD5(content).toString(); };