From eaccbf5adad5fdc81c294c8da9c860ac89df802d Mon Sep 17 00:00:00 2001 From: xion Date: Mon, 12 May 2025 04:30:17 +0800 Subject: [PATCH] add sync download --- kevisual.json | 5 ++ package.json | 3 +- pnpm-lock.yaml | 8 +++ src/command/deploy.ts | 8 +-- src/command/sync.ts | 18 ------- src/command/sync/modules/base.ts | 87 ++++++++++++++++++++++++++++++++ src/command/sync/modules/type.ts | 21 ++++++++ src/command/sync/sync.ts | 33 ++++++++++++ src/index.ts | 2 +- src/module/download/install.ts | 6 ++- src/module/download/upload.ts | 17 ++++++- src/module/logger.ts | 5 ++ src/scripts/upload.ts | 27 ++++++++-- src/uitls/hash.ts | 11 ++++ 14 files changed, 219 insertions(+), 32 deletions(-) create mode 100644 kevisual.json delete mode 100644 src/command/sync.ts create mode 100644 src/command/sync/modules/base.ts create mode 100644 src/command/sync/modules/type.ts create mode 100644 src/command/sync/sync.ts create mode 100644 src/module/logger.ts create mode 100644 src/uitls/hash.ts diff --git a/kevisual.json b/kevisual.json new file mode 100644 index 0000000..8b998a6 --- /dev/null +++ b/kevisual.json @@ -0,0 +1,5 @@ +{ + "sync": { + "build/01-summary.md": "https://kevisual.xiongxiao.me/root/ai/kevisual/01-summary.md" + } +} \ No newline at end of file diff --git a/package.json b/package.json index 89840de..d15aa7b 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "bun.config.mjs" ], "scripts": { - "dev": "bun src/run.ts ", + "dev": "NODE_ENV=development bun src/run.ts ", "dev:tsx": "tsx src/run.ts ", "build": "rimraf dist && bun run bun.config.mjs", "postbuild": "cd assistant && pnpm build", @@ -42,6 +42,7 @@ }, "devDependencies": { "@kevisual/load": "^0.0.6", + "@kevisual/logger": "^0.0.2", "@kevisual/query": "0.0.17", "@kevisual/query-login": "0.0.5", "@types/bun": "^1.2.13", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 554d5b4..1a5a9b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,6 +15,9 @@ importers: '@kevisual/load': specifier: ^0.0.6 version: 0.0.6 + '@kevisual/logger': + specifier: ^0.0.2 + version: 0.0.2 '@kevisual/query': specifier: 0.0.17 version: 0.0.17(encoding@0.1.13)(ws@8.18.0) @@ -609,6 +612,9 @@ packages: '@kevisual/use-config': ^1.0.11 pm2: ^5.4.3 + '@kevisual/logger@0.0.2': + resolution: {integrity: sha512-4NVdNsOHmMRg+OuZPoNNdI3p7jRII7lMJHRar1IoBck7fFIV7YGMNQirrrjk07MHv+Eh+U+uUljjgEWbse92RA==} + '@kevisual/query-login@0.0.5': resolution: {integrity: sha512-389cMMWAisjQoafxX+cUEa2z41S5koDjiyHkucfCkhRoP4M6g0iqbBMavLKmLOWSKx3R8e3ZmXT6RfsYGBb8Ww==} peerDependencies: @@ -2513,6 +2519,8 @@ snapshots: '@kevisual/use-config': 1.0.11(dotenv@16.5.0) pm2: 6.0.5(supports-color@10.0.0) + '@kevisual/logger@0.0.2': {} + '@kevisual/query-login@0.0.5(@kevisual/query@0.0.17(@kevisual/ws@8.0.0)(encoding@0.1.13))(rollup@4.40.2)(typescript@5.8.2)': dependencies: '@kevisual/cache': 0.0.2(rollup@4.40.2)(tslib@2.8.1)(typescript@5.8.2) diff --git a/src/command/deploy.ts b/src/command/deploy.ts index 350b720..36b98ef 100644 --- a/src/command/deploy.ts +++ b/src/command/deploy.ts @@ -9,9 +9,8 @@ import inquirer from 'inquirer'; import { packLib, unpackLib } from './publish.ts'; import chalk from 'chalk'; import { installDeps } from '@/uitls/npm.ts'; -import cryptojs from 'crypto-js'; import { upload } from '@/module/download/upload.ts'; -const MD5 = cryptojs.MD5; +import { getHash } from '@/uitls/hash.ts'; /** * 获取package.json 中的 basename, version, user, appKey * @returns @@ -150,10 +149,7 @@ const command = new Command('deploy') console.error('error', error); } }); -export const getHash = (file: string) => { - const content = fs.readFileSync(file, 'utf-8'); - return MD5(content).toString(); -}; + type UploadFileOptions = { key: string; version: string; diff --git a/src/command/sync.ts b/src/command/sync.ts deleted file mode 100644 index b924666..0000000 --- a/src/command/sync.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { program as app, Command } from '@/program.ts'; - -const command = new Command('sync') - .option('-d --dir ') - .description('同步项目') - .action(() => { - console.log('同步项目'); - }); -const syncUpload = new Command('upload').description('上传项目').action(() => { - console.log('上传项目'); -}); -const syncDownload = new Command('download').description('下载项目').action(() => { - console.log('下载项目'); -}); - -command.addCommand(syncUpload); -command.addCommand(syncDownload); -app.addCommand(command); diff --git a/src/command/sync/modules/base.ts b/src/command/sync/modules/base.ts new file mode 100644 index 0000000..8551cfb --- /dev/null +++ b/src/command/sync/modules/base.ts @@ -0,0 +1,87 @@ +import path from 'node:path'; +import fs from 'node:fs'; +import { Config, SyncList } from './type.ts'; +import { fileIsExist } from '@/uitls/file.ts'; + +export type SyncOptions = { + dir?: string; + configFilename?: string; + baseURL?: string; +}; +export class SyncBase { + config: Config; + #filename: string; + #dir: string; + baseURL: string; + constructor(opts?: SyncOptions) { + const filename = opts?.configFilename || 'kevisual.json'; + const dir = opts?.dir || process.cwd(); + this.#filename = filename; + this.#dir = path.resolve(dir); + this.baseURL = opts?.baseURL ?? ''; + this.init(); + } + async init() { + try { + const dir = this.#dir; + const filename = this.#filename; + const filepath = path.join(dir, filename); + if (!fileIsExist(filepath)) throw new Error('config file not found'); + const config = JSON.parse(fs.readFileSync(filepath, 'utf-8')); + this.config = config; + return config; + } catch (err) { + this.config = {} as Config; + return {} as Config; + } + } + + async getSyncList(): Promise { + const config = this.config!; + const sync = config?.sync || {}; + const syncKeys = Object.keys(sync); + const baseURL = this.baseURL; + const syncList = syncKeys.map((key) => { + const value = sync[key]; + const filepath = path.join(this.#dir, key); // 文件的路径 + + const checkAuth = (value: string = '', baseURL: string = '') => { + if (value.startsWith(baseURL)) { + return true; + } + return false; + }; + if (typeof value === 'string') { + return { + filepath, + url: value, + auth: checkAuth(value, baseURL), + }; + } + return { + filepath, + ...value, + auth: checkAuth(value.url, baseURL), + }; + }); + + return syncList; + } + async getDir(filepath: string, check = false) { + const dir = path.dirname(filepath); + if (check) { + if (!fileIsExist(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } + return dir; + } + async download() { + // const syncList = await this.getSyncList(); + // for (const item of syncList) { + // } + } + async upload() { + // need check permission + } +} diff --git a/src/command/sync/modules/type.ts b/src/command/sync/modules/type.ts new file mode 100644 index 0000000..9f7646b --- /dev/null +++ b/src/command/sync/modules/type.ts @@ -0,0 +1,21 @@ +export type SyncConfig = { + type?: 'sync'; // 是否可以同步 + url: string; // 文件具体的 url 的地址 +}; +export interface Config { + name?: string; // 项目名称 + version?: string; // 项目版本号 + ignore?: string[]; // 忽略的目录或则文件，默认忽略 node_modules 使用 fast-glob 去匹配 + + sync: { + [key: string]: SyncConfig | string; + }; +} + +export type SyncList = { + filepath: string; + /** + * 是否需要鉴权, baseURL 为 kevisual 服务时，需要鉴权 + */ + auth?: boolean; +} & SyncConfig; diff --git a/src/command/sync/sync.ts b/src/command/sync/sync.ts new file mode 100644 index 0000000..a0a485a --- /dev/null +++ b/src/command/sync/sync.ts @@ -0,0 +1,33 @@ +import { program as app, Command } from '@/program.ts'; +import { SyncBase } from './modules/base.ts'; +import { baseURL } from '@/module/query.ts'; +import { fetchLink } from '@/module/download/install.ts'; +import fs from 'node:fs'; + +const command = new Command('sync') + .option('-d --dir ') + .description('同步项目') + .action(() => { + console.log('同步项目'); + }); +const syncUpload = new Command('upload').description('上传项目').action(() => { + console.log('上传项目'); +}); +const syncDownload = new Command('download') + .option('-d --dir ', '配置目录') + .description('下载项目') + .action(async () => { + console.log('下载项目'); + const sync = new SyncBase({ baseURL: baseURL }); + const syncList = await sync.getSyncList(); + console.log(syncList); + for (const item of syncList) { + const { content } = await fetchLink(item.url, { setToken: item.auth, returnContent: true }); + await sync.getDir(item.filepath, true); + fs.writeFileSync(item.filepath, content); + } + }); + +command.addCommand(syncUpload); +command.addCommand(syncDownload); +app.addCommand(command); diff --git a/src/index.ts b/src/index.ts index 07505df..d1da22b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ import './command/npm.ts'; import './command/publish.ts'; import './command/init.ts'; import './command/proxy.ts'; -import './command/sync.ts'; +import './command/sync/sync.ts'; import './command/app/index.ts'; diff --git a/src/module/download/install.ts b/src/module/download/install.ts index 461a436..87e8811 100644 --- a/src/module/download/install.ts +++ b/src/module/download/install.ts @@ -23,21 +23,25 @@ export type Package = { type Options = { check?: boolean; returnContent?: boolean; + setToken?: boolean; [key: string]: any; }; export const fetchLink = async (url: string, opts?: Options) => { const token = process.env.KEVISUAL_TOKEN || storage.getItem('token'); const fetchURL = new URL(url); const check = opts?.check ?? false; + const setToken = opts?.setToken ?? true; if (check) { if (!url.startsWith(baseURL)) { throw new Error('url must start with ' + baseURL); } } - if (token) { + if (token && setToken) { fetchURL.searchParams.set('token', token); } fetchURL.searchParams.set('download', 'true'); + console.log('fetchURL', fetchURL.toString()); + const res = await fetch(fetchURL.toString()); const blob = await res.blob(); const type = blob.type; diff --git a/src/module/download/upload.ts b/src/module/download/upload.ts index d430cf4..05ab46b 100644 --- a/src/module/download/upload.ts +++ b/src/module/download/upload.ts @@ -1,3 +1,4 @@ +import { getBufferHash, getHash } from '@/uitls/hash.ts'; import FormData from 'form-data'; export const handleResponse = async (err: any, res: any) => { return new Promise((resolve) => { @@ -37,6 +38,7 @@ export const getFormParams = (opts: UploadOptions, headers: any): FormData.Submi ...headers, }, }; + console.log('getFormParams', value); return value; }; type UploadOptions = { @@ -44,14 +46,25 @@ type UploadOptions = { file?: string | Buffer | File; token?: string; form?: FormData; + needHash?: boolean; }; export const upload = (opts: UploadOptions): Promise<{ code?: number; message?: string; [key: string]: any }> => { const form = opts?.form || new FormData(); if (!opts.form) { + let hash = ''; + let value: any; + let type = 'string'; if (typeof opts.file === 'string') { - form.append('file', Buffer.from(opts.file)); + value = Buffer.from(opts.file); } else { - form.append('file', opts.file); + type = 'buffer'; + value = opts.file; + } + form.append('file', value); + if (opts.needHash) { + hash = getBufferHash(value); + opts.url = new URL(opts.url.toString()); + opts.url.searchParams.append('hash', hash); } } const headers = form.getHeaders(); diff --git a/src/module/logger.ts b/src/module/logger.ts new file mode 100644 index 0000000..41c028e --- /dev/null +++ b/src/module/logger.ts @@ -0,0 +1,5 @@ +import { Logger } from '@kevisual/logger/node'; + +export const logger = new Logger({ + level: 'info', +}); diff --git a/src/scripts/upload.ts b/src/scripts/upload.ts index ae0ade2..5b28440 100644 --- a/src/scripts/upload.ts +++ b/src/scripts/upload.ts @@ -3,9 +3,11 @@ import { baseURL, storage } from '@/module/query.ts'; import { upload } from '@/module/download/upload.ts'; import fs from 'node:fs'; import path from 'node:path'; - +import { getHash, getBufferHash } from '@/uitls/hash.ts'; +import { logger } from '@/module/logger.ts'; const scriptPath = path.join(process.cwd(), 'src', 'scripts'); const sum = 'https://kevisual.xiongxiao.me/root/ai/kevisual/01-summary.md'; +const sum2 = 'https://kevisual.xiongxiao.me/root/resources/ai/1.0.0/kevisual/01-summary.md'; const download = async () => { const { content } = await fetchLink(sum, { returnContent: true }); console.log(content.toString()); @@ -16,9 +18,28 @@ const download = async () => { const uploadTest = async () => { const file = fs.readFileSync(path.join(scriptPath, './summary.md')); const token = storage.getItem('token'); + const url = new URL(sum); // const res = await upload({ url: sum, file: '# 汇总 123', token }); - const res = await upload({ url: sum, file: file, token }); - console.log(res); + url.searchParams.append('force', 'true'); + // url.searchParams.append('meta', encodeURIComponent(JSON.stringify({ m: 'meta-test' }))); + const res = await upload({ url: url, file: file, token, needHash: true }); + logger.info('上传成功', res); }; uploadTest(); +const hashCheck = () => { + const filepath = path.join(scriptPath, './summary.md'); + const file = fs.readFileSync(filepath); + console.log(getHash(filepath)); + console.log(getBufferHash(file)); +}; + +// hashCheck(); + +// const buf = Buffer.from('123'); +// const abc = { +// a: 1, +// b: 2, +// c: 3, +// }; +// console.log(typeof buf, buf instanceof Buffer, abc instanceof Buffer); diff --git a/src/uitls/hash.ts b/src/uitls/hash.ts new file mode 100644 index 0000000..e9c03d5 --- /dev/null +++ b/src/uitls/hash.ts @@ -0,0 +1,11 @@ +import MD5 from 'crypto-js/md5.js'; +import fs from 'node:fs'; + +export const getHash = (file: string) => { + const content = fs.readFileSync(file, 'utf-8'); + return MD5(content).toString(); +}; + +export const getBufferHash = (buffer: Buffer) => { + return MD5(buffer.toString()).toString(); +};