kevisual/kevisual-oss
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: kevisual:8150bb6177a42cdc5c4d9f3771ae0c35fff3ca7c
Branches Tags
kevisual:main
...
compare: kevisual:main
Branches Tags
kevisual:main

6 Commits
8150bb6177 ... main

Author SHA1 Message Date
abearxiong
151a74fad1 feat: 添加 listObjects 方法的 getMeta 选项以获取对象元数据 2026-02-01 17:37:48 +08:00
abearxiong
d4016e5680 refactor: remove copy-object and type definitions; migrate to new S3 structure 
- Deleted copy-object.ts and type.ts files as part of the refactor.
- Updated index.ts to export from new S3 core and type files.
- Refactored OssBase class in core.ts to handle file uploads and object management.
- Enhanced error handling for object retrieval and metadata management.
- Introduced new methods for handling object streams and metadata filtering.
- Updated download utility functions to align with new structure.
 2026-01-31 05:12:59 +08:00
abearxiong
0443ecdad3 update 2026-01-08 15:39:28 +08:00
abearxiong
1c6c045666 update 2026-01-08 11:05:40 +08:00
abearxiong
f512d97e09 chore(config): 更新环境变量示例及npm身份验证令牌 
- 在.env.example中添加S3_ENDPOINT环境变量示例配置
- 将.npmrc中npmjs.org的_authToken改为固定令牌
- 升级package.json版本号到0.0.15
- 调整测试文件中putObject调用的对象路径为'lanzhoub/a.json'
 2026-01-08 01:31:38 +08:00
abearxiong
6411e42b3a chore(build): 优化依赖和配置文件管理 
- .gitignore 中新增 .env 文件忽略规则并保留示例文件
- package.json 中新增 src 文件夹纳入发布文件范围
- 升级 devDependencies 中部分包版本并新增 AWS SDK 和 es-toolkit 等依赖
- 移除 lodash 改用 es-toolkit 的 omit 函数以减小包体积
- 使用 dotenv 的解构导入替换默认导入，规范配置加载方式
- util 模块导出新增 extract-standard-headers 功能模块接口
 2026-01-08 00:45:48 +08:00
20 changed files with 2683 additions and 467 deletions
Expand all files Collapse all files

5
.env.example Normal file
View File

@@ -0,0 +1,5 @@
S3_ACCESS_KEY_ID=your_access_key_id
S3_ACCESS_KEY_SECRET=your_secret_access_key
S3_REGION=your_region
S3_BUCKET_NAME=your_bucket_name
S3_ENDPOINT=your_endpoint

3
.gitignore vendored
View File

@@ -1,3 +1,6 @@
node_modules
dist
.turbo
.env
!.env*example

2
.npmrc
View File

@@ -1,2 +1,2 @@
//npm.xiongxiao.me/:_authToken=${ME_NPM_TOKEN}
//registry.npmjs.org/:_authToken=${NPM_TOKEN}
//registry.npmjs.org/:_authToken=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

37
bun.config.ts
View File

@@ -1,18 +1,23 @@
import glob from 'fast-glob';
import dts from 'bun-plugin-dts';
import { resolvePath } from '@kevisual/use-config';
import { execSync } from 'node:child_process';
const services = await glob('src/services/*.ts');
const buildFn = async (opts: { entry?: string, naming?: string }) => {
const entry = opts.entry || 'src/index.ts';
const naming = opts.naming || 'app';
const external: string[] = ["bun"];
await Bun.build({
target: 'node',
format: 'esm',
entrypoints: [resolvePath(entry, { meta: import.meta })],
outdir: resolvePath('./dist', { meta: import.meta }),
naming: {
entry: `${naming}.js`,
},
external,
});
const cmd = `dts -i ${entry} -o ${naming}.d.ts`;
execSync(cmd);
};
await Bun.build({
entrypoints: ['src/index.ts', ...services],
outdir: './dist',
target: 'node',
format: 'esm',
splitting: false,
sourcemap: 'none',
minify: false,
external: ['minio'],
plugins: [dts()],
});
console.log('Build completed!');
await buildFn({ naming: 'index', entry: 'src/index.ts' });
await buildFn({ naming: 'services', entry: 'src/services/index.ts' });

33
package.json
View File

@@ -1,45 +1,40 @@
{
"name": "@kevisual/oss",
"version": "0.0.13",
"version": "0.0.19",
"main": "dist/index.js",
"scripts": {
"build": "bun run bun.config.ts"
},
"files": [
"dist"
"dist",
"src"
],
"keywords": [],
"author": "abearxiong <xiongxiao@xiongxiao.me>",
"license": "MIT",
"type": "module",
"devDependencies": {
"@types/bun": "^1.3.3",
"@types/node": "^24.10.1",
"@aws-sdk/client-s3": "^3.978.0",
"@kevisual/use-config": "^1.0.28",
"@types/bun": "^1.3.8",
"@types/node": "^25.1.0",
"bun-plugin-dts": "^0.3.0",
"dotenv": "^16.5.0",
"minio": "^8.0.5",
"tsup": "^8.4.0"
"dotenv": "^17.2.3",
"es-toolkit": "^1.44.0",
"fast-glob": "^3.3.3"
},
"exports": {
".": {
"import": "./dist/index.js",
"types": "./dist/index.d.ts"
},
"./config": {
"import": "./dist/services/config.js",
"types": "./dist/services/config.d.ts"
},
"./services": {
"import": "./dist/services/index.js",
"types": "./dist/services/index.d.ts"
}
"import": "./dist/services.js",
"types": "./dist/services.d.ts"
},
"./s3.ts": "./src/s3/core.ts"
},
"publishConfig": {
"access": "public"
},
"dependencies": {
"@types/lodash": "^4.17.21",
"fast-glob": "^3.3.3",
"lodash": "^4.17.21"
}
}

1687
pnpm-lock.yaml generated Normal file
View File

File diff suppressed because it is too large Load Diff

26
src/core/copy-object.ts
View File

@@ -1,26 +0,0 @@
import { Client, CopyDestinationOptions, CopySourceOptions } from 'minio';
type CopyObjectOpts = {
bucketName: string;
newMetadata: Record<string, string>;
objectName: string;
client: Client;
};
/**
* 复制对象 REPLACE 替换
* @param param0
* @returns
*/
export const copyObject = async ({ bucketName, newMetadata, objectName, client }: CopyObjectOpts) => {
const source = new CopySourceOptions({ Bucket: bucketName, Object: objectName });
const stat = await client.statObject(bucketName, objectName);
const sourceMetadata = stat.metaData;
const destination = new CopyDestinationOptions({
Bucket: bucketName,
Object: objectName,
UserMetadata: { ...sourceMetadata, ...newMetadata },
MetadataDirective: 'REPLACE',
});
const copyResult = await client.copyObject(source, destination);
return copyResult;
};

87
src/core/type.ts
View File

@@ -1,87 +0,0 @@
import { ItemBucketMetadata, Client } from 'minio';
export type UploadedObjectInfo = {
etag: string;
lastModified?: Date;
size?: number;
versionId: string;
metadata?: ItemBucketMetadata;
};
export type StatObjectResult = {
size: number;
etag: string;
lastModified: Date;
metaData: ItemBucketMetadata;
versionId?: string | null;
};
export type ListFileObject = {
name: string;
size: number;
lastModified: Date;
etag: string;
};
export type ListDirectoryObject = {
prefix: string;
size: number;
};
export type ListObjectResult = ListFileObject | ListDirectoryObject;
export interface OssBaseOperation {
prefix: string;
setPrefix(prefix: string): void;
/**
* 获取对象
* @param objectName 对象名
*/
getObject(objectName: string): Promise<any>;
/**
* 上传对象
* @param objectName 对象名
* @param data 数据
*/
putObject(objectName: string, data: Buffer | string, metaData?: ItemBucketMetadata): Promise<UploadedObjectInfo>;
/**
* 上传文件
* @param objectName 对象名
* @param filePath 文件路径
*/
fPutObject(objectName: string, filePath: string, metaData?: ItemBucketMetadata): Promise<UploadedObjectInfo>;
/**
* 获取对象信息
* @param objectName 对象名
*/
statObject(objectName: string): Promise<StatObjectResult>;
/**
* 删除对象
* @param objectName 对象名
*/
deleteObject(objectName: string): Promise<any>;
/**
* 列出对象
* @param objectName 对象名
* @param opts 选项
* @param opts.recursive 是否递归
* @param opts.startAfter 开始位置
*/
listObjects(
objectName: string,
opts?: {
/**
* 是否递归
*/
recursive?: boolean;
/**
* 开始位置
*/
startAfter?: string;
},
): Promise<ListObjectResult[]>;
/**
* 复制对象
* @param sourceObject 源对象
* @param targetObject 目标对象
*/
copyObject: Client['copyObject'];
}
export interface OssService extends OssBaseOperation {
owner: string;
}

245
src/index.ts
View File

@@ -1,244 +1,3 @@
import { Client, ItemBucketMetadata } from 'minio';
import { ListFileObject, ListObjectResult, OssBaseOperation } from './core/type.ts';
import { hash } from './util/hash.ts';
import { copyObject } from './core/copy-object.ts';
import omit from 'lodash/omit.js';
import { getContentType } from './util/get-content-type.ts';
export type OssBaseOptions<T = { [key: string]: any }> = {
/**
* 已经初始化好的minio client
*/
client: Client;
/**
* 桶名
*/
bucketName: string;
/**
* 前缀
*/
prefix?: string;
} & T;
export * from './s3/core.ts'
export class OssBase implements OssBaseOperation {
client?: Client;
bucketName: string;
prefix = '';
/**
* 计算字符串或者对象的的md5值
*/
hash = hash;
constructor(opts: OssBaseOptions) {
if (!opts.client) {
throw new Error('client is required');
}
this.bucketName = opts.bucketName;
this.client = opts.client;
this.prefix = opts?.prefix ?? '';
}
setPrefix(prefix: string) {
this.prefix = prefix;
}
async getObject(objectName: string) {
const bucketName = this.bucketName;
const obj = await this.client.getObject(bucketName, `${this.prefix}${objectName}`);
return obj;
}
async getJson(objectName: string): Promise<Record<string, any>> {
const obj = await this.getObject(objectName);
return new Promise((resolve, reject) => {
let data = '';
obj.on('data', (chunk) => {
data += chunk;
});
obj.on('end', () => {
try {
const jsonData = JSON.parse(data);
resolve(jsonData);
} catch (error) {
reject(new Error('Failed to parse JSON'));
}
});
obj.on('error', (err) => {
reject(err);
});
});
}
/**
* 上传文件, 当是流的时候，中断之后的etag会变，所以传递的时候不要嵌套async await，例如 busboy 监听文件流内部的时候，不要用check
* @param objectName
* @param data
* @param metaData
* @param options 如果文件本身存在，则复制原有的meta的内容
* @returns
*/
async putObject(
objectName: string,
data: Buffer | string | Object,
metaData: ItemBucketMetadata = {},
opts?: { check?: boolean; isStream?: boolean; size?: number },
) {
let putData: Buffer | string;
let size: number = opts?.size;
const isStream = opts?.isStream;
if (!isStream) {
if (typeof data === 'string') {
putData = data;
size = putData.length;
} else {
putData = JSON.stringify(data);
size = putData.length;
}
} else {
putData = data as any;
// 对于流式上传，如果没有提供 size，会导致多部分上传，ETag 会是 ****-1 格式
// 必须提供准确的 size 才能得到标准的 MD5 格式 ETag
if (!size) {
throw new Error('Stream upload requires size parameter to avoid multipart upload and get standard MD5 ETag');
}
}
if (opts?.check) {
const obj = await this.statObject(objectName, true);
if (obj) {
const omitMeda = ['size', 'content-type', 'cache-control', 'app-source'];
const objMeta = JSON.parse(JSON.stringify(omit(obj.metaData, omitMeda)));
metaData = {
...objMeta,
...metaData,
};
}
}
const bucketName = this.bucketName;
const obj = await this.client.putObject(bucketName, `${this.prefix}${objectName}`, putData, size, metaData);
return obj;
}
async deleteObject(objectName: string) {
const bucketName = this.bucketName;
const obj = await this.client.removeObject(bucketName, `${this.prefix}${objectName}`);
return obj;
}
async listObjects<IS_FILE = false>(objectName: string, opts?: { recursive?: boolean; startAfter?: string }) {
const bucketName = this.bucketName;
const prefix = `${this.prefix}${objectName}`;
const res = await new Promise((resolve, reject) => {
let res: any[] = [];
let hasError = false;
this.client
.listObjectsV2(bucketName, prefix, opts?.recursive ?? false, opts?.startAfter)
.on('data', (data) => {
res.push(data);
})
.on('error', (err) => {
console.error('minio error', prefix, err);
hasError = true;
})
.on('end', () => {
if (hasError) {
reject();
return;
} else {
resolve(res);
}
});
});
return res as IS_FILE extends true ? ListFileObject[] : ListObjectResult[];
}
async fPutObject(objectName: string, filePath: string, metaData?: ItemBucketMetadata) {
const bucketName = this.bucketName;
const obj = await this.client.fPutObject(bucketName, `${this.prefix}${objectName}`, filePath, metaData);
return obj as any;
}
/**
* 获取完整的对象名称
* @param objectName
* @returns
*/
async getObjectName(objectName: string) {
return `${this.prefix}${objectName}`;
}
async statObject(objectName: string, checkFile = true) {
const bucketName = this.bucketName;
try {
const obj = await this.client.statObject(bucketName, `${this.prefix}${objectName}`);
return obj;
} catch (e) {
if (e.code === 'NotFound') {
return null;
}
throw e;
}
}
/**
* 检查文件hash是否一致
* @param objectName
* @param hash
* @returns
*/
async checkObjectHash(
objectName: string,
hash: string,
meta?: ItemBucketMetadata,
): Promise<{ success: boolean; metaData: ItemBucketMetadata | null; obj: any; equalMeta?: boolean }> {
const obj = await this.statObject(`${this.prefix}${objectName}`, true);
if (!obj) {
return { success: false, metaData: null, obj: null, equalMeta: false };
}
let metaData: ItemBucketMetadata = {};
const omitMeda = ['content-type', 'cache-control', 'app-source'];
const objMeta = omit(obj.metaData, omitMeda);
metaData = {
...objMeta,
};
let equalMeta = false;
if (meta) {
equalMeta = JSON.stringify(metaData) === JSON.stringify(meta);
}
return { success: obj.etag === hash, metaData, obj, equalMeta };
}
getMetadata(pathname: string, meta: ItemBucketMetadata = { 'app-source': 'user-app' }) {
const isHtml = pathname.endsWith('.html');
if (isHtml) {
meta = {
...meta,
'content-type': 'text/html; charset=utf-8',
'cache-control': 'no-cache',
};
} else {
meta = {
...meta,
'content-type': getContentType(pathname),
'cache-control': 'max-age=31536000, immutable',
};
}
return meta;
}
async copyObject(sourceObject: any, targetObject: any) {
const bucketName = this.bucketName;
const obj = await this.client.copyObject(bucketName, sourceObject, targetObject);
return obj;
}
async replaceObject(objectName: string, meta: { [key: string]: string }) {
const { bucketName, client } = this;
return copyObject({ bucketName, client, objectName: `${this.prefix}${objectName}`, newMetadata: meta });
}
static create<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, opts: OssBaseOptions<U>): T {
return new this(opts);
}
static fromBase<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, createOpts: { oss: OssBase; opts: Partial<OssBaseOptions<U>> }): T {
const base = createOpts.oss;
const opts = createOpts.opts as any;
return new this({
client: base.client,
bucketName: base.bucketName,
...opts,
});
}
}
export * from './s3/type.ts'

46
src/s3/copy-object.ts Normal file
View File

@@ -0,0 +1,46 @@
import { S3Client, CopyObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
import { extractStandardHeaders } from '../util/extract-standard-headers.ts';
type CopyObjectOpts = {
bucketName: string;
newMetadata: Record<string, string>;
objectName: string;
client: S3Client;
};
/**
* 复制对象 REPLACE 替换（使用 AWS SDK 实现）
* @param opts 复制选项
* @returns 复制结果
*/
export const copyObject = async ({ bucketName, newMetadata, objectName, client }: CopyObjectOpts) => {
// 获取当前对象的元数据
const headCommand = new HeadObjectCommand({
Bucket: bucketName,
Key: objectName,
});
const headResponse = await client.send(headCommand);
const sourceMetadata = headResponse.Metadata || {};
// 合并元数据
const mergedMeta = { ...sourceMetadata, ...newMetadata };
const { standardHeaders, customMetadata } = extractStandardHeaders(mergedMeta);
// 执行复制操作（同一对象，用于更新元数据）
const copyCommand = new CopyObjectCommand({
Bucket: bucketName,
CopySource: `${bucketName}/${objectName}`,
Key: objectName,
ContentType: standardHeaders.ContentType,
CacheControl: standardHeaders.CacheControl,
ContentDisposition: standardHeaders.ContentDisposition,
ContentEncoding: standardHeaders.ContentEncoding,
ContentLanguage: standardHeaders.ContentLanguage,
Expires: standardHeaders.Expires,
Metadata: customMetadata,
MetadataDirective: 'REPLACE',
});
const copyResult = await client.send(copyCommand);
return copyResult;
};

490
src/s3/core.ts Normal file
View File

@@ -0,0 +1,490 @@
import {
S3Client,
GetObjectCommand,
PutObjectCommand,
DeleteObjectCommand,
ListObjectsV2Command,
HeadObjectCommand,
CopyObjectCommand,
type GetObjectCommandOutput,
} from '@aws-sdk/client-s3';
import type { Readable } from 'node:stream';
import fs from 'node:fs';
import { omit } from 'es-toolkit';
import {
OssBaseOperation,
ItemBucketMetadata,
UploadedObjectInfo,
StatObjectResult,
ListObjectResult,
ListFileObject,
} from './type.ts';
import { hash } from '../util/hash.ts';
import { getContentType } from '../util/get-content-type.ts';
import { extractStandardHeaders } from '../util/extract-standard-headers.ts';
export type OssBaseOptions<T = { [key: string]: any }> = {
/**
* 已经初始化好的 S3Client
*/
client: S3Client;
/**
* 桶名
*/
bucketName: string;
/**
* 前缀
*/
prefix?: string;
} & T;
export class OssBase implements OssBaseOperation {
client: S3Client;
bucketName: string;
prefix = '';
/**
* 计算字符串或者对象的的md5值
*/
hash = hash;
constructor(opts: OssBaseOptions) {
if (!opts.client) {
throw new Error('client is required');
}
this.bucketName = opts.bucketName;
this.client = opts.client;
this.prefix = opts?.prefix ?? '';
}
setPrefix(prefix: string) {
this.prefix = prefix;
}
/**
* 获取对象
* @param objectName 对象名
*/
async getObject(objectName: string): Promise<GetObjectCommandOutput> {
const command = new GetObjectCommand({
Bucket: this.bucketName,
Key: `${this.prefix}${objectName}`,
});
const response = await this.client.send(command);
return response;
}
/**
* 获取对象内容为字符串
* @param objectName 对象名
*/
async getObjectAsString(objectName: string): Promise<string> {
const response = await this.getObject(objectName);
if (response.Body) {
return await response.Body.transformToString();
}
throw new Error('Object body is empty');
}
/**
* 获取对象内容为 JSON
* @param objectName 对象名
*/
async getJson(objectName: string): Promise<Record<string, any>> {
const str = await this.getObjectAsString(objectName);
try {
return JSON.parse(str);
} catch (error) {
throw new Error('Failed to parse JSON');
}
}
/**
* 上传对象
* @param objectName 对象名
* @param data 数据
* @param metaData 元数据
* @param opts 选项
*/
async putObject(
objectName: string,
data: Buffer | string | Object | Readable,
metaData: ItemBucketMetadata = {},
opts?: { check?: boolean; isStream?: boolean; size?: number; contentType?: string },
): Promise<UploadedObjectInfo> {
let putData: Buffer | string | Readable;
let contentLength: number | undefined = opts?.size;
const isStream = opts?.isStream;
if (!isStream) {
if (typeof data === 'string') {
putData = data;
contentLength = Buffer.byteLength(data);
} else if (Buffer.isBuffer(data)) {
putData = data;
contentLength = data.length;
} else {
putData = JSON.stringify(data);
contentLength = Buffer.byteLength(putData);
}
} else {
// Stream 上传：自动读取到 Buffer 以获取 contentLength
if (!contentLength) {
const chunks: Buffer[] = [];
for await (const chunk of data as Readable) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
putData = Buffer.concat(chunks);
contentLength = putData.length;
} else {
putData = data as Readable;
}
}
// 检查现有对象并合并元数据
if (opts?.check) {
const obj = await this.statObject(objectName, true);
if (obj) {
const omitMeta = ['size', 'content-type', 'cache-control', 'app-source'];
const objMeta = JSON.parse(JSON.stringify(omit(obj.metaData, omitMeta)));
metaData = {
...objMeta,
...metaData,
};
}
}
const { standardHeaders, customMetadata } = extractStandardHeaders(metaData);
const command = new PutObjectCommand({
Bucket: this.bucketName,
Key: `${this.prefix}${objectName}`,
Body: putData,
ContentLength: contentLength,
ContentType: opts?.contentType || standardHeaders.ContentType || getContentType(objectName),
CacheControl: standardHeaders.CacheControl,
ContentDisposition: standardHeaders.ContentDisposition,
ContentEncoding: standardHeaders.ContentEncoding,
ContentLanguage: standardHeaders.ContentLanguage,
Expires: standardHeaders.Expires,
Metadata: customMetadata,
});
const response = await this.client.send(command);
return {
etag: response.ETag?.replace(/"/g, '') || '',
versionId: response.VersionId || '',
};
}
/**
* 上传文件
* @param objectName 对象名
* @param filePath 文件路径
* @param metaData 元数据
*/
async fPutObject(
objectName: string,
filePath: string,
metaData?: ItemBucketMetadata,
): Promise<UploadedObjectInfo> {
const stat = fs.statSync(filePath);
const { standardHeaders, customMetadata } = extractStandardHeaders(metaData || {});
const THRESHOLD = 5 * 1024 * 1024; // 5MB
let command: PutObjectCommand;
if (stat.size < THRESHOLD) {
// 小文件：读取到内存再上传，避免流式传输导致的 IncompleteBody 错误
const fileBuffer = await fs.promises.readFile(filePath);
command = new PutObjectCommand({
Bucket: this.bucketName,
Key: `${this.prefix}${objectName}`,
Body: fileBuffer,
ContentLength: fileBuffer.length,
ContentType: standardHeaders.ContentType || getContentType(filePath),
CacheControl: standardHeaders.CacheControl,
ContentDisposition: standardHeaders.ContentDisposition,
ContentEncoding: standardHeaders.ContentEncoding,
ContentLanguage: standardHeaders.ContentLanguage,
Expires: standardHeaders.Expires,
Metadata: customMetadata,
});
} else {
// 大文件：使用流式上传
const fileStream = fs.createReadStream(filePath);
command = new PutObjectCommand({
Bucket: this.bucketName,
Key: `${this.prefix}${objectName}`,
Body: fileStream,
ContentLength: stat.size,
ContentType: standardHeaders.ContentType || getContentType(filePath),
CacheControl: standardHeaders.CacheControl,
ContentDisposition: standardHeaders.ContentDisposition,
ContentEncoding: standardHeaders.ContentEncoding,
ContentLanguage: standardHeaders.ContentLanguage,
Expires: standardHeaders.Expires,
Metadata: customMetadata,
});
}
const response = await this.client.send(command);
return {
etag: response.ETag?.replace(/"/g, '') || '',
versionId: response.VersionId || '',
};
}
/**
* 删除对象
* @param objectName 对象名
*/
async deleteObject(objectName: string): Promise<void> {
const command = new DeleteObjectCommand({
Bucket: this.bucketName,
Key: `${this.prefix}${objectName}`,
});
await this.client.send(command);
}
async deleteObjects(objectNameList: string[]): Promise<void> {
for (const objectName of objectNameList) {
await this.deleteObject(objectName);
}
}
/**
* 列出对象
* @param objectName 前缀
* @param opts 选项
*/
async listObjects<IS_FILE = false>(
objectName: string,
opts?: { recursive?: boolean; startAfter?: string; maxKeys?: number, getMeta?: boolean },
): Promise<IS_FILE extends true ? ListFileObject[] : ListObjectResult[]> {
const prefix = `${this.prefix}${objectName}`;
const results: ListObjectResult[] = [];
let continuationToken: string | undefined;
const getMeta = opts?.getMeta ?? false;
do {
const command = new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: prefix,
Delimiter: opts?.recursive ? undefined : '/',
StartAfter: opts?.startAfter,
MaxKeys: opts?.maxKeys || 1000,
ContinuationToken: continuationToken,
});
const response = await this.client.send(command);
// 处理文件对象
if (response.Contents) {
for (const item of response.Contents) {
const result: ListFileObject = {
name: item.Key || '',
size: item.Size || 0,
lastModified: item.LastModified || new Date(),
etag: item.ETag?.replace(/"/g, '') || '',
}
if (getMeta) {
const stat = await this.statObject(item.Key || '', false);
if (stat?.metaData) {
result.metaData = stat.metaData;
}
}
results.push(result);
}
}
// 处理目录（CommonPrefixes）
if (response.CommonPrefixes && !opts?.recursive) {
for (const prefix of response.CommonPrefixes) {
results.push({
prefix: prefix.Prefix || '',
size: 0,
});
}
}
continuationToken = response.NextContinuationToken;
} while (continuationToken);
return results as IS_FILE extends true ? ListFileObject[] : ListObjectResult[];
}
/**
* 获取对象信息
* @param objectName 对象名
* @param checkFile 是否检查文件存在（不存在返回null而非抛错）
*/
async statObject(objectName: string, checkFile = true): Promise<StatObjectResult | null> {
try {
const command = new HeadObjectCommand({
Bucket: this.bucketName,
Key: `${this.prefix}${objectName}`,
});
const response = await this.client.send(command);
return {
standardHeaders: {
contentType: response.ContentType,
cacheControl: response.CacheControl,
contentDisposition: response.ContentDisposition,
contentEncoding: response.ContentEncoding,
contentLanguage: response.ContentLanguage,
},
size: response.ContentLength || 0,
etag: response.ETag?.replace?.(/"/g, '') || '',
lastModified: response.LastModified || new Date(),
metaData: (response.Metadata as ItemBucketMetadata) || {},
versionId: response.VersionId || null,
};
} catch (e: any) {
// 检查是否是 404 错误 - 支持多种 S3 兼容存储的错误格式
const isNotFound =
e.name === 'NotFound' ||
e.name === 'NoSuchBucket' ||
e.name === 'NoSuchKey' ||
e.code === 'NotFound' ||
e.code === 'NoSuchBucket' ||
e.code === 'NoSuchKey' ||
e.$metadata?.httpStatusCode === 404;
if (checkFile && isNotFound) {
return null;
}
console.error('statObject error', e);
throw e;
}
}
/**
* 获取完整的对象名称
* @param objectName 对象名
*/
getObjectName(objectName: string): string {
return `${this.prefix}${objectName}`;
}
/**
* 检查文件hash是否一致
* @param objectName 对象名
* @param hash hash值
* @param meta 元数据
*/
async checkObjectHash(
objectName: string,
hash: string,
meta?: ItemBucketMetadata,
): Promise<{ success: boolean; metaData: ItemBucketMetadata | null; obj: StatObjectResult | null; equalMeta?: boolean }> {
const obj = await this.statObject(objectName, true);
if (!obj) {
return { success: false, metaData: null, obj: null, equalMeta: false };
}
const omitMeta = ['Content-Type', 'Cache-Control', 'app-source'];
const metaData = omit(obj.metaData, omitMeta);
let equalMeta = false;
if (meta) {
equalMeta = JSON.stringify(metaData) === JSON.stringify(meta);
}
return { success: obj.etag === hash, metaData, obj, equalMeta };
}
/**
* 获取元数据
* @param pathname 路径名
* @param meta 元数据
*/
getMetadata(pathname: string, meta: ItemBucketMetadata = { 'app-source': 'user-app' }): ItemBucketMetadata {
const isHtml = pathname.endsWith('.html');
if (isHtml) {
meta = {
...meta,
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'no-cache',
};
} else {
meta = {
...meta,
'Content-Type': getContentType(pathname),
'Cache-Control': 'max-age=31536000, immutable',
};
}
return meta;
}
/**
* 复制对象
* @param sourceObject 源对象
* @param targetObject 目标对象
*/
async copyObject(sourceObject: string, targetObject: string): Promise<any> {
const command = new CopyObjectCommand({
Bucket: this.bucketName,
CopySource: `${this.bucketName}/${this.prefix}${sourceObject}`,
Key: `${this.prefix}${targetObject}`,
});
const response = await this.client.send(command);
return response;
}
/**
* 替换对象元数据
* @param objectName 对象名
* @param meta 新元数据
*/
async replaceObject(objectName: string, meta: ItemBucketMetadata): Promise<any> {
const key = `${this.prefix}${objectName}`;
// 获取当前对象的元数据
const stat = await this.statObject(objectName, false);
const sourceMetadata = stat?.metaData || {};
const mergedMeta = { ...sourceMetadata, ...meta };
const { standardHeaders, customMetadata } = extractStandardHeaders(mergedMeta);
const command = new CopyObjectCommand({
Bucket: this.bucketName,
CopySource: `${this.bucketName}/${key}`,
Key: key,
ContentType: standardHeaders.ContentType,
CacheControl: standardHeaders.CacheControl,
ContentDisposition: standardHeaders.ContentDisposition,
ContentEncoding: standardHeaders.ContentEncoding,
ContentLanguage: standardHeaders.ContentLanguage,
Expires: standardHeaders.Expires,
Metadata: customMetadata,
MetadataDirective: 'REPLACE',
});
const response = await this.client.send(command);
return response;
}
/**
* 创建实例
*/
static create<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, opts: OssBaseOptions<U>): T {
return new this(opts);
}
/**
* 从已有实例创建
*/
static fromBase<T extends OssBase, U>(
this: new (opts: OssBaseOptions<U>) => T,
createOpts: { oss: OssBase; opts: Partial<OssBaseOptions<U>> },
): T {
const base = createOpts.oss;
const opts = createOpts.opts as any;
return new this({
client: base.client,
bucketName: base.bucketName,
...opts,
});
}
}
export const getStram = (data: GetObjectCommandOutput) => {
if (data.Body) {
return data.Body as Readable;
}
throw new Error('Object body is empty');
}

165
src/s3/type.ts Normal file
View File

@@ -0,0 +1,165 @@
import { CopyObjectCommandOutput } from '@aws-sdk/client-s3';
import { Readable } from 'node:stream';
export type ItemBucketMetadata = Record<string, string>;
export type UploadedObjectInfo = {
etag: string;
lastModified?: Date;
size?: number;
versionId: string;
metadata?: ItemBucketMetadata;
};
export type StatObjectResult = {
size: number;
etag: string;
lastModified: Date;
metaData: ItemBucketMetadata;
versionId?: string | null;
standardHeaders: {
contentType?: string;
cacheControl?: string;
contentDisposition?: string;
contentEncoding?: string;
contentLanguage?: string;
}
};
export type ListFileObject = {
name: string;
size: number;
lastModified: Date;
etag: string;
metaData?: ItemBucketMetadata;
};
export type ListDirectoryObject = {
prefix: string;
size: number;
};
export type ListObjectResult = ListFileObject | ListDirectoryObject;
export interface OssBaseOperation {
prefix: string;
/**
* 设置前缀
* @param prefix 前缀
*/
setPrefix(prefix: string): void;
/**
* 获取对象
* @param objectName 对象名
*/
getObject(objectName: string): Promise<any>;
/**
* 获取对象内容为字符串
* @param objectName 对象名
*/
getObjectAsString?(objectName: string): Promise<string>;
/**
* 获取对象内容为 JSON
* @param objectName 对象名
*/
getJson?(objectName: string): Promise<Record<string, any>>;
/**
* 上传对象
* @param objectName 对象名
* @param data 数据
* @param metaData 元数据
* @param opts 选项
*/
putObject(
objectName: string,
data: Buffer | string | Object | Readable,
metaData?: ItemBucketMetadata,
opts?: { check?: boolean; isStream?: boolean; size?: number; contentType?: string },
): Promise<UploadedObjectInfo>;
/**
* 上传文件
* @param objectName 对象名
* @param filePath 文件路径
* @param metaData 元数据
*/
fPutObject(objectName: string, filePath: string, metaData?: ItemBucketMetadata): Promise<UploadedObjectInfo>;
/**
* 获取对象信息
* @param objectName 对象名
* @param checkFile 是否检查文件存在（不存在返回null而非抛错）
*/
statObject(objectName: string, checkFile?: boolean): Promise<StatObjectResult | null>;
/**
* 删除对象
* @param objectName 对象名
*/
deleteObject(objectName: string): Promise<any>;
/**
* 列出对象
* @param objectName 前缀
* @param opts 选项
*/
listObjects(
objectName: string,
opts?: {
/** 是否递归 */
recursive?: boolean;
/** 开始位置 */
startAfter?: string;
/** 最大返回数量 */
maxKeys?: number;
},
): Promise<ListObjectResult[]>;
/**
* 获取完整的对象名称
* @param objectName 对象名
*/
getObjectName?(objectName: string): string;
/**
* 检查文件hash是否一致
* @param objectName 对象名
* @param hash hash值
* @param meta 元数据
*/
checkObjectHash?(
objectName: string,
hash: string,
meta?: ItemBucketMetadata,
): Promise<{ success: boolean; metaData: ItemBucketMetadata | null; obj: StatObjectResult | null; equalMeta?: boolean }>;
/**
* 获取元数据
* @param pathname 路径名
* @param meta 元数据
*/
getMetadata?(pathname: string, meta?: ItemBucketMetadata): ItemBucketMetadata;
/**
* 复制对象
* @param sourceObject 源对象
* @param targetObject 目标对象
*/
copyObject(sourceObject: string, targetObject: string): Promise<CopyObjectCommandOutput>;
/**
* 替换对象元数据
* @param objectName 对象名
* @param meta 新元数据
*/
replaceObject?(objectName: string, meta: ItemBucketMetadata): Promise<any>;
}
export interface OssService extends OssBaseOperation {
owner: string;
}

4
src/services/config.ts
View File

@@ -1,5 +1,5 @@
import { OssBase, OssBaseOptions } from '../index.ts';
import { OssService } from '../core/type.ts';
import { OssBase, OssBaseOptions } from '../s3/core.ts';
import { OssService } from '../s3/type.ts';
import * as util from '../util/index.ts';
export class ConfigOssService extends OssBase implements OssService {

2
src/services/index.ts
View File

@@ -6,4 +6,4 @@ export * from '../util/download.ts';
export * from '../util/index.ts';
export * from '../core/type.ts';
export * from '../s3/type.ts';

103
src/test/common.ts Normal file
View File

@@ -0,0 +1,103 @@
import * as dotenv from 'dotenv';
dotenv.config();
import { S3Client, ListObjectsV2Command, GetBucketMetadataConfigurationCommand, HeadObjectCommand, CopyObjectCommand } from '@aws-sdk/client-s3';
export const s3Client = new S3Client({
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY_ID || '',
secretAccessKey: process.env.S3_ACCESS_KEY_SECRET || '',
},
region: process.env.S3_REGION,
endpoint: 'https://tos-s3-cn-shanghai.volces.com',
});
export const bucketName = process.env.S3_BUCKET_NAME;
export async function listS3Objects() {
const command = new ListObjectsV2Command({
Bucket: process.env.S3_BUCKET_NAME,
});
try {
const result = await s3Client.send(command);
console.log('S3 Objects:', result.Contents);
return result.Contents;
} catch (error) {
console.error('Error listing S3 objects:', error);
throw error;
}
}
// listS3Objects();
export async function getMetaData(key = 'readme.md') {
const command = new HeadObjectCommand({
Bucket: process.env.S3_BUCKET_NAME,
Key: key,
});
try {
const result = await s3Client.send(command);
// 解码 TOS 返回的 URL 编码的元数据值
const decodedMetadata: Record<string, string> = {};
if (result.Metadata) {
for (const [key, value] of Object.entries(result.Metadata)) {
decodedMetadata[key] = decodeURIComponent(value);
}
}
console.log('Metadata for', result);
return decodedMetadata;
} catch (error) {
console.error('Error getting metadata for', key, ':', error);
throw error;
}
}
// const metadata = await getMetaData();
// console.log('metadata', metadata);
export async function setMetaData(key = 'readme.md', metadata: Record<string, string>) {
// 注意：S3 不支持直接更新对象的元数据。必须通过复制对象到自身来实现元数据的更新。
const copySource = `${process.env.S3_BUCKET_NAME}/${key}`;
// 分离标准 HTTP 头和自定义元数据
// 标准头应作为顶层参数，自定义元数据才放在 Metadata 中
const standardHeaders: Record<string, any> = {};
const customMetadata: Record<string, string> = {};
const standardHeaderKeys = ['content-type', 'cache-control', 'content-disposition', 'content-encoding', 'content-language', 'expires'];
for (const [key, value] of Object.entries(metadata)) {
const lowerKey = key.toLowerCase();
if (standardHeaderKeys.includes(lowerKey)) {
// 使用驼峰命名
const camelKey = lowerKey.split('-').map((word, index) =>
index === 0 ? word.charAt(0).toUpperCase() + word.slice(1) : word.charAt(0).toUpperCase() + word.slice(1)
).join('');
standardHeaders[camelKey] = value;
} else {
customMetadata[key] = value;
}
}
const command = new CopyObjectCommand({
Bucket: process.env.S3_BUCKET_NAME,
Key: key,
CopySource: copySource,
...standardHeaders, // 标准头作为顶层参数
Metadata: customMetadata, // 只有自定义元数据放在这里
MetadataDirective: 'REPLACE', // 指定替换元数据
});
try {
const result = await s3Client.send(command);
console.log('Metadata updated successfully for', key);
return result;
} catch (error) {
console.error('Error setting metadata for', key, ':', error);
throw error;
}
}
// setMetaData('readme.md', { 'type': 'app', 'Content-Type': 'text/html' });

4
src/test/config-admin.ts
View File

@@ -1,10 +1,10 @@
import dotenv from 'dotenv';
import { config } from 'dotenv';
import { ConfigOssService } from '../services/index.ts';
import { Client } from 'minio';
import path from 'path';
import { downloadObject } from '../util/download.ts';
const cwd = process.cwd();
dotenv.config({ path: path.resolve(cwd, '..', '..', '.env.dev') });
config({ path: path.resolve(cwd, '..', '..', '.env.dev') });
console.log(
'config',

25
src/test/test-s3.ts Normal file
View File

@@ -0,0 +1,25 @@
import { OssBase } from "@/s3/core.ts";
import { S3Client } from '@aws-sdk/client-s3';
import { s3Client, bucketName } from './common.ts';
const oss = new OssBase({
client: s3Client,
bucketName: bucketName,
});
// const list = await oss.listObjects('');
// console.log(list);
// const obj = await oss.getObjectAsString('readme.md');
// console.log(obj);
let putJson = {
name: 'test',
age: 18,
}
const objPut = await oss.putObject('lanzhoub/a.json', putJson)
console.log(objPut);

138
src/util/download.ts
View File

@@ -1,73 +1,73 @@
import { ServerResponse } from 'node:http';
import { BucketItemStat } from 'minio';
import fs from 'node:fs';
import path from 'node:path';
// import { ServerResponse } from 'node:http';
// import { BucketItemStat } from 'minio';
// import fs from 'node:fs';
// import path from 'node:path';
const viewableExtensions = ['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp', 'mp4', 'webm', 'mp3', 'wav', 'ogg', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx'];
import { OssBase } from '../index.ts';
/**
* 过滤 metaData 中的 key, 去除 password, accesskey, secretkey，
* 并返回过滤后的 metaData
* @param metaData
* @returns
*/
export const filterMetaDataKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
const keys = Object.keys(metaData);
// remove X-Amz- meta data
const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
return filteredKeys.reduce((acc, key) => {
acc[key] = metaData[key];
return acc;
}, {} as Record<string, string>);
};
type SendObjectOptions = {
res: ServerResponse;
client: OssBase;
objectName: string;
isDownload?: boolean;
};
export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
res.writeHead(code, { 'Content-Type': 'text/plain' });
res.end(msg || 'Not Found File');
return;
};
export const sendObject = async ({ res, objectName, client, isDownload = false }: SendObjectOptions) => {
let stat: BucketItemStat;
try {
stat = await client.statObject(objectName);
} catch (e) {
} finally {
if (!stat || stat.size === 0) {
return NotFoundFile(res);
}
const contentLength = stat.size;
const etag = stat.etag;
const lastModified = stat.lastModified.toISOString();
const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
const filteredMetaData = filterMetaDataKeys(stat.metaData, ['size', 'etag', 'last-modified']);
const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
// const viewableExtensions = ['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp', 'mp4', 'webm', 'mp3', 'wav', 'ogg', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx'];
// import { OssBase } from '../index.ts';
// /**
// * 过滤 metaData 中的 key, 去除 password, accesskey, secretkey，
// * 并返回过滤后的 metaData
// * @param metaData
// * @returns
// */
// export const filterMetaDataKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
// const keys = Object.keys(metaData);
// // remove X-Amz- meta data
// const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
// const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
// return filteredKeys.reduce((acc, key) => {
// acc[key] = metaData[key];
// return acc;
// }, {} as Record<string, string>);
// };
// type SendObjectOptions = {
// res: ServerResponse;
// client: OssBase;
// objectName: string;
// isDownload?: boolean;
// };
// export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
// res.writeHead(code, { 'Content-Type': 'text/plain' });
// res.end(msg || 'Not Found File');
// return;
// };
// export const sendObject = async ({ res, objectName, client, isDownload = false }: SendObjectOptions) => {
// let stat: BucketItemStat;
// try {
// stat = await client.statObject(objectName);
// } catch (e) {
// } finally {
// if (!stat || stat.size === 0) {
// return NotFoundFile(res);
// }
// const contentLength = stat.size;
// const etag = stat.etag;
// const lastModified = stat.lastModified.toISOString();
// const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
// const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
// const filteredMetaData = filterMetaDataKeys(stat.metaData, ['size', 'etag', 'last-modified']);
// const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
res.writeHead(200, {
'Content-Length': contentLength,
etag,
'last-modified': lastModified,
'Content-Disposition': contentDisposition,
...filteredMetaData,
});
const objectStream = await client.getObject(objectName);
// res.writeHead(200, {
// 'Content-Length': contentLength,
// etag,
// 'last-modified': lastModified,
// 'Content-Disposition': contentDisposition,
// ...filteredMetaData,
// });
// const objectStream = await client.getObject(objectName);
objectStream.pipe(res, { end: true });
}
};
// objectStream.pipe(res, { end: true });
// }
// };
export const downloadObject = async ({ objectName, client, filePath }: Pick<SendObjectOptions, 'objectName' | 'client'> & { filePath: string }) => {
const objectStream = await client.getObject(objectName);
const dir = path.dirname(filePath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
objectStream.pipe(fs.createWriteStream(filePath));
return objectStream;
};
// export const downloadObject = async ({ objectName, client, filePath }: Pick<SendObjectOptions, 'objectName' | 'client'> & { filePath: string }) => {
// const objectStream = await client.getObject(objectName);
// const dir = path.dirname(filePath);
// if (!fs.existsSync(dir)) {
// fs.mkdirSync(dir, { recursive: true });
// }
// objectStream.pipe(fs.createWriteStream(filePath));
// return objectStream;
// };

44
src/util/extract-standard-headers.ts Normal file
View File

@@ -0,0 +1,44 @@
export const standardHeaderKeys = ['content-type', 'cache-control', 'content-disposition', 'content-encoding', 'content-language', 'expires'];
export type StandardHeaders = {
ContentType?: string;
CacheControl?: string;
ContentDisposition?: string;
ContentEncoding?: string;
ContentLanguage?: string;
Expires?: Date;
};
/**
* 从元数据中提取标准头部和自定义元数据
* @param metaData 原始元数据
* @returns 标准头部和自定义元数据
*/
export function extractStandardHeaders(metaData: Record<string, string>): {
standardHeaders: StandardHeaders;
customMetadata: Record<string, string>;
} {
const standardHeaders: StandardHeaders = {};
const customMetadata: Record<string, string> = {};
for (const [key, value] of Object.entries(metaData)) {
const lowerKey = key.toLowerCase();
if (lowerKey === 'content-type') {
standardHeaders.ContentType = value;
} else if (lowerKey === 'cache-control') {
standardHeaders.CacheControl = value;
} else if (lowerKey === 'content-disposition') {
standardHeaders.ContentDisposition = value;
} else if (lowerKey === 'content-encoding') {
standardHeaders.ContentEncoding = value;
} else if (lowerKey === 'content-language') {
standardHeaders.ContentLanguage = value;
} else if (lowerKey === 'expires') {
standardHeaders.Expires = new Date(value);
} else {
customMetadata[key] = value;
}
}
return { standardHeaders, customMetadata };
}

2
src/util/index.ts
View File

@@ -1,3 +1,5 @@
export * from './hash.ts';
export * from './get-content-type.ts';
export * from './extract-standard-headers.ts';