feat(plugin-fm): add option for storage to remove file physically or not (#2005)

* feat(plugin-fm): add server side destroy action for removing files of attachments

* feat(plugin-fm): add client option for storage
This commit is contained in:
Junyi 2023-06-07 19:44:16 +07:00 committed by GitHub
parent a1872fa75b
commit 0c150eaf9b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 639 additions and 196 deletions

View File

@ -43,6 +43,7 @@
"no-explicit-any": "off", "no-explicit-any": "off",
"@typescript-eslint/no-explicit-any": "off", "@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/ban-ts-comment": "off", "@typescript-eslint/ban-ts-comment": "off",
"@typescript-eslint/no-var-requires": "off" "@typescript-eslint/no-var-requires": "off",
"promise/always-return": "off"
} }
} }

View File

@ -27,4 +27,5 @@ export default {
Filename: '文件名', Filename: '文件名',
'Will be used for API': '将用于 API', 'Will be used for API': '将用于 API',
'Default storage will be used when not selected': '留空将使用默认存储空间', 'Default storage will be used when not selected': '留空将使用默认存储空间',
'Keep file in storage when destroy record': '删除记录时保留文件',
}; };

View File

@ -76,6 +76,16 @@ const collection = {
'x-component': 'Checkbox', 'x-component': 'Checkbox',
} as ISchema, } as ISchema,
}, },
{
type: 'boolean',
name: 'paranoid',
interface: 'boolean',
uiSchema: {
title: `{{t("Keep file in storage when destroy record", { ns: "${NAMESPACE}" })}}`,
type: 'boolean',
'x-component': 'Checkbox',
} as ISchema,
},
], ],
}; };
@ -184,6 +194,12 @@ export const storageSchema: ISchema = {
title: '', title: '',
'x-content': `{{t("Default storage", { ns: "${NAMESPACE}" })}}`, 'x-content': `{{t("Default storage", { ns: "${NAMESPACE}" })}}`,
}, },
paranoid: {
title: '',
'x-component': 'CollectionField',
'x-decorator': 'FormItem',
'x-content': `{{t("Keep file in storage when destroy record", { ns: "${NAMESPACE}" })}}`,
},
footer: { footer: {
type: 'void', type: 'void',
'x-component': 'Action.Drawer.Footer', 'x-component': 'Action.Drawer.Footer',
@ -318,7 +334,13 @@ export const storageSchema: ISchema = {
title: '', title: '',
'x-component': 'CollectionField', 'x-component': 'CollectionField',
'x-decorator': 'FormItem', 'x-decorator': 'FormItem',
'x-content': '{{t("Default storage")}}', 'x-content': `{{t("Default storage", { ns: "${NAMESPACE}" })}}`,
},
paranoid: {
title: '',
'x-component': 'CollectionField',
'x-decorator': 'FormItem',
'x-content': `{{t("Keep file in storage when destroy record", { ns: "${NAMESPACE}" })}}`,
}, },
footer: { footer: {
type: 'void', type: 'void',

View File

@ -11,7 +11,8 @@ describe('action', () => {
let app; let app;
let agent; let agent;
let db; let db;
let StorageModel; let StorageRepo;
let AttachmentRepo;
beforeEach(async () => { beforeEach(async () => {
app = await getApp({ app = await getApp({
@ -20,13 +21,17 @@ describe('action', () => {
agent = app.agent(); agent = app.agent();
db = app.db; db = app.db;
StorageModel = db.getCollection('storages').model; AttachmentRepo = db.getCollection('attachments').repository;
await StorageModel.create({ StorageRepo = db.getCollection('storages').repository;
name: 'local1', await StorageRepo.create({
type: STORAGE_TYPE_LOCAL, values: {
baseUrl: DEFAULT_LOCAL_BASE_URL, name: 'local1',
rules: { type: STORAGE_TYPE_LOCAL,
size: 1024, baseUrl: DEFAULT_LOCAL_BASE_URL,
rules: {
size: 1024,
},
paranoid: true,
}, },
}); });
}); });
@ -35,133 +40,167 @@ describe('action', () => {
await db.close(); await db.close();
}); });
describe('default storage', () => { describe('create / upload', () => {
it('upload file should be ok', async () => { describe('default storage', () => {
const { body } = await agent.resource('attachments').create({ it('upload file should be ok', async () => {
[FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'), const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'),
});
const matcher = {
title: 'text',
extname: '.txt',
path: '',
size: 13,
mimetype: 'text/plain',
meta: {},
storageId: 1,
};
// 文件上传和解析是否正常
expect(body.data).toMatchObject(matcher);
// 文件的 url 是否正常生成
expect(body.data.url).toBe(`${DEFAULT_LOCAL_BASE_URL}${body.data.path}/${body.data.filename}`);
const Attachment = db.getModel('attachments');
const attachment = await Attachment.findOne({
where: { id: body.data.id },
include: ['storage'],
});
// 文件的数据是否正常保存
expect(attachment).toMatchObject(matcher);
// 关联的存储引擎是否正确
const storage = await attachment.getStorage();
expect(storage).toMatchObject({
type: 'local',
options: { documentRoot: LOCAL_STORAGE_DEST },
rules: {},
path: '',
baseUrl: DEFAULT_LOCAL_BASE_URL,
default: true,
});
const { documentRoot = 'storage/uploads' } = storage.options || {};
const destPath = path.resolve(
path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot),
storage.path,
);
const file = await fs.readFile(`${destPath}/${attachment.filename}`);
// 文件是否保存到指定路径
expect(file.toString()).toBe('Hello world!\n');
// 通过 url 是否能正确访问
const url = attachment.url.replace(`http://localhost:${APP_PORT}`, '');
const content = await agent.get(url);
expect(content.text).toBe('Hello world!\n');
});
});
describe('specific storage', () => {
it('fail as 400 because file size greater than rules', async () => {
db.collection({
name: 'customers',
fields: [
{
name: 'avatar',
type: 'belongsTo',
target: 'attachments',
storage: 'local1',
},
],
});
const response = await agent.resource('attachments').create({
attachmentField: 'customers.avatar',
file: path.resolve(__dirname, './files/image.jpg'),
});
expect(response.status).toBe(400);
}); });
const matcher = { it('fail as 400 because file mimetype does not match', async () => {
title: 'text', const textStorage = await StorageRepo.create({
extname: '.txt', values: {
path: '', name: 'local2',
size: 13, type: STORAGE_TYPE_LOCAL,
mimetype: 'text/plain', baseUrl: DEFAULT_LOCAL_BASE_URL,
meta: {}, rules: {
storageId: 1, mimetype: ['text/*'],
}; },
},
});
// 文件上传和解析是否正常 db.collection({
expect(body.data).toMatchObject(matcher); name: 'customers',
// 文件的 url 是否正常生成 fields: [
expect(body.data.url).toBe(`${DEFAULT_LOCAL_BASE_URL}${body.data.path}/${body.data.filename}`); {
name: 'avatar',
type: 'belongsTo',
target: 'attachments',
storage: textStorage.name,
},
],
});
const Attachment = db.getModel('attachments'); // await db.sync();
const attachment = await Attachment.findOne({
where: { id: body.data.id },
include: ['storage'],
});
// 文件的数据是否正常保存
expect(attachment).toMatchObject(matcher);
// 关联的存储引擎是否正确 const response = await agent.resource('attachments').create({
const storage = await attachment.getStorage(); attachmentField: 'customers.avatar',
expect(storage).toMatchObject({ file: path.resolve(__dirname, './files/image.jpg'),
type: 'local', });
options: { documentRoot: LOCAL_STORAGE_DEST },
rules: {}, expect(response.status).toBe(400);
path: '',
baseUrl: DEFAULT_LOCAL_BASE_URL,
default: true,
}); });
const { documentRoot = 'storage/uploads' } = storage.options || {}; it('upload to storage which is not default', async () => {
const destPath = path.resolve( const BASE_URL = `http://localhost:${APP_PORT}/another-uploads`;
path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot), const urlPath = 'test/path';
storage.path,
);
const file = await fs.readFile(`${destPath}/${attachment.filename}`);
// 文件是否保存到指定路径
expect(file.toString()).toBe('Hello world!\n');
// 通过 url 是否能正确访问 // 动态添加 storage
const url = attachment.url.replace(`http://localhost:${APP_PORT}`, ''); const storage = await StorageRepo.create({
const content = await agent.get(url); values: {
expect(content.text).toBe('Hello world!\n'); name: 'local_private',
type: STORAGE_TYPE_LOCAL,
rules: {
mimetype: ['text/*'],
},
path: urlPath,
baseUrl: BASE_URL,
options: {
documentRoot: 'uploads/another',
},
},
});
db.collection({
name: 'customers',
fields: [
{
name: 'file',
type: 'belongsTo',
target: 'attachments',
storage: storage.name,
},
],
});
const { body } = await agent.resource('attachments').create({
attachmentField: 'customers.file',
file: path.resolve(__dirname, './files/text.txt'),
});
// 文件的 url 是否正常生成
expect(body.data.url).toBe(`${BASE_URL}/${urlPath}/${body.data.filename}`);
console.log(body.data.url);
const url = body.data.url.replace(`http://localhost:${APP_PORT}`, '');
const content = await agent.get(url);
expect(content.text).toBe('Hello world!\n');
});
}); });
}); });
describe('specific storage', () => { describe('destroy', () => {
it('fail as 400 because file size greater than rules', async () => { it('destroy one existing file with `paranoid`', async () => {
db.collection({
name: 'customers',
fields: [
{
name: 'avatar',
type: 'belongsTo',
target: 'attachments',
storage: 'local1',
},
],
});
const response = await agent.resource('attachments').create({
attachmentField: 'customers.avatar',
file: path.resolve(__dirname, './files/image.jpg'),
});
expect(response.status).toBe(400);
});
it('fail as 400 because file mimetype does not match', async () => {
const textStorage = await StorageModel.create({
name: 'local2',
type: STORAGE_TYPE_LOCAL,
baseUrl: DEFAULT_LOCAL_BASE_URL,
rules: {
mimetype: ['text/*'],
},
});
db.collection({
name: 'customers',
fields: [
{
name: 'avatar',
type: 'belongsTo',
target: 'attachments',
storage: textStorage.name,
},
],
});
// await db.sync();
const response = await agent.resource('attachments').create({
attachmentField: 'customers.avatar',
file: path.resolve(__dirname, './files/image.jpg'),
});
expect(response.status).toBe(400);
});
it('upload to storage which is not default', async () => {
const BASE_URL = `http://localhost:${APP_PORT}/another-uploads`;
const urlPath = 'test/path';
// 动态添加 storage
const storage = await StorageModel.create({
name: 'local_private',
type: STORAGE_TYPE_LOCAL,
rules: {
mimetype: ['text/*'],
},
path: urlPath,
baseUrl: BASE_URL,
options: {
documentRoot: 'uploads/another',
},
});
db.collection({ db.collection({
name: 'customers', name: 'customers',
fields: [ fields: [
@ -169,22 +208,125 @@ describe('action', () => {
name: 'file', name: 'file',
type: 'belongsTo', type: 'belongsTo',
target: 'attachments', target: 'attachments',
storage: storage.name, storage: 'local1',
}, },
], ],
}); });
await db.sync();
const { body } = await agent.resource('attachments').create({ const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'),
attachmentField: 'customers.file', attachmentField: 'customers.file',
file: path.resolve(__dirname, './files/text.txt'),
}); });
// 文件的 url 是否正常生成 const { data: attachment } = body;
expect(body.data.url).toBe(`${BASE_URL}/${urlPath}/${body.data.filename}`);
console.log(body.data.url); // 关联的存储引擎是否正确
const url = body.data.url.replace(`http://localhost:${APP_PORT}`, ''); const storage = await StorageRepo.findById(attachment.storageId);
const content = await agent.get(url);
expect(content.text).toBe('Hello world!\n'); const { documentRoot = 'storage/uploads' } = storage.options || {};
const destPath = path.resolve(
path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot),
storage.path,
);
const file = await fs.stat(path.join(destPath, attachment.filename));
expect(file).toBeTruthy();
const res2 = await agent.resource('attachments').destroy({ filterByTk: attachment.id });
const attachmentExists = await AttachmentRepo.findById(attachment.id);
expect(attachmentExists).toBeNull();
const fileExists = await fs.stat(path.join(destPath, attachment.filename)).catch(() => false);
expect(fileExists).toBeTruthy();
});
it('destroy one existing file', async () => {
const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'),
});
const { data: attachment } = body;
const storage = await StorageRepo.findById(attachment.storageId);
const { documentRoot = path.join('storage', 'uploads') } = storage.options || {};
const destPath = path.resolve(
path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot),
storage.path,
);
const file = await fs.stat(path.join(destPath, attachment.filename));
expect(file).toBeTruthy();
const res2 = await agent.resource('attachments').destroy({ filterByTk: attachment.id });
const attachmentExists = await AttachmentRepo.findById(attachment.id);
expect(attachmentExists).toBeNull();
const fileExists = await fs.stat(path.join(destPath, attachment.filename)).catch(() => false);
expect(fileExists).toBeFalsy();
});
it('destroy multiple existing files', async () => {
const { body: f1 } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'),
});
const { body: f2 } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'),
});
const storage = await StorageRepo.findOne({
filter: {
name: 'local1',
},
});
const { documentRoot = path.join('storage', 'uploads') } = storage.options || {};
const destPath = path.resolve(
path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot),
storage.path,
);
const file1 = await fs.stat(path.join(destPath, f1.data.filename));
expect(file1).toBeTruthy();
const res2 = await agent.resource('attachments').destroy({ filter: { id: [f1.data.id, f2.data.id] } });
const attachmentExists = await AttachmentRepo.count();
expect(attachmentExists).toBe(0);
const file1Exists = await fs.stat(path.join(destPath, f1.data.filename)).catch(() => false);
expect(file1Exists).toBeFalsy();
const file2Exists = await fs.stat(path.join(destPath, f2.data.filename)).catch(() => false);
expect(file2Exists).toBeFalsy();
});
it('destroy record without file exists should be ok', async () => {
const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'),
});
const { data: attachment } = body;
const storage = await StorageRepo.findById(attachment.storageId);
const { documentRoot = path.join('storage', 'uploads') } = storage.options || {};
const destPath = path.resolve(
path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot),
storage.path,
);
const filePath = path.join(destPath, attachment.filename);
const file = await fs.stat(filePath);
expect(file).toBeTruthy();
await fs.unlink(filePath);
const res2 = await agent.resource('attachments').destroy({ filterByTk: attachment.id });
expect(res2.status).toBe(200);
const attachmentExists = await AttachmentRepo.findById(attachment.id);
expect(attachmentExists).toBeNull();
}); });
}); });
}); });

View File

@ -11,6 +11,8 @@ describe('storage:ali-oss', () => {
let app: MockServer; let app: MockServer;
let agent; let agent;
let db: Database; let db: Database;
let AttachmentRepo;
let StorageRepo;
let storage; let storage;
beforeEach(async () => { beforeEach(async () => {
@ -18,12 +20,16 @@ describe('storage:ali-oss', () => {
agent = app.agent(); agent = app.agent();
db = app.db; db = app.db;
const Storage = db.getCollection('storages').model; AttachmentRepo = db.getCollection('attachments').repository;
storage = await Storage.create({ StorageRepo = db.getCollection('storages').repository;
...aliossStorage.defaults(),
name: 'ali-oss', storage = await StorageRepo.create({
default: true, values: {
path: 'test/path', ...aliossStorage.defaults(),
name: 'ali-oss',
default: true,
path: 'test/path',
},
}); });
}); });
@ -31,7 +37,7 @@ describe('storage:ali-oss', () => {
await db.close(); await db.close();
}); });
describe('direct attachment', () => { describe('upload', () => {
itif('upload file should be ok', async () => { itif('upload file should be ok', async () => {
const { body } = await agent.resource('attachments').create({ const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'), [FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'),
@ -66,4 +72,56 @@ describe('storage:ali-oss', () => {
expect(content.text).toBe('Hello world!\n'); expect(content.text).toBe('Hello world!\n');
}); });
}); });
describe('destroy', () => {
itif('destroy record should also delete file', async () => {
const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'),
});
// 通过 url 是否能正确访问
const content1 = await requestFile(body.data.url, agent);
expect(content1.text).toBe('Hello world!\n');
const res = await agent.resource('attachments').destroy({
filterByTk: body.data.id,
});
expect(res.statusCode).toBe(200);
const count = await AttachmentRepo.count();
expect(count).toBe(0);
const content2 = await requestFile(body.data.url, agent);
expect(content2.status).toBe(404);
});
itif('destroy record should not delete file when paranoid', async () => {
const paranoidStorage = await StorageRepo.create({
values: {
...aliossStorage.defaults(),
name: 'ali-oss-2',
path: 'test/nocobase',
paranoid: true,
default: true,
},
});
const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'),
});
// 通过 url 是否能正确访问
const content1 = await requestFile(body.data.url, agent);
expect(content1.text).toBe('Hello world!\n');
const res = await agent.resource('attachments').destroy({
filterByTk: body.data.id,
});
expect(res.statusCode).toBe(200);
const count = await AttachmentRepo.count();
expect(count).toBe(0);
const content2 = await requestFile(body.data.url, agent);
expect(content2.status).toBe(200);
});
});
}); });

View File

@ -11,6 +11,8 @@ describe('storage:s3', () => {
let app: MockServer; let app: MockServer;
let agent; let agent;
let db: Database; let db: Database;
let AttachmentRepo;
let StorageRepo;
let storage; let storage;
beforeEach(async () => { beforeEach(async () => {
@ -18,12 +20,16 @@ describe('storage:s3', () => {
agent = app.agent(); agent = app.agent();
db = app.db; db = app.db;
const Storage = db.getCollection('storages').model; AttachmentRepo = db.getCollection('attachments').repository;
storage = await Storage.create({ StorageRepo = db.getCollection('storages').repository;
...s3Storage.defaults(),
name: 's3', storage = await StorageRepo.create({
default: true, values: {
path: 'test/path', ...s3Storage.defaults(),
name: 's3',
default: true,
path: 'test/path',
},
}); });
}); });
@ -37,10 +43,9 @@ describe('storage:s3', () => {
[FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'), [FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'),
}); });
const Attachment = db.getCollection('attachments').model; const attachment = await AttachmentRepo.findOne({
const attachment = await Attachment.findOne<any>({ filterByTk: body.data.id,
where: { id: body.data.id }, appends: ['storage'],
include: ['storage'],
}); });
const matcher = { const matcher = {
@ -65,4 +70,57 @@ describe('storage:s3', () => {
expect(content.text).toBe('Hello world!\n'); expect(content.text).toBe('Hello world!\n');
}); });
}); });
describe('destroy', () => {
itif('destroy record should also delete file', async () => {
const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'),
});
// 通过 url 是否能正确访问
const content1 = await requestFile(body.data.url, agent);
expect(content1.text).toBe('Hello world!\n');
const res = await agent.resource('attachments').destroy({
filterByTk: body.data.id,
});
expect(res.statusCode).toBe(200);
const count = await AttachmentRepo.count();
expect(count).toBe(0);
const content2 = await requestFile(body.data.url, agent);
console.log(content2.status, body.data.url);
expect(content2.status).toBe(403);
});
itif('destroy record should not delete file when paranoid', async () => {
const paranoidStorage = await StorageRepo.create({
values: {
...s3Storage.defaults(),
name: 's3-2',
path: 'test/nocobase',
paranoid: true,
default: true,
},
});
const { body } = await agent.resource('attachments').create({
[FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'),
});
// 通过 url 是否能正确访问
const content1 = await requestFile(body.data.url, agent);
expect(content1.text).toBe('Hello world!\n');
const res = await agent.resource('attachments').destroy({
filterByTk: body.data.id,
});
expect(res.statusCode).toBe(200);
const count = await AttachmentRepo.count();
expect(count).toBe(0);
const content2 = await requestFile(body.data.url, agent);
expect(content2.status).toBe(200);
});
});
}); });

View File

@ -1,5 +1,5 @@
import multer from '@koa/multer'; import multer from '@koa/multer';
import { Context, Next } from '@nocobase/actions'; import actions, { Context, Next } from '@nocobase/actions';
import path from 'path'; import path from 'path';
import { DEFAULT_MAX_FILE_SIZE, FILE_FIELD_NAME, LIMIT_FILES } from '../constants'; import { DEFAULT_MAX_FILE_SIZE, FILE_FIELD_NAME, LIMIT_FILES } from '../constants';
@ -48,42 +48,16 @@ function getFileData(ctx: Context) {
}; };
} }
export async function middleware(ctx: Context, next: Next) {
const { resourceName, actionName } = ctx.action;
const { attachmentField } = ctx.action.params;
const collection = ctx.db.getCollection(resourceName);
if (collection?.options?.template !== 'file' || !['upload', 'create'].includes(actionName)) {
return next();
}
const storageName = ctx.db.getFieldByPath(attachmentField)?.options?.storage || collection.options.storage;
const StorageRepo = ctx.db.getRepository('storages');
const storage = await StorageRepo.findOne({ filter: storageName ? { name: storageName } : { default: true } });
ctx.storage = storage;
await multipart(ctx, async () => {
const values = getFileData(ctx);
ctx.action.mergeParams({
values,
});
await next();
});
}
async function multipart(ctx: Context, next: Next) { async function multipart(ctx: Context, next: Next) {
const { storage } = ctx; const { storage } = ctx;
if (!storage) { if (!storage) {
console.error('[file-manager] no linked or default storage provided'); ctx.logger.error('[file-manager] no linked or default storage provided');
return ctx.throw(500); return ctx.throw(500);
} }
const storageConfig = getStorageConfig(storage.type); const storageConfig = getStorageConfig(storage.type);
if (!storageConfig) { if (!storageConfig) {
console.error(`[file-manager] storage type "${storage.type}" is not defined`); ctx.logger.error(`[file-manager] storage type "${storage.type}" is not defined`);
return ctx.throw(500); return ctx.throw(500);
} }
@ -107,5 +81,95 @@ async function multipart(ctx: Context, next: Next) {
return ctx.throw(500); return ctx.throw(500);
} }
const values = getFileData(ctx);
ctx.action.mergeParams({
values,
});
await next();
}
export async function createMiddleware(ctx: Context, next: Next) {
const { resourceName, actionName } = ctx.action;
const { attachmentField } = ctx.action.params;
const collection = ctx.db.getCollection(resourceName);
if (collection?.options?.template !== 'file' || !['upload', 'create'].includes(actionName)) {
return next();
}
const storageName = ctx.db.getFieldByPath(attachmentField)?.options?.storage || collection.options.storage;
const StorageRepo = ctx.db.getRepository('storages');
const storage = await StorageRepo.findOne({ filter: storageName ? { name: storageName } : { default: true } });
ctx.storage = storage;
await multipart(ctx, next);
}
export async function destroyMiddleware(ctx: Context, next: Next) {
const { resourceName, actionName } = ctx.action;
const collection = ctx.db.getCollection(resourceName);
if (collection?.options?.template !== 'file' || actionName !== 'destroy') {
return next();
}
const repository = ctx.db.getRepository(resourceName);
const { filterByTk, filter } = ctx.action.params;
const records = await repository.find({
filterByTk,
filter,
context: ctx,
});
const storageIds = new Set(records.map((record) => record.storageId));
const storageGroupedRecords = records.reduce((result, record) => {
const storageId = record.storageId;
if (!result[storageId]) {
result[storageId] = [];
}
result[storageId].push(record);
return result;
}, {});
const storages = await ctx.db.getRepository('storages').find({
filter: {
id: [...storageIds] as any[],
paranoid: {
$ne: true,
},
},
});
let count = 0;
const undeleted = [];
await storages.reduce(
(promise, storage) =>
promise.then(async () => {
const storageConfig = getStorageConfig(storage.type);
const result = await storageConfig.delete(storage, storageGroupedRecords[storage.id]);
count += result[0];
undeleted.push(...result[1]);
}),
Promise.resolve(),
);
if (undeleted.length) {
const ids = undeleted.map((record) => record.id);
ctx.action.mergeParams({
filter: {
id: {
$notIn: ids,
},
},
});
ctx.logger.error('[file-manager] some of attachment files are not successfully deleted: ', { ids });
}
await next(); await next();
} }

View File

@ -1,7 +1,9 @@
import actions from '@nocobase/actions'; import actions from '@nocobase/actions';
import { middleware } from './attachments'; import { createMiddleware, destroyMiddleware } from './attachments';
export default function ({ app }) { export default function ({ app }) {
app.resourcer.use(middleware); app.resourcer.use(createMiddleware);
app.resourcer.registerActionHandler('upload', actions.create); app.resourcer.registerActionHandler('upload', actions.create);
app.resourcer.use(destroyMiddleware);
} }

View File

@ -56,8 +56,9 @@ export default {
defaultValue: false, defaultValue: false,
}, },
{ {
type: 'hasMany', type: 'boolean',
name: 'attachments', name: 'paranoid',
defaultValue: false,
}, },
], ],
} as CollectionOptions; } as CollectionOptions;

View File

@ -4,6 +4,8 @@ import initActions from './actions';
import { STORAGE_TYPE_LOCAL } from './constants'; import { STORAGE_TYPE_LOCAL } from './constants';
import { getStorageConfig } from './storages'; import { getStorageConfig } from './storages';
export { default as storageTypes } from './storages';
export default class PluginFileManager extends Plugin { export default class PluginFileManager extends Plugin {
storageType() { storageType() {
return process.env.DEFAULT_STORAGE_TYPE ?? 'local'; return process.env.DEFAULT_STORAGE_TYPE ?? 'local';

View File

@ -1,3 +1,4 @@
import { AttachmentModel } from '.';
import { STORAGE_TYPE_ALI_OSS } from '../constants'; import { STORAGE_TYPE_ALI_OSS } from '../constants';
import { cloudFilenameGetter } from '../utils'; import { cloudFilenameGetter } from '../utils';
@ -23,4 +24,12 @@ export default {
}, },
}; };
}, },
async delete(storage, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]> {
const { client } = this.make(storage);
const { deleted } = await client.deleteMulti(records.map((record) => `${record.path}/${record.filename}`));
return [
deleted.length,
records.filter((record) => !deleted.find((item) => item.Key === `${record.path}/${record.filename}`)),
];
},
}; };

View File

@ -1,3 +1,7 @@
import { StorageEngine } from 'multer';
import Application from '@nocobase/server';
import { Registry } from '@nocobase/utils';
import local from './local'; import local from './local';
import oss from './ali-oss'; import oss from './ali-oss';
import s3 from './s3'; import s3 from './s3';
@ -5,20 +9,38 @@ import cos from './tx-cos';
import { STORAGE_TYPE_LOCAL, STORAGE_TYPE_ALI_OSS, STORAGE_TYPE_S3, STORAGE_TYPE_TX_COS } from '../constants'; import { STORAGE_TYPE_LOCAL, STORAGE_TYPE_ALI_OSS, STORAGE_TYPE_S3, STORAGE_TYPE_TX_COS } from '../constants';
export interface StorageModel {
title: string;
type: string;
name: string;
baseUrl: string;
options: { [key: string]: string };
deleteFileOnDestroy?: boolean;
}
export interface AttachmentModel {
title: string;
filename: string;
path: string;
}
export interface IStorage { export interface IStorage {
filenameKey?: string; filenameKey?: string;
middleware?: Function; middleware?(app: Application): void;
getFileData?: Function; getFileData?(file: { [key: string]: any }): { [key: string]: any };
make: Function; make(storage: StorageModel): StorageEngine;
defaults: Function; defaults(): StorageModel;
delete(storage: StorageModel, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]>;
} }
const map = new Map<string, IStorage>(); const storageTypes = new Registry<IStorage>();
map.set(STORAGE_TYPE_LOCAL, local); storageTypes.register(STORAGE_TYPE_LOCAL, local);
map.set(STORAGE_TYPE_ALI_OSS, oss); storageTypes.register(STORAGE_TYPE_ALI_OSS, oss);
map.set(STORAGE_TYPE_S3, s3); storageTypes.register(STORAGE_TYPE_S3, s3);
map.set(STORAGE_TYPE_TX_COS, cos); storageTypes.register(STORAGE_TYPE_TX_COS, cos);
export function getStorageConfig(key: string): IStorage { export function getStorageConfig(key: string): IStorage {
return map.get(key); return storageTypes.get(key);
} }
export default storageTypes;

View File

@ -1,12 +1,15 @@
import path from 'path';
import fs from 'fs/promises';
import Application from '@nocobase/server'; import Application from '@nocobase/server';
import serve from 'koa-static'; import serve from 'koa-static';
import mkdirp from 'mkdirp'; import mkdirp from 'mkdirp';
import multer from 'multer'; import multer from 'multer';
import path from 'path';
import { Transactionable } from 'sequelize/types'; import { Transactionable } from 'sequelize/types';
import { URL } from 'url'; import { URL } from 'url';
import { STORAGE_TYPE_LOCAL } from '../constants'; import { STORAGE_TYPE_LOCAL } from '../constants';
import { getFilename } from '../utils'; import { getFilename } from '../utils';
import { AttachmentModel } from '.';
// use koa-mount match logic // use koa-mount match logic
function match(basePath: string, pathname: string): boolean { function match(basePath: string, pathname: string): boolean {
@ -49,7 +52,7 @@ function createLocalServerUpdateHook(app, storages) {
} }
function getDocumentRoot(storage): string { function getDocumentRoot(storage): string {
const { documentRoot = 'storage/uploads' } = storage.options || {}; const { documentRoot = process.env.LOCAL_STORAGE_DEST || 'storage/uploads' } = storage.options || {};
// TODO(feature): 后面考虑以字符串模板的方式使用,可注入 req/action 相关变量,以便于区分文件夹 // TODO(feature): 后面考虑以字符串模板的方式使用,可注入 req/action 相关变量,以便于区分文件夹
return path.resolve(path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot)); return path.resolve(path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot));
} }
@ -137,4 +140,29 @@ export default {
}, },
}; };
}, },
async delete(storage, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]> {
const documentRoot = getDocumentRoot(storage);
let count = 0;
const undeleted = [];
await records.reduce(
(promise, record) =>
promise.then(async () => {
try {
await fs.unlink(path.join(documentRoot, record.path, record.filename));
count += 1;
} catch (ex) {
if (ex.code === 'ENOENT') {
console.warn(ex.message);
count += 1;
} else {
console.error(ex);
undeleted.push(record);
}
}
}),
Promise.resolve(),
);
return [count, undeleted];
},
}; };

View File

@ -1,3 +1,4 @@
import { AttachmentModel } from '.';
import { STORAGE_TYPE_S3 } from '../constants'; import { STORAGE_TYPE_S3 } from '../constants';
import { cloudFilenameGetter } from '../utils'; import { cloudFilenameGetter } from '../utils';
@ -44,4 +45,21 @@ export default {
}, },
}; };
}, },
async delete(storage, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]> {
const { DeleteObjectsCommand } = require('@aws-sdk/client-s3');
const { s3 } = this.make(storage);
const { Deleted } = await s3.send(
new DeleteObjectsCommand({
Bucket: storage.options.bucket,
Delete: {
Objects: records.map((record) => ({ Key: `${record.path}/${record.filename}` })),
},
}),
);
return [
Deleted.length,
records.filter((record) => !Deleted.find((item) => item.Key === `${record.path}/${record.filename}`)),
];
},
}; };

View File

@ -1,3 +1,6 @@
import { promisify } from 'util';
import { AttachmentModel } from '.';
import { STORAGE_TYPE_TX_COS } from '../constants'; import { STORAGE_TYPE_TX_COS } from '../constants';
import { cloudFilenameGetter } from '../utils'; import { cloudFilenameGetter } from '../utils';
@ -24,4 +27,16 @@ export default {
}, },
}; };
}, },
async delete(storage, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]> {
const { cos } = this.make(storage);
const { Deleted } = await promisify(cos.deleteMultipleObject)({
Region: storage.options.Region,
Bucket: storage.options.Bucket,
Objects: records.map((record) => ({ Key: `${record.path}/${record.filename}` })),
});
return [
Deleted.length,
records.filter((record) => !Deleted.find((item) => item.Key === `${record.path}/${record.filename}`)),
];
},
}; };