From 0c150eaf9b7f547121cbdf97f3897909113d2638 Mon Sep 17 00:00:00 2001 From: Junyi Date: Wed, 7 Jun 2023 19:44:16 +0700 Subject: [PATCH] feat(plugin-fm): add option for storage to remove file physically or not (#2005) * feat(plugin-fm): add server side destroy action for removing files of attachments * feat(plugin-fm): add client option for storage --- .eslintrc | 3 +- .../file-manager/src/client/locale/zh-CN.ts | 1 + .../src/client/schemas/storage.ts | 24 +- .../src/server/__tests__/action.test.ts | 406 ++++++++++++------ .../server/__tests__/storages/ali-oss.test.ts | 72 +++- .../src/server/__tests__/storages/s3.test.ts | 78 +++- .../src/server/actions/attachments.ts | 122 ++++-- .../file-manager/src/server/actions/index.ts | 6 +- .../src/server/collections/storages.ts | 5 +- .../plugins/file-manager/src/server/server.ts | 2 + .../src/server/storages/ali-oss.ts | 9 + .../file-manager/src/server/storages/index.ts | 42 +- .../file-manager/src/server/storages/local.ts | 32 +- .../file-manager/src/server/storages/s3.ts | 18 + .../src/server/storages/tx-cos.ts | 15 + 15 files changed, 639 insertions(+), 196 deletions(-) diff --git a/.eslintrc b/.eslintrc index f2f9c58ed..805d21c4d 100755 --- a/.eslintrc +++ b/.eslintrc @@ -43,6 +43,7 @@ "no-explicit-any": "off", "@typescript-eslint/no-explicit-any": "off", "@typescript-eslint/ban-ts-comment": "off", - "@typescript-eslint/no-var-requires": "off" + "@typescript-eslint/no-var-requires": "off", + "promise/always-return": "off" } } diff --git a/packages/plugins/file-manager/src/client/locale/zh-CN.ts b/packages/plugins/file-manager/src/client/locale/zh-CN.ts index f88948865..0b897cb25 100644 --- a/packages/plugins/file-manager/src/client/locale/zh-CN.ts +++ b/packages/plugins/file-manager/src/client/locale/zh-CN.ts @@ -27,4 +27,5 @@ export default { Filename: '文件名', 'Will be used for API': '将用于 API', 'Default storage will be used when not selected': '留空将使用默认存储空间', + 'Keep file in storage when destroy record': '删除记录时保留文件', }; diff --git a/packages/plugins/file-manager/src/client/schemas/storage.ts b/packages/plugins/file-manager/src/client/schemas/storage.ts index c78c7d446..cc2d552c9 100644 --- a/packages/plugins/file-manager/src/client/schemas/storage.ts +++ b/packages/plugins/file-manager/src/client/schemas/storage.ts @@ -76,6 +76,16 @@ const collection = { 'x-component': 'Checkbox', } as ISchema, }, + { + type: 'boolean', + name: 'paranoid', + interface: 'boolean', + uiSchema: { + title: `{{t("Keep file in storage when destroy record", { ns: "${NAMESPACE}" })}}`, + type: 'boolean', + 'x-component': 'Checkbox', + } as ISchema, + }, ], }; @@ -184,6 +194,12 @@ export const storageSchema: ISchema = { title: '', 'x-content': `{{t("Default storage", { ns: "${NAMESPACE}" })}}`, }, + paranoid: { + title: '', + 'x-component': 'CollectionField', + 'x-decorator': 'FormItem', + 'x-content': `{{t("Keep file in storage when destroy record", { ns: "${NAMESPACE}" })}}`, + }, footer: { type: 'void', 'x-component': 'Action.Drawer.Footer', @@ -318,7 +334,13 @@ export const storageSchema: ISchema = { title: '', 'x-component': 'CollectionField', 'x-decorator': 'FormItem', - 'x-content': '{{t("Default storage")}}', + 'x-content': `{{t("Default storage", { ns: "${NAMESPACE}" })}}`, + }, + paranoid: { + title: '', + 'x-component': 'CollectionField', + 'x-decorator': 'FormItem', + 'x-content': `{{t("Keep file in storage when destroy record", { ns: "${NAMESPACE}" })}}`, }, footer: { type: 'void', diff --git a/packages/plugins/file-manager/src/server/__tests__/action.test.ts b/packages/plugins/file-manager/src/server/__tests__/action.test.ts index fe38e4a13..3c019c351 100644 --- a/packages/plugins/file-manager/src/server/__tests__/action.test.ts +++ b/packages/plugins/file-manager/src/server/__tests__/action.test.ts @@ -11,7 +11,8 @@ describe('action', () => { let app; let agent; let db; - let StorageModel; + let StorageRepo; + let AttachmentRepo; beforeEach(async () => { app = await getApp({ @@ -20,13 +21,17 @@ describe('action', () => { agent = app.agent(); db = app.db; - StorageModel = db.getCollection('storages').model; - await StorageModel.create({ - name: 'local1', - type: STORAGE_TYPE_LOCAL, - baseUrl: DEFAULT_LOCAL_BASE_URL, - rules: { - size: 1024, + AttachmentRepo = db.getCollection('attachments').repository; + StorageRepo = db.getCollection('storages').repository; + await StorageRepo.create({ + values: { + name: 'local1', + type: STORAGE_TYPE_LOCAL, + baseUrl: DEFAULT_LOCAL_BASE_URL, + rules: { + size: 1024, + }, + paranoid: true, }, }); }); @@ -35,133 +40,167 @@ describe('action', () => { await db.close(); }); - describe('default storage', () => { - it('upload file should be ok', async () => { - const { body } = await agent.resource('attachments').create({ - [FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'), + describe('create / upload', () => { + describe('default storage', () => { + it('upload file should be ok', async () => { + const { body } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'), + }); + + const matcher = { + title: 'text', + extname: '.txt', + path: '', + size: 13, + mimetype: 'text/plain', + meta: {}, + storageId: 1, + }; + + // 文件上传和解析是否正常 + expect(body.data).toMatchObject(matcher); + // 文件的 url 是否正常生成 + expect(body.data.url).toBe(`${DEFAULT_LOCAL_BASE_URL}${body.data.path}/${body.data.filename}`); + + const Attachment = db.getModel('attachments'); + const attachment = await Attachment.findOne({ + where: { id: body.data.id }, + include: ['storage'], + }); + // 文件的数据是否正常保存 + expect(attachment).toMatchObject(matcher); + + // 关联的存储引擎是否正确 + const storage = await attachment.getStorage(); + expect(storage).toMatchObject({ + type: 'local', + options: { documentRoot: LOCAL_STORAGE_DEST }, + rules: {}, + path: '', + baseUrl: DEFAULT_LOCAL_BASE_URL, + default: true, + }); + + const { documentRoot = 'storage/uploads' } = storage.options || {}; + const destPath = path.resolve( + path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot), + storage.path, + ); + const file = await fs.readFile(`${destPath}/${attachment.filename}`); + // 文件是否保存到指定路径 + expect(file.toString()).toBe('Hello world!\n'); + + // 通过 url 是否能正确访问 + const url = attachment.url.replace(`http://localhost:${APP_PORT}`, ''); + const content = await agent.get(url); + expect(content.text).toBe('Hello world!\n'); + }); + }); + + describe('specific storage', () => { + it('fail as 400 because file size greater than rules', async () => { + db.collection({ + name: 'customers', + fields: [ + { + name: 'avatar', + type: 'belongsTo', + target: 'attachments', + storage: 'local1', + }, + ], + }); + + const response = await agent.resource('attachments').create({ + attachmentField: 'customers.avatar', + file: path.resolve(__dirname, './files/image.jpg'), + }); + expect(response.status).toBe(400); }); - const matcher = { - title: 'text', - extname: '.txt', - path: '', - size: 13, - mimetype: 'text/plain', - meta: {}, - storageId: 1, - }; + it('fail as 400 because file mimetype does not match', async () => { + const textStorage = await StorageRepo.create({ + values: { + name: 'local2', + type: STORAGE_TYPE_LOCAL, + baseUrl: DEFAULT_LOCAL_BASE_URL, + rules: { + mimetype: ['text/*'], + }, + }, + }); - // 文件上传和解析是否正常 - expect(body.data).toMatchObject(matcher); - // 文件的 url 是否正常生成 - expect(body.data.url).toBe(`${DEFAULT_LOCAL_BASE_URL}${body.data.path}/${body.data.filename}`); + db.collection({ + name: 'customers', + fields: [ + { + name: 'avatar', + type: 'belongsTo', + target: 'attachments', + storage: textStorage.name, + }, + ], + }); - const Attachment = db.getModel('attachments'); - const attachment = await Attachment.findOne({ - where: { id: body.data.id }, - include: ['storage'], - }); - // 文件的数据是否正常保存 - expect(attachment).toMatchObject(matcher); + // await db.sync(); - // 关联的存储引擎是否正确 - const storage = await attachment.getStorage(); - expect(storage).toMatchObject({ - type: 'local', - options: { documentRoot: LOCAL_STORAGE_DEST }, - rules: {}, - path: '', - baseUrl: DEFAULT_LOCAL_BASE_URL, - default: true, + const response = await agent.resource('attachments').create({ + attachmentField: 'customers.avatar', + file: path.resolve(__dirname, './files/image.jpg'), + }); + + expect(response.status).toBe(400); }); - const { documentRoot = 'storage/uploads' } = storage.options || {}; - const destPath = path.resolve( - path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot), - storage.path, - ); - const file = await fs.readFile(`${destPath}/${attachment.filename}`); - // 文件是否保存到指定路径 - expect(file.toString()).toBe('Hello world!\n'); + it('upload to storage which is not default', async () => { + const BASE_URL = `http://localhost:${APP_PORT}/another-uploads`; + const urlPath = 'test/path'; - // 通过 url 是否能正确访问 - const url = attachment.url.replace(`http://localhost:${APP_PORT}`, ''); - const content = await agent.get(url); - expect(content.text).toBe('Hello world!\n'); + // 动态添加 storage + const storage = await StorageRepo.create({ + values: { + name: 'local_private', + type: STORAGE_TYPE_LOCAL, + rules: { + mimetype: ['text/*'], + }, + path: urlPath, + baseUrl: BASE_URL, + options: { + documentRoot: 'uploads/another', + }, + }, + }); + + db.collection({ + name: 'customers', + fields: [ + { + name: 'file', + type: 'belongsTo', + target: 'attachments', + storage: storage.name, + }, + ], + }); + + const { body } = await agent.resource('attachments').create({ + attachmentField: 'customers.file', + file: path.resolve(__dirname, './files/text.txt'), + }); + + // 文件的 url 是否正常生成 + expect(body.data.url).toBe(`${BASE_URL}/${urlPath}/${body.data.filename}`); + console.log(body.data.url); + const url = body.data.url.replace(`http://localhost:${APP_PORT}`, ''); + const content = await agent.get(url); + expect(content.text).toBe('Hello world!\n'); + }); }); }); - describe('specific storage', () => { - it('fail as 400 because file size greater than rules', async () => { - db.collection({ - name: 'customers', - fields: [ - { - name: 'avatar', - type: 'belongsTo', - target: 'attachments', - storage: 'local1', - }, - ], - }); - - const response = await agent.resource('attachments').create({ - attachmentField: 'customers.avatar', - file: path.resolve(__dirname, './files/image.jpg'), - }); - expect(response.status).toBe(400); - }); - - it('fail as 400 because file mimetype does not match', async () => { - const textStorage = await StorageModel.create({ - name: 'local2', - type: STORAGE_TYPE_LOCAL, - baseUrl: DEFAULT_LOCAL_BASE_URL, - rules: { - mimetype: ['text/*'], - }, - }); - - db.collection({ - name: 'customers', - fields: [ - { - name: 'avatar', - type: 'belongsTo', - target: 'attachments', - storage: textStorage.name, - }, - ], - }); - - // await db.sync(); - - const response = await agent.resource('attachments').create({ - attachmentField: 'customers.avatar', - file: path.resolve(__dirname, './files/image.jpg'), - }); - - expect(response.status).toBe(400); - }); - - it('upload to storage which is not default', async () => { - const BASE_URL = `http://localhost:${APP_PORT}/another-uploads`; - const urlPath = 'test/path'; - - // 动态添加 storage - const storage = await StorageModel.create({ - name: 'local_private', - type: STORAGE_TYPE_LOCAL, - rules: { - mimetype: ['text/*'], - }, - path: urlPath, - baseUrl: BASE_URL, - options: { - documentRoot: 'uploads/another', - }, - }); - + describe('destroy', () => { + it('destroy one existing file with `paranoid`', async () => { db.collection({ name: 'customers', fields: [ @@ -169,22 +208,125 @@ describe('action', () => { name: 'file', type: 'belongsTo', target: 'attachments', - storage: storage.name, + storage: 'local1', }, ], }); + await db.sync(); + const { body } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'), attachmentField: 'customers.file', - file: path.resolve(__dirname, './files/text.txt'), }); - // 文件的 url 是否正常生成 - expect(body.data.url).toBe(`${BASE_URL}/${urlPath}/${body.data.filename}`); - console.log(body.data.url); - const url = body.data.url.replace(`http://localhost:${APP_PORT}`, ''); - const content = await agent.get(url); - expect(content.text).toBe('Hello world!\n'); + const { data: attachment } = body; + + // 关联的存储引擎是否正确 + const storage = await StorageRepo.findById(attachment.storageId); + + const { documentRoot = 'storage/uploads' } = storage.options || {}; + const destPath = path.resolve( + path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot), + storage.path, + ); + const file = await fs.stat(path.join(destPath, attachment.filename)); + expect(file).toBeTruthy(); + + const res2 = await agent.resource('attachments').destroy({ filterByTk: attachment.id }); + + const attachmentExists = await AttachmentRepo.findById(attachment.id); + expect(attachmentExists).toBeNull(); + + const fileExists = await fs.stat(path.join(destPath, attachment.filename)).catch(() => false); + expect(fileExists).toBeTruthy(); + }); + + it('destroy one existing file', async () => { + const { body } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'), + }); + + const { data: attachment } = body; + + const storage = await StorageRepo.findById(attachment.storageId); + + const { documentRoot = path.join('storage', 'uploads') } = storage.options || {}; + const destPath = path.resolve( + path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot), + storage.path, + ); + const file = await fs.stat(path.join(destPath, attachment.filename)); + expect(file).toBeTruthy(); + + const res2 = await agent.resource('attachments').destroy({ filterByTk: attachment.id }); + + const attachmentExists = await AttachmentRepo.findById(attachment.id); + expect(attachmentExists).toBeNull(); + + const fileExists = await fs.stat(path.join(destPath, attachment.filename)).catch(() => false); + expect(fileExists).toBeFalsy(); + }); + + it('destroy multiple existing files', async () => { + const { body: f1 } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'), + }); + + const { body: f2 } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'), + }); + + const storage = await StorageRepo.findOne({ + filter: { + name: 'local1', + }, + }); + + const { documentRoot = path.join('storage', 'uploads') } = storage.options || {}; + const destPath = path.resolve( + path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot), + storage.path, + ); + const file1 = await fs.stat(path.join(destPath, f1.data.filename)); + expect(file1).toBeTruthy(); + + const res2 = await agent.resource('attachments').destroy({ filter: { id: [f1.data.id, f2.data.id] } }); + + const attachmentExists = await AttachmentRepo.count(); + expect(attachmentExists).toBe(0); + + const file1Exists = await fs.stat(path.join(destPath, f1.data.filename)).catch(() => false); + expect(file1Exists).toBeFalsy(); + + const file2Exists = await fs.stat(path.join(destPath, f2.data.filename)).catch(() => false); + expect(file2Exists).toBeFalsy(); + }); + + it('destroy record without file exists should be ok', async () => { + const { body } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, './files/text.txt'), + }); + + const { data: attachment } = body; + + const storage = await StorageRepo.findById(attachment.storageId); + + const { documentRoot = path.join('storage', 'uploads') } = storage.options || {}; + const destPath = path.resolve( + path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot), + storage.path, + ); + const filePath = path.join(destPath, attachment.filename); + const file = await fs.stat(filePath); + expect(file).toBeTruthy(); + await fs.unlink(filePath); + + const res2 = await agent.resource('attachments').destroy({ filterByTk: attachment.id }); + expect(res2.status).toBe(200); + + const attachmentExists = await AttachmentRepo.findById(attachment.id); + expect(attachmentExists).toBeNull(); }); }); }); diff --git a/packages/plugins/file-manager/src/server/__tests__/storages/ali-oss.test.ts b/packages/plugins/file-manager/src/server/__tests__/storages/ali-oss.test.ts index c0f986489..d2ca58dcb 100644 --- a/packages/plugins/file-manager/src/server/__tests__/storages/ali-oss.test.ts +++ b/packages/plugins/file-manager/src/server/__tests__/storages/ali-oss.test.ts @@ -11,6 +11,8 @@ describe('storage:ali-oss', () => { let app: MockServer; let agent; let db: Database; + let AttachmentRepo; + let StorageRepo; let storage; beforeEach(async () => { @@ -18,12 +20,16 @@ describe('storage:ali-oss', () => { agent = app.agent(); db = app.db; - const Storage = db.getCollection('storages').model; - storage = await Storage.create({ - ...aliossStorage.defaults(), - name: 'ali-oss', - default: true, - path: 'test/path', + AttachmentRepo = db.getCollection('attachments').repository; + StorageRepo = db.getCollection('storages').repository; + + storage = await StorageRepo.create({ + values: { + ...aliossStorage.defaults(), + name: 'ali-oss', + default: true, + path: 'test/path', + }, }); }); @@ -31,7 +37,7 @@ describe('storage:ali-oss', () => { await db.close(); }); - describe('direct attachment', () => { + describe('upload', () => { itif('upload file should be ok', async () => { const { body } = await agent.resource('attachments').create({ [FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'), @@ -66,4 +72,56 @@ describe('storage:ali-oss', () => { expect(content.text).toBe('Hello world!\n'); }); }); + + describe('destroy', () => { + itif('destroy record should also delete file', async () => { + const { body } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'), + }); + // 通过 url 是否能正确访问 + const content1 = await requestFile(body.data.url, agent); + expect(content1.text).toBe('Hello world!\n'); + + const res = await agent.resource('attachments').destroy({ + filterByTk: body.data.id, + }); + + expect(res.statusCode).toBe(200); + const count = await AttachmentRepo.count(); + expect(count).toBe(0); + + const content2 = await requestFile(body.data.url, agent); + expect(content2.status).toBe(404); + }); + + itif('destroy record should not delete file when paranoid', async () => { + const paranoidStorage = await StorageRepo.create({ + values: { + ...aliossStorage.defaults(), + name: 'ali-oss-2', + path: 'test/nocobase', + paranoid: true, + default: true, + }, + }); + + const { body } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'), + }); + // 通过 url 是否能正确访问 + const content1 = await requestFile(body.data.url, agent); + expect(content1.text).toBe('Hello world!\n'); + + const res = await agent.resource('attachments').destroy({ + filterByTk: body.data.id, + }); + + expect(res.statusCode).toBe(200); + const count = await AttachmentRepo.count(); + expect(count).toBe(0); + + const content2 = await requestFile(body.data.url, agent); + expect(content2.status).toBe(200); + }); + }); }); diff --git a/packages/plugins/file-manager/src/server/__tests__/storages/s3.test.ts b/packages/plugins/file-manager/src/server/__tests__/storages/s3.test.ts index 3712d5900..ee59c5b38 100644 --- a/packages/plugins/file-manager/src/server/__tests__/storages/s3.test.ts +++ b/packages/plugins/file-manager/src/server/__tests__/storages/s3.test.ts @@ -11,6 +11,8 @@ describe('storage:s3', () => { let app: MockServer; let agent; let db: Database; + let AttachmentRepo; + let StorageRepo; let storage; beforeEach(async () => { @@ -18,12 +20,16 @@ describe('storage:s3', () => { agent = app.agent(); db = app.db; - const Storage = db.getCollection('storages').model; - storage = await Storage.create({ - ...s3Storage.defaults(), - name: 's3', - default: true, - path: 'test/path', + AttachmentRepo = db.getCollection('attachments').repository; + StorageRepo = db.getCollection('storages').repository; + + storage = await StorageRepo.create({ + values: { + ...s3Storage.defaults(), + name: 's3', + default: true, + path: 'test/path', + }, }); }); @@ -37,10 +43,9 @@ describe('storage:s3', () => { [FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'), }); - const Attachment = db.getCollection('attachments').model; - const attachment = await Attachment.findOne({ - where: { id: body.data.id }, - include: ['storage'], + const attachment = await AttachmentRepo.findOne({ + filterByTk: body.data.id, + appends: ['storage'], }); const matcher = { @@ -65,4 +70,57 @@ describe('storage:s3', () => { expect(content.text).toBe('Hello world!\n'); }); }); + + describe('destroy', () => { + itif('destroy record should also delete file', async () => { + const { body } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'), + }); + // 通过 url 是否能正确访问 + const content1 = await requestFile(body.data.url, agent); + expect(content1.text).toBe('Hello world!\n'); + + const res = await agent.resource('attachments').destroy({ + filterByTk: body.data.id, + }); + + expect(res.statusCode).toBe(200); + const count = await AttachmentRepo.count(); + expect(count).toBe(0); + + const content2 = await requestFile(body.data.url, agent); + console.log(content2.status, body.data.url); + expect(content2.status).toBe(403); + }); + + itif('destroy record should not delete file when paranoid', async () => { + const paranoidStorage = await StorageRepo.create({ + values: { + ...s3Storage.defaults(), + name: 's3-2', + path: 'test/nocobase', + paranoid: true, + default: true, + }, + }); + + const { body } = await agent.resource('attachments').create({ + [FILE_FIELD_NAME]: path.resolve(__dirname, '../files/text.txt'), + }); + // 通过 url 是否能正确访问 + const content1 = await requestFile(body.data.url, agent); + expect(content1.text).toBe('Hello world!\n'); + + const res = await agent.resource('attachments').destroy({ + filterByTk: body.data.id, + }); + + expect(res.statusCode).toBe(200); + const count = await AttachmentRepo.count(); + expect(count).toBe(0); + + const content2 = await requestFile(body.data.url, agent); + expect(content2.status).toBe(200); + }); + }); }); diff --git a/packages/plugins/file-manager/src/server/actions/attachments.ts b/packages/plugins/file-manager/src/server/actions/attachments.ts index 8dbf8f780..99e26a993 100644 --- a/packages/plugins/file-manager/src/server/actions/attachments.ts +++ b/packages/plugins/file-manager/src/server/actions/attachments.ts @@ -1,5 +1,5 @@ import multer from '@koa/multer'; -import { Context, Next } from '@nocobase/actions'; +import actions, { Context, Next } from '@nocobase/actions'; import path from 'path'; import { DEFAULT_MAX_FILE_SIZE, FILE_FIELD_NAME, LIMIT_FILES } from '../constants'; @@ -48,42 +48,16 @@ function getFileData(ctx: Context) { }; } -export async function middleware(ctx: Context, next: Next) { - const { resourceName, actionName } = ctx.action; - const { attachmentField } = ctx.action.params; - const collection = ctx.db.getCollection(resourceName); - - if (collection?.options?.template !== 'file' || !['upload', 'create'].includes(actionName)) { - return next(); - } - - const storageName = ctx.db.getFieldByPath(attachmentField)?.options?.storage || collection.options.storage; - const StorageRepo = ctx.db.getRepository('storages'); - const storage = await StorageRepo.findOne({ filter: storageName ? { name: storageName } : { default: true } }); - - ctx.storage = storage; - - await multipart(ctx, async () => { - const values = getFileData(ctx); - - ctx.action.mergeParams({ - values, - }); - - await next(); - }); -} - async function multipart(ctx: Context, next: Next) { const { storage } = ctx; if (!storage) { - console.error('[file-manager] no linked or default storage provided'); + ctx.logger.error('[file-manager] no linked or default storage provided'); return ctx.throw(500); } const storageConfig = getStorageConfig(storage.type); if (!storageConfig) { - console.error(`[file-manager] storage type "${storage.type}" is not defined`); + ctx.logger.error(`[file-manager] storage type "${storage.type}" is not defined`); return ctx.throw(500); } @@ -107,5 +81,95 @@ async function multipart(ctx: Context, next: Next) { return ctx.throw(500); } + const values = getFileData(ctx); + + ctx.action.mergeParams({ + values, + }); + + await next(); +} + +export async function createMiddleware(ctx: Context, next: Next) { + const { resourceName, actionName } = ctx.action; + const { attachmentField } = ctx.action.params; + const collection = ctx.db.getCollection(resourceName); + + if (collection?.options?.template !== 'file' || !['upload', 'create'].includes(actionName)) { + return next(); + } + + const storageName = ctx.db.getFieldByPath(attachmentField)?.options?.storage || collection.options.storage; + const StorageRepo = ctx.db.getRepository('storages'); + const storage = await StorageRepo.findOne({ filter: storageName ? { name: storageName } : { default: true } }); + + ctx.storage = storage; + + await multipart(ctx, next); +} + +export async function destroyMiddleware(ctx: Context, next: Next) { + const { resourceName, actionName } = ctx.action; + const collection = ctx.db.getCollection(resourceName); + + if (collection?.options?.template !== 'file' || actionName !== 'destroy') { + return next(); + } + + const repository = ctx.db.getRepository(resourceName); + + const { filterByTk, filter } = ctx.action.params; + + const records = await repository.find({ + filterByTk, + filter, + context: ctx, + }); + + const storageIds = new Set(records.map((record) => record.storageId)); + const storageGroupedRecords = records.reduce((result, record) => { + const storageId = record.storageId; + if (!result[storageId]) { + result[storageId] = []; + } + result[storageId].push(record); + return result; + }, {}); + + const storages = await ctx.db.getRepository('storages').find({ + filter: { + id: [...storageIds] as any[], + paranoid: { + $ne: true, + }, + }, + }); + + let count = 0; + const undeleted = []; + await storages.reduce( + (promise, storage) => + promise.then(async () => { + const storageConfig = getStorageConfig(storage.type); + const result = await storageConfig.delete(storage, storageGroupedRecords[storage.id]); + count += result[0]; + undeleted.push(...result[1]); + }), + Promise.resolve(), + ); + + if (undeleted.length) { + const ids = undeleted.map((record) => record.id); + ctx.action.mergeParams({ + filter: { + id: { + $notIn: ids, + }, + }, + }); + + ctx.logger.error('[file-manager] some of attachment files are not successfully deleted: ', { ids }); + } + await next(); } diff --git a/packages/plugins/file-manager/src/server/actions/index.ts b/packages/plugins/file-manager/src/server/actions/index.ts index 3332756bb..05a6e3175 100644 --- a/packages/plugins/file-manager/src/server/actions/index.ts +++ b/packages/plugins/file-manager/src/server/actions/index.ts @@ -1,7 +1,9 @@ import actions from '@nocobase/actions'; -import { middleware } from './attachments'; +import { createMiddleware, destroyMiddleware } from './attachments'; export default function ({ app }) { - app.resourcer.use(middleware); + app.resourcer.use(createMiddleware); app.resourcer.registerActionHandler('upload', actions.create); + + app.resourcer.use(destroyMiddleware); } diff --git a/packages/plugins/file-manager/src/server/collections/storages.ts b/packages/plugins/file-manager/src/server/collections/storages.ts index d03270752..836094118 100644 --- a/packages/plugins/file-manager/src/server/collections/storages.ts +++ b/packages/plugins/file-manager/src/server/collections/storages.ts @@ -56,8 +56,9 @@ export default { defaultValue: false, }, { - type: 'hasMany', - name: 'attachments', + type: 'boolean', + name: 'paranoid', + defaultValue: false, }, ], } as CollectionOptions; diff --git a/packages/plugins/file-manager/src/server/server.ts b/packages/plugins/file-manager/src/server/server.ts index 7317a0183..6ce533efc 100644 --- a/packages/plugins/file-manager/src/server/server.ts +++ b/packages/plugins/file-manager/src/server/server.ts @@ -4,6 +4,8 @@ import initActions from './actions'; import { STORAGE_TYPE_LOCAL } from './constants'; import { getStorageConfig } from './storages'; +export { default as storageTypes } from './storages'; + export default class PluginFileManager extends Plugin { storageType() { return process.env.DEFAULT_STORAGE_TYPE ?? 'local'; diff --git a/packages/plugins/file-manager/src/server/storages/ali-oss.ts b/packages/plugins/file-manager/src/server/storages/ali-oss.ts index 8b7b9b560..b1a6aeb9d 100644 --- a/packages/plugins/file-manager/src/server/storages/ali-oss.ts +++ b/packages/plugins/file-manager/src/server/storages/ali-oss.ts @@ -1,3 +1,4 @@ +import { AttachmentModel } from '.'; import { STORAGE_TYPE_ALI_OSS } from '../constants'; import { cloudFilenameGetter } from '../utils'; @@ -23,4 +24,12 @@ export default { }, }; }, + async delete(storage, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]> { + const { client } = this.make(storage); + const { deleted } = await client.deleteMulti(records.map((record) => `${record.path}/${record.filename}`)); + return [ + deleted.length, + records.filter((record) => !deleted.find((item) => item.Key === `${record.path}/${record.filename}`)), + ]; + }, }; diff --git a/packages/plugins/file-manager/src/server/storages/index.ts b/packages/plugins/file-manager/src/server/storages/index.ts index 677163bae..ee90f5a5a 100644 --- a/packages/plugins/file-manager/src/server/storages/index.ts +++ b/packages/plugins/file-manager/src/server/storages/index.ts @@ -1,3 +1,7 @@ +import { StorageEngine } from 'multer'; +import Application from '@nocobase/server'; +import { Registry } from '@nocobase/utils'; + import local from './local'; import oss from './ali-oss'; import s3 from './s3'; @@ -5,20 +9,38 @@ import cos from './tx-cos'; import { STORAGE_TYPE_LOCAL, STORAGE_TYPE_ALI_OSS, STORAGE_TYPE_S3, STORAGE_TYPE_TX_COS } from '../constants'; +export interface StorageModel { + title: string; + type: string; + name: string; + baseUrl: string; + options: { [key: string]: string }; + deleteFileOnDestroy?: boolean; +} + +export interface AttachmentModel { + title: string; + filename: string; + path: string; +} + export interface IStorage { filenameKey?: string; - middleware?: Function; - getFileData?: Function; - make: Function; - defaults: Function; + middleware?(app: Application): void; + getFileData?(file: { [key: string]: any }): { [key: string]: any }; + make(storage: StorageModel): StorageEngine; + defaults(): StorageModel; + delete(storage: StorageModel, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]>; } -const map = new Map(); -map.set(STORAGE_TYPE_LOCAL, local); -map.set(STORAGE_TYPE_ALI_OSS, oss); -map.set(STORAGE_TYPE_S3, s3); -map.set(STORAGE_TYPE_TX_COS, cos); +const storageTypes = new Registry(); +storageTypes.register(STORAGE_TYPE_LOCAL, local); +storageTypes.register(STORAGE_TYPE_ALI_OSS, oss); +storageTypes.register(STORAGE_TYPE_S3, s3); +storageTypes.register(STORAGE_TYPE_TX_COS, cos); export function getStorageConfig(key: string): IStorage { - return map.get(key); + return storageTypes.get(key); } + +export default storageTypes; diff --git a/packages/plugins/file-manager/src/server/storages/local.ts b/packages/plugins/file-manager/src/server/storages/local.ts index 04307047e..9bc1ed844 100644 --- a/packages/plugins/file-manager/src/server/storages/local.ts +++ b/packages/plugins/file-manager/src/server/storages/local.ts @@ -1,12 +1,15 @@ +import path from 'path'; +import fs from 'fs/promises'; + import Application from '@nocobase/server'; import serve from 'koa-static'; import mkdirp from 'mkdirp'; import multer from 'multer'; -import path from 'path'; import { Transactionable } from 'sequelize/types'; import { URL } from 'url'; import { STORAGE_TYPE_LOCAL } from '../constants'; import { getFilename } from '../utils'; +import { AttachmentModel } from '.'; // use koa-mount match logic function match(basePath: string, pathname: string): boolean { @@ -49,7 +52,7 @@ function createLocalServerUpdateHook(app, storages) { } function getDocumentRoot(storage): string { - const { documentRoot = 'storage/uploads' } = storage.options || {}; + const { documentRoot = process.env.LOCAL_STORAGE_DEST || 'storage/uploads' } = storage.options || {}; // TODO(feature): 后面考虑以字符串模板的方式使用，可注入 req/action 相关变量，以便于区分文件夹 return path.resolve(path.isAbsolute(documentRoot) ? documentRoot : path.join(process.cwd(), documentRoot)); } @@ -137,4 +140,29 @@ export default { }, }; }, + async delete(storage, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]> { + const documentRoot = getDocumentRoot(storage); + let count = 0; + const undeleted = []; + await records.reduce( + (promise, record) => + promise.then(async () => { + try { + await fs.unlink(path.join(documentRoot, record.path, record.filename)); + count += 1; + } catch (ex) { + if (ex.code === 'ENOENT') { + console.warn(ex.message); + count += 1; + } else { + console.error(ex); + undeleted.push(record); + } + } + }), + Promise.resolve(), + ); + + return [count, undeleted]; + }, }; diff --git a/packages/plugins/file-manager/src/server/storages/s3.ts b/packages/plugins/file-manager/src/server/storages/s3.ts index 92430a405..66f89d801 100644 --- a/packages/plugins/file-manager/src/server/storages/s3.ts +++ b/packages/plugins/file-manager/src/server/storages/s3.ts @@ -1,3 +1,4 @@ +import { AttachmentModel } from '.'; import { STORAGE_TYPE_S3 } from '../constants'; import { cloudFilenameGetter } from '../utils'; @@ -44,4 +45,21 @@ export default { }, }; }, + async delete(storage, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]> { + const { DeleteObjectsCommand } = require('@aws-sdk/client-s3'); + const { s3 } = this.make(storage); + const { Deleted } = await s3.send( + new DeleteObjectsCommand({ + Bucket: storage.options.bucket, + Delete: { + Objects: records.map((record) => ({ Key: `${record.path}/${record.filename}` })), + }, + }), + ); + + return [ + Deleted.length, + records.filter((record) => !Deleted.find((item) => item.Key === `${record.path}/${record.filename}`)), + ]; + }, }; diff --git a/packages/plugins/file-manager/src/server/storages/tx-cos.ts b/packages/plugins/file-manager/src/server/storages/tx-cos.ts index eb6ebf6e3..ecb3cc434 100644 --- a/packages/plugins/file-manager/src/server/storages/tx-cos.ts +++ b/packages/plugins/file-manager/src/server/storages/tx-cos.ts @@ -1,3 +1,6 @@ +import { promisify } from 'util'; + +import { AttachmentModel } from '.'; import { STORAGE_TYPE_TX_COS } from '../constants'; import { cloudFilenameGetter } from '../utils'; @@ -24,4 +27,16 @@ export default { }, }; }, + async delete(storage, records: AttachmentModel[]): Promise<[number, AttachmentModel[]]> { + const { cos } = this.make(storage); + const { Deleted } = await promisify(cos.deleteMultipleObject)({ + Region: storage.options.Region, + Bucket: storage.options.Bucket, + Objects: records.map((record) => ({ Key: `${record.path}/${record.filename}` })), + }); + return [ + Deleted.length, + records.filter((record) => !Deleted.find((item) => item.Key === `${record.path}/${record.filename}`)), + ]; + }, };