From fa97d0a642546f2bf02b2e1f330790f8a3c2dd13 Mon Sep 17 00:00:00 2001 From: ChengLei Shao Date: Mon, 8 Jan 2024 18:59:56 +0800 Subject: [PATCH] feat: application backup and restore (#3268) * fix: perform load action on boot main app * feat: add dataType option in collection duplicator * chore: reset optional dumpable config * chore: dump command * chore: dump & restore command * chore: delay restore * fix: dump test * chore: restore command * chore: dump command action * chore: dumpable collection api * chore: client collection option * feat: backup& restore client * chore: content disposition header in dump response * chore: download backup field * feat: collection origin option * fix: test * chore: collection manager collection origin * chore: upload backup field * chore: upload restore file * chore: upload restore file * fix: test * chore: backup and restore support learn more * refactor: upload restore file * refactor: upload restore file * fix: test * fix: test * chore: dumpable collection with title * chore: pg only test * chore: test * fix: test * chore: test sleep * style: locale improve * refactor: download backup file * refactor: start restore * fix: restore key name * refactor: start restore * refactor: start restore * refactor: start restore * refactor: start restore * refactor: start restore * refactor: start restore * chore: unify duplicator option * fix: dump empty collection * chore: test * chore: test * style: style improve * refactor: locale improve * chore: dumpalbe collection orders * style: style improve * style: style improve * style: icon adjust * chore: nginx body size * chore: get file status * feat: run dump task * feat: download api * chore: backup files resourcer * feat: restore destroy api * chore: backup files resoucer * feat: list backup files action * chore: get collection meta from dumped file * fix: dump file name * fix: test * chore: backup and restore ui * chore: swagger api for backup & restore * chore: api doc * chore: api doc * chore: api doc * chore: backup and restore ui * chore: backup and restore ui * chore: backup and restore ui * chore: backup and restore ui * chore: backup and restore ui * fix: restore values * style: style improve * fix: download field respontype * fix: restore form local file * refactor: local improve * refactor: delete backup file * fix: in progress status * refactor: locale improve * refactor: locale improve * refactor: style improve * refactor: style improve * refactor: style improve * test: dump collection table attribute * chore: dump collection with table attributes * chore: test * chore: create new table in restore * fix: import error * chore: restore table from backup file * chore: sync collection after restore collections * fix: restore json data * style: style improve * chore: restore with fields * chore: test * fix: test * fix: test with underscored * style: style improve * fix: lock file state * chore: add test file * refactor: backup & restore plugin * fix: mysql test * chore: skip import view collection * chore: restore collection with inherits topo order * fix: import * style: style improve * fix: restore sequence fields * fix: themeConfig collection duplicator option * fix: restore with dialectOnly meta * fix: throw error * fix: restore * fix: import backup file created in postgres into mysql * fix: repeated items in inherits * chore: upgrade after restore * feat: check database env before restore * feat: handle autoincr val in postgres * chore: sqlite & mysql queryInterface * chore: test * fix: test * chore: test * fix: build * fix: pg test * fix: restore with date field * chore: theme-config collection * chore: chage import collections method to support collection origin * chore: fallback get autoincr value in mysql * fix: dataType normalize * chore: delay restore * chore: test * fix: build * feat: collectin onDump * feat: collection onDump interface * chore: dump with view collection * chore: sync in restore * refactor: locale improve * refactor: code improve * fix: test * fix: data sync * chore: rename backup & restore plugin * chore: skip test * style: style improve * style: style improve * style: style improve * style: style improve * chore: import version check * chore: backup file dir * chore: build * fix: bugs * fix: error * fix: pageSize * fix: import origin * fix: improve code * fix: remove namespace * chore: dump rules config * fix: dump custom collection * chore: version * fix: test * fix: test * fix: test * fix: test * chore: test * fix: load custom collection * fix: client * fix: translation * chore: code * fix: bug * fix: support shared option * fix: roles collection dumpRules * chore: test * fix: define collections * chore: collection group * fix: translation * fix: translation * fix: restore options * chore: restore command * chore: dump error * fix: too many open files --------- Co-authored-by: katherinehhh Co-authored-by: chenos --- .gitignore | 1 + .../client/src/collection-manager/types.ts | 30 +- .../collection-group-manager.test.ts | 49 ++ .../core/database/src/collection-factory.ts | 1 + .../database/src/collection-group-manager.ts | 122 ++- .../core/database/src/collection-importer.ts | 2 + packages/core/database/src/collection.ts | 55 +- packages/core/database/src/database.ts | 90 ++- .../src/fields/belongs-to-many-field.ts | 4 + packages/core/database/src/fields/field.ts | 4 +- .../query-interface/mysql-query-interface.ts | 43 +- .../postgres-query-interface.ts | 66 ++ .../src/query-interface/query-interface.ts | 15 +- .../query-interface/sqlite-query-interface.ts | 45 ++ packages/core/database/src/sync-runner.ts | 1 + packages/core/server/src/app-supervisor.ts | 2 +- packages/core/server/src/application.ts | 1 - packages/core/server/src/errors/handler.ts | 4 + packages/core/server/src/gateway/index.ts | 4 +- .../server/src/helpers/application-version.ts | 4 +- .../src/plugin-manager/options/collection.ts | 3 +- packages/core/server/src/plugin.ts | 30 +- packages/core/test/src/server/index.ts | 1 + .../src/server/collections/roles-users.ts | 11 +- .../src/server/collections/roles.ts | 12 +- .../src/server/collections/rolesResources.ts | 9 +- .../collections/rolesResourcesActions.ts | 9 +- .../collections/rolesResourcesScopes.ts | 9 +- .../src/server/collections/users.ts | 4 +- .../@nocobase/plugin-acl/src/server/server.ts | 7 +- .../src/client/Configuration/index.tsx | 2 +- .../collections/{api-keys.ts => apiKeys.ts} | 8 +- .../plugin-api-keys/src/collections/index.ts | 1 - .../plugin-api-keys/src/server/plugin.ts | 4 +- .../src/server/collections/auditChanges.ts | 6 +- .../src/server/collections/auditLogs.ts | 6 +- .../plugin-audit-logs/src/server/index.ts | 4 +- .../src/server/collections/authenticators.ts | 13 +- .../src/server/collections/token-blacklist.ts | 12 +- .../collections/users-authenticators.ts | 17 +- .../plugin-auth/src/server/plugin.ts | 4 +- .../.npmignore | 0 .../LICENSE | 0 .../README.md | 0 .../README.zh-CN.md | 0 .../client.d.ts | 0 .../client.js | 0 .../package.json | 15 +- .../server.d.ts | 0 .../server.js | 0 .../src/client/Configuration.tsx | 476 ++++++++++++ .../src/client/DuplicatorProvider.tsx | 12 + .../src/client/index.tsx | 17 + .../src/client/locale/index.ts | 18 + .../src/index.ts | 0 .../src/locale/en-US.json | 44 ++ .../plugin-backup-restore/src/locale/ja-JP.ts | 24 + .../plugin-backup-restore/src/locale/pt-BR.ts | 27 + .../src/locale/zh-CN.json | 50 ++ .../src/server/__tests__/api.test.ts | 177 +++++ .../src/server/__tests__/dump-action.test.ts | 48 -- .../src/server/__tests__/dumper.test.ts | 729 ++++++++++++++++++ .../backup_20231111_112233.nbdump.lock | 0 .../__tests__/fixtures/dump.nbdump.fixture | Bin .../fixtures/files/backup1.nbdump.lock | 0 .../src/server/__tests__/index.ts | 9 +- .../src/server/__tests__/on-dump.test.ts | 71 ++ .../src/server/app-migrator.ts | 41 + .../src/server/collection-group-manager.ts | 5 + .../src/server/commands/restore-command.ts | 67 ++ .../src/server/dumper.ts | 543 +++++++++++++ .../src/server/errors/restore-check-error.ts | 6 + .../src/server/field-value-writer.ts | 11 +- .../src/server/index.ts | 0 .../src/server/locale/zh-CN.ts | 0 .../src/server/resourcers/backup-files.ts | 212 +++++ .../src/server/restorer.ts | 405 ++++++++++ .../src/server/server.ts | 13 + .../src/server/utils.ts | 0 .../src/swagger/index.ts | 433 +++++++++++ .../src/server/collections/chartsQueries.ts | 4 +- .../plugin-charts/src/server/plugin.ts | 4 +- .../src/server/collections/chinaRegions.ts | 4 +- .../plugin-china-region/src/server/index.ts | 4 +- .../__tests__/collections.repository.test.ts | 48 +- .../collections/collectionCategories.ts | 7 +- .../src/server/collections/collections.ts | 5 +- .../src/server/collections/fields.ts | 4 +- .../src/server/models/collection.ts | 10 + .../src/server/server.ts | 4 +- .../src/server/collections/customRequest.ts | 10 +- .../server/collections/customRequestsRoles.ts | 10 +- .../src/server/plugin.ts | 4 +- .../plugin-duplicator/src/client/index.tsx | 7 - .../plugin-duplicator/src/locale/en-US.json | 8 - .../plugin-duplicator/src/locale/zh-CN.json | 8 - .../src/server/__tests__/api.test.ts | 73 -- .../collection-group-manager.test.ts | 37 - .../src/server/__tests__/dumper.test.ts | 35 - .../src/server/actions/dump-action.ts | 25 - .../actions/dumpable-collections-action.ts | 13 - .../src/server/actions/get-app.ts | 14 - .../src/server/actions/get-dict-action.ts | 36 - .../src/server/actions/restore-action.ts | 43 -- .../src/server/app-migrator.ts | 94 --- .../src/server/collection-group-manager.ts | 28 - .../src/server/commands/dump-command.ts | 50 -- .../commands/inquire-question-builder.ts | 72 -- .../src/server/commands/restore-command.ts | 94 --- .../plugin-duplicator/src/server/dumper.ts | 300 ------- .../plugin-duplicator/src/server/restorer.ts | 380 --------- .../plugin-duplicator/src/server/server.ts | 50 -- .../src/server/collections/attachments.ts | 13 +- .../src/server/collections/storages.ts | 11 +- .../src/server/collections/graphPositions.ts | 4 +- .../src/server/index.ts | 4 +- .../src/server/collections/iframe-html.ts | 3 +- .../plugin-iframe-block/src/server/plugin.ts | 5 +- .../server/collections/localization-texts.ts | 13 +- .../collections/localization-translations.ts | 12 +- .../src/server/plugin.ts | 4 +- .../server/collections/mapConfiguration.ts | 13 +- .../@nocobase/plugin-map/src/server/plugin.ts | 6 +- .../src/server/collections/applications.ts | 6 +- .../src/server/server.ts | 4 +- .../server/__tests__/collection-sync.test.ts | 8 +- .../src/server/plugin.ts | 31 +- .../src/server/index.ts | 4 +- .../src/server/index.ts | 4 +- .../src/server/index.ts | 4 +- .../src/server/index.ts | 4 +- .../src/server/collections/sequences.ts | 23 +- .../server/collections/collectionsHistory.ts | 11 +- .../src/server/collections/fieldsHistory.ts | 11 +- .../src/server/plugin.ts | 4 +- .../src/server/collections/systemSettings.ts | 3 +- .../src/server/server.ts | 4 +- .../src/server/collections/theme-config.ts | 31 + .../plugin-theme-editor/src/server/plugin.ts | 32 +- .../src/server/collections/uiRoutes.ts | 5 +- .../server/collections/uiSchemaServerHooks.ts | 3 +- .../server/collections/uiSchemaTemplates.ts | 3 +- .../server/collections/uiSchemaTreePath.ts | 3 +- .../src/server/collections/uiSchemas.ts | 4 +- .../src/server/collections/users.ts | 13 +- .../plugin-users/src/server/server.ts | 4 +- .../src/server/collections/verifications.ts | 12 +- .../collections/verifications_providers.ts | 12 +- .../src/server/Plugin.ts | 8 +- .../{users_jobs.ts => 1-users_jobs.ts} | 12 +- .../server/collections/{jobs.ts => 2-jobs.ts} | 6 +- .../collections/{users.ts => 3-users.ts} | 6 +- .../plugin-workflow-test/src/server/index.ts | 4 +- .../plugin-workflow/src/server/Plugin.ts | 4 +- .../src/server/collections/executions.ts | 6 +- .../src/server/collections/flow_nodes.ts | 4 +- .../src/server/collections/jobs.ts | 6 +- .../src/server/collections/workflows.ts | 4 +- packages/presets/nocobase/package.json | 2 +- packages/presets/nocobase/src/server/index.ts | 2 +- yarn.lock | 2 +- 161 files changed, 4151 insertions(+), 1899 deletions(-) create mode 100644 packages/core/database/src/__tests__/collection-group-manager.test.ts rename packages/plugins/@nocobase/plugin-api-keys/src/collections/{api-keys.ts => apiKeys.ts} (94%) delete mode 100644 packages/plugins/@nocobase/plugin-api-keys/src/collections/index.ts rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/.npmignore (100%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/LICENSE (100%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/README.md (100%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/README.zh-CN.md (100%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/client.d.ts (100%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/client.js (100%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/package.json (76%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/server.d.ts (100%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/server.js (100%) create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/client/Configuration.tsx create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/client/DuplicatorProvider.tsx create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/client/index.tsx create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/client/locale/index.ts rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/src/index.ts (100%) create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/locale/en-US.json create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/locale/ja-JP.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/locale/pt-BR.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/locale/zh-CN.json create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/api.test.ts rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/src/server/__tests__/dump-action.test.ts (73%) create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/dumper.test.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/fixtures/backup_20231111_112233.nbdump.lock rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/src/server/__tests__/fixtures/dump.nbdump.fixture (100%) create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/fixtures/files/backup1.nbdump.lock rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/src/server/__tests__/index.ts (70%) create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/on-dump.test.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/app-migrator.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/collection-group-manager.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/commands/restore-command.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/dumper.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/errors/restore-check-error.ts rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/src/server/field-value-writer.ts (87%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/src/server/index.ts (100%) rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/src/server/locale/zh-CN.ts (100%) create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/resourcers/backup-files.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/restorer.ts create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/server/server.ts rename packages/plugins/@nocobase/{plugin-duplicator => plugin-backup-restore}/src/server/utils.ts (100%) create mode 100644 packages/plugins/@nocobase/plugin-backup-restore/src/swagger/index.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/client/index.tsx delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/locale/en-US.json delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/locale/zh-CN.json delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/api.test.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/collection-group-manager.test.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/dumper.test.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/actions/dump-action.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/actions/dumpable-collections-action.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/actions/get-app.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/actions/get-dict-action.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/actions/restore-action.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/app-migrator.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/collection-group-manager.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/commands/dump-command.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/commands/inquire-question-builder.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/commands/restore-command.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/dumper.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/restorer.ts delete mode 100644 packages/plugins/@nocobase/plugin-duplicator/src/server/server.ts create mode 100644 packages/plugins/@nocobase/plugin-theme-editor/src/server/collections/theme-config.ts rename packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/{users_jobs.ts => 1-users_jobs.ts} (81%) rename packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/{jobs.ts => 2-jobs.ts} (74%) rename packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/{users.ts => 3-users.ts} (69%) diff --git a/.gitignore b/.gitignore index 6e14faa3d..d1badae95 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ docker/**/storage cache/diskstore-* *.nbdump storage/duplicator/* +storage/backups/* **/.dumi/tmp **/.dumi/tmp-test **/.dumi/tmp-production diff --git a/packages/core/client/src/collection-manager/types.ts b/packages/core/client/src/collection-manager/types.ts index d1bf64f22..b50daa2f9 100644 --- a/packages/core/client/src/collection-manager/types.ts +++ b/packages/core/client/src/collection-manager/types.ts @@ -4,6 +4,27 @@ import { ReactNode } from 'react'; type dumpable = 'required' | 'optional' | 'skip'; type CollectionSortable = string | boolean | { name?: string; scopeKey?: string }; +type MetaDataType = 'meta'; +type ConfigDataType = 'config'; +type BusinessDataType = 'business'; + +type DumpDataType = MetaDataType | ConfigDataType | BusinessDataType; +type Dumpable = 'required' | 'optional' | 'skip'; + +type BaseDuplicatorObject = { + with?: string[] | string; + delayRestore?: any; +}; + +type Duplicator = + | Dumpable + | ({ + dumpable?: Dumpable; + } & BaseDuplicatorObject) + | ({ + dataType?: DumpDataType; + } & BaseDuplicatorObject); + export interface CollectionOptions { name: string; title?: string; @@ -16,14 +37,7 @@ export interface CollectionOptions { * @prop {string[] | string} [with] - Collections dumped with this collection * @prop {any} [delayRestore] - A function to execute after all collections are restored */ - duplicator?: - | dumpable - | { - dumpable: dumpable; - with?: string[] | string; - delayRestore?: any; - }; - + duplicator?: Duplicator; tableName?: string; inherits?: string[] | string; viewName?: string; diff --git a/packages/core/database/src/__tests__/collection-group-manager.test.ts b/packages/core/database/src/__tests__/collection-group-manager.test.ts new file mode 100644 index 000000000..2e482d605 --- /dev/null +++ b/packages/core/database/src/__tests__/collection-group-manager.test.ts @@ -0,0 +1,49 @@ +import { CollectionGroupManager } from '../collection-group-manager'; + +describe('collection group manager', () => { + it('should unify duplicator option', async () => { + expect(CollectionGroupManager.unifyDumpRules('skipped')).toMatchObject({ + group: 'skipped', + }); + + expect(CollectionGroupManager.unifyDumpRules('required')).toMatchObject({ + group: 'required', + }); + + expect( + CollectionGroupManager.unifyDumpRules({ + required: true, + }), + ).toMatchObject({ + group: 'required', + }); + + expect( + CollectionGroupManager.unifyDumpRules({ + skipped: true, + }), + ).toMatchObject({ + group: 'skipped', + }); + + expect( + CollectionGroupManager.unifyDumpRules({ + group: 'required', + delayRestore: {}, + }), + ).toMatchObject({ + group: 'required', + delayRestore: {}, + }); + + expect( + CollectionGroupManager.unifyDumpRules({ + group: 'logs', + delayRestore: {}, + }), + ).toMatchObject({ + group: 'logs', + delayRestore: {}, + }); + }); +}); diff --git a/packages/core/database/src/collection-factory.ts b/packages/core/database/src/collection-factory.ts index 5d3324e39..14098c4c1 100644 --- a/packages/core/database/src/collection-factory.ts +++ b/packages/core/database/src/collection-factory.ts @@ -5,6 +5,7 @@ import { Model } from './model'; type CollectionTypeOptions = { condition: (options: CollectionOptions) => boolean; onSync?: (model: typeof Model, options: any) => Promise; + onDump?: (dumper: any, collection: Collection) => Promise; }; export class CollectionFactory { diff --git a/packages/core/database/src/collection-group-manager.ts b/packages/core/database/src/collection-group-manager.ts index 12420a8b5..0240c360c 100644 --- a/packages/core/database/src/collection-group-manager.ts +++ b/packages/core/database/src/collection-group-manager.ts @@ -1,94 +1,62 @@ +import { BaseDumpRules, DumpRules } from './collection'; import Database from './database'; -import { isString, castArray } from 'lodash'; +type RequiredGroup = 'required'; +type SkippedGroup = 'skipped'; + +export type BuiltInGroup = RequiredGroup | SkippedGroup; + +export type DumpRulesGroupType = BuiltInGroup | string; + +// Collection Group is a collection of collections, which can be dumped and restored together. export interface CollectionGroup { - namespace: string; collections: string[]; function: string; - - dumpable: 'required' | 'optional' | 'skip'; + dataType: DumpRulesGroupType; delayRestore?: any; } +export interface CollectionGroupWithCollectionTitle extends Omit { + collections: Array<{ + name: string; + title: string; + }>; +} + export class CollectionGroupManager { constructor(public db: Database) {} - getGroups() { - const collections = [...this.db.collections.values()]; - const groups = new Map(); - - const skipped = []; - - for (const collection of collections) { - const groupKey = collection.options.namespace; - - if (!groupKey) { - continue; - } - - const [namespace, groupFunc] = groupKey.split('.'); - - if (!groupFunc) { - skipped.push({ - name: collection.name, - reason: 'no-group-function', - }); - - continue; - } - - if (!groups.has(groupKey)) { - const dumpable = (() => { - if (!collection.options.duplicator) { - return undefined; - } - - if (isString(collection.options.duplicator)) { - return { - dumpable: collection.options.duplicator, - }; - } - - return collection.options.duplicator; - })(); - - if (!dumpable) { - skipped.push({ - name: collection.name, - reason: 'no-dumpable', - }); - continue; - } - - const group: CollectionGroup = { - namespace, - function: groupFunc, - collections: dumpable.with ? castArray(dumpable.with) : [], - dumpable: dumpable.dumpable, - }; - - if (dumpable.delayRestore) { - group.delayRestore = dumpable.delayRestore; - } - - groups.set(groupKey, group); - } - - const group = groups.get(groupKey); - group.collections.push(collection.name); + static unifyDumpRules(dumpRules: DumpRules): + | (BaseDumpRules & { + group: DumpRulesGroupType; + }) + | undefined { + if (!dumpRules) { + return undefined; } - const results = [...groups.values()]; - const groupCollections = results.map((i) => i.collections).flat(); - - for (const skipItem of skipped) { - if (groupCollections.includes(skipItem.name)) { - continue; - } - - this.db.logger.warn(`collection ${skipItem.name} is not in any collection group, reason: ${skipItem.reason}.`); + if (typeof dumpRules === 'string') { + return { + group: dumpRules, + }; } - return results; + if ('required' in dumpRules && (dumpRules as { required: true }).required) { + return { + ...dumpRules, + group: 'required', + }; + } + + if ('skipped' in dumpRules && (dumpRules as { skipped: true }).skipped) { + return { + ...dumpRules, + group: 'skipped', + }; + } + + return dumpRules as BaseDumpRules & { + group: DumpRulesGroupType; + }; } } diff --git a/packages/core/database/src/collection-importer.ts b/packages/core/database/src/collection-importer.ts index 7036dbe61..969312885 100644 --- a/packages/core/database/src/collection-importer.ts +++ b/packages/core/database/src/collection-importer.ts @@ -27,11 +27,13 @@ export class ImporterReader { const files = await readdir(this.directory, { encoding: 'utf-8', }); + const modules = files .filter((fileName) => { if (fileName.endsWith('.d.ts')) { return false; } + const ext = path.parse(fileName).ext.replace('.', ''); return this.extensions.has(ext); }) diff --git a/packages/core/database/src/collection.ts b/packages/core/database/src/collection.ts index ad46b72f5..8666e5982 100644 --- a/packages/core/database/src/collection.ts +++ b/packages/core/database/src/collection.ts @@ -16,12 +16,20 @@ import { Model } from './model'; import { AdjacencyListRepository } from './repositories/tree-repository/adjacency-list-repository'; import { Repository } from './repository'; import { checkIdentifier, md5, snakeCase } from './utils'; +import { BuiltInGroup } from './collection-group-manager'; export type RepositoryType = typeof Repository; -export type CollectionSortable = string | boolean | { name?: string; scopeKey?: string }; +export type CollectionSortable = + | string + | boolean + | { + name?: string; + scopeKey?: string; + }; type dumpable = 'required' | 'optional' | 'skip'; +type dumpableType = 'meta' | 'business' | 'config'; function EnsureAtomicity(target: any, propertyKey: string, descriptor: PropertyDescriptor) { const originalMethod = descriptor.value; @@ -53,6 +61,16 @@ function EnsureAtomicity(target: any, propertyKey: string, descriptor: PropertyD return descriptor; } +export type BaseDumpRules = { + delayRestore?: any; +}; + +export type DumpRules = + | BuiltInGroup + | ({ required: true } & BaseDumpRules) + | ({ skipped: true } & BaseDumpRules) + | ({ group: BuiltInGroup | string } & BaseDumpRules); + export interface CollectionOptions extends Omit { name: string; title?: string; @@ -65,13 +83,7 @@ export interface CollectionOptions extends Omit * @prop {string[] | string} [with] - Collections dumped with this collection * @prop {any} [delayRestore] - A function to execute after all collections are restored */ - duplicator?: - | dumpable - | { - dumpable: dumpable; - with?: string[] | string; - delayRestore?: any; - }; + dumpRules?: DumpRules; tableName?: string; inherits?: string[] | string; @@ -91,11 +103,19 @@ export interface CollectionOptions extends Omit * @default 'options' */ magicAttribute?: string; - tree?: string; - template?: string; + /** + * where is the collection from + * + * values + * - 'plugin' - collection is from plugin + * - 'core' - collection is from core + * - 'user' - collection is from user + */ + origin?: string; + [key: string]: any; } @@ -152,6 +172,10 @@ export class Collection< return this.options.name; } + get origin() { + return this.options.origin || 'core'; + } + get titleField() { return (this.options.titleField as string) || this.model.primaryKeyAttribute; } @@ -564,7 +588,16 @@ export class Collection< this.setField(options.name || name, options); } - addIndex(index: string | string[] | { fields: string[]; unique?: boolean; [key: string]: any }) { + addIndex( + index: + | string + | string[] + | { + fields: string[]; + unique?: boolean; + [key: string]: any; + }, + ) { if (!index) { return; } diff --git a/packages/core/database/src/database.ts b/packages/core/database/src/database.ts index 0e85d304a..92cc85762 100644 --- a/packages/core/database/src/database.ts +++ b/packages/core/database/src/database.ts @@ -1,5 +1,6 @@ -import { Logger, LoggerOptions, createConsoleLogger, createLogger } from '@nocobase/logger'; +import { createConsoleLogger, createLogger, Logger, LoggerOptions } from '@nocobase/logger'; import { applyMixins, AsyncEmitter } from '@nocobase/utils'; +import chalk from 'chalk'; import merge from 'deepmerge'; import { EventEmitter } from 'events'; import { backOff } from 'exponential-backoff'; @@ -22,6 +23,7 @@ import { } from 'sequelize'; import { SequelizeStorage, Umzug } from 'umzug'; import { Collection, CollectionOptions, RepositoryType } from './collection'; +import { CollectionFactory } from './collection-factory'; import { CollectionGroupManager } from './collection-group-manager'; import { ImporterReader, ImportFileExtension } from './collection-importer'; import DatabaseUtils from './database-utils'; @@ -30,6 +32,7 @@ import { referentialIntegrityCheck } from './features/referential-integrity-chec import { ArrayFieldRepository } from './field-repository/array-field-repository'; import * as FieldTypes from './fields'; import { Field, FieldContext, RelationField } from './fields'; +import { checkDatabaseVersion } from './helpers'; import { InheritedCollection } from './inherited-collection'; import InheritanceMap from './inherited-map'; import { registerBuiltInListeners } from './listeners'; @@ -72,9 +75,6 @@ import { import { patchSequelizeQueryInterface, snakeCase } from './utils'; import { BaseValueParser, registerFieldValueParsers } from './value-parsers'; import { ViewCollection } from './view-collection'; -import { CollectionFactory } from './collection-factory'; -import chalk from 'chalk'; -import { checkDatabaseVersion } from './helpers'; export type MergeOptions = merge.Options; @@ -194,7 +194,7 @@ export class Database extends EventEmitter implements AsyncEmitter { utils = new DatabaseUtils(this); referenceMap = new ReferencesMap(); inheritanceMap = new InheritanceMap(); - importedFrom = new Map>(); + importedFrom = new Map>(); modelHook: ModelHook; version: DatabaseVersion; delayCollectionExtend = new Map(); @@ -307,8 +307,8 @@ export class Database extends EventEmitter implements AsyncEmitter { name: 'migrations', autoGenId: false, timestamps: false, - namespace: 'core.migration', - duplicator: 'required', + dumpRules: 'required', + origin: 'core', fields: [{ type: 'string', name: 'name', primaryKey: true }], }); @@ -336,26 +336,6 @@ export class Database extends EventEmitter implements AsyncEmitter { return this._instanceId; } - registerCollectionType() { - this.collectionFactory.registerCollectionType(InheritedCollection, { - condition: (options) => { - return options.inherits && lodash.castArray(options.inherits).length > 0; - }, - }); - - this.collectionFactory.registerCollectionType(ViewCollection, { - condition: (options) => { - return options.viewName || options.view; - }, - }); - - this.collectionFactory.registerCollectionType(SqlCollection, { - condition: (options) => { - return options.sql; - }, - }); - } - setContext(context: any) { this.context = context; } @@ -459,6 +439,15 @@ export class Database extends EventEmitter implements AsyncEmitter { } }); + this.on('afterDefineCollection', async (collection: Collection) => { + const options = collection.options; + if (options.origin) { + const existsSet = this.importedFrom.get(options.origin) || new Set(); + existsSet.add(collection.name); + this.importedFrom.set(options.origin, existsSet); + } + }); + registerBuiltInListeners(this); } @@ -912,11 +901,10 @@ export class Database extends EventEmitter implements AsyncEmitter { if (module.extend) { this.extendCollection(module.collectionOptions, module.mergeOptions); } else { - const collection = this.collection(module); - - if (options.from) { - this.importedFrom.set(options.from, [...(this.importedFrom.get(options.from) || []), collection.name]); - } + const collection = this.collection({ + ...module, + origin: options.from, + }); result.set(collection.name, collection); } @@ -924,6 +912,44 @@ export class Database extends EventEmitter implements AsyncEmitter { return result; } + + private registerCollectionType() { + this.collectionFactory.registerCollectionType(InheritedCollection, { + condition: (options) => { + return options.inherits && lodash.castArray(options.inherits).length > 0; + }, + }); + + this.collectionFactory.registerCollectionType(ViewCollection, { + condition: (options) => { + return options.viewName || options.view; + }, + + async onSync() { + return; + }, + + async onDump(dumper, collection: Collection) { + const viewDef = await collection.db.queryInterface.viewDef(collection.getTableNameWithSchemaAsString()); + + dumper.writeSQLContent(`view-${collection.name}`, { + sql: [ + `DROP VIEW IF EXISTS ${collection.getTableNameWithSchemaAsString()}`, + `CREATE VIEW ${collection.getTableNameWithSchemaAsString()} AS ${viewDef}`, + ], + group: 'required', + }); + + return; + }, + }); + + this.collectionFactory.registerCollectionType(SqlCollection, { + condition: (options) => { + return options.sql; + }, + }); + } } export function extendCollection(collectionOptions: CollectionOptions, mergeOptions?: MergeOptions) { diff --git a/packages/core/database/src/fields/belongs-to-many-field.ts b/packages/core/database/src/fields/belongs-to-many-field.ts index bd38d5ef4..7ba7fece9 100644 --- a/packages/core/database/src/fields/belongs-to-many-field.ts +++ b/packages/core/database/src/fields/belongs-to-many-field.ts @@ -71,6 +71,10 @@ export class BelongsToManyField extends RelationField { name: through, }; + if (this.collection.options.dumpRules) { + throughCollectionOptions['dumpRules'] = this.collection.options.dumpRules; + } + // set through collection schema if (this.collection.collectionSchema()) { throughCollectionOptions['schema'] = this.collection.collectionSchema(); diff --git a/packages/core/database/src/fields/field.ts b/packages/core/database/src/fields/field.ts index f26b23617..2e8a423c2 100644 --- a/packages/core/database/src/fields/field.ts +++ b/packages/core/database/src/fields/field.ts @@ -152,8 +152,10 @@ export abstract class Field { toSequelize(): any { const opts = _.omit(this.options, ['name']); + if (this.dataType) { - Object.assign(opts, { type: this.dataType }); + // @ts-ignore + Object.assign(opts, { type: this.database.sequelize.normalizeDataType(this.dataType) }); } return opts; diff --git a/packages/core/database/src/query-interface/mysql-query-interface.ts b/packages/core/database/src/query-interface/mysql-query-interface.ts index 116b8a531..ef49653eb 100644 --- a/packages/core/database/src/query-interface/mysql-query-interface.ts +++ b/packages/core/database/src/query-interface/mysql-query-interface.ts @@ -1,4 +1,4 @@ -import { Transactionable } from 'sequelize'; +import { Transaction, Transactionable } from 'sequelize'; import { Collection } from '../collection'; import sqlParser from '../sql-parser'; import QueryInterface, { TableInfo } from './query-interface'; @@ -86,4 +86,45 @@ export default class MysqlQueryInterface extends QueryInterface { return results[0]['Create Table']; } + + async getAutoIncrementInfo(options: { tableInfo: TableInfo; fieldName: string }): Promise<{ + seqName?: string; + currentVal: number; + }> { + const { tableInfo, fieldName } = options; + + const sql = `SELECT AUTO_INCREMENT as currentVal + FROM information_schema.tables + WHERE table_schema = DATABASE() + AND table_name = '${tableInfo.tableName}';`; + + const results = await this.db.sequelize.query(sql, { type: 'SELECT' }); + + let currentVal = results[0]['currentVal'] as number; + + if (currentVal === null) { + // use max value of field instead + const maxSql = `SELECT MAX(${fieldName}) as currentVal + FROM ${tableInfo.tableName};`; + const maxResults = await this.db.sequelize.query(maxSql, { type: 'SELECT' }); + currentVal = maxResults[0]['currentVal'] as number; + } + + return { + currentVal, + }; + } + + async setAutoIncrementVal(options: { + tableInfo: TableInfo; + columnName: string; + seqName?: string; + currentVal: number; + transaction?: Transaction; + }): Promise { + const { tableInfo, columnName, seqName, currentVal, transaction } = options; + + const sql = `ALTER TABLE ${tableInfo.tableName} AUTO_INCREMENT = ${currentVal};`; + await this.db.sequelize.query(sql, { transaction }); + } } diff --git a/packages/core/database/src/query-interface/postgres-query-interface.ts b/packages/core/database/src/query-interface/postgres-query-interface.ts index c68944b9a..699fdfffd 100644 --- a/packages/core/database/src/query-interface/postgres-query-interface.ts +++ b/packages/core/database/src/query-interface/postgres-query-interface.ts @@ -2,12 +2,78 @@ import lodash from 'lodash'; import { Collection } from '../collection'; import sqlParser from '../sql-parser/postgres'; import QueryInterface, { TableInfo } from './query-interface'; +import { Transaction } from 'sequelize'; export default class PostgresQueryInterface extends QueryInterface { constructor(db) { super(db); } + async setAutoIncrementVal(options: { + tableInfo: TableInfo; + columnName: string; + seqName?: string; + currentVal?: number; + transaction?: Transaction; + }): Promise { + const { tableInfo, columnName, seqName, currentVal, transaction } = options; + if (!seqName) { + throw new Error('seqName is required to set auto increment val in postgres'); + } + + await this.db.sequelize.query( + `alter table ${this.db.utils.quoteTable({ + tableName: tableInfo.tableName, + schema: tableInfo.schema, + })} + alter column "${columnName}" set default nextval('${seqName}')`, + { + transaction, + }, + ); + + if (currentVal) { + await this.db.sequelize.query(`select setval('${seqName}', ${currentVal})`, { + transaction, + }); + } + } + + async getAutoIncrementInfo(options: { + tableInfo: TableInfo; + fieldName: string; + }): Promise<{ seqName?: string; currentVal: number }> { + const fieldName = options.fieldName || 'id'; + const tableInfo = options.tableInfo; + + const sequenceNameResult = await this.db.sequelize.query( + `SELECT column_default + FROM information_schema.columns + WHERE table_name = '${tableInfo.tableName}' + and table_schema = '${tableInfo.schema || 'public'}' + and "column_name" = '${fieldName}';`, + ); + + const columnDefault = sequenceNameResult[0][0]['column_default']; + + const regex = new RegExp(/nextval\('(.*)'::regclass\)/); + const match = regex.exec(columnDefault); + + const sequenceName = match[1]; + + const sequenceCurrentValResult = await this.db.sequelize.query( + `select last_value + from ${sequenceName}`, + ); + + const sequenceCurrentVal = parseInt(sequenceCurrentValResult[0][0]['last_value']); + + return { + seqName: sequenceName, + currentVal: sequenceCurrentVal, + }; + } + async collectionTableExists(collection: Collection, options?) { const transaction = options?.transaction; diff --git a/packages/core/database/src/query-interface/query-interface.ts b/packages/core/database/src/query-interface/query-interface.ts index b1c7b0d5a..2aa07a278 100644 --- a/packages/core/database/src/query-interface/query-interface.ts +++ b/packages/core/database/src/query-interface/query-interface.ts @@ -1,4 +1,4 @@ -import { QueryInterface as SequelizeQueryInterface, Transactionable } from 'sequelize'; +import { QueryInterface as SequelizeQueryInterface, Transaction, Transactionable } from 'sequelize'; import { Collection } from '../collection'; import Database from '../database'; @@ -51,4 +51,17 @@ export default abstract class QueryInterface { await this.db.sequelize.getQueryInterface().dropAllTables(options); } + + abstract getAutoIncrementInfo(options: { + tableInfo: TableInfo; + fieldName: string; + }): Promise<{ seqName?: string; currentVal: number }>; + + abstract setAutoIncrementVal(options: { + tableInfo: TableInfo; + columnName: string; + seqName?: string; + currentVal: number; + transaction?: Transaction; + }): Promise; } diff --git a/packages/core/database/src/query-interface/sqlite-query-interface.ts b/packages/core/database/src/query-interface/sqlite-query-interface.ts index a1bdd0375..113a5d0a8 100644 --- a/packages/core/database/src/query-interface/sqlite-query-interface.ts +++ b/packages/core/database/src/query-interface/sqlite-query-interface.ts @@ -1,6 +1,7 @@ import { Collection } from '../collection'; import sqlParser from '../sql-parser'; import QueryInterface, { TableInfo } from './query-interface'; +import { Transaction } from 'sequelize'; export default class SqliteQueryInterface extends QueryInterface { constructor(db) { @@ -90,4 +91,48 @@ export default class SqliteQueryInterface extends QueryInterface { showTableDefinition(tableInfo: TableInfo): Promise { return Promise.resolve(undefined); } + + async getAutoIncrementInfo(options: { tableInfo: TableInfo; fieldName: string }): Promise<{ + seqName?: string; + currentVal: number; + }> { + const { tableInfo } = options; + + const tableName = tableInfo.tableName; + + const sql = `SELECT seq + FROM sqlite_sequence + WHERE name = '${tableName}';`; + + const results = await this.db.sequelize.query(sql, { type: 'SELECT' }); + + const row = results[0]; + + if (!row) { + return { + currentVal: 0, + }; + } + + return { + currentVal: row['seq'], + }; + } + + async setAutoIncrementVal(options: { + tableInfo: TableInfo; + columnName: string; + seqName?: string; + currentVal: number; + transaction?: Transaction; + }): Promise { + const { tableInfo, columnName, seqName, currentVal, transaction } = options; + + const tableName = tableInfo.tableName; + + const sql = `UPDATE sqlite_sequence + SET seq = ${currentVal} + WHERE name = '${tableName}';`; + await this.db.sequelize.query(sql, { transaction }); + } } diff --git a/packages/core/database/src/sync-runner.ts b/packages/core/database/src/sync-runner.ts index 7fd87eb08..11e940fc9 100644 --- a/packages/core/database/src/sync-runner.ts +++ b/packages/core/database/src/sync-runner.ts @@ -28,6 +28,7 @@ export class SyncRunner { const tableName = inheritedCollection.getTableNameWithSchema(); const attributes = model.tableAttributes; + const childAttributes = lodash.pickBy(attributes, (value) => { return !value.inherit; }); diff --git a/packages/core/server/src/app-supervisor.ts b/packages/core/server/src/app-supervisor.ts index 1f20da23b..8a9903a2f 100644 --- a/packages/core/server/src/app-supervisor.ts +++ b/packages/core/server/src/app-supervisor.ts @@ -264,7 +264,7 @@ export class AppSupervisor extends EventEmitter implements AsyncEmitter { if ( maintainingStatus && - ['install', 'upgrade', 'pm.add', 'pm.update', 'pm.enable', 'pm.disable', 'pm.remove'].includes( + ['install', 'upgrade', 'pm.add', 'pm.update', 'pm.enable', 'pm.disable', 'pm.remove', 'restore'].includes( maintainingStatus.command.name, ) && !startOptions.recover diff --git a/packages/core/server/src/application.ts b/packages/core/server/src/application.ts index 5ee9fe481..ad2d053b3 100644 --- a/packages/core/server/src/application.ts +++ b/packages/core/server/src/application.ts @@ -500,7 +500,6 @@ export class Application exten return command; } catch (error) { - console.log({ error }); if (!this.activatedCommand) { this.activatedCommand = { name: 'unknown', diff --git a/packages/core/server/src/errors/handler.ts b/packages/core/server/src/errors/handler.ts index bee5be442..aaa3ee4dc 100644 --- a/packages/core/server/src/errors/handler.ts +++ b/packages/core/server/src/errors/handler.ts @@ -12,5 +12,9 @@ export function getErrorLevel(e: Error): ErrorLevel { return 'warn'; } + if (e.name === 'RestoreCheckError') { + return 'warn'; + } + return 'fatal'; } diff --git a/packages/core/server/src/gateway/index.ts b/packages/core/server/src/gateway/index.ts index 7b05b215e..77d0cffb3 100644 --- a/packages/core/server/src/gateway/index.ts +++ b/packages/core/server/src/gateway/index.ts @@ -14,7 +14,7 @@ import { parse } from 'url'; import xpipe from 'xpipe'; import { AppSupervisor } from '../app-supervisor'; import { ApplicationOptions } from '../application'; -import { PLUGIN_STATICS_PATH, getPackageDirByExposeUrl, getPackageNameByExposeUrl } from '../plugin-manager'; +import { getPackageDirByExposeUrl, getPackageNameByExposeUrl, PLUGIN_STATICS_PATH } from '../plugin-manager'; import { applyErrorWithArgs, getErrorWithCode } from './errors'; import { IPCSocketClient } from './ipc-socket-client'; import { IPCSocketServer } from './ipc-socket-server'; @@ -326,6 +326,8 @@ export class Gateway extends EventEmitter { const mainApp = AppSupervisor.getInstance().bootMainApp(options.mainAppOptions); + await mainApp.load(); + mainApp .runAsCLI(process.argv, { throwError: true, diff --git a/packages/core/server/src/helpers/application-version.ts b/packages/core/server/src/helpers/application-version.ts index 60cb634c5..dd8f00835 100644 --- a/packages/core/server/src/helpers/application-version.ts +++ b/packages/core/server/src/helpers/application-version.ts @@ -11,9 +11,9 @@ export class ApplicationVersion { if (!app.db.hasCollection('applicationVersion')) { app.db.collection({ name: 'applicationVersion', - namespace: 'core.applicationVersion', - duplicator: 'required', + dataType: 'meta', timestamps: false, + dumpRules: 'required', fields: [{ name: 'value', type: 'string' }], }); } diff --git a/packages/core/server/src/plugin-manager/options/collection.ts b/packages/core/server/src/plugin-manager/options/collection.ts index b6c607be3..0f09d456e 100644 --- a/packages/core/server/src/plugin-manager/options/collection.ts +++ b/packages/core/server/src/plugin-manager/options/collection.ts @@ -2,8 +2,7 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ name: 'applicationPlugins', - namespace: 'core.applicationPlugins', - duplicator: 'required', + dumpRules: 'required', repository: 'PluginManagerRepository', fields: [ { type: 'string', name: 'name', unique: true }, diff --git a/packages/core/server/src/plugin.ts b/packages/core/server/src/plugin.ts index 0690a84df..51c0b74fb 100644 --- a/packages/core/server/src/plugin.ts +++ b/packages/core/server/src/plugin.ts @@ -4,7 +4,7 @@ import fs from 'fs'; import type { TFuncKey, TOptions } from 'i18next'; import { resolve } from 'path'; import { Application } from './application'; -import { InstallOptions, getExposeChangelogUrl, getExposeReadmeUrl } from './plugin-manager'; +import { getExposeChangelogUrl, getExposeReadmeUrl, InstallOptions } from './plugin-manager'; import { checkAndGetCompatible } from './plugin-manager/utils'; export interface PluginInterface { @@ -109,7 +109,7 @@ export abstract class Plugin implements PluginInterface { async importCollections(collectionsPath: string) { await this.db.import({ directory: collectionsPath, - from: this.getName(), + from: `plugin:${this.getName()}`, }); } @@ -129,20 +129,30 @@ export abstract class Plugin implements PluginInterface { ...this.options, }; } - const file = await fs.promises.realpath(resolve(process.env.NODE_MODULES_PATH, packageName)); - const lastUpdated = (await fs.promises.stat(file)).ctime; - const others = await checkAndGetCompatible(packageName); - return { + + const results = { ...this.options, - ...others, readmeUrl: getExposeReadmeUrl(packageName, locale), changelogUrl: getExposeChangelogUrl(packageName), - lastUpdated, - file, - updatable: file.startsWith(process.env.PLUGIN_STORAGE_PATH), displayName: packageJson[`displayName.${locale}`] || packageJson.displayName || name, description: packageJson[`description.${locale}`] || packageJson.description, }; + + if (!options.withOutOpenFile) { + const file = await fs.promises.realpath( + resolve(process.env.NODE_MODULES_PATH || resolve(process.cwd(), 'node_modules'), packageName), + ); + + return { + ...results, + ...(await checkAndGetCompatible(packageName)), + lastUpdated: (await fs.promises.stat(file)).ctime, + file, + updatable: file.startsWith(process.env.PLUGIN_STORAGE_PATH), + }; + } + + return results; } } diff --git a/packages/core/test/src/server/index.ts b/packages/core/test/src/server/index.ts index 832c9ead2..43453084d 100644 --- a/packages/core/test/src/server/index.ts +++ b/packages/core/test/src/server/index.ts @@ -6,6 +6,7 @@ export { default as supertest } from 'supertest'; export * from './mockServer'; export const pgOnly: () => any = () => (process.env.DB_DIALECT == 'postgres' ? describe : describe.skip); +export const isPg = () => process.env.DB_DIALECT == 'postgres'; export function randomStr() { // create random string diff --git a/packages/plugins/@nocobase/plugin-acl/src/server/collections/roles-users.ts b/packages/plugins/@nocobase/plugin-acl/src/server/collections/roles-users.ts index 9d7a79074..c82a431df 100644 --- a/packages/plugins/@nocobase/plugin-acl/src/server/collections/roles-users.ts +++ b/packages/plugins/@nocobase/plugin-acl/src/server/collections/roles-users.ts @@ -1,8 +1,9 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { +export default defineCollection({ name: 'rolesUsers', - duplicator: 'optional', - namespace: 'acl.acl', + dumpRules: { + group: 'user', + }, fields: [{ type: 'boolean', name: 'default' }], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-acl/src/server/collections/roles.ts b/packages/plugins/@nocobase/plugin-acl/src/server/collections/roles.ts index e48ee4297..41daae367 100644 --- a/packages/plugins/@nocobase/plugin-acl/src/server/collections/roles.ts +++ b/packages/plugins/@nocobase/plugin-acl/src/server/collections/roles.ts @@ -1,11 +1,7 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'acl.acl', - duplicator: { - dumpable: 'required', - with: 'uiSchemas', - }, +export default defineCollection({ + dumpRules: 'required', name: 'roles', title: '{{t("Roles")}}', autoGenId: false, @@ -98,4 +94,4 @@ export default { through: 'rolesUsers', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResources.ts b/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResources.ts index 9387aec79..237c34bfa 100644 --- a/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResources.ts +++ b/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResources.ts @@ -1,8 +1,7 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'acl.acl', - duplicator: 'required', +export default defineCollection({ + dumpRules: 'required', name: 'rolesResources', model: 'RoleResourceModel', indexes: [ @@ -30,4 +29,4 @@ export default { target: 'rolesResourcesActions', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResourcesActions.ts b/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResourcesActions.ts index e34aeacde..05da21007 100644 --- a/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResourcesActions.ts +++ b/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResourcesActions.ts @@ -1,8 +1,7 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'acl.acl', - duplicator: 'required', +export default defineCollection({ + dumpRules: 'required', name: 'rolesResourcesActions', model: 'RoleResourceActionModel', fields: [ @@ -28,4 +27,4 @@ export default { onDelete: 'RESTRICT', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResourcesScopes.ts b/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResourcesScopes.ts index cd4abddc2..1ff4ea708 100644 --- a/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResourcesScopes.ts +++ b/packages/plugins/@nocobase/plugin-acl/src/server/collections/rolesResourcesScopes.ts @@ -1,8 +1,7 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'acl.acl', - duplicator: 'required', +export default defineCollection({ + dumpRules: 'required', name: 'rolesResourcesScopes', fields: [ { @@ -22,4 +21,4 @@ export default { name: 'scope', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-acl/src/server/collections/users.ts b/packages/plugins/@nocobase/plugin-acl/src/server/collections/users.ts index 46872bc46..00a9056b4 100644 --- a/packages/plugins/@nocobase/plugin-acl/src/server/collections/users.ts +++ b/packages/plugins/@nocobase/plugin-acl/src/server/collections/users.ts @@ -1,6 +1,6 @@ -import { extend } from '@nocobase/database'; +import { extendCollection } from '@nocobase/database'; -export default extend({ +export default extendCollection({ name: 'users', fields: [ { diff --git a/packages/plugins/@nocobase/plugin-acl/src/server/server.ts b/packages/plugins/@nocobase/plugin-acl/src/server/server.ts index c7c67c254..d73586369 100644 --- a/packages/plugins/@nocobase/plugin-acl/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-acl/src/server/server.ts @@ -1,5 +1,6 @@ import { NoPermissionError } from '@nocobase/acl'; import { Context, utils as actionUtils } from '@nocobase/actions'; +import { Cache } from '@nocobase/cache'; import { Collection, RelationField, snakeCase } from '@nocobase/database'; import { Plugin } from '@nocobase/server'; import { Mutex } from 'async-mutex'; @@ -13,7 +14,6 @@ import { setCurrentRole } from './middlewares/setCurrentRole'; import { RoleModel } from './model/RoleModel'; import { RoleResourceActionModel } from './model/RoleResourceActionModel'; import { RoleResourceModel } from './model/RoleResourceModel'; -import { Cache } from '@nocobase/cache'; export interface AssociationFieldAction { associationActions: string[]; @@ -891,10 +891,11 @@ export class PluginACL extends Plugin { async load() { await this.importCollections(resolve(__dirname, 'collections')); + this.db.extendCollection({ name: 'rolesUischemas', - namespace: 'acl.acl', - duplicator: 'required', + dumpRules: 'required', + origin: `plugin:${this.name}`, }); } } diff --git a/packages/plugins/@nocobase/plugin-api-keys/src/client/Configuration/index.tsx b/packages/plugins/@nocobase/plugin-api-keys/src/client/Configuration/index.tsx index 5ef9748e0..9b248c572 100644 --- a/packages/plugins/@nocobase/plugin-api-keys/src/client/Configuration/index.tsx +++ b/packages/plugins/@nocobase/plugin-api-keys/src/client/Configuration/index.tsx @@ -1,7 +1,7 @@ import { RecursionField } from '@formily/react'; import { CollectionManagerProvider, SchemaComponentOptions, useCurrentRoles } from '@nocobase/client'; import React from 'react'; -import apiKeysCollection from '../../collections'; +import apiKeysCollection from '../../collections/apiKeys'; import { ExpiresSelect } from './ExpiresSelect'; import { configurationSchema } from './schema'; diff --git a/packages/plugins/@nocobase/plugin-api-keys/src/collections/api-keys.ts b/packages/plugins/@nocobase/plugin-api-keys/src/collections/apiKeys.ts similarity index 94% rename from packages/plugins/@nocobase/plugin-api-keys/src/collections/api-keys.ts rename to packages/plugins/@nocobase/plugin-api-keys/src/collections/apiKeys.ts index 5e155b58d..2a75a757f 100644 --- a/packages/plugins/@nocobase/plugin-api-keys/src/collections/api-keys.ts +++ b/packages/plugins/@nocobase/plugin-api-keys/src/collections/apiKeys.ts @@ -2,12 +2,12 @@ import type { CollectionOptions } from '@nocobase/database'; import { generateNTemplate } from '../locale'; export default { - namespace: 'api-keys', - duplicator: 'optional', + dumpRules: { + group: 'user', + }, + shared: true, name: 'apiKeys', - title: '{{t("API keys", {"ns": "api-keys"})}}', sortable: 'sort', - model: 'ApiKeyModel', createdBy: true, updatedAt: false, updatedBy: false, diff --git a/packages/plugins/@nocobase/plugin-api-keys/src/collections/index.ts b/packages/plugins/@nocobase/plugin-api-keys/src/collections/index.ts deleted file mode 100644 index dc988885c..000000000 --- a/packages/plugins/@nocobase/plugin-api-keys/src/collections/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from './api-keys'; diff --git a/packages/plugins/@nocobase/plugin-api-keys/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-api-keys/src/server/plugin.ts index a33b6a533..77f67586b 100644 --- a/packages/plugins/@nocobase/plugin-api-keys/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-api-keys/src/server/plugin.ts @@ -34,9 +34,7 @@ export default class ApiKeysPlugin extends Plugin { } async load() { - await this.db.import({ - directory: resolve(__dirname, '../collections'), - }); + await this.importCollections(resolve(__dirname, '../collections')); this.app.resourcer.use(async (ctx, next) => { const { resourceName, actionName } = ctx.action.params; diff --git a/packages/plugins/@nocobase/plugin-audit-logs/src/server/collections/auditChanges.ts b/packages/plugins/@nocobase/plugin-audit-logs/src/server/collections/auditChanges.ts index eb814a36f..c0713a451 100644 --- a/packages/plugins/@nocobase/plugin-audit-logs/src/server/collections/auditChanges.ts +++ b/packages/plugins/@nocobase/plugin-audit-logs/src/server/collections/auditChanges.ts @@ -1,14 +1,16 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'audit-logs.auditLogs', - duplicator: 'optional', + dumpRules: { + group: 'log', + }, name: 'auditChanges', title: '变动值', createdBy: false, updatedBy: false, createdAt: false, updatedAt: false, + shared: true, fields: [ { type: 'json', diff --git a/packages/plugins/@nocobase/plugin-audit-logs/src/server/collections/auditLogs.ts b/packages/plugins/@nocobase/plugin-audit-logs/src/server/collections/auditLogs.ts index 88c7d8fab..76609b423 100644 --- a/packages/plugins/@nocobase/plugin-audit-logs/src/server/collections/auditLogs.ts +++ b/packages/plugins/@nocobase/plugin-audit-logs/src/server/collections/auditLogs.ts @@ -1,12 +1,14 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'audit-logs.auditLogs', - duplicator: 'optional', + dumpRules: { + group: 'log', + }, name: 'auditLogs', createdBy: false, updatedBy: false, updatedAt: false, + shared: true, fields: [ { type: 'date', diff --git a/packages/plugins/@nocobase/plugin-audit-logs/src/server/index.ts b/packages/plugins/@nocobase/plugin-audit-logs/src/server/index.ts index 6bddf1d4b..7093d6227 100644 --- a/packages/plugins/@nocobase/plugin-audit-logs/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-audit-logs/src/server/index.ts @@ -10,9 +10,7 @@ export default class PluginActionLogs extends Plugin { } async load() { - await this.db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); this.db.addMigrations({ namespace: 'audit-logs', diff --git a/packages/plugins/@nocobase/plugin-auth/src/server/collections/authenticators.ts b/packages/plugins/@nocobase/plugin-auth/src/server/collections/authenticators.ts index 64a6e3b52..e77e07e69 100644 --- a/packages/plugins/@nocobase/plugin-auth/src/server/collections/authenticators.ts +++ b/packages/plugins/@nocobase/plugin-auth/src/server/collections/authenticators.ts @@ -1,14 +1,15 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; /** * Collection for extended authentication methods, */ -export default { - namespace: 'auth.auth', - duplicator: 'optional', +export default defineCollection({ + dumpRules: { + group: 'third-party', + }, + shared: true, name: 'authenticators', sortable: true, - title: '{{t("Authenticators")}}', model: 'AuthModel', createdBy: true, updatedBy: true, @@ -95,4 +96,4 @@ export default { through: 'usersAuthenticators', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-auth/src/server/collections/token-blacklist.ts b/packages/plugins/@nocobase/plugin-auth/src/server/collections/token-blacklist.ts index fd388e278..d3bb3d866 100644 --- a/packages/plugins/@nocobase/plugin-auth/src/server/collections/token-blacklist.ts +++ b/packages/plugins/@nocobase/plugin-auth/src/server/collections/token-blacklist.ts @@ -1,8 +1,10 @@ -import { CollectionOptions } from '@nocobase/client'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'auth.token-black', - duplicator: 'optional', +export default defineCollection({ + dumpRules: { + group: 'log', + }, + shared: true, name: 'tokenBlacklist', model: 'TokenBlacklistModel', fields: [ @@ -16,4 +18,4 @@ export default { name: 'expiration', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-auth/src/server/collections/users-authenticators.ts b/packages/plugins/@nocobase/plugin-auth/src/server/collections/users-authenticators.ts index 395092da9..3111710a3 100644 --- a/packages/plugins/@nocobase/plugin-auth/src/server/collections/users-authenticators.ts +++ b/packages/plugins/@nocobase/plugin-auth/src/server/collections/users-authenticators.ts @@ -1,20 +1,15 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; /** * Collection for user information of extended authentication methods, * such as saml, oicd, oauth, sms, etc. */ -export default { - namespace: 'auth.auth', - duplicator: { - dumpable: 'optional', - /** - * When dump this collection, the users collection is required to be dumped. - */ - with: 'users', +export default defineCollection({ + dumpRules: { + group: 'user', }, + shared: true, name: 'usersAuthenticators', - title: '{{t("Users Authenticators")}}', model: 'UserAuthModel', createdBy: true, updatedBy: true, @@ -70,4 +65,4 @@ export default { defaultValue: {}, }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts index 92b6a3fce..9856744ff 100644 --- a/packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts @@ -24,9 +24,7 @@ export class AuthPlugin extends Plugin { async load() { // Set up database - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); this.db.addMigrations({ namespace: 'auth', directory: resolve(__dirname, 'migrations'), diff --git a/packages/plugins/@nocobase/plugin-duplicator/.npmignore b/packages/plugins/@nocobase/plugin-backup-restore/.npmignore similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/.npmignore rename to packages/plugins/@nocobase/plugin-backup-restore/.npmignore diff --git a/packages/plugins/@nocobase/plugin-duplicator/LICENSE b/packages/plugins/@nocobase/plugin-backup-restore/LICENSE similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/LICENSE rename to packages/plugins/@nocobase/plugin-backup-restore/LICENSE diff --git a/packages/plugins/@nocobase/plugin-duplicator/README.md b/packages/plugins/@nocobase/plugin-backup-restore/README.md similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/README.md rename to packages/plugins/@nocobase/plugin-backup-restore/README.md diff --git a/packages/plugins/@nocobase/plugin-duplicator/README.zh-CN.md b/packages/plugins/@nocobase/plugin-backup-restore/README.zh-CN.md similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/README.zh-CN.md rename to packages/plugins/@nocobase/plugin-backup-restore/README.zh-CN.md diff --git a/packages/plugins/@nocobase/plugin-duplicator/client.d.ts b/packages/plugins/@nocobase/plugin-backup-restore/client.d.ts similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/client.d.ts rename to packages/plugins/@nocobase/plugin-backup-restore/client.d.ts diff --git a/packages/plugins/@nocobase/plugin-duplicator/client.js b/packages/plugins/@nocobase/plugin-backup-restore/client.js similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/client.js rename to packages/plugins/@nocobase/plugin-backup-restore/client.js diff --git a/packages/plugins/@nocobase/plugin-duplicator/package.json b/packages/plugins/@nocobase/plugin-backup-restore/package.json similarity index 76% rename from packages/plugins/@nocobase/plugin-duplicator/package.json rename to packages/plugins/@nocobase/plugin-backup-restore/package.json index b7909586c..899e255fb 100644 --- a/packages/plugins/@nocobase/plugin-duplicator/package.json +++ b/packages/plugins/@nocobase/plugin-backup-restore/package.json @@ -1,5 +1,5 @@ { - "name": "@nocobase/plugin-duplicator", + "name": "@nocobase/plugin-backup-restore", "displayName": "App backup & restore", "displayName.zh-CN": "应用的备份与还原", "description": "Can be used for application replication, migration, upgrade, etc", @@ -10,7 +10,7 @@ "repository": { "type": "git", "url": "git+https://github.com/nocobase/nocobase.git", - "directory": "packages/plugins/duplicator" + "directory": "packages/plugins/plugin-backup-restore" }, "devDependencies": { "@koa/multer": "^3.0.2", @@ -23,14 +23,17 @@ "koa-send": "^5.0.1", "mkdirp": "^1.0.4", "react": "^18.2.0", - "tar": "^6.1.13" + "tar": "^6.1.13", + "object-path": "^0.11.8", + "content-disposition": "^0.5.4", + "semver": "^7.5.4" }, "peerDependencies": { "@nocobase/client": "0.x", "@nocobase/database": "0.x", "@nocobase/server": "0.x", "@nocobase/test": "0.x", - "@nocobase/utils": "0.x" - }, - "gitHead": "979a9c59a98c61a2287dd847580746a9b597cbde" + "@nocobase/utils": "0.x", + "@nocobase/actions": "0.x" + } } diff --git a/packages/plugins/@nocobase/plugin-duplicator/server.d.ts b/packages/plugins/@nocobase/plugin-backup-restore/server.d.ts similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/server.d.ts rename to packages/plugins/@nocobase/plugin-backup-restore/server.d.ts diff --git a/packages/plugins/@nocobase/plugin-duplicator/server.js b/packages/plugins/@nocobase/plugin-backup-restore/server.js similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/server.js rename to packages/plugins/@nocobase/plugin-backup-restore/server.js diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/client/Configuration.tsx b/packages/plugins/@nocobase/plugin-backup-restore/src/client/Configuration.tsx new file mode 100644 index 000000000..8c60570f6 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/client/Configuration.tsx @@ -0,0 +1,476 @@ +import { InboxOutlined, PlusOutlined, ReloadOutlined, UploadOutlined } from '@ant-design/icons'; +import { FormItem } from '@formily/antd-v5'; +import { Checkbox, DatePicker, useAPIClient, useCompile } from '@nocobase/client'; +import { Alert, App, Button, Card, Divider, Modal, Space, Spin, Table, Tabs, Upload, UploadProps, message } from 'antd'; +import { saveAs } from 'file-saver'; +import React, { useEffect, useMemo, useState } from 'react'; +import { useDuplicatorTranslation } from './locale'; + +const { Dragger } = Upload; + +function useUploadProps(props: UploadProps): any { + const onChange = (param) => { + props.onChange?.(param); + }; + + const api = useAPIClient(); + + return { + ...props, + customRequest({ action, data, file, filename, headers, onError, onProgress, onSuccess, withCredentials }) { + const formData = new FormData(); + if (data) { + Object.keys(data).forEach((key) => { + formData.append(key, data[key]); + }); + } + formData.append(filename, file); + // eslint-disable-next-line promise/catch-or-return + api.axios + .post(action, formData, { + withCredentials, + headers, + onUploadProgress: ({ total, loaded }) => { + onProgress({ percent: Math.round((loaded / total) * 100).toFixed(2) }, file); + }, + }) + .then(({ data }) => { + onSuccess(data, file); + }) + .catch(onError) + .finally(() => {}); + + return { + abort() { + console.log('upload progress is aborted.'); + }, + }; + }, + onChange, + }; +} + +const LearnMore: any = (props: { collectionsData?: any; isBackup?: boolean }) => { + const { collectionsData } = props; + const { t } = useDuplicatorTranslation(); + const [isModalOpen, setIsModalOpen] = useState(false); + const [dataSource, setDataSource] = useState(collectionsData); + useEffect(() => { + setDataSource(collectionsData); + }, [collectionsData]); + const apiClient = useAPIClient(); + const compile = useCompile(); + const resource = useMemo(() => { + return apiClient.resource('backupFiles'); + }, [apiClient]); + const showModal = async () => { + if (props.isBackup) { + const data = await resource.dumpableCollections(); + setDataSource(data?.data); + setIsModalOpen(true); + } + setIsModalOpen(true); + }; + + const handleOk = () => { + setIsModalOpen(false); + }; + + const handleCancel = () => { + setIsModalOpen(false); + }; + const columns = [ + { + title: t('Collection'), + dataIndex: 'collection', + key: 'collection', + render: (_, data) => { + return ( +
+ {compile(data.title)} +
+
{data.name}
+
+ ); + }, + }, + { + title: t('Origin'), + dataIndex: 'plugin', + key: 'origin', + width: '50%', + render: (_, data) => { + const { origin } = data; + return ( +
+ {origin.title} +
+
{origin.name}
+
+ ); + }, + }, + ]; + const items = Object.keys(dataSource || {}).map((item) => { + return { + key: item, + label: t(`${item}.title`), + children: ( + <> + + + + ), + }; + }); + + return ( + <> + {t('Learn more')} + + + + + ); +}; + +const Restore: React.FC = ({ ButtonComponent = Button, title, upload = false, fileData }) => { + const { t } = useDuplicatorTranslation(); + const [dataTypes, setDataTypes] = useState(['required']); + const [isModalOpen, setIsModalOpen] = useState(false); + const [restoreData, setRestoreData] = useState(null); + const [loading, setLoading] = useState(false); + const apiClient = useAPIClient(); + const resource = useMemo(() => { + return apiClient.resource('backupFiles'); + }, [apiClient]); + const [dataSource, setDataSource] = useState([]); + + useEffect(() => { + setDataSource( + Object.keys(restoreData?.dumpableCollectionsGroupByGroup || []).map((key) => ({ + value: key, + label: t(`${key}.title`), + disabled: ['required', 'skipped'].includes(key), + })), + ); + }, [restoreData]); + + const showModal = async () => { + setIsModalOpen(true); + if (!upload) { + setLoading(true); + const { data } = await resource.get({ filterByTk: fileData.name }); + setDataSource( + Object.keys(data?.data?.meta?.dumpableCollectionsGroupByGroup || []).map((key) => ({ + value: key, + label: t(`${key}.title`), + disabled: ['required', 'skipped'].includes(key), + })), + ); + setRestoreData(data?.data?.meta); + setLoading(false); + } + }; + const handleOk = () => { + resource.restore({ + values: { + dataTypes, + filterByTk: fileData?.name, + key: restoreData?.key, + }, + }); + setIsModalOpen(false); + }; + + const handleCancel = () => { + setIsModalOpen(false); + setRestoreData(null); + setDataTypes(['required']); + }; + return ( + <> + {title} + + + {upload && !restoreData && } + {(!upload || restoreData) && [ + + {t('Select the data to be restored')} ( + + ): + , +
+ + setDataTypes(checkValue)} + /> + +
, + ]} + + + + ); +}; + +const NewBackup: React.FC = ({ ButtonComponent = Button, refresh }) => { + const { t } = useDuplicatorTranslation(); + const [isModalOpen, setIsModalOpen] = useState(false); + const [dataTypes, setBackupData] = useState(['required']); + const apiClient = useAPIClient(); + const [dataSource, setDataSource] = useState([]); + + const showModal = async () => { + const { data } = await apiClient.resource('backupFiles').dumpableCollections(); + setDataSource( + Object.keys(data || []).map((key) => ({ + value: key, + label: t(`${key}.title`), + disabled: ['required', 'skipped'].includes(key), + })), + ); + setIsModalOpen(true); + }; + + const handleOk = () => { + apiClient.request({ + url: 'backupFiles:create', + method: 'post', + data: { + dataTypes, + }, + }); + setIsModalOpen(false); + setBackupData(['required']); + setTimeout(() => { + refresh(); + }, 500); + }; + + const handleCancel = () => { + setIsModalOpen(false); + setBackupData(['required']); + }; + + return ( + <> + } type="primary" onClick={showModal}> + {t('New backup')} + + + + {t('Select the data to be backed up')} ( + + ): + +
+ setBackupData(checkValue)} + value={dataTypes} + /> +
+ + + ); +}; + +const RestoreUpload: React.FC = (props: any) => { + const { t } = useDuplicatorTranslation(); + const uploadProps: UploadProps = { + multiple: false, + action: '/backupFiles:upload', + onChange(info) { + if (info.fileList.length > 1) { + info.fileList.splice(0, info.fileList.length - 1); // 只保留一个文件 + } + const { status } = info.file; + if (status === 'done') { + message.success(`${info.file.name} ` + t('file uploaded successfully')); + props.setRestoreData({ ...info.file.response?.data?.meta, key: info.file.response?.data.key }); + } else if (status === 'error') { + message.error(`${info.file.name} ` + t('file upload failed')); + } + }, + onDrop(e) { + console.log('Dropped files', e.dataTransfer.files); + }, + }; + + return ( + +

+ +

+

{t('Click or drag file to this area to upload')}

+ + ); +}; + +export const BackupAndRestoreList = () => { + const { t } = useDuplicatorTranslation(); + const apiClient = useAPIClient(); + const [dataSource, setDataSource] = useState([]); + const [loading, setLoading] = useState(false); + const [downloadTarget, setDownloadTarget] = useState(false); + const { modal } = App.useApp(); + const resource = useMemo(() => { + return apiClient.resource('backupFiles'); + }, [apiClient]); + + useEffect(() => { + queryFieldList(); + }, []); + const queryFieldList = async () => { + setLoading(true); + const { data } = await resource.list(); + setDataSource(data.data); + setLoading(false); + }; + const handleDownload = async (fileData) => { + setDownloadTarget(fileData.name); + const data = await apiClient.request({ + url: 'backupFiles:download', + method: 'get', + params: { + filterByTk: fileData.name, + }, + responseType: 'blob', + }); + setDownloadTarget(false); + const blob = new Blob([data.data]); + saveAs(blob, fileData.name); + }; + const handleRefresh = async () => { + await queryFieldList(); + }; + const handleDestory = (fileData) => { + modal.confirm({ + title: t('Delete record', { ns: 'client' }), + content: t('Are you sure you want to delete it?', { ns: 'client' }), + onOk: async () => { + await resource.destroy({ filterByTk: fileData.name }); + await queryFieldList(); + message.success(t('Deleted successfully')); + }, + }); + }; + + return ( +
+ + + + + {t('Restore backup from local')} + + } + /> + + +
{ + return data.inProgress + ? { + colSpan: 4, + } + : {}; + }, + render: (name, data) => + data.inProgress ? ( +
+ {name}({t('Backing up')}...) +
+ ) : ( +
{name}
+ ), + }, + { + title: t('File size'), + dataIndex: 'fileSize', + onCell: (data) => { + return data.inProgress + ? { + colSpan: 0, + } + : {}; + }, + }, + { + title: t('Created at', { ns: 'client' }), + dataIndex: 'createdAt', + onCell: (data) => { + return data.inProgress + ? { + colSpan: 0, + } + : {}; + }, + render: (value) => { + return ; + }, + }, + { + title: t('Actions', { ns: 'client' }), + dataIndex: 'actions', + onCell: (data) => { + return data.inProgress + ? { + colSpan: 0, + } + : {}; + }, + render: (_, record) => ( + }> + + handleDownload(record)}> + {t('Download')} + + handleDestory(record)}>{t('Delete')} + + ), + }, + ]} + /> + + + ); +}; diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/client/DuplicatorProvider.tsx b/packages/plugins/@nocobase/plugin-backup-restore/src/client/DuplicatorProvider.tsx new file mode 100644 index 000000000..4ea6cb9d1 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/client/DuplicatorProvider.tsx @@ -0,0 +1,12 @@ +import { SchemaComponentOptions, CurrentAppInfoProvider } from '@nocobase/client'; +import React, { FC } from 'react'; + +export const DuplicatorProvider: FC = function (props) { + return ( + + {props.children} + + ); +}; + +DuplicatorProvider.displayName = 'DuplicatorProvider'; diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/client/index.tsx b/packages/plugins/@nocobase/plugin-backup-restore/src/client/index.tsx new file mode 100644 index 000000000..87b73f914 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/client/index.tsx @@ -0,0 +1,17 @@ +import { Plugin } from '@nocobase/client'; +import { BackupAndRestoreList } from './Configuration'; +import { DuplicatorProvider } from './DuplicatorProvider'; +import { NAMESPACE } from './locale'; +export class DuplicatorPlugin extends Plugin { + async load() { + this.app.use(DuplicatorProvider); + this.app.pluginSettingsManager.add(NAMESPACE, { + title: `{{t("Backup & Restore", { ns: "${NAMESPACE}" })}}`, + icon: 'CloudServerOutlined', + Component: BackupAndRestoreList, + aclSnippet: 'pm.backup.restore', + }); + } +} + +export default DuplicatorPlugin; diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/client/locale/index.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/client/locale/index.ts new file mode 100644 index 000000000..8f2d42f15 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/client/locale/index.ts @@ -0,0 +1,18 @@ +import { i18n } from '@nocobase/client'; +import { useTranslation } from 'react-i18next'; + +export const NAMESPACE = 'backup-restore'; + +export function lang(key: string) { + return i18n.t(key, { ns: NAMESPACE }); +} + +export function generateNTemplate(key: string) { + return `{{t('${key}', { ns: '${NAMESPACE}', nsMode: 'fallback' })}}`; +} + +export function useDuplicatorTranslation() { + return useTranslation(NAMESPACE, { + nsMode: 'fallback', + }); +} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/index.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/index.ts similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/src/index.ts rename to packages/plugins/@nocobase/plugin-backup-restore/src/index.ts diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/locale/en-US.json b/packages/plugins/@nocobase/plugin-backup-restore/src/locale/en-US.json new file mode 100644 index 000000000..7c7612345 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/locale/en-US.json @@ -0,0 +1,44 @@ +{ + "Backup file": "Backup file", + "System metadata": "System metadata", + "System config": "System config", + "Business data": "Business data", + "Backup & Restore": "Backup & Restore", + "Backup": "Backup", + "Restore": "Restore", + "Configuration": "Configuration", + "Select the data to be backed up": "Select the data to be backed up", + "Select the data to be restored": "Select the data to be restored", + "Click or drag file to this area to upload": "Click or drag file to this area to upload", + "Learn more": "Learn more", + "Start backup": "Start backup", + "Start restore": "Start restore", + "Backed up successfully": "Backed up successfully", + "Plugin": "Plugin", + "file uploaded successfully": "file uploaded successfully", + "file upload failed": "file upload failed", + "Download": "Download", + "Restore backup from local": "Restore backup from local", + "Backup instructions": "Backup instructions", + "File size": "File size", + "New backup": "New backup", + "Origin": "Origin", + "Backing up": "Backing up", + "Refresh": "Refresh", + "Delete": "Delete", + "Deleted successfully": "Deleted successfully", + "required.title": "Required data", + "user.title": "User data", + "log.title": "Log data", + "custom.title": "Custom collection data", + "skipped.title": "Skipped data", + "unknown.title": "Unknown", + "third-party.title": "Third-party service information", + "required.description": "Required data", + "user.description": "User data", + "log.description": "Log data", + "custom.description": "Custom collection data", + "skipped.description": "Skipped data", + "unknown.description": "Data without configured dump rules", + "third-party.description": "Third-party service information" +} diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/locale/ja-JP.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/locale/ja-JP.ts new file mode 100644 index 000000000..048e92d2a --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/locale/ja-JP.ts @@ -0,0 +1,24 @@ +export default { + 'System metadata': 'システムメタデータ', + 'System config': 'システム構成', + 'Business data': 'ビジネスデータ', + 'Backup & Restore': 'バックアップとリストア', + Backup: 'バックアップ', + Restore: 'リストア', + Configuration: '構成', + 'Select the data to be backed up': 'バックアップされたデータの選択', + 'Select the data to be restored': 'リストアされたデータの選択', + 'Learn more': '詳細', + 'Start backup': 'バックアップの開始', + 'Start restore': 'リストアを開始', + 'Backed up successfully': 'バックアップ成功', + Plugin: 'プラグイン', + 'file uploaded successfully': 'ファイルのアップロードに成功しました', + Download: 'ダウンロード', + 'Restore backup from local':'ローカルからバックアップに返信', + 'Backup instructions':'バックアップの説明', + 'File size':'ファイルサイズ', + 'New backup':'新規バックアップ', + 'Origin':'ソース', + 'Backing up':'バックアップ中' +}; diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/locale/pt-BR.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/locale/pt-BR.ts new file mode 100644 index 000000000..b8dd555d2 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/locale/pt-BR.ts @@ -0,0 +1,27 @@ +const locale = { + 'System metadata': 'Metadados do Sistema', + 'System config': 'configuração do sistema', + 'Business data': 'Dados comerciais', + 'Backup & Restore': 'Cópia de Segurança e Restauração', + Backup: 'cópias de segurança', + Restore: 'redução', + Configuration: 'atribuição', + 'Select the data to be backed up': 'Seleccionar os dados de cópia de segurança', + 'Select the data to be restored': 'Seleccionar os dados restaurados', + 'Learn more': 'Saiba mais', + 'Start restore': 'Iniciar a Restauração', + 'Start backup': 'Iniciar a cópia de segurança', + 'Backed up successfully': 'Cópia de segurança bem sucedida', + 'Plugin':'Plugins', + 'file uploaded successfully': 'O ficheiro foi enviado com sucesso', + 'file upload failed': 'O envio do ficheiro falhou', + 'Download':'download', + 'Restore backup from local':'Restaurar a cópia de segurança localmente', + 'Backup instructions':'Instruções de cópia de segurança', + 'File size':'tamanho do ficheiro', + 'New backup':'Nova Cópia de Segurança', + 'Origin':'fonte', + 'Backing up':'Cópia de segurança em curso' +}; + +export default locale; diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/locale/zh-CN.json b/packages/plugins/@nocobase/plugin-backup-restore/src/locale/zh-CN.json new file mode 100644 index 000000000..b7b559d12 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/locale/zh-CN.json @@ -0,0 +1,50 @@ +{ + "Backup file": "备份文件", + "Select Import data": "请选择导入数据", + "Select Import Plugins": "请选择导入插件", + "Select User Collections": "请选择用户数据", + "Basic Data": "基础数据", + "Optional Data": "可选数据", + "User Data": "用户数据", + "System metadata": "系统元数据", + "System config": "系统配置", + "Business data": "业务数据", + "Backup & Restore": "备份和还原", + "Backup": "备份", + "Restore": "还原", + "Configuration": "配置", + "Select the data to be backed up": "选择备份的数据", + "Select the data to be restored": "选择还原的数据", + "Click or drag file to this area to upload": "点击或拖拽文件至此区域即可上传", + "Learn more": "了解更多", + "Start backup": "开始备份", + "Start restore": "开始还原", + "Backed up successfully": "备份成功", + "Deleted successfully": "删除成功", + "Plugin": "插件", + "file uploaded successfully": "文件上传成功", + "file upload failed": "文件上传失败", + "Download": "下载", + "Restore backup from local": "从本地恢复备份", + "Backup instructions": "备份说明", + "File size":"文件大小", + "New backup":"新建备份", + "Origin":"来源", + "Backing up":"备份中", + "Refresh": "刷新", + "Delete": "删除", + "required.title": "必备数据", + "user.title": "用户数据", + "log.title": "日志数据", + "custom.title": "自建表数据", + "skipped.title": "跳过的数据", + "unknown.title": "未知数据", + "third-party.title": "第三方服务信息", + "required.description": "必备数据", + "user.description": "用户数据", + "log.description": "日志数据", + "custom.description": "自建表数据", + "skipped.description": "跳过的数据", + "unknown.description": "没有配置备份规则的数据", + "third-party.description": "第三方服务信息" +} diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/api.test.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/api.test.ts new file mode 100644 index 000000000..d60b4343f --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/api.test.ts @@ -0,0 +1,177 @@ +import { MockServer, waitSecond } from '@nocobase/test'; +import createApp from './index'; +import { Dumper } from '../dumper'; + +describe('backup files', () => { + let app: MockServer; + + beforeEach(async () => { + app = await createApp(); + }); + + afterEach(async () => { + await app.destroy(); + }); + + it('should create dump file', async () => { + const createResponse = await app + .agent() + .resource('backupFiles') + .create({ + dataTypes: ['meta', 'config', 'business'], + }); + + expect(createResponse.status).toBe(200); + + const dumpKey = createResponse.body.data.key; + expect(dumpKey).toBeDefined(); + + const promise = Dumper.getTaskPromise(dumpKey); + + await promise; + }); + + describe('resource action', () => { + let dumpKey: string; + + let dumper: Dumper; + + beforeEach(async () => { + dumper = new Dumper(app); + + dumpKey = await dumper.runDumpTask({ + groups: new Set(['meta', 'config', 'business']), + }); + + const promise = Dumper.getTaskPromise(dumpKey); + + await promise; + }); + + it('should list backup file with in progress status', async () => { + await waitSecond(1000); + const fileName = Dumper.generateFileName(); + await dumper.writeLockFile(fileName); + const listResponse = await app.agent().resource('backupFiles').list(); + + expect(listResponse.status).toBe(200); + + const body = listResponse.body; + + const firstItem = body.data[0]; + expect(firstItem.status).toEqual('in_progress'); + }); + + it('should list backup file', async () => { + const listResponse = await app.agent().resource('backupFiles').list(); + + expect(listResponse.status).toBe(200); + + const body = listResponse.body; + + expect(body.meta.count).toBeDefined(); + expect(body.meta.totalPage).toBeDefined(); + }); + + it('should get backup file', async () => { + const getResponse = await app.agent().resource('backupFiles').get({ + filterByTk: dumpKey, + }); + + expect(getResponse.status).toBe(200); + + expect(getResponse.body.data.name).toEqual(dumpKey); + + console.log({ getResponse: getResponse.body.data }); + }); + + it('should restore from file name', async () => { + const restoreResponse = await app + .agent() + .resource('backupFiles') + .restore({ + values: { + filterByTk: dumpKey, + dataTypes: ['meta', 'config', 'business'], + }, + }); + + expect(restoreResponse.status).toBe(200); + }); + + it('should destroy dump file', async () => { + const destroyResponse = await app.agent().resource('backupFiles').destroy({ + filterByTk: dumpKey, + }); + + expect(destroyResponse.status).toBe(200); + + const getResponse = await app.agent().resource('backupFiles').get({ + filterByTk: dumpKey, + }); + + expect(getResponse.status).toBe(404); + }); + + it('should restore from upload file', async () => { + const filePath = dumper.backUpFilePath(dumpKey); + const packageInfoResponse = await app.agent().post('/backupFiles:upload').attach('file', filePath); + + expect(packageInfoResponse.status).toBe(200); + const data = packageInfoResponse.body.data; + + expect(data['key']).toBeTruthy(); + expect(data['meta']).toBeTruthy(); + + const restoreResponse = await app + .agent() + .resource('backupFiles') + .restore({ + values: { + key: data['key'], + dataTypes: ['meta', 'config', 'business'], + }, + }); + + expect(restoreResponse.status).toBe(200); + }); + }); + + it('should get dumpable collections', async () => { + await app.db.getCollection('collections').repository.create({ + values: { + name: 'test', + title: '测试', + fields: [ + { + name: 'title', + type: 'string', + title: '标题', + }, + ], + }, + context: {}, + }); + + const response = await app.agent().get('/backupFiles:dumpableCollections'); + + expect(response.status).toBe(200); + + const body = response.body; + + expect(body['required']).toBeTruthy(); + expect(body['third-party']).toBeTruthy(); + expect(body['custom']).toBeTruthy(); + + const testCollectionInfo = body['custom'].find((item: any) => item.name === 'test'); + + expect(testCollectionInfo).toMatchObject({ + name: 'test', + title: '测试', + group: 'custom', + origin: { + name: '@nocobase/plugin-collection-manager', + }, + }); + }); +}); diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/dump-action.test.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/dump-action.test.ts similarity index 73% rename from packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/dump-action.test.ts rename to packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/dump-action.test.ts index eace87249..f97fe4fee 100644 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/dump-action.test.ts +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/dump-action.test.ts @@ -120,52 +120,4 @@ describe('dump', () => { await db.sequelize.query(sql, { type: 'INSERT' }); }); - - it('should dump user defined functions', async () => { - if (db.sequelize.getDialect() !== 'postgres') { - return; - } - - await db.sequelize.query(` - CREATE OR REPLACE FUNCTION add(integer, integer) RETURNS integer - AS 'select $1 + $2;' - LANGUAGE SQL - IMMUTABLE - RETURNS NULL ON NULL INPUT; - `); - - await db.sequelize.query(` -CREATE OR REPLACE FUNCTION trigger_function() - RETURNS TRIGGER - LANGUAGE PLPGSQL -AS $$ -BEGIN - -- trigger logic -END; -$$`); - - await db.sequelize.query(` -CREATE TRIGGER last_name_changes - BEFORE UPDATE - ON ${app.db.getCollection('users').quotedTableName()} - FOR EACH ROW - EXECUTE PROCEDURE trigger_function(); - `); - - await db.sequelize.query(` - CREATE OR REPLACE VIEW vistaView AS SELECT 'Hello World' as hello; - `); - - const dumper = new Dumper(app, { - workDir: testDir, - }); - - await dumper.dumpDb(); - - const restorer = new Restorer(app, { - workDir: testDir, - }); - - await restorer.importDb(); - }); }); diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/dumper.test.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/dumper.test.ts new file mode 100644 index 000000000..d5f232edf --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/dumper.test.ts @@ -0,0 +1,729 @@ +import { MockServer } from '@nocobase/test'; +import createApp from './index'; +import { Dumper } from '../dumper'; +import { Restorer } from '../restorer'; +import path from 'path'; +import fs from 'fs'; +import { Database } from '@nocobase/database'; + +describe('dumper', () => { + let app: MockServer; + let db: Database; + + beforeEach(async () => { + app = await createApp(); + db = app.db; + }); + + afterEach(async () => { + await app.destroy(); + }); + + it.skip('should restore from file', async () => { + const file = '/home/chareice/Downloads/backup_20231121_100606_4495.nbdump'; + const restorer = new Restorer(app, { + backUpFilePath: file, + }); + + await restorer.restore({ + groups: new Set(['meta', 'business']), + }); + }); + + it('should write sql content', async () => { + const dumper = new Dumper(app); + + const result = await dumper.dump({ + groups: new Set(['required']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.restore({ + groups: new Set(['required']), + }); + }); + + it('should dump and restore date field', async () => { + await db.getRepository('collections').create({ + values: { + name: 'tests', + fields: [ + { + type: 'date', + name: 'test_data', + }, + ], + }, + context: {}, + }); + + await db.getRepository('tests').create({ + values: { + date: new Date(), + }, + }); + + const dumper = new Dumper(app); + + const result = await dumper.dump({ + groups: new Set(['required', 'custom']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.restore({ + groups: new Set(['required', 'custom']), + }); + + const testCollection = app.db.getCollection('tests'); + const items = await testCollection.repository.find(); + expect(items.length).toBe(1); + }); + + describe('id seq', () => { + let allGroups; + + beforeEach(async () => { + await db.getRepository('collections').create({ + values: { + name: 'tests', + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }, + context: {}, + }); + + const Test = db.getCollection('tests'); + + for (let i = 0; i < 10; i++) { + await Test.repository.create({ + values: { + name: `test${i}`, + }, + }); + } + + const dumper = new Dumper(app); + + const collections = await dumper.collectionsGroupByDataTypes(); + allGroups = Object.keys(collections); + + const result = await dumper.dump({ + groups: new Set(allGroups), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.restore({ + groups: new Set(allGroups), + }); + }); + + it('should reset id seq after restore collection', async () => { + const testCollection = app.db.getCollection('tests'); + + await testCollection.repository.create({ + values: { + name: 'test', + }, + }); + }); + }); + + it('should restore parent collection', async () => { + if (!db.inDialect('postgres')) { + return; + } + + await db.getRepository('collections').create({ + values: { + name: 'parent', + fields: [ + { + type: 'string', + name: 'parentName', + }, + ], + }, + context: {}, + }); + + await db.getRepository('collections').create({ + values: { + name: 'child', + inherits: ['parent'], + fields: [ + { + type: 'string', + name: 'childName', + }, + ], + }, + context: {}, + }); + + await db.getRepository('parent').create({ + values: { + parentName: 'parentName', + }, + }); + + await db.getRepository('child').create({ + values: { + childName: 'childName', + }, + }); + + expect(await app.db.getRepository('parent').count()).toEqual(2); + + const dumper = new Dumper(app); + const result = await dumper.dump({ + groups: new Set(['required', 'custom']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.restore({ + groups: new Set(['required', 'custom']), + }); + + expect(await app.db.getRepository('parent').count()).toEqual(2); + }); + + it('should restore with audit logs', async () => { + await app.runCommand('pm', 'enable', 'audit-logs'); + + await app.db.getRepository('collections').create({ + values: { + name: 'tests', + logging: true, + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }, + context: {}, + }); + + const Post = app.db.getCollection('tests').model; + const post = await Post.create({ name: '123456' }); + await post.update({ name: '223456' }); + await post.destroy(); + const auditLogs = await app.db.getCollection('auditLogs').repository.find({ + appends: ['changes'], + }); + + expect(auditLogs.length).toBe(3); + + const dumper = new Dumper(app); + const result = await dumper.dump({ + groups: new Set(['required', 'log']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.restore({ + groups: new Set(['required', 'log']), + }); + + const log = await app.db.getCollection('auditLogs').repository.findOne({ + filter: { type: 'update' }, + appends: ['changes'], + }); + + const changes = log.get('changes'); + expect(typeof changes[0].before).toBe('string'); + }); + + it('should sort collections by inherits', async () => { + const collections = [ + { + name: 'parent1', + inherits: [], + }, + { + name: 'parent2', + inherits: [], + }, + { + name: 'child3', + inherits: ['child1', 'child2'], + }, + { + name: 'child1', + inherits: ['parent1', 'parent2'], + }, + { + name: 'child2', + inherits: ['parent1'], + }, + ]; + + const sorted = Restorer.sortCollectionsByInherits(collections); + + expect(sorted[0].name).toBe('parent1'); + expect(sorted[1].name).toBe('parent2'); + expect(sorted[2].name).toBe('child1'); + expect(sorted[3].name).toBe('child2'); + expect(sorted[4].name).toBe('child3'); + }); + + it('should handle inherited collection order', async () => { + if (!db.inDialect('postgres')) { + return; + } + + await db.getRepository('collections').create({ + values: { + name: 'parent1', + fields: [ + { + type: 'string', + name: 'parent1Name', + }, + ], + }, + context: {}, + }); + + await db.getRepository('collections').create({ + values: { + name: 'parent2', + fields: [ + { + type: 'string', + name: 'parent2Name', + }, + ], + }, + context: {}, + }); + + await db.getRepository('collections').create({ + values: { + name: 'child1', + inherits: ['parent1', 'parent2'], + fields: [ + { + type: 'string', + name: 'child1Name', + }, + ], + }, + context: {}, + }); + + await db.getRepository('parent1').create({ + values: { + parent1Name: 'parent1Name', + }, + }); + + await db.getRepository('parent2').create({ + values: { + parent2Name: 'parent2Name', + }, + }); + + await db.getRepository('child1').create({ + values: { + child1Name: 'child1Name', + }, + }); + + const dumper = new Dumper(app); + const result = await dumper.dump({ + groups: new Set(['required', 'custom']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + const meta = await restorer.parseBackupFile(); + + const businessCollections = meta.dumpableCollectionsGroupByGroup.custom; + const child1 = businessCollections.find(({ name }) => name === 'child1'); + + expect(child1.inherits).toEqual(['parent1', 'parent2']); + + await restorer.restore({ + groups: new Set(['required', 'custom']), + }); + }); + + it.skip('should list dumped files', async () => { + const dumper = new Dumper(app); + const list = await dumper.allBackUpFilePaths({ + includeInProgress: true, + dir: path.join(__dirname, './fixtures/files'), + }); + console.log({ list }); + expect(list.length).toBe(2); + }); + + it('should dump and restore with view collection', async () => { + await db.getRepository('collections').create({ + values: { + name: 'tests', + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }, + context: {}, + }); + + const testCollection = db.getCollection('tests'); + + const viewName = 'test_view'; + + const dropViewSQL = `DROP VIEW IF EXISTS ${viewName}`; + await db.sequelize.query(dropViewSQL); + + const viewSQL = `CREATE VIEW ${viewName} as SELECT * FROM ${testCollection.quotedTableName()}`; + + await db.sequelize.query(viewSQL); + + await db.getRepository('collections').create({ + values: { + name: viewName, + view: true, + schema: db.inDialect('postgres') ? 'public' : undefined, + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }, + context: {}, + }); + + const dumper = new Dumper(app); + const result = await dumper.dump({ + groups: new Set(['required', 'custom']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.restore({ + groups: new Set(['required', 'custom']), + }); + }); + + it('should dump & restore sequence data', async () => { + await db.getRepository('collections').create({ + values: { + name: 'tests', + fields: [ + { + type: 'sequence', + name: 'name', + patterns: [ + { + type: 'integer', + options: { key: 1 }, + }, + ], + }, + ], + }, + context: {}, + }); + + const Test = db.getCollection('tests'); + + const sequenceCollection = db.getCollection('sequences'); + expect(await sequenceCollection.repository.count()).toBe(1); + + const dumper = new Dumper(app); + const result = await dumper.dump({ + groups: new Set(['required', 'custom']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.restore({ + groups: new Set(['required', 'custom']), + }); + + expect(await app.db.getCollection('sequences').repository.count()).toBe(1); + }); + + it('should dump and restore map file', async () => { + const data = { + polygon: [ + [114.081074, 22.563646], + [114.147335, 22.559207], + [114.134975, 22.531621], + [114.09103, 22.520045], + [114.033695, 22.575376], + [114.025284, 22.55461], + [114.033523, 22.533048], + ], + point: [114.048868, 22.554927], + circle: [114.058996, 22.549695, 4171], + lineString: [ + [114.047323, 22.534158], + [114.120966, 22.544146], + ], + }; + + await app.runAsCLI(['pm', 'enable', 'map'], { from: 'user' }); + + const fields = [ + { + type: 'point', + name: 'point', + }, + { + type: 'polygon', + name: 'polygon', + }, + { + type: 'circle', + name: 'circle', + }, + { + type: 'lineString', + name: 'lineString', + }, + ]; + + await app.db.getRepository('collections').create({ + values: { + name: 'tests', + fields, + }, + context: {}, + }); + + await app.db.getRepository('tests').create({ + values: { + ...data, + }, + }); + + const dumper = new Dumper(app); + const result = await dumper.dump({ + groups: new Set(['required', 'custom']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.restore({ + groups: new Set(['required', 'custom']), + }); + + const testCollection = app.db.getCollection('tests'); + const tableInfo = await app.db.sequelize.getQueryInterface().describeTable(testCollection.getTableNameWithSchema()); + + expect(tableInfo.point).toBeDefined(); + }); + + it('should dump collection meta', async () => { + await db.getRepository('collections').create({ + values: { + name: 'tests', + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }, + context: {}, + }); + + await db.getRepository('tests').create({ + values: [ + { + name: 'test1', + }, + { + name: 'test2', + }, + ], + }); + + const dumper = new Dumper(app); + await dumper.dumpCollection({ + name: 'tests', + }); + + const collectionDir = path.resolve(dumper.workDir, 'collections', 'tests'); + const metaFile = path.resolve(collectionDir, 'meta'); + const meta = JSON.parse(fs.readFileSync(metaFile, 'utf8')); + + expect(meta.name).toBe('tests'); + const autoIncrement = meta.autoIncrement; + expect(autoIncrement).toBeDefined(); + }); + + it('should save dump meta to dump file', async () => { + const dumper = new Dumper(app); + const result = await dumper.dump({ + groups: new Set(['required']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + const meta = await restorer.parseBackupFile(); + expect(meta.dumpableCollectionsGroupByGroup.required).toBeTruthy(); + + expect(meta.DB_UNDERSCORED).toBeDefined(); + }); + + describe('get file status', function () { + it('should get in progress status', async () => { + const fileName = 'backup_20231111_112233.nbdump'; + const fullPath = path.resolve(__dirname, './fixtures', fileName); + + const status = await Dumper.getFileStatus(fullPath); + expect(status['inProgress']).toBeTruthy(); + }); + + it('should get ok status', async () => { + const dumper = new Dumper(app); + const result = await dumper.dump({ + groups: new Set(['required']), + }); + + const status = await Dumper.getFileStatus(result.filePath); + expect(status['inProgress']).toBeFalsy(); + }); + + it('should throw error when file not exists', async () => { + await expect(Dumper.getFileStatus('not_exists_file')).rejects.toThrowError(); + }); + }); + + it('should run dump task', async () => { + const dumper = new Dumper(app); + + const taskId = await dumper.runDumpTask({ + groups: new Set(['meta']), + }); + + expect(taskId).toBeDefined(); + + const promise = Dumper.getTaskPromise(taskId); + expect(promise).toBeDefined(); + + await promise; + }); + + it('should create dump file name', async () => { + expect(Dumper.generateFileName()).toMatch(/^backup_\d{8}_\d{6}_\d{4}\.nbdump$/); + }); + + it('should get dumped collections by data types', async () => { + await app.db.getRepository('collections').create({ + values: { + name: 'test_collection', + fields: [ + { + name: 'test_field1', + type: 'string', + }, + ], + }, + context: {}, + }); + + const dumper = new Dumper(app); + const collections = await dumper.getCollectionsByDataTypes(new Set(['custom'])); + expect(collections.includes('test_collection')).toBeTruthy(); + }); + + it('should dump collection table structure', async () => { + await app.db.getRepository('collections').create({ + values: { + name: 'test_collection', + fields: [ + { + name: 'test_field1', + type: 'string', + }, + ], + }, + context: {}, + }); + + const dumper = new Dumper(app); + await dumper.dumpCollection({ + name: 'test_collection', + }); + + const collectionDir = path.resolve(dumper.workDir, 'collections', 'test_collection'); + const metaFile = path.resolve(collectionDir, 'meta'); + const meta = JSON.parse(fs.readFileSync(metaFile, 'utf8')); + + const attributes = meta.attributes; + expect(attributes).toBeDefined(); + expect(attributes.id.isCollectionField).toBeFalsy(); + expect(attributes.id.type).toBe('BIGINT'); + + expect(attributes['test_field1'].isCollectionField).toBeTruthy(); + expect(attributes['test_field1'].type).toBe('string'); + }); + + it('should get dumped collections with origin option', async () => { + const dumper = new Dumper(app); + const dumpableCollections = await dumper.dumpableCollections(); + const applicationPlugins = dumpableCollections.find(({ name }) => name === 'applicationPlugins'); + + expect(applicationPlugins.origin).toMatchObject({ + title: 'core', + name: 'core', + }); + }); + + it('should get custom collections group', async () => { + await app.db.getRepository('collections').create({ + values: { + name: 'test_collection', + fields: [ + { + name: 'test_field1', + type: 'string', + }, + ], + }, + context: {}, + }); + + const dumper = new Dumper(app); + const dumpableCollections = await dumper.collectionsGroupByDataTypes(); + + expect(dumpableCollections.custom).toBeDefined(); + }); +}); diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/fixtures/backup_20231111_112233.nbdump.lock b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/fixtures/backup_20231111_112233.nbdump.lock new file mode 100644 index 000000000..e69de29bb diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/fixtures/dump.nbdump.fixture b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/fixtures/dump.nbdump.fixture similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/fixtures/dump.nbdump.fixture rename to packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/fixtures/dump.nbdump.fixture diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/fixtures/files/backup1.nbdump.lock b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/fixtures/files/backup1.nbdump.lock new file mode 100644 index 000000000..e69de29bb diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/index.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/index.ts similarity index 70% rename from packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/index.ts rename to packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/index.ts index c62a78a84..97be68f6c 100644 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/index.ts +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/index.ts @@ -1,10 +1,13 @@ import { mockServer } from '@nocobase/test'; export default async function createApp() { - const app = mockServer(); + const app = mockServer({ + plugins: ['nocobase'], + }); + + await app.cleanDb(); app.plugin((await import('../server')).default, { name: 'duplicator' }); - app.plugin('error-handler'); - app.plugin('collection-manager'); + await app.loadAndInstall({ clean: true }); return app; diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/on-dump.test.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/on-dump.test.ts new file mode 100644 index 000000000..aace1da75 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/__tests__/on-dump.test.ts @@ -0,0 +1,71 @@ +import { MockServer } from '@nocobase/test'; +import { Collection, Database } from '@nocobase/database'; +import createApp from './index'; +import { Dumper } from '../dumper'; +import { Restorer } from '../restorer'; +import path from 'path'; +import fs from 'fs'; + +describe('on dump', () => { + let app: MockServer; + let db: Database; + + beforeEach(async () => { + app = await createApp(); + db = app.db; + }); + + afterEach(async () => { + await app.destroy(); + }); + + it('should handle collection onDump api', async () => { + class OnDumpCollection extends Collection {} + + app.db.collectionFactory.registerCollectionType(OnDumpCollection, { + condition(options) { + return options.onDump; + }, + + async onSync(model, options) { + return; + }, + + async onDump(dumper: Dumper, collection) { + dumper.writeSQLContent('onDumpCollection', { + sql: `CREATE TABLE ${collection.getTableNameWithSchemaAsString()} (id int);`, + group: 'required', + }); + }, + }); + + await app.db.getCollection('collections').repository.create({ + values: { + name: 'onDumpCollection', + title: 'onDumpCollection', + onDump: true, + }, + context: {}, + }); + + const dumper = new Dumper(app); + + const result = await dumper.dump({ + groups: new Set(['required']), + }); + + const restorer = new Restorer(app, { + backUpFilePath: result.filePath, + }); + + await restorer.parseBackupFile(); + + const sqlContentPath = path.resolve(restorer.workDir, 'sql-content.json'); + const sqlContent = JSON.parse(await fs.promises.readFile(sqlContentPath, 'utf8')); + + expect(sqlContent).toBeDefined(); + await restorer.restore({ + groups: new Set(['required', 'custom']), + }); + }); +}); diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/app-migrator.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/app-migrator.ts new file mode 100644 index 000000000..49a1eacf1 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/app-migrator.ts @@ -0,0 +1,41 @@ +import { Application } from '@nocobase/server'; +import { applyMixins, AsyncEmitter } from '@nocobase/utils'; +import crypto from 'crypto'; +import EventEmitter from 'events'; +import fsPromises from 'fs/promises'; +import * as os from 'os'; +import path from 'path'; + +export type AppMigratorOptions = { + workDir?: string; +}; +abstract class AppMigrator extends EventEmitter { + public readonly workDir: string; + public app: Application; + + abstract direction: 'restore' | 'dump'; + + declare emitAsync: (event: string | symbol, ...args: any[]) => Promise; + + constructor(app: Application, options?: AppMigratorOptions) { + super(); + + this.app = app; + this.workDir = options?.workDir || this.tmpDir(); + } + + tmpDir() { + return path.resolve(os.tmpdir(), `nocobase-${crypto.randomUUID()}`); + } + + async rmDir(dir: string) { + await fsPromises.rm(dir, { recursive: true, force: true }); + } + + async clearWorkDir() { + await this.rmDir(this.workDir); + } +} + +applyMixins(AppMigrator, [AsyncEmitter]); +export { AppMigrator }; diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/collection-group-manager.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/collection-group-manager.ts new file mode 100644 index 000000000..40bda315f --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/collection-group-manager.ts @@ -0,0 +1,5 @@ +import { CollectionGroup } from '@nocobase/database'; + +export class CollectionGroupManager { + static collectionGroups: CollectionGroup[] = []; +} diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/commands/restore-command.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/commands/restore-command.ts new file mode 100644 index 000000000..0753b5966 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/commands/restore-command.ts @@ -0,0 +1,67 @@ +import { Application, AppSupervisor } from '@nocobase/server'; +import { Restorer } from '../restorer'; +import { DumpRulesGroupType } from '@nocobase/database'; + +export default function addRestoreCommand(app: Application) { + app + .command('restore') + .ipc() + .argument('', 'restore file path') + .option('-a, --app ', 'sub app name if you want to restore into a sub app') + .option('-f, --force', 'force restore') + .option( + '-g, --groups ', + 'groups to restore', + (value, previous) => { + return previous.concat([value]); + }, + [], + ) + .action(async (restoreFilePath, options) => { + // should confirm data will be overwritten + if (!options.force) { + app.log.warn('This action will overwrite your current data, please make sure you have a backup❗️❗️'); + return; + } + + let importApp = app; + + if (options.app) { + if ( + !(await app.db.getCollection('applications').repository.findOne({ + filter: { name: options.app }, + })) + ) { + // create sub app if not exists + await app.db.getCollection('applications').repository.create({ + values: { + name: options.app, + }, + }); + } + + const subApp = await AppSupervisor.getInstance().getApp(options.app); + + if (!subApp) { + app.log.error(`app ${options.app} not found`); + await app.stop(); + return; + } + + importApp = subApp; + } + + const groups: Set = new Set(options.groups); + groups.add('required'); + + const restorer = new Restorer(importApp, { + backUpFilePath: restoreFilePath, + }); + + await restorer.restore({ + groups, + }); + + await app.restart(); + }); +} diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/dumper.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/dumper.ts new file mode 100644 index 000000000..9f56b6f14 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/dumper.ts @@ -0,0 +1,543 @@ +import { Collection, CollectionGroupManager as DBCollectionGroupManager, DumpRulesGroupType } from '@nocobase/database'; +import archiver from 'archiver'; +import dayjs from 'dayjs'; +import fs from 'fs'; +import fsPromises from 'fs/promises'; +import { default as _, default as lodash } from 'lodash'; +import mkdirp from 'mkdirp'; +import path from 'path'; +import * as process from 'process'; +import stream from 'stream'; +import util from 'util'; +import { AppMigrator } from './app-migrator'; +import { FieldValueWriter } from './field-value-writer'; +import { DUMPED_EXTENSION, humanFileSize, sqlAdapter } from './utils'; + +const finished = util.promisify(stream.finished); + +type DumpOptions = { + groups: Set; + fileName?: string; +}; + +type BackUpStatusOk = { + name: string; + createdAt: Date; + fileSize: string; + status: 'ok'; +}; + +type BackUpStatusDoing = { + name: string; + inProgress: true; + status: 'in_progress'; +}; + +export class Dumper extends AppMigrator { + static dumpTasks: Map> = new Map(); + + direction = 'dump' as const; + + sqlContent: { + [key: string]: { + sql: string | string[]; + group: DumpRulesGroupType; + }; + } = {}; + + static getTaskPromise(taskId: string): Promise | undefined { + return this.dumpTasks.get(taskId); + } + + static async getFileStatus(filePath: string): Promise { + const lockFile = filePath + '.lock'; + const fileName = path.basename(filePath); + + return fs.promises + .stat(lockFile) + .then((lockFileStat) => { + if (lockFileStat.isFile()) { + return { + name: fileName, + inProgress: true, + status: 'in_progress', + } as BackUpStatusDoing; + } else { + throw new Error('Lock file is not a file'); + } + }) + .catch((error) => { + // 如果 Lock 文件不存在，检查备份文件 + if (error.code === 'ENOENT') { + return fs.promises.stat(filePath).then((backupFileStat) => { + if (backupFileStat.isFile()) { + return { + name: fileName, + createdAt: backupFileStat.birthtime, + fileSize: humanFileSize(backupFileStat.size), + status: 'ok', + } as BackUpStatusOk; + } else { + throw new Error('Path is not a file'); + } + }); + } + // 其他错误直接抛出 + throw error; + }); + } + + static generateFileName() { + return `backup_${dayjs().format(`YYYYMMDD_HHmmss_${Math.floor(1000 + Math.random() * 9000)}`)}.${DUMPED_EXTENSION}`; + } + + writeSQLContent( + key: string, + data: { + sql: string | string[]; + group: DumpRulesGroupType; + }, + ) { + this.sqlContent[key] = data; + } + + getSQLContent(key: string) { + return this.sqlContent[key]; + } + + async getCollectionsByDataTypes(groups: Set): Promise { + const dumpableCollectionsGroupByDataTypes = await this.collectionsGroupByDataTypes(); + + return [...groups].reduce((acc, key) => { + return acc.concat(dumpableCollectionsGroupByDataTypes[key] || []); + }, []); + } + + async dumpableCollections() { + return ( + await Promise.all( + [...this.app.db.collections.values()].map(async (c) => { + try { + const dumpRules = DBCollectionGroupManager.unifyDumpRules(c.options.dumpRules); + let origin = c.origin; + let originTitle = origin; + + // plugin collections + if (origin.startsWith('plugin:')) { + const plugin = this.app.pm.get(origin.replace(/^plugin:/, '')); + const pluginInfo = await plugin.toJSON({ + withOutOpenFile: true, + }); + + originTitle = pluginInfo.displayName; + origin = pluginInfo.packageName; + } + + // user collections + if (origin === 'collection-manager') { + originTitle = 'user'; + origin = 'user'; + } + + const options: any = { + name: c.name, + title: c.options.title || c.name, + options: c.options, + group: dumpRules?.group, + isView: c.isView(), + origin: { + name: origin, + title: originTitle, + }, + }; + + if (c.options.inherits && c.options.inherits.length > 0) { + options.inherits = c.options.inherits; + } + + return options; + } catch (e) { + console.error(e); + throw new Error(`collection ${c.name} has invalid dumpRules option`, { cause: e }); + } + }), + ) + ).map((item) => { + if (!item.group) { + item.group = 'unknown'; + } + + return item; + }); + } + + async collectionsGroupByDataTypes() { + const grouped = lodash.groupBy(await this.dumpableCollections(), 'group'); + + return Object.fromEntries(Object.entries(grouped).map(([key, value]) => [key, value.map((item) => item.name)])); + } + + backUpStorageDir() { + return path.resolve(process.cwd(), 'storage', 'backups'); + } + + async allBackUpFilePaths(options?: { includeInProgress?: boolean; dir?: string }) { + const dirname = options?.dir || this.backUpStorageDir(); + const includeInProgress = options?.includeInProgress; + + try { + const files = await fsPromises.readdir(dirname); + + const lockFilesSet = new Set( + files.filter((file) => path.extname(file) === '.lock').map((file) => path.basename(file, '.lock')), + ); + + const filteredFiles = files + .filter((file) => { + const baseName = path.basename(file); + const isLockFile = path.extname(file) === '.lock'; + const isDumpFile = path.extname(file) === `.${DUMPED_EXTENSION}`; + + return (includeInProgress && isLockFile) || (isDumpFile && !lockFilesSet.has(baseName)); + }) + .map(async (file) => { + const filePath = path.resolve(dirname, file); + const stats = await fsPromises.stat(filePath); + return { filePath, birthtime: stats.birthtime.getTime() }; + }); + + const filesData = await Promise.all(filteredFiles); + + filesData.sort((a, b) => b.birthtime - a.birthtime); + + return filesData.map((fileData) => fileData.filePath); + } catch (error) { + if (!error.message.includes('no such file or directory')) { + console.error('Error reading directory:', error); + } + return []; + } + } + + backUpFilePath(fileName: string) { + const dirname = this.backUpStorageDir(); + return path.resolve(dirname, fileName); + } + + lockFilePath(fileName: string) { + const lockFile = fileName + '.lock'; + const dirname = this.backUpStorageDir(); + return path.resolve(dirname, lockFile); + } + + async writeLockFile(fileName: string) { + const dirname = this.backUpStorageDir(); + await mkdirp(dirname); + + const filePath = this.lockFilePath(fileName); + await fsPromises.writeFile(filePath, 'lock', 'utf8'); + } + + async cleanLockFile(fileName: string) { + const filePath = this.lockFilePath(fileName); + await fsPromises.unlink(filePath); + } + + async runDumpTask(options: Omit) { + const backupFileName = Dumper.generateFileName(); + await this.writeLockFile(backupFileName); + + const promise = this.dump({ + groups: options.groups, + fileName: backupFileName, + }).finally(() => { + this.cleanLockFile(backupFileName); + Dumper.dumpTasks.delete(backupFileName); + }); + + Dumper.dumpTasks.set(backupFileName, promise); + + return backupFileName; + } + + async dumpableCollectionsGroupByGroup() { + return _(await this.dumpableCollections()) + .map((c) => _.pick(c, ['name', 'group', 'origin', 'title', 'isView', 'inherits'])) + .groupBy('group') + .mapValues((items) => _.sortBy(items, (item) => item.name)) + .value(); + } + + async dump(options: DumpOptions) { + const dumpingGroups = options.groups; + dumpingGroups.add('required'); + + const delayCollections = new Set(); + const dumpedCollections = await this.getCollectionsByDataTypes(dumpingGroups); + + for (const collectionName of dumpedCollections) { + const collection = this.app.db.getCollection(collectionName); + if (lodash.get(collection.options, 'duplicator.delayRestore')) { + delayCollections.add(collectionName); + } + + await this.dumpCollection({ + name: collectionName, + }); + } + + await this.dumpMeta({ + dumpableCollectionsGroupByGroup: lodash.pick(await this.dumpableCollectionsGroupByGroup(), [...dumpingGroups]), + dumpedGroups: [...dumpingGroups], + delayCollections: [...delayCollections], + }); + + await this.dumpDb(options); + + const backupFileName = options.fileName || Dumper.generateFileName(); + const filePath = await this.packDumpedDir(backupFileName); + await this.clearWorkDir(); + return filePath; + } + + async dumpDb(options: DumpOptions) { + for (const collection of this.app.db.collections.values()) { + const collectionOnDumpOption = this.app.db.collectionFactory.collectionTypes.get( + collection.constructor as typeof Collection, + )?.onDump; + + if (collectionOnDumpOption) { + await collectionOnDumpOption(this, collection); + } + } + + if (this.hasSqlContent()) { + const dbDumpPath = path.resolve(this.workDir, 'sql-content.json'); + + await fsPromises.writeFile( + dbDumpPath, + JSON.stringify( + Object.keys(this.sqlContent) + .filter((key) => options.groups.has(this.sqlContent[key].group)) + .reduce((acc, key) => { + acc[key] = this.sqlContent[key]; + return acc; + }, {}), + ), + 'utf8', + ); + } + } + + hasSqlContent() { + return Object.keys(this.sqlContent).length > 0; + } + + async dumpMeta(additionalMeta: object = {}) { + const metaPath = path.resolve(this.workDir, 'meta'); + + const metaObj = { + version: await this.app.version.get(), + dialect: this.app.db.sequelize.getDialect(), + DB_UNDERSCORED: process.env.DB_UNDERSCORED, + DB_TABLE_PREFIX: process.env.DB_TABLE_PREFIX, + DB_SCHEMA: process.env.DB_SCHEMA, + COLLECTION_MANAGER_SCHEMA: process.env.COLLECTION_MANAGER_SCHEMA, + ...additionalMeta, + }; + + if (this.app.db.inDialect('postgres')) { + if (this.app.db.inheritanceMap.nodes.size > 0) { + metaObj['dialectOnly'] = true; + } + } + + if (this.hasSqlContent()) { + metaObj['dialectOnly'] = true; + } + + await fsPromises.writeFile(metaPath, JSON.stringify(metaObj), 'utf8'); + } + + async dumpCollection(options: { name: string }) { + const app = this.app; + const dir = this.workDir; + + const collectionName = options.name; + app.log.info(`dumping collection ${collectionName}`); + + const collection = app.db.getCollection(collectionName); + + if (!collection) { + this.app.log.warn(`collection ${collectionName} not found`); + return; + } + + const collectionOnDumpOption = this.app.db.collectionFactory.collectionTypes.get( + collection.constructor as typeof Collection, + )?.onDump; + + if (collectionOnDumpOption) { + return; + } + + // @ts-ignore + const attributes = collection.model.tableAttributes; + + // @ts-ignore + const columns: string[] = [...new Set(lodash.map(attributes, 'field'))]; + + const collectionDataDir = path.resolve(dir, 'collections', collectionName); + + await fsPromises.mkdir(collectionDataDir, { recursive: true }); + + let count = 0; + + if (columns.length !== 0) { + // write collection data + const dataFilePath = path.resolve(collectionDataDir, 'data'); + const dataStream = fs.createWriteStream(dataFilePath); + + const rows = await app.db.sequelize.query( + sqlAdapter( + app.db, + `SELECT * + FROM ${collection.isParent() ? 'ONLY' : ''} ${collection.quotedTableName()}`, + ), + { + type: 'SELECT', + }, + ); + + for (const row of rows) { + const rowData = JSON.stringify( + columns.map((col) => { + const val = row[col]; + const field = collection.getField(col); + + return field ? FieldValueWriter.toDumpedValue(field, val) : val; + }), + ); + + dataStream.write(rowData + '\r\n', 'utf8'); + } + + dataStream.end(); + await finished(dataStream); + + count = rows.length; + } + + const metaAttributes = lodash.mapValues(attributes, (attr, key) => { + const collectionField = collection.getField(key); + const fieldOptionKeys = ['field', 'primaryKey', 'autoIncrement', 'allowNull', 'defaultValue', 'unique']; + + if (collectionField) { + // is a field + return { + field: attr.field, + isCollectionField: true, + type: collectionField.type, + typeOptions: collectionField.options, + }; + } + + return { + ...lodash.pick(attr, fieldOptionKeys), + type: attr.type.constructor.toString(), + isCollectionField: false, + typeOptions: attr.type.options, + }; + }); + + const meta = { + name: collectionName, + tableName: collection.getTableNameWithSchema(), + count, + columns, + attributes: metaAttributes, + }; + + if (collection.options.inherits) { + meta['inherits'] = lodash.uniq(collection.options.inherits); + } + + // @ts-ignore + const autoIncrAttr = collection.model.autoIncrementAttribute; + + if ( + autoIncrAttr && + collection.model.rawAttributes[autoIncrAttr] && + collection.model.rawAttributes[autoIncrAttr].autoIncrement + ) { + const queryInterface = app.db.queryInterface; + const autoIncrInfo = await queryInterface.getAutoIncrementInfo({ + tableInfo: { + tableName: collection.model.tableName, + schema: collection.collectionSchema(), + }, + fieldName: autoIncrAttr, + }); + + meta['autoIncrement'] = { + ...autoIncrInfo, + fieldName: autoIncrAttr, + }; + } + + // write meta file + await fsPromises.writeFile(path.resolve(collectionDataDir, 'meta'), JSON.stringify(meta), 'utf8'); + } + + async packDumpedDir(fileName: string) { + const dirname = this.backUpStorageDir(); + await mkdirp(dirname); + + const filePath = path.resolve(dirname, fileName); + const output = fs.createWriteStream(filePath); + + const archive = archiver('zip', { + zlib: { level: 9 }, + }); + + // Create a promise that resolves when the 'close' event is fired + const onClose = new Promise((resolve, reject) => { + output.on('close', function () { + console.log('dumped file size: ' + humanFileSize(archive.pointer(), true)); + resolve(true); + }); + + output.on('end', function () { + console.log('Data has been drained'); + }); + + archive.on('warning', function (err) { + if (err.code === 'ENOENT') { + // log warning + } else { + // throw error + reject(err); + } + }); + + archive.on('error', function (err) { + reject(err); + }); + }); + + archive.pipe(output); + + archive.directory(this.workDir, false); + + // Finalize the archive + await archive.finalize(); + + // Wait for the 'close' event + await onClose; + + return { + filePath, + dirname, + }; + } +} diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/errors/restore-check-error.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/errors/restore-check-error.ts new file mode 100644 index 000000000..a09086e96 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/errors/restore-check-error.ts @@ -0,0 +1,6 @@ +export class RestoreCheckError extends Error { + constructor(message) { + super(message); + this.name = 'RestoreCheckError'; + } +} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/field-value-writer.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/field-value-writer.ts similarity index 87% rename from packages/plugins/@nocobase/plugin-duplicator/src/server/field-value-writer.ts rename to packages/plugins/@nocobase/plugin-backup-restore/src/server/field-value-writer.ts index 05b1139b6..946a679c7 100644 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/field-value-writer.ts +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/field-value-writer.ts @@ -58,9 +58,18 @@ export class FieldValueWriter { } } +function isJSONObjectOrArrayString(str) { + try { + const parsed = JSON.parse(str); + return typeof parsed === 'object' && parsed !== null; + } catch (e) { + return false; + } +} + FieldValueWriter.registerWriter([DataTypes.JSON.toString(), DataTypes.JSONB.toString()], (val) => { try { - return lodash.isString(val) ? JSON.parse(val) : val; + return isJSONObjectOrArrayString(val) ? JSON.parse(val) : val; } catch (err) { if (err instanceof SyntaxError && err.message.includes('Unexpected')) { return val; diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/index.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/index.ts similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/src/server/index.ts rename to packages/plugins/@nocobase/plugin-backup-restore/src/server/index.ts diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/locale/zh-CN.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/locale/zh-CN.ts similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/src/server/locale/zh-CN.ts rename to packages/plugins/@nocobase/plugin-backup-restore/src/server/locale/zh-CN.ts diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/resourcers/backup-files.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/resourcers/backup-files.ts new file mode 100644 index 000000000..e8f5a5bb7 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/resourcers/backup-files.ts @@ -0,0 +1,212 @@ +import { Dumper } from '../dumper'; +import { DumpRulesGroupType } from '@nocobase/database'; +import fs from 'fs'; +import { koaMulter as multer } from '@nocobase/utils'; +import os from 'os'; +import path from 'path'; +import fsPromises from 'fs/promises'; +import { Restorer } from '../restorer'; +import { DEFAULT_PAGE, DEFAULT_PER_PAGE } from '@nocobase/actions'; + +export default { + name: 'backupFiles', + middleware: async (ctx, next) => { + if (ctx.action.actionName !== 'upload') { + return next(); + } + + const storage = multer.diskStorage({ + destination: os.tmpdir(), + filename: function (req, file, cb) { + const randomName = Date.now().toString() + Math.random().toString().slice(2); // 随机生成文件名 + cb(null, randomName); + }, + }); + + const upload = multer({ storage }).single('file'); + return upload(ctx, next); + }, + actions: { + async list(ctx, next) { + const { page = DEFAULT_PAGE, pageSize = DEFAULT_PER_PAGE } = ctx.action.params; + + const dumper = new Dumper(ctx.app); + const backupFiles = await dumper.allBackUpFilePaths({ + includeInProgress: true, + }); + + // handle pagination + const count = backupFiles.length; + + const rows = await Promise.all( + backupFiles.slice((page - 1) * pageSize, page * pageSize).map(async (file) => { + // if file is lock file, remove lock extension + return await Dumper.getFileStatus(file.endsWith('.lock') ? file.replace('.lock', '') : file); + }), + ); + + ctx.body = { + count, + rows, + page: Number(page), + pageSize: Number(pageSize), + totalPage: Math.ceil(count / pageSize), + }; + + await next(); + }, + async get(ctx, next) { + const { filterByTk } = ctx.action.params; + const dumper = new Dumper(ctx.app); + const filePath = dumper.backUpFilePath(filterByTk); + + async function sendError(message, status = 404) { + ctx.body = { status: 'error', message }; + ctx.status = status; + } + + try { + const fileState = await Dumper.getFileStatus(filePath); + if (fileState.status !== 'ok') { + await sendError(`Backup file ${filterByTk} not found`); + } else { + const restorer = new Restorer(ctx.app, { + backUpFilePath: filePath, + }); + + const restoreMeta = await restorer.parseBackupFile(); + + ctx.body = { + ...fileState, + meta: restoreMeta, + }; + } + } catch (e) { + if (e.code === 'ENOENT') { + await sendError(`Backup file ${filterByTk} not found`); + } + } + + await next(); + }, + + /** + * create dump task + * @param ctx + * @param next + */ + async create(ctx, next) { + const data = < + { + dataTypes: string[]; + } + >ctx.request.body; + + const dumper = new Dumper(ctx.app); + + const taskId = await dumper.runDumpTask({ + groups: new Set(data.dataTypes) as Set, + }); + + ctx.body = { + key: taskId, + }; + + await next(); + }, + + /** + * download backup file + * @param ctx + * @param next + */ + async download(ctx, next) { + const { filterByTk } = ctx.action.params; + const dumper = new Dumper(ctx.app); + + const filePath = dumper.backUpFilePath(filterByTk); + + const fileState = await Dumper.getFileStatus(filePath); + + if (fileState.status !== 'ok') { + throw new Error(`Backup file ${filterByTk} not found`); + } + + ctx.attachment(filePath); + ctx.body = fs.createReadStream(filePath); + await next(); + }, + + async restore(ctx, next) { + const { dataTypes, filterByTk, key } = ctx.action.params.values; + + const filePath = (() => { + if (key) { + const tmpDir = os.tmpdir(); + return path.resolve(tmpDir, key); + } + + if (filterByTk) { + const dumper = new Dumper(ctx.app); + return dumper.backUpFilePath(filterByTk); + } + })(); + + if (!filePath) { + throw new Error(`Backup file ${filterByTk} not found`); + } + + const args = ['restore', '-f', filePath]; + + for (const dataType of dataTypes) { + args.push('-g', dataType); + } + + await ctx.app.runCommand(...args); + + await next(); + }, + + async destroy(ctx, next) { + const { filterByTk } = ctx.action.params; + const dumper = new Dumper(ctx.app); + const filePath = dumper.backUpFilePath(filterByTk); + + await fsPromises.unlink(filePath); + + // remove file + ctx.body = { + status: 'ok', + }; + await next(); + }, + + async upload(ctx, next) { + const file = ctx.file; + const fileName = file.filename; + + const restorer = new Restorer(ctx.app, { + backUpFilePath: file.path, + }); + + const restoreMeta = await restorer.parseBackupFile(); + + ctx.body = { + key: fileName, + meta: restoreMeta, + }; + + await next(); + }, + + async dumpableCollections(ctx, next) { + ctx.withoutDataWrapping = true; + + const dumper = new Dumper(ctx.app); + + ctx.body = await dumper.dumpableCollectionsGroupByGroup(); + + await next(); + }, + }, +}; diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/restorer.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/restorer.ts new file mode 100644 index 000000000..2331fec26 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/restorer.ts @@ -0,0 +1,405 @@ +import decompress from 'decompress'; +import fs from 'fs'; +import fsPromises from 'fs/promises'; +import path from 'path'; +import { AppMigrator, AppMigratorOptions } from './app-migrator'; +import { readLines } from './utils'; +import { Application } from '@nocobase/server'; +import { DataTypes, DumpRulesGroupType } from '@nocobase/database'; +import lodash, { isPlainObject } from 'lodash'; +import { FieldValueWriter } from './field-value-writer'; +import * as Topo from '@hapi/topo'; +import { RestoreCheckError } from './errors/restore-check-error'; +import semver from 'semver'; + +type RestoreOptions = { + groups: Set; +}; + +export class Restorer extends AppMigrator { + direction = 'restore' as const; + backUpFilePath: string; + decompressed = false; + importedCollections: string[] = []; + + constructor( + app: Application, + options: AppMigratorOptions & { + backUpFilePath?: string; + }, + ) { + super(app, options); + const { backUpFilePath } = options; + + if (backUpFilePath) { + this.setBackUpFilePath(backUpFilePath); + } + } + + static sortCollectionsByInherits( + collections: Array<{ + name: string; + inherits: string[]; + }>, + ): any { + const sorter = new Topo.Sorter(); + + for (const collection of collections) { + const options: any = { + group: collection.name, + }; + + if (collection.inherits?.length) { + options.after = collection.inherits; + } + sorter.add(collection, options); + } + + return sorter.sort(); + } + + setBackUpFilePath(backUpFilePath: string) { + if (path.isAbsolute(backUpFilePath)) { + this.backUpFilePath = backUpFilePath; + } else if (path.basename(backUpFilePath) === backUpFilePath) { + const dirname = path.resolve(process.cwd(), 'storage', 'duplicator'); + this.backUpFilePath = path.resolve(dirname, backUpFilePath); + } else { + this.backUpFilePath = path.resolve(process.cwd(), backUpFilePath); + } + } + + async parseBackupFile() { + await this.decompressBackup(this.backUpFilePath); + return await this.getImportMeta(); + } + + async restore(options: RestoreOptions) { + await this.decompressBackup(this.backUpFilePath); + await this.checkMeta(); + await this.importCollections(options); + await this.importDb(options); + await this.upgradeApp(); + await this.clearWorkDir(); + } + + async getImportMeta() { + const metaFile = path.resolve(this.workDir, 'meta'); + return JSON.parse(await fsPromises.readFile(metaFile, 'utf8')) as any; + } + + async checkMeta() { + const meta = await this.getImportMeta(); + + if (meta['dialectOnly'] && !this.app.db.inDialect(meta['dialect'])) { + throw new RestoreCheckError(`this backup file can only be imported in database ${meta['dialect']}`); + } + + const checkEnv = (envName: string) => { + const valueInPackage = meta[envName] || ''; + const valueInEnv = process.env[envName] || ''; + + if (valueInPackage && valueInEnv !== valueInPackage) { + throw new RestoreCheckError(`for use this backup file, please set ${envName}=${valueInPackage}`); + } + }; + + for (const envName of ['DB_UNDERSCORED', 'DB_SCHEMA', 'COLLECTION_MANAGER_SCHEMA', 'DB_TABLE_PREFIX']) { + checkEnv(envName); + } + + const version = meta['version']; + if (semver.lt(version, '0.18.0-alpha.2')) { + throw new RestoreCheckError(`this backup file can only be imported in nocobase ${version}`); + } + } + + async importCollections(options: RestoreOptions) { + const importCollection = async (collectionName: string) => { + await this.importCollection({ + name: collectionName, + }); + }; + + const { dumpableCollectionsGroupByGroup, delayCollections } = await this.parseBackupFile(); + + // import plugins + await importCollection('applicationPlugins'); + await this.app.reload(); + + // import required collections + const metaCollections = dumpableCollectionsGroupByGroup.required; + + for (const collection of metaCollections) { + if (collection.name === 'applicationPlugins') { + continue; + } + + if (delayCollections.includes(collection.name)) { + continue; + } + + await importCollection(collection.name); + } + + options.groups.delete('required'); + + // import other groups + const importGroups = [...options.groups]; + for (const group of importGroups) { + const collections = dumpableCollectionsGroupByGroup[group]; + if (!collections) { + this.app.log.warn(`group ${group} not found`); + continue; + } + for (const collection of Restorer.sortCollectionsByInherits(collections)) { + await importCollection(collection.name); + } + } + + await this.app.reload(); + + await (this.app.db.getRepository('collections') as any).load(); + + // sync new plugins and new collections from backup file + await this.app.db.sync(); + + for (const collectionName of delayCollections) { + const delayRestore = this.app.db.getCollection(collectionName).options.dumpRules['delayRestore']; + await delayRestore(this); + } + + await this.emitAsync('restoreCollectionsFinished'); + } + + async decompressBackup(backupFilePath: string) { + if (!this.decompressed) await decompress(backupFilePath, this.workDir); + } + + async readCollectionMeta(collectionName: string) { + const dir = this.workDir; + const collectionMetaPath = path.resolve(dir, 'collections', collectionName, 'meta'); + const metaContent = await fsPromises.readFile(collectionMetaPath, 'utf8'); + return JSON.parse(metaContent); + } + + async importCollection(options: { + name: string; + insert?: boolean; + clear?: boolean; + rowCondition?: (row: any) => boolean; + }) { + const app = this.app; + const db = app.db; + + const collectionName = options.name; + + if (!collectionName) { + throw new Error('collection name is required'); + } + + const dir = this.workDir; + + const collectionDataPath = path.resolve(dir, 'collections', collectionName, 'data'); + const collectionMetaPath = path.resolve(dir, 'collections', collectionName, 'meta'); + + try { + await fsPromises.stat(collectionMetaPath); + } catch (e) { + app.logger.info(`${collectionName} has no meta`); + return; + } + + const metaContent = await fsPromises.readFile(collectionMetaPath, 'utf8'); + const meta = JSON.parse(metaContent); + + let addSchemaTableName: any = meta.tableName; + + if (!this.app.db.inDialect('postgres') && isPlainObject(addSchemaTableName)) { + addSchemaTableName = addSchemaTableName.tableName; + } + + const columns = meta['columns']; + + if (columns.length == 0) { + app.logger.info(`${collectionName} has no columns`); + return; + } + + const fieldAttributes = lodash.mapValues(meta.attributes, (attr) => { + if (attr.isCollectionField) { + const fieldClass = db.fieldTypes.get(attr.type); + if (!fieldClass) throw new Error(`field type ${attr.type} not found`); + + return new fieldClass(attr.typeOptions, { + database: db, + }); + } + + return undefined; + }); + + const rawAttributes = lodash.mapValues(meta.attributes, (attr, key) => { + if (attr.isCollectionField) { + const field = fieldAttributes[key]; + return { + ...field.toSequelize(), + field: attr.field, + }; + } + + const DataTypeClass = DataTypes[db.options.dialect as string][attr.type] || DataTypes[attr.type]; + + const obj = { + ...attr, + type: new DataTypeClass(), + }; + + if (attr.defaultValue && ['JSON', 'JSONB', 'JSONTYPE'].includes(attr.type)) { + obj.defaultValue = JSON.stringify(attr.defaultValue); + } + + return obj; + }); + + if (options.clear !== false) { + // drop table + await db.sequelize.getQueryInterface().dropTable(addSchemaTableName, { + cascade: true, + }); + + // create table + await db.sequelize.getQueryInterface().createTable(addSchemaTableName, rawAttributes); + + if (meta.inherits) { + for (const inherit of lodash.uniq(meta.inherits)) { + const parentMeta = await this.readCollectionMeta(inherit as string); + const sql = `ALTER TABLE ${app.db.utils.quoteTable(addSchemaTableName)} INHERIT ${app.db.utils.quoteTable( + parentMeta.tableName, + )};`; + await db.sequelize.query(sql); + } + } + } + + // read file content from collection data + const rows = await readLines(collectionDataPath); + + if (rows.length == 0) { + app.logger.info(`${collectionName} has no data to import`); + this.importedCollections.push(collectionName); + return; + } + + const rowsWithMeta = rows + .map((row) => + JSON.parse(row) + .map((val, index) => [columns[index], val]) + .reduce((carry, [column, val]) => { + const field = fieldAttributes[column]; + carry[column] = field ? FieldValueWriter.write(field, val) : val; + + return carry; + }, {}), + ) + .filter((row) => { + if (options.rowCondition) { + return options.rowCondition(row); + } + + return true; + }); + + if (rowsWithMeta.length == 0) { + app.logger.info(`${collectionName} has no data to import`); + this.importedCollections.push(collectionName); + return; + } + + const insertGeneratorAttributes = lodash.mapKeys(rawAttributes, (value, key) => { + return value.field; + }); + + //@ts-ignore + const sql = db.sequelize.queryInterface.queryGenerator.bulkInsertQuery( + addSchemaTableName, + rowsWithMeta, + {}, + insertGeneratorAttributes, + ); + + if (options.insert === false) { + return sql; + } + + await app.db.sequelize.query(sql, { + type: 'INSERT', + }); + + app.logger.info(`${collectionName} imported with ${rowsWithMeta.length} rows`); + + if (meta.autoIncrement) { + const queryInterface = app.db.queryInterface; + await queryInterface.setAutoIncrementVal({ + tableInfo: isPlainObject(meta.tableName) + ? meta.tableName + : { + schema: 'public', + tableName: meta.tableName, + }, + columnName: meta.autoIncrement.fieldName, + seqName: meta.autoIncrement.seqName, + currentVal: meta.autoIncrement.currentVal, + }); + } + + this.importedCollections.push(collectionName); + } + + async importDb(options: RestoreOptions) { + const sqlContentPath = path.resolve(this.workDir, 'sql-content.json'); + + // if db.sql file not exists, skip import + if (!fs.existsSync(sqlContentPath)) { + return; + } + + // read file content from db.sql + const sqlData = JSON.parse(await fsPromises.readFile(sqlContentPath, 'utf8')); + + const sqlContent = Object.keys(sqlData) + .filter((key) => options.groups.has(sqlData[key].group)) + .reduce((acc, key) => { + acc[key] = sqlData[key]; + return acc; + }, {}); + + const queries = Object.values( + sqlContent as { + [key: string]: { + sql: string | string[]; + group: DumpRulesGroupType; + }; + }, + ); + + for (const sqlData of queries) { + try { + this.app.log.info(`import sql: ${sqlData.sql}`); + for (const sql of lodash.castArray(sqlData.sql)) { + await this.app.db.sequelize.query(sql); + } + } catch (e) { + if (e.name === 'SequelizeDatabaseError') { + this.app.logger.error(e.message); + } else { + throw e; + } + } + } + } + + async upgradeApp() { + await this.app.runCommand('upgrade'); + } +} diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/server/server.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/server.ts new file mode 100644 index 000000000..601508c6d --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/server/server.ts @@ -0,0 +1,13 @@ +import { Plugin } from '@nocobase/server'; +import backupFilesResourcer from './resourcers/backup-files'; +import addRestoreCommand from './commands/restore-command'; + +export default class Duplicator extends Plugin { + beforeLoad() { + addRestoreCommand(this.app); + } + + async load() { + this.app.resourcer.define(backupFilesResourcer); + } +} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/utils.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/server/utils.ts similarity index 100% rename from packages/plugins/@nocobase/plugin-duplicator/src/server/utils.ts rename to packages/plugins/@nocobase/plugin-backup-restore/src/server/utils.ts diff --git a/packages/plugins/@nocobase/plugin-backup-restore/src/swagger/index.ts b/packages/plugins/@nocobase/plugin-backup-restore/src/swagger/index.ts new file mode 100644 index 000000000..0f082b075 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-backup-restore/src/swagger/index.ts @@ -0,0 +1,433 @@ +export default { + info: { + title: 'NocoBase API - Backup & Restore plugin', + }, + tags: [], + paths: { + '/backupFiles:create': { + post: { + summary: 'Create a new backup file', + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/DumpOptions', + }, + }, + }, + }, + responses: { + '200': { + description: 'Dump successful', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + key: { + type: 'string', + }, + }, + }, + }, + }, + }, + }, + }, + }, + '/backupFiles:list': { + get: { + summary: 'Get backup file list', + parameters: [ + { + name: 'page', + in: 'query', + description: 'Page number of item to retrieve', + required: false, + schema: { + type: 'integer', + format: 'int32', + default: 1, + }, + }, + { + name: 'pageSize', + in: 'query', + description: 'Number of item to retrieve per page', + required: false, + schema: { + type: 'integer', + format: 'int32', + default: 10, + }, + }, + ], + + responses: { + '200': { + description: 'A paged array of backup statuses', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + data: { + type: 'array', + items: { + oneOf: [ + { + $ref: '#/components/schemas/BackUpStatusOk', + }, + { + $ref: '#/components/schemas/BackUpStatusDoing', + }, + ], + }, + }, + meta: { + type: 'object', + properties: { + page: { + type: 'integer', + format: 'int32', + }, + pageSize: { + type: 'integer', + format: 'int32', + }, + count: { + type: 'integer', + format: 'int64', + }, + totalPage: { + type: 'integer', + format: 'int32', + }, + }, + }, + }, + required: ['data', 'meta'], + }, + }, + }, + }, + }, + }, + }, + '/backupFiles:get': { + get: { + summary: 'Get backup file info', + parameters: [ + { + name: 'filterByTk', + in: 'query', + required: true, + schema: { + type: 'string', + }, + }, + ], + + responses: { + '200': { + description: 'Status of the backup operation', + content: { + 'application/json': { + schema: { + oneOf: [ + { + $ref: '#/components/schemas/BackUpStatusOk', + }, + { + $ref: '#/components/schemas/BackUpStatusDoing', + }, + ], + }, + }, + }, + }, + }, + }, + }, + '/backupFiles:download': { + get: { + summary: 'Download a backup file', + parameters: [ + { + name: 'filterByTk', + in: 'query', + required: true, + schema: { + type: 'string', + }, + }, + ], + responses: { + '200': { + description: 'Download successful', + content: { + 'application/octet-stream': { + schema: { + type: 'string', + format: 'binary', + }, + }, + }, + }, + }, + }, + }, + '/backupFiles:destroy': { + post: { + summary: 'Destroy a backup file', + parameters: [ + { + name: 'filterByTk', + in: 'query', + required: true, + schema: { + type: 'string', + }, + }, + ], + responses: { + '200': { + description: 'Destroy successful', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + }, + }, + }, + }, + }, + }, + }, + }, + '/backupFiles:upload': { + post: { + summary: 'Upload a backup file', + requestBody: { + required: true, + content: { + 'multipart/form-data': { + schema: { + type: 'object', + properties: { + file: { + type: 'string', + format: 'binary', + }, + }, + }, + }, + }, + }, + + responses: { + 200: { + description: 'Upload successful', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + key: { + type: 'string', + }, + }, + }, + }, + }, + }, + }, + }, + }, + + '/backupFiles:restore': { + post: { + summary: 'Restore from a backup file', + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + filterByTk: { + type: 'string', + }, + dataTypes: { + type: 'array', + items: { + $ref: '#/components/schemas/DumpDataType', + }, + uniqueItems: true, + }, + key: { + type: 'string', + }, + }, + oneOf: [ + { + required: ['filterByTk', 'dataTypes'], + }, + { + required: ['key', 'dataTypes'], + }, + ], + }, + }, + }, + }, + + responses: { + '200': { + description: 'Restore successful', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + }, + }, + }, + }, + }, + }, + }, + }, + '/backupFiles:dumpableCollections': { + get: { + summary: 'Get dumpable collections', + responses: { + '200': { + description: 'A list of dumpable collections', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + meta: { + type: 'array', + items: { + type: 'object', + properties: { + name: { + type: 'string', + }, + title: { + type: 'string', + }, + }, + }, + }, + config: { + type: 'array', + items: { + type: 'object', + properties: { + name: { + type: 'string', + }, + title: { + type: 'string', + }, + }, + }, + }, + business: { + type: 'array', + items: { + type: 'object', + properties: { + name: { + type: 'string', + }, + title: { + type: 'string', + }, + }, + }, + }, + }, + required: ['meta', 'config', 'business'], + }, + }, + }, + }, + }, + }, + }, + }, + + components: { + schemas: { + BackUpStatusOk: { + type: 'object', + properties: { + name: { + type: 'string', + }, + createdAt: { + type: 'string', + format: 'date-time', + }, + fileSize: { + type: 'string', + }, + status: { + type: 'string', + enum: ['ok'], + }, + }, + required: ['name', 'createdAt', 'fileSize', 'status'], + }, + + BackUpStatusDoing: { + type: 'object', + properties: { + name: { + type: 'string', + }, + inProgress: { + type: 'boolean', + enum: [true], + }, + status: { + type: 'string', + enum: ['in_progress'], + }, + }, + required: ['name', 'inProgress', 'status'], + }, + + DumpDataType: { + type: 'string', + enum: ['meta', 'config', 'business'], + }, + DumpOptions: { + type: 'object', + properties: { + dataTypes: { + type: 'array', + items: { + $ref: '#/components/schemas/DumpDataType', + }, + uniqueItems: true, + }, + }, + required: ['dataTypes'], + }, + }, + }, +}; diff --git a/packages/plugins/@nocobase/plugin-charts/src/server/collections/chartsQueries.ts b/packages/plugins/@nocobase/plugin-charts/src/server/collections/chartsQueries.ts index 5a39527c1..1b07e0be5 100644 --- a/packages/plugins/@nocobase/plugin-charts/src/server/collections/chartsQueries.ts +++ b/packages/plugins/@nocobase/plugin-charts/src/server/collections/chartsQueries.ts @@ -1,9 +1,9 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'charts.chartsQueries', - duplicator: 'optional', + dumpRules: 'required', name: 'chartsQueries', + shared: true, fields: [ { name: 'title', diff --git a/packages/plugins/@nocobase/plugin-charts/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-charts/src/server/plugin.ts index 0e1cdc160..84ad0ed3a 100644 --- a/packages/plugins/@nocobase/plugin-charts/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-charts/src/server/plugin.ts @@ -30,9 +30,7 @@ export class ChartsPlugin extends Plugin { } async load() { - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); this.app.resourcer.registerActionHandlers({ 'chartsQueries:getData': getData, diff --git a/packages/plugins/@nocobase/plugin-china-region/src/server/collections/chinaRegions.ts b/packages/plugins/@nocobase/plugin-china-region/src/server/collections/chinaRegions.ts index 38cf7a5b4..d84ead5cc 100644 --- a/packages/plugins/@nocobase/plugin-china-region/src/server/collections/chinaRegions.ts +++ b/packages/plugins/@nocobase/plugin-china-region/src/server/collections/chinaRegions.ts @@ -1,10 +1,8 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'china-region.china-region', - duplicator: 'skip', + dumpRules: 'skipped', name: 'chinaRegions', - title: '中国行政区划', autoGenId: false, fields: [ // 如使用代码作为 id 可能更节省，但由于代码数字最长为 12 字节，除非使用 bigint(64) 才够放置 diff --git a/packages/plugins/@nocobase/plugin-china-region/src/server/index.ts b/packages/plugins/@nocobase/plugin-china-region/src/server/index.ts index 640ab6dea..49cb059d7 100644 --- a/packages/plugins/@nocobase/plugin-china-region/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-china-region/src/server/index.ts @@ -16,9 +16,7 @@ export class PluginChinaRegion extends Plugin { } async load() { - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); this.app.acl.allow('chinaRegions', 'list', 'loggedIn'); diff --git a/packages/plugins/@nocobase/plugin-collection-manager/src/server/__tests__/collections.repository.test.ts b/packages/plugins/@nocobase/plugin-collection-manager/src/server/__tests__/collections.repository.test.ts index 6c7372bc5..e2f098c43 100644 --- a/packages/plugins/@nocobase/plugin-collection-manager/src/server/__tests__/collections.repository.test.ts +++ b/packages/plugins/@nocobase/plugin-collection-manager/src/server/__tests__/collections.repository.test.ts @@ -1,4 +1,4 @@ -import Database, { Collection as DBCollection, HasManyRepository } from '@nocobase/database'; +import Database, { Collection as DBCollection, CollectionGroupManager, HasManyRepository } from '@nocobase/database'; import Application from '@nocobase/server'; import { createApp } from '.'; import CollectionManagerPlugin, { CollectionRepository } from '../index'; @@ -120,6 +120,52 @@ describe('collections repository', () => { }); }); + it('should create collection with optional duplicator option', async () => { + await Collection.repository.create({ + values: { + name: 'tests', + dumpRules: { + group: 'business', + }, + fields: [ + { + type: 'string', + name: 'title', + }, + ], + }, + context: {}, + }); + + const testsCollection = db.getCollection('tests'); + + const duplicator = CollectionGroupManager.unifyDumpRules(testsCollection.options.dumpRules); + expect(duplicator.group).toEqual('business'); + }); + + it('should create collection with required duplicator option', async () => { + await Collection.repository.create({ + values: { + name: 'tests', + dumpRules: { + group: 'required', + }, + fields: [ + { + type: 'string', + name: 'title', + }, + ], + }, + context: {}, + }); + + const testsCollection = db.getCollection('tests'); + + const duplicator = CollectionGroupManager.unifyDumpRules(testsCollection.options.dumpRules); + expect(duplicator.group).toEqual('required'); + }); + it('should create collection with sortable option', async () => { await Collection.repository.create({ values: { diff --git a/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/collectionCategories.ts b/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/collectionCategories.ts index 8fad4a1a8..3f415cdb0 100644 --- a/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/collectionCategories.ts +++ b/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/collectionCategories.ts @@ -1,11 +1,10 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'collection-manager.collections', - duplicator: { - dumpable: 'required', - with: 'collectionCategory', + dumpRules: { + group: 'required', }, + shared: true, name: 'collectionCategories', autoGenId: true, sortable: true, diff --git a/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/collections.ts b/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/collections.ts index d9946567a..9375202d3 100644 --- a/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/collections.ts +++ b/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/collections.ts @@ -1,10 +1,9 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'collection-manager.collections', - duplicator: 'required', + dumpRules: 'required', + shared: true, name: 'collections', - title: '数据表配置', sortable: 'sort', autoGenId: false, model: 'CollectionModel', diff --git a/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/fields.ts b/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/fields.ts index 54d6390b4..4bf585ba7 100644 --- a/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/fields.ts +++ b/packages/plugins/@nocobase/plugin-collection-manager/src/server/collections/fields.ts @@ -1,8 +1,8 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'collection-manager.collections', - duplicator: 'required', + dumpRules: 'required', + shared: true, name: 'fields', autoGenId: false, model: 'FieldModel', diff --git a/packages/plugins/@nocobase/plugin-collection-manager/src/server/models/collection.ts b/packages/plugins/@nocobase/plugin-collection-manager/src/server/models/collection.ts index f861dfc16..bdd772410 100644 --- a/packages/plugins/@nocobase/plugin-collection-manager/src/server/models/collection.ts +++ b/packages/plugins/@nocobase/plugin-collection-manager/src/server/models/collection.ts @@ -22,10 +22,16 @@ export class CollectionModel extends MagicAttributeModel { let collection: Collection; const collectionOptions = { + namespace: 'collections.business', + origin: 'plugin:collection-manager', ...this.get(), fields: [], }; + if (!this.db.inDialect('postgres') && collectionOptions.schema) { + delete collectionOptions.schema; + } + if (this.db.hasCollection(name)) { collection = this.db.getCollection(name); @@ -39,6 +45,10 @@ export class CollectionModel extends MagicAttributeModel { collection.updateOptions(collectionOptions); } else { + if (!collectionOptions.dumpRules) { + lodash.set(collectionOptions, 'dumpRules.group', 'custom'); + } + collection = this.db.collection(collectionOptions); } diff --git a/packages/plugins/@nocobase/plugin-collection-manager/src/server/server.ts b/packages/plugins/@nocobase/plugin-collection-manager/src/server/server.ts index 5930629ba..3981d03de 100644 --- a/packages/plugins/@nocobase/plugin-collection-manager/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-collection-manager/src/server/server.ts @@ -361,8 +361,8 @@ export class CollectionManagerPlugin extends Plugin { this.app.db.extendCollection({ name: 'collectionCategory', - namespace: 'collection-manager', - duplicator: 'required', + dumpRules: 'required', + origin: `plugin:${this.name}`, }); } } diff --git a/packages/plugins/@nocobase/plugin-custom-request/src/server/collections/customRequest.ts b/packages/plugins/@nocobase/plugin-custom-request/src/server/collections/customRequest.ts index a487a2fac..3e14cd9df 100644 --- a/packages/plugins/@nocobase/plugin-custom-request/src/server/collections/customRequest.ts +++ b/packages/plugins/@nocobase/plugin-custom-request/src/server/collections/customRequest.ts @@ -1,10 +1,8 @@ -import { CollectionOptions } from '@nocobase/client'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'custom-requests', - duplicator: 'optional', +export default defineCollection({ + dumpRules: 'required', name: 'customRequests', - title: '{{t("Custom request")}}', fields: [ { type: 'uid', @@ -27,4 +25,4 @@ export default { name: 'options', // 配置的请求参数都放这里 }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-custom-request/src/server/collections/customRequestsRoles.ts b/packages/plugins/@nocobase/plugin-custom-request/src/server/collections/customRequestsRoles.ts index a02c68012..e35b58a54 100644 --- a/packages/plugins/@nocobase/plugin-custom-request/src/server/collections/customRequestsRoles.ts +++ b/packages/plugins/@nocobase/plugin-custom-request/src/server/collections/customRequestsRoles.ts @@ -1,8 +1,6 @@ -import { CollectionOptions } from '@nocobase/client'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'custom-requests', - duplicator: 'optional', +export default defineCollection({ + dumpRules: 'required', name: 'customRequestsRoles', - title: '{{t("Custom request")}}', -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-custom-request/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-custom-request/src/server/plugin.ts index 0bd07553c..21c69a631 100644 --- a/packages/plugins/@nocobase/plugin-custom-request/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-custom-request/src/server/plugin.ts @@ -24,9 +24,7 @@ export class CustomRequestPlugin extends Plugin { } async load() { - await this.db.import({ - directory: resolve(__dirname, './collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); this.app.resource({ name: 'customRequests', diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/client/index.tsx b/packages/plugins/@nocobase/plugin-duplicator/src/client/index.tsx deleted file mode 100644 index fe8d025af..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/client/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import { Plugin } from '@nocobase/client'; - -export class DuplicatorPlugin extends Plugin { - async load() {} -} - -export default DuplicatorPlugin; diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/locale/en-US.json b/packages/plugins/@nocobase/plugin-duplicator/src/locale/en-US.json deleted file mode 100644 index 52a6b3e64..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/locale/en-US.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Select Import data": "Select Import data", - "Select Import Plugins": "Select Import Plugins", - "Select User Collections": "Select User Collections", - "Basic Data": "Basic Data", - "Optional Data": "Optional Data", - "User Data": "User Data" -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/locale/zh-CN.json b/packages/plugins/@nocobase/plugin-duplicator/src/locale/zh-CN.json deleted file mode 100644 index 1d92696fc..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/locale/zh-CN.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Select Import data": "请选择导入数据", - "Select Import Plugins": "请选择导入插件", - "Select User Collections": "请选择用户数据", - "Basic Data": "基础数据", - "Optional Data": "可选数据", - "User Data": "用户数据" -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/api.test.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/api.test.ts deleted file mode 100644 index c024f1cfb..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/api.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { mockServer, MockServer } from '@nocobase/test'; -import path from 'path'; - -describe('duplicator api', () => { - let app: MockServer; - beforeEach(async () => { - app = mockServer(); - app.plugin((await import('../server')).default, { name: 'duplicator' }); - app.plugin('error-handler'); - app.plugin('collection-manager'); - await app.loadAndInstall({ clean: true }); - }); - - afterEach(async () => { - await app.destroy(); - }); - - it('should get collection groups', async () => { - await app.db.getRepository('collections').create({ - values: { - name: 'test_collection', - title: '测试Collection', - fields: [ - { - name: 'test_field1', - type: 'string', - }, - ], - }, - context: {}, - }); - - const collectionGroupsResponse = await app.agent().resource('duplicator').dumpableCollections(); - expect(collectionGroupsResponse.status).toBe(200); - - const data = collectionGroupsResponse.body; - - expect(data['requiredGroups']).toBeTruthy(); - expect(data['optionalGroups']).toBeTruthy(); - expect(data['userCollections']).toBeTruthy(); - }); - - it('should request dump api', async () => { - const dumpResponse = await app.agent().post('/duplicator:dump').send({ - selectedCollectionGroups: [], - selectedUserCollections: [], - }); - - expect(dumpResponse.status).toBe(200); - }); - - it('should request restore api', async () => { - const packageInfoResponse = await app - .agent() - .post('/duplicator:upload') - .attach('file', path.resolve(__dirname, './fixtures/dump.nbdump.fixture')); - - console.log(packageInfoResponse.body); - expect(packageInfoResponse.status).toBe(200); - const data = packageInfoResponse.body.data; - - expect(data['key']).toBeTruthy(); - expect(data['meta']).toBeTruthy(); - - const restoreResponse = await app.agent().post('/duplicator:restore').send({ - restoreKey: data['key'], - selectedOptionalGroups: [], - selectedUserCollections: [], - }); - - expect(restoreResponse.status).toBe(200); - }); -}); diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/collection-group-manager.test.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/collection-group-manager.test.ts deleted file mode 100644 index b1711143c..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/collection-group-manager.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { mockServer, MockServer } from '@nocobase/test'; -import { CollectionGroupManager } from '../collection-group-manager'; - -describe('collection group manager', () => { - let app: MockServer; - beforeEach(async () => { - app = mockServer({ - plugins: ['error-handler', 'collection-manager'], - }); - - await app.loadAndInstall({ - clean: true, - }); - }); - - afterEach(async () => { - await app.destroy(); - }); - - it('should list collection groups from db collections', async () => { - const collectionGroups = CollectionGroupManager.getGroups(app); - - expect(collectionGroups.map((i) => i.function)).toMatchObject([ - 'migration', - 'applicationPlugins', - 'applicationVersion', - 'collections', - ]); - - expect(collectionGroups.find((i) => i.function === 'collections')).toMatchObject({ - namespace: 'collection-manager', - function: 'collections', - collections: ['collectionCategory', 'collectionCategories', 'collections', 'fields'], - dumpable: 'required', - }); - }); -}); diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/dumper.test.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/dumper.test.ts deleted file mode 100644 index c8d5dc87d..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/__tests__/dumper.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { MockServer } from '@nocobase/test'; -import createApp from './index'; -import { Dumper } from '../dumper'; - -describe('dumper', () => { - let app: MockServer; - beforeEach(async () => { - app = await createApp(); - }); - - afterEach(async () => { - await app.destroy(); - }); - - it('should get collection groups', async () => { - await app.db.getRepository('collections').create({ - values: { - name: 'test_collection', - fields: [ - { - name: 'test_field1', - type: 'string', - }, - ], - }, - context: {}, - }); - - const dump = new Dumper(app); - const dumpableCollections = await dump.dumpableCollections(); - - expect((dumpableCollections.requiredGroups || []).length).toBeGreaterThan(0); - expect(dumpableCollections.userCollections[0]['name']).toEqual('test_collection'); - }); -}); diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/dump-action.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/dump-action.ts deleted file mode 100644 index 112773da6..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/dump-action.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Dumper } from '../dumper'; -import send from 'koa-send'; -import { getApp } from './get-app'; - -export default async function dumpAction(ctx, next) { - const data = < - { - selectedOptionalGroupNames: string[]; - selectedUserCollections: string[]; - app?: string; - } - >ctx.request.body; - - const app = await getApp(ctx, data.app); - - const dumper = new Dumper(app); - - const { filePath, dirname } = await dumper.dump(data); - - await send(ctx, filePath.replace(dirname, ''), { - root: dirname, - }); - - await next(); -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/dumpable-collections-action.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/dumpable-collections-action.ts deleted file mode 100644 index d759ba631..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/dumpable-collections-action.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Dumper } from '../dumper'; -import { getApp } from './get-app'; - -export default async function dumpableCollections(ctx, next) { - ctx.withoutDataWrapping = true; - - const app = await getApp(ctx, ctx.request.query.app); - const dumper = new Dumper(app); - - ctx.body = await dumper.dumpableCollections(); - - await next(); -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/get-app.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/get-app.ts deleted file mode 100644 index 8da93d9d3..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/get-app.ts +++ /dev/null @@ -1,14 +0,0 @@ -export async function getApp(ctx, subAppName) { - let app = ctx.app; - - if (subAppName) { - const subApp = await app.appManager.getApplication(subAppName); - if (!subApp) { - throw new Error(`app ${subAppName} not found`); - } - - app = subApp; - } - - return app; -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/get-dict-action.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/get-dict-action.ts deleted file mode 100644 index ad686a087..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/get-dict-action.ts +++ /dev/null @@ -1,36 +0,0 @@ -export default async function getDictAction(ctx, next) { - ctx.withoutDataWrapping = true; - let collectionNames = await ctx.db.getRepository('collections').find(); - collectionNames = collectionNames.map((item) => item.get('name')); - const collections: any[] = []; - for (const [name, collection] of ctx.db.collections) { - const columns: any[] = []; - for (const key in collection.model.rawAttributes) { - if (Object.prototype.hasOwnProperty.call(collection.model.rawAttributes, key)) { - const attribute = collection.model.rawAttributes[key]; - columns.push({ - realName: attribute.field, - name: key, - }); - } - } - const item = { - name, - tableName: collection.model.tableName, - title: collection.options.title, - namespace: collection.options.namespace, - duplicator: collection.options.duplicator, - schema: collection.options.schema || process.env.DB_SCHEMA, - // columns, - }; - if (!item.namespace && collectionNames.includes(name)) { - item.namespace = 'collection-manager'; - if (!item.duplicator) { - item.duplicator = 'optional'; - } - } - collections.push(item); - } - ctx.body = collections; - await next(); -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/restore-action.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/restore-action.ts deleted file mode 100644 index 1526b9d73..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/actions/restore-action.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Restorer } from '../restorer'; -import * as os from 'os'; -import path from 'path'; -import { getApp } from './get-app'; - -export async function restoreAction(ctx, next) { - const { restoreKey, selectedOptionalGroups, selectedUserCollections } = ctx.request.body; - const appName = ctx.request.body.app; - - const tmpDir = os.tmpdir(); - const filePath = path.resolve(tmpDir, restoreKey); - - const app = await getApp(ctx, appName); - - const restorer = new Restorer(app, { - backUpFilePath: filePath, - }); - - await restorer.restore({ - selectedOptionalGroupNames: selectedOptionalGroups, - selectedUserCollections, - }); - - await next(); -} - -export const getPackageContent = async (ctx, next) => { - const file = ctx.file; - const fileName = file.filename; - - const restorer = new Restorer(ctx.app, { - backUpFilePath: file.path, - }); - - const restoreMeta = await restorer.parseBackupFile(); - - ctx.body = { - key: fileName, - meta: restoreMeta, - }; - - await next(); -}; diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/app-migrator.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/app-migrator.ts deleted file mode 100644 index 0ede98347..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/app-migrator.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { Application } from '@nocobase/server'; -import { applyMixins, AsyncEmitter } from '@nocobase/utils'; -import crypto from 'crypto'; -import EventEmitter from 'events'; -import fsPromises from 'fs/promises'; -import lodash from 'lodash'; -import * as os from 'os'; -import path from 'path'; -import { CollectionGroupManager } from './collection-group-manager'; - -export type AppMigratorOptions = { - workDir?: string; -}; -abstract class AppMigrator extends EventEmitter { - protected workDir: string; - public app: Application; - - abstract direction: 'restore' | 'dump'; - - declare emitAsync: (event: string | symbol, ...args: any[]) => Promise; - - constructor(app, options?: AppMigratorOptions) { - super(); - - this.app = app; - this.workDir = options?.workDir || this.tmpDir(); - } - - tmpDir() { - return path.resolve(os.tmpdir(), `nocobase-${crypto.randomUUID()}`); - } - - getPluginCollections(plugins: string | string[]) { - return lodash - .castArray(plugins) - .map((pluginName) => { - return this.app.db.importedFrom.get(pluginName) || []; - }) - .flat(); - } - - async getAppPlugins() { - const plugins = await this.app.db.getCollection('applicationPlugins').repository.find(); - - return lodash.uniq(['core', ...this.app.pm.getAliases(), ...plugins.map((plugin) => plugin.get('name'))]); - } - - async getAppPluginCollectionGroups() { - const plugins = await this.getAppPlugins(); - return CollectionGroupManager.collectionGroups.filter((collectionGroup) => - plugins.includes(collectionGroup.namespace), - ); - } - - async getCustomCollections() { - const collections = await this.app.db.getCollection('collections').repository.find(); - return collections.filter((collection) => !collection.get('isThrough')).map((collection) => collection.get('name')); - } - - async rmDir(dir: string) { - await fsPromises.rm(dir, { recursive: true, force: true }); - } - - async clearWorkDir() { - await this.rmDir(this.workDir); - } - - findThroughCollections(collections: string[]) { - return [ - ...new Set( - collections - .map((collectionName) => this.app.db.getCollection(collectionName)) - .map((collection) => - [...collection.fields.values()].filter((field) => field.through).map((field) => field.through), - ) - .flat(), - ), - ]; - } - - findSequenceFields(collections: string[]) { - return [ - ...new Set( - collections - .map((collectionName) => this.app.db.getCollection(collectionName)) - .map((collection) => [...collection.fields.values()].filter((field) => field.type === 'sequence')) - .flat(), - ), - ]; - } -} - -applyMixins(AppMigrator, [AsyncEmitter]); -export { AppMigrator }; diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/collection-group-manager.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/collection-group-manager.ts deleted file mode 100644 index 368298ae6..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/collection-group-manager.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Application } from '@nocobase/server'; -import { CollectionGroup } from '@nocobase/database'; - -export class CollectionGroupManager { - static collectionGroups: CollectionGroup[] = []; - - static getGroups(app: Application) { - return app.db.collectionGroupManager.getGroups(); - } - - static getGroupsCollections(groups: CollectionGroup[]) { - if (!groups || groups.length == 0) { - return []; - } - - return groups.map((collectionGroup) => collectionGroup.collections).flat(); - } - - static classifyCollectionGroups(collectionGroups: CollectionGroup[]) { - const requiredGroups = collectionGroups.filter((collectionGroup) => collectionGroup.dumpable === 'required'); - const optionalGroups = collectionGroups.filter((collectionGroup) => collectionGroup.dumpable === 'optional'); - - return { - requiredGroups, - optionalGroups, - }; - } -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/dump-command.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/dump-command.ts deleted file mode 100644 index fc044eeb2..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/dump-command.ts +++ /dev/null @@ -1,50 +0,0 @@ -import inquirer from 'inquirer'; -import { Application, AppSupervisor } from '@nocobase/server'; -import { Dumper } from '../dumper'; -import InquireQuestionBuilder from './inquire-question-builder'; - -export default function addDumpCommand(app: Application) { - app - .command('dump') - .option('-a, --app ', 'sub app name if you dump sub app in multiple apps') - .action(async (options) => { - let dumpApp = app; - - if (options.app) { - const subApp = await AppSupervisor.getInstance().getApp(options.app); - - if (!subApp) { - app.log.error(`app ${options.app} not found`); - await app.stop(); - return; - } - - dumpApp = subApp; - } - - await dumpCommandAction(dumpApp); - }); -} - -async function dumpCommandAction(app) { - const dumper = new Dumper(app); - const { requiredGroups, optionalGroups, userCollections } = await dumper.dumpableCollections(); - - const questions = InquireQuestionBuilder.buildInquirerQuestions({ - requiredGroups, - optionalGroups, - optionalCollections: userCollections, - direction: 'dump', - }); - - const results = await inquirer.prompt(questions); - - const { filePath } = await dumper.dump({ - selectedOptionalGroupNames: results.collectionGroups, - selectedUserCollections: results.userCollections, - }); - - app.log.info(`dumped to ${filePath}`); - - await app.stop(); -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/inquire-question-builder.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/inquire-question-builder.ts deleted file mode 100644 index 57f56a32e..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/inquire-question-builder.ts +++ /dev/null @@ -1,72 +0,0 @@ -import inquirer from 'inquirer'; -import { CollectionGroup } from '@nocobase/database'; - -export default class InquireQuestionBuilder { - static buildInquirerQuestions(options: { - requiredGroups: CollectionGroup[]; - optionalGroups: CollectionGroup[]; - optionalCollections: { - name: string; - title: string; - }[]; - direction: 'dump' | 'restore'; - }) { - const { requiredGroups, optionalGroups, optionalCollections, direction } = options; - const questions = [this.buildInquirerPluginQuestion(requiredGroups, optionalGroups, direction)]; - - if (optionalCollections.length > 0) { - questions.push(this.buildInquirerCollectionQuestion(optionalCollections, direction)); - } - - return questions; - } - - static buildInquirerPluginQuestion(requiredGroups, optionalGroups, direction: 'dump' | 'restore') { - return { - type: 'checkbox', - name: 'collectionGroups', - message: `Select the plugin collections to be ${direction === 'dump' ? 'dumped' : 'restored'}`, - loop: false, - pageSize: 20, - choices: [ - new inquirer.Separator('== Required =='), - ...requiredGroups.map((collectionGroup) => ({ - name: `${collectionGroup.function} (${collectionGroup.namespace})`, - value: `${collectionGroup.namespace}.${collectionGroup.function}`, - checked: true, - disabled: true, - })), - - new inquirer.Separator('== Optional =='), - ...optionalGroups.map((collectionGroup) => ({ - name: `${collectionGroup.function} (${collectionGroup.namespace})`, - value: `${collectionGroup.namespace}.${collectionGroup.function}`, - checked: direction === 'dump', - })), - ], - }; - } - - static buildInquirerCollectionQuestion( - collections: { - name: string; - title: string; - }[], - direction: 'dump' | 'restore', - ) { - return { - type: 'checkbox', - name: 'userCollections', - message: `Select the collection records to be ${direction === 'dump' ? 'dumped' : 'restored'}`, - loop: false, - pageSize: 30, - choices: collections.map((collection) => { - return { - name: collection.title, - value: collection.name, - checked: direction === 'dump', - }; - }), - }; - } -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/restore-command.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/restore-command.ts deleted file mode 100644 index 985dbee3d..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/commands/restore-command.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { Application, AppSupervisor } from '@nocobase/server'; -import { Restorer } from '../restorer'; -import inquirer from 'inquirer'; -import InquireQuestionBuilder from './inquire-question-builder'; - -export default function addRestoreCommand(app: Application) { - app - .command('restore') - .argument('', 'restore file path') - .option('-a, --app ', 'sub app name if you want to restore into a sub app') - .option('-f, --force', 'force restore without warning') - .action(async (restoreFilePath, options) => { - let importApp = app; - - if (options.app) { - if ( - !(await app.db.getCollection('applications').repository.findOne({ - filter: { name: options.app }, - })) - ) { - // create sub app if not exists - await app.db.getCollection('applications').repository.create({ - values: { - name: options.app, - }, - }); - } - - const subApp = await AppSupervisor.getInstance().getApp(options.app); - - if (!subApp) { - app.log.error(`app ${options.app} not found`); - await app.stop(); - return; - } - - importApp = subApp; - } - - // should confirm data will be overwritten - if (!options.force && !(await restoreWarning())) { - return; - } - - await restoreActionCommand(importApp, restoreFilePath); - }); -} - -interface RestoreContext { - app: Application; - dir: string; -} - -async function restoreWarning() { - const results = await inquirer.prompt([ - { - type: 'confirm', - name: 'confirm', - message: 'Danger !!! This action will overwrite your current data, please make sure you have a backup❗️❗️', - default: false, - }, - ]); - - return results.confirm; -} - -async function restoreActionCommand(app: Application, restoreFilePath: string) { - const restorer = new Restorer(app, { - backUpFilePath: restoreFilePath, - }); - const restoreMeta = await restorer.parseBackupFile(); - - const { requiredGroups, selectedOptionalGroups, selectedUserCollections } = restoreMeta; - - const questions = InquireQuestionBuilder.buildInquirerQuestions({ - requiredGroups, - optionalGroups: selectedOptionalGroups, - optionalCollections: await Promise.all( - selectedUserCollections.map(async (name) => { - return { name, title: await restorer.getImportCollectionTitle(name) }; - }), - ), - direction: 'restore', - }); - - const results = await inquirer.prompt(questions); - - await restorer.restore({ - selectedOptionalGroupNames: results.collectionGroups, - selectedUserCollections: results.userCollections, - }); - - await app.stop(); -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/dumper.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/dumper.ts deleted file mode 100644 index ad679d05c..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/dumper.ts +++ /dev/null @@ -1,300 +0,0 @@ -import { CollectionGroup } from '@nocobase/database'; -import archiver from 'archiver'; -import dayjs from 'dayjs'; -import fs from 'fs'; -import fsPromises from 'fs/promises'; -import lodash from 'lodash'; -import mkdirp from 'mkdirp'; -import path from 'path'; -import stream from 'stream'; -import util from 'util'; -import { AppMigrator } from './app-migrator'; -import { CollectionGroupManager } from './collection-group-manager'; -import { FieldValueWriter } from './field-value-writer'; -import { DUMPED_EXTENSION, humanFileSize, sqlAdapter } from './utils'; - -const finished = util.promisify(stream.finished); - -export class Dumper extends AppMigrator { - direction = 'dump' as const; - - async dumpableCollections(): Promise<{ - requiredGroups: CollectionGroup[]; - optionalGroups: CollectionGroup[]; - userCollections: Array<{ - name: string; - title: string; - }>; - }> { - const appCollectionGroups = CollectionGroupManager.getGroups(this.app); - - const { requiredGroups, optionalGroups } = CollectionGroupManager.classifyCollectionGroups(appCollectionGroups); - const pluginsCollections = CollectionGroupManager.getGroupsCollections(appCollectionGroups); - - const userCollections = await this.getCustomCollections(); - return lodash.cloneDeep({ - requiredGroups, - optionalGroups, - userCollections: await Promise.all( - userCollections - .filter((collection) => !pluginsCollections.includes(collection)) //remove collection that is in plugins - .map(async (name) => { - // map user collection to { name, title } - - const collectionInstance = await this.app.db.getRepository('collections').findOne({ - filterByTk: name, - }); - - return { - name, - title: collectionInstance.get('title'), - }; - }), - ), - }); - } - - async dump(options: { selectedOptionalGroupNames: string[]; selectedUserCollections: string[] }) { - const { requiredGroups, optionalGroups } = await this.dumpableCollections(); - const { selectedOptionalGroupNames, selectedUserCollections = [] } = options; - - const throughCollections = this.findThroughCollections(selectedUserCollections); - - const selectedOptionalGroups = optionalGroups.filter((group) => { - return selectedOptionalGroupNames.some((selectedOptionalGroupName) => { - const [namespace, functionKey] = selectedOptionalGroupName.split('.'); - return group.function === functionKey && group.namespace === namespace; - }); - }); - - const dumpedCollections = lodash.uniq( - [ - CollectionGroupManager.getGroupsCollections(requiredGroups), - CollectionGroupManager.getGroupsCollections(selectedOptionalGroups), - selectedUserCollections, - throughCollections, - ].flat(), - ); - - for (const collection of dumpedCollections) { - await this.dumpCollection({ - name: collection, - }); - } - - const mapGroupToMetaJson = (groups) => - groups.map((group: CollectionGroup) => { - const data = { - ...group, - }; - - if (group.delayRestore) { - data['delayRestore'] = true; - } - - return data; - }); - - await this.dumpMeta({ - requiredGroups: mapGroupToMetaJson(requiredGroups), - selectedOptionalGroups: mapGroupToMetaJson(selectedOptionalGroups), - selectedUserCollections: selectedUserCollections, - }); - - await this.dumpDb(); - - const filePath = await this.packDumpedDir(); - await this.clearWorkDir(); - return filePath; - } - - async dumpDb() { - const db = this.app.db; - const dialect = db.sequelize.getDialect(); - const sqlContent = []; - if (dialect === 'postgres') { - // get user defined functions in postgres - const functions = await db.sequelize.query( - `SELECT n.nspname AS function_schema, - p.proname AS function_name, - pg_get_functiondef(p.oid) AS def - FROM pg_proc p - LEFT JOIN pg_namespace n ON p.pronamespace = n.oid - WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') - ORDER BY function_schema, - function_name;`, - { - type: 'SELECT', - }, - ); - - for (const f of functions) { - sqlContent.push(f['def']); - } - - // get user defined triggers in postgres - const triggers = await db.sequelize.query( - `select pg_get_triggerdef(oid) - from pg_trigger`, - { - type: 'SELECT', - }, - ); - - for (const t of triggers) { - sqlContent.push(t['pg_get_triggerdef']); - } - - // get user defined views in postgres - const views = await db.sequelize.query( - `SELECT n.nspname AS schema_name, - v.viewname AS view_name, - v.definition AS view_definition - FROM pg_views v - JOIN - pg_namespace n ON v.schemaname = n.nspname - WHERE n.nspname NOT IN ('information_schema', 'pg_catalog') - ORDER BY schema_name, - view_name;`, - { - type: 'SELECT', - }, - ); - - for (const v of views) { - sqlContent.push(`CREATE OR REPLACE VIEW ${v['view_name']} AS ${v['view_definition']}`); - } - } - - if (sqlContent.length > 0) { - const dbDumpPath = path.resolve(this.workDir, 'db.sql'); - await fsPromises.writeFile(dbDumpPath, JSON.stringify(sqlContent), 'utf8'); - } - } - - async dumpMeta(additionalMeta: object = {}) { - const metaPath = path.resolve(this.workDir, 'meta'); - - await fsPromises.writeFile( - metaPath, - JSON.stringify({ - version: await this.app.version.get(), - dialect: this.app.db.sequelize.getDialect(), - ...additionalMeta, - }), - 'utf8', - ); - } - - async dumpCollection(options: { name: string }) { - const app = this.app; - const dir = this.workDir; - - const collectionName = options.name; - app.log.info(`dumping collection ${collectionName}`); - - const collection = app.db.getCollection(collectionName); - - if (!collection) { - this.app.log.warn(`collection ${collectionName} not found`); - return; - } - - // @ts-ignore - const columns: string[] = [...new Set(lodash.map(collection.model.tableAttributes, 'field'))]; - - if (columns.length == 0) { - this.app.log.warn(`collection ${collectionName} has no columns`); - return; - } - - const collectionDataDir = path.resolve(dir, 'collections', collectionName); - - await fsPromises.mkdir(collectionDataDir, { recursive: true }); - - // write collection data - const dataFilePath = path.resolve(collectionDataDir, 'data'); - const dataStream = fs.createWriteStream(dataFilePath); - - const rows = await app.db.sequelize.query( - sqlAdapter( - app.db, - `SELECT * - FROM ${collection.isParent() ? 'ONLY' : ''} ${collection.quotedTableName()}`, - ), - { - type: 'SELECT', - }, - ); - - for (const row of rows) { - const rowData = JSON.stringify( - columns.map((col) => { - const val = row[col]; - const field = collection.getField(col); - - return field ? FieldValueWriter.toDumpedValue(field, val) : val; - }), - ); - - dataStream.write(rowData + '\r\n', 'utf8'); - } - - dataStream.end(); - await finished(dataStream); - - const meta = { - name: collectionName, - tableName: collection.model.tableName, - count: rows.length, - columns, - }; - - // write meta file - await fsPromises.writeFile(path.resolve(collectionDataDir, 'meta'), JSON.stringify(meta), 'utf8'); - } - - async packDumpedDir() { - const dirname = path.resolve(process.cwd(), 'storage', 'duplicator'); - mkdirp.sync(dirname); - const filePath = path.resolve(dirname, `dump-${dayjs().format('YYYYMMDDTHHmmss')}.${DUMPED_EXTENSION}`); - - const output = fs.createWriteStream(filePath); - - const archive = archiver('zip', { - zlib: { level: 9 }, - }); - - output.on('close', function () { - console.log('dumped file size: ' + humanFileSize(archive.pointer(), true)); - }); - - output.on('end', function () { - console.log('Data has been drained'); - }); - - archive.on('warning', function (err) { - if (err.code === 'ENOENT') { - // log warning - } else { - // throw error - throw err; - } - }); - - archive.on('error', function (err) { - throw err; - }); - - archive.pipe(output); - - archive.directory(this.workDir, false); - - await archive.finalize(); - console.log('dumped to', filePath); - return { - filePath, - dirname, - }; - } -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/restorer.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/restorer.ts deleted file mode 100644 index a59df739b..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/restorer.ts +++ /dev/null @@ -1,380 +0,0 @@ -import decompress from 'decompress'; -import fs from 'fs'; -import fsPromises from 'fs/promises'; -import path from 'path'; -import { AppMigrator, AppMigratorOptions } from './app-migrator'; -import { CollectionGroupManager } from './collection-group-manager'; -import { FieldValueWriter } from './field-value-writer'; -import { readLines, sqlAdapter } from './utils'; -import { Application } from '@nocobase/server'; - -export class Restorer extends AppMigrator { - direction = 'restore' as const; - backUpFilePath: string; - decompressed = false; - importedCollections: string[] = []; - - constructor( - app: Application, - options: AppMigratorOptions & { - backUpFilePath?: string; - }, - ) { - super(app, options); - const { backUpFilePath } = options; - - if (backUpFilePath) { - this.setBackUpFilePath(backUpFilePath); - } - } - - setBackUpFilePath(backUpFilePath) { - if (path.isAbsolute(backUpFilePath)) { - this.backUpFilePath = backUpFilePath; - } else if (path.basename(backUpFilePath) === backUpFilePath) { - const dirname = path.resolve(process.cwd(), 'storage', 'duplicator'); - this.backUpFilePath = path.resolve(dirname, backUpFilePath); - } else { - this.backUpFilePath = path.resolve(process.cwd(), backUpFilePath); - } - } - - async parseBackupFile() { - await this.decompressBackup(this.backUpFilePath); - return await this.getImportMeta(); - } - - async restore(options: { selectedOptionalGroupNames: string[]; selectedUserCollections: string[] }) { - await this.decompressBackup(this.backUpFilePath); - await this.importCollections(options); - await this.importDb(); - await this.clearWorkDir(); - } - - async getImportPlugins() { - const meta = await this.getImportCollectionMeta('applicationPlugins'); - const nameIndex = meta.columns.indexOf('name'); - - const plugins = await this.getImportCollectionData('applicationPlugins'); - return ['core', ...plugins.map((plugin) => JSON.parse(plugin)[nameIndex])]; - } - - async getImportCustomCollections() { - const collections = await this.getImportCollections(); - const meta = await this.getImportCollectionMeta('collections'); - const data = await this.getImportCollectionData('collections'); - - return data - .map((row) => JSON.parse(row)[meta.columns.indexOf('name')]) - .filter((name) => collections.includes(name)); - } - - async getImportCollectionTitle(collectionName) { - const meta = await this.getImportCollectionMeta('collections'); - const data = await this.getImportCollectionData('collections'); - - const index = meta.columns.indexOf('name'); - const row = data.find((row) => JSON.parse(row)[index] === collectionName); - - if (!row) { - throw new Error(`Collection ${collectionName} not found`); - } - - const titleIndex = meta.columns.indexOf('title'); - - return JSON.parse(row)[titleIndex]; - } - - async getImportCollections() { - const collectionsDir = path.resolve(this.workDir, 'collections'); - return await fsPromises.readdir(collectionsDir); - } - - async getImportCollectionData(collectionName) { - const dataFile = path.resolve(this.workDir, 'collections', collectionName, 'data'); - return await readLines(dataFile); - } - - async getImportCollectionMeta(collectionName) { - const metaData = path.resolve(this.workDir, 'collections', collectionName, 'meta'); - return JSON.parse(await fsPromises.readFile(metaData, 'utf8')); - } - - async getImportMeta() { - const metaFile = path.resolve(this.workDir, 'meta'); - return JSON.parse(await fsPromises.readFile(metaFile, 'utf8')) as any; - } - - async importCollections(options: { - ignore?: string | string[]; - selectedOptionalGroupNames: string[]; - selectedUserCollections: string[]; - }) { - const importCollection = async (collectionName: string) => { - const collectionMetaPath = path.resolve(this.workDir, 'collections', collectionName, 'meta'); - - const metaContent = await fsPromises.readFile(collectionMetaPath, 'utf8'); - const meta = JSON.parse(metaContent); - const tableName = this.app.db.utils.quoteTable(meta.tableName); - - try { - // disable trigger - if (this.app.db.inDialect('postgres')) { - await this.app.db.sequelize.query(`ALTER TABLE IF EXISTS ${tableName} DISABLE TRIGGER ALL`); - } - - await this.importCollection({ - name: collectionName, - }); - } catch (err) { - this.app.log.warn(`import collection ${collectionName} failed`, { - err, - }); - } finally { - if (this.app.db.inDialect('postgres')) { - await this.app.db.sequelize.query(`ALTER TABLE IF EXISTS ${tableName} ENABLE TRIGGER ALL`); - } - } - }; - - // import applicationPlugins first - await importCollection('applicationPlugins'); - // reload app - await this.app.reload(); - - const { requiredGroups, selectedOptionalGroups } = await this.parseBackupFile(); - - const delayGroups = [...requiredGroups, ...selectedOptionalGroups].filter((group) => group.delay); - const delayCollections = CollectionGroupManager.getGroupsCollections(delayGroups); - - // import required plugins collections - for (const collectionName of CollectionGroupManager.getGroupsCollections(requiredGroups).filter( - (i) => !delayCollections.includes(i) && i != 'applicationPlugins', - )) { - await importCollection(collectionName); - } - - // load imported collections into database object - await (this.app.db.getRepository('collections') as any).load(); - - // sync database - await this.app.db.sync({ - force: false, - alter: { - drop: false, - }, - }); - - const userCollections = options.selectedUserCollections || []; - const throughCollections = this.findThroughCollections(userCollections); - - const customCollections = [ - ...CollectionGroupManager.getGroupsCollections( - selectedOptionalGroups.filter((group) => { - return options.selectedOptionalGroupNames.some((selectedOptionalGroupName) => { - const [namespace, functionKey] = selectedOptionalGroupName.split('.'); - return group.function === functionKey && group.namespace === namespace; - }); - }), - ), - ...userCollections, - ...throughCollections, - ]; - - // import custom collections - for (const collectionName of customCollections) { - await importCollection(collectionName); - } - - // import delay groups - const appGroups = CollectionGroupManager.getGroups(this.app); - - for (const collectionGroup of delayGroups) { - const appCollectionGroup = appGroups.find( - (group) => group.namespace === collectionGroup.name && group.function === collectionGroup.function, - ); - await appCollectionGroup.delayRestore(this); - } - - await this.emitAsync('restoreCollectionsFinished'); - } - - async decompressBackup(backupFilePath: string) { - if (!this.decompressed) await decompress(backupFilePath, this.workDir); - } - - async importCollection(options: { - name: string; - insert?: boolean; - clear?: boolean; - rowCondition?: (row: any) => boolean; - }) { - const app = this.app; - const db = app.db; - - const collectionName = options.name; - const dir = this.workDir; - const collection = app.db.getCollection(collectionName); - const collectionDataPath = path.resolve(dir, 'collections', collectionName, 'data'); - const collectionMetaPath = path.resolve(dir, 'collections', collectionName, 'meta'); - - const metaContent = await fsPromises.readFile(collectionMetaPath, 'utf8'); - const meta = JSON.parse(metaContent); - app.log.info(`collection meta ${metaContent}`); - - const addSchemaTableName = db.utils.addSchema(meta.tableName); - const tableName = db.utils.quoteTable(meta.tableName); - - if (options.clear !== false) { - // truncate old data - let sql = `TRUNCATE TABLE ${tableName}`; - - if (app.db.inDialect('sqlite')) { - sql = `DELETE - FROM ${tableName}`; - } - - await app.db.sequelize.query(sqlAdapter(app.db, sql)); - } - - // read file content from collection data - const rows = await readLines(collectionDataPath); - - if (rows.length == 0) { - app.logger.info(`${collectionName} has no data to import`); - return; - } - - const columns = meta['columns']; - - const fields = columns - .map((column) => [column, collection.getField(column)]) - .reduce((carry, [column, type]) => { - carry[column] = type; - return carry; - }, {}); - - const rowsWithMeta = rows - .map((row) => - JSON.parse(row) - .map((val, index) => [columns[index], val]) - .reduce((carry, [column, val]) => { - const field = fields[column]; - - carry[column] = field ? FieldValueWriter.write(field, val) : val; - - return carry; - }, {}), - ) - .filter((row) => { - if (options.rowCondition) { - return options.rowCondition(row); - } - - return true; - }); - - if (rowsWithMeta.length == 0) { - app.logger.info(`${collectionName} has no data to import`); - return; - } - - const model = collection.model; - - const fieldMappedAttributes = {}; - // @ts-ignore - for (const attr in model.tableAttributes) { - fieldMappedAttributes[model.rawAttributes[attr].field || attr] = model.rawAttributes[attr]; - } - - //@ts-ignore - const sql = collection.model.queryInterface.queryGenerator.bulkInsertQuery( - addSchemaTableName, - rowsWithMeta, - {}, - fieldMappedAttributes, - ); - - if (options.insert === false) { - return sql; - } - - await app.db.sequelize.query(sql, { - type: 'INSERT', - }); - - const primaryKeyAttribute = collection.model.rawAttributes[collection.model.primaryKeyAttribute]; - - if (primaryKeyAttribute && primaryKeyAttribute.autoIncrement) { - this.on('restoreCollectionsFinished', async () => { - if (this.app.db.inDialect('postgres')) { - const sequenceNameResult = await app.db.sequelize.query( - `SELECT column_default - FROM information_schema.columns - WHERE table_name = '${collection.model.tableName}' - and "column_name" = 'id' - and table_schema = '${app.db.options.schema || 'public'}';`, - ); - - if (sequenceNameResult[0].length) { - const columnDefault = sequenceNameResult[0][0]['column_default']; - if (columnDefault.includes(`${collection.model.tableName}_id_seq`)) { - const regex = new RegExp(/nextval\('(.*)'::regclass\)/); - const match = regex.exec(columnDefault); - const sequenceName = match[1]; - - const maxVal = await app.db.sequelize.query( - `SELECT MAX("${primaryKeyAttribute.field}") - FROM ${tableName}`, - { - type: 'SELECT', - }, - ); - - const updateSeqSQL = `SELECT setval('${sequenceName}', ${maxVal[0]['max']})`; - await app.db.sequelize.query(updateSeqSQL); - } - } - } - - if (this.app.db.inDialect('sqlite')) { - await app.db.sequelize.query( - `UPDATE sqlite_sequence - set seq = (SELECT MAX("${primaryKeyAttribute.field}") FROM "${collection.model.tableName}") - WHERE name = "${collection.model.tableName}"`, - ); - } - }); - } - - app.logger.info(`${collectionName} imported with ${rowsWithMeta.length} rows`); - - this.importedCollections.push(collection.name); - } - - async importDb() { - const sqlFilePath = path.resolve(this.workDir, 'db.sql'); - // if db.sql file not exists, skip import - if (!fs.existsSync(sqlFilePath)) { - return; - } - - // read file content from db.sql - const queriesContent = await fsPromises.readFile(sqlFilePath, 'utf8'); - - const queries = JSON.parse(queriesContent); - - for (const sql of queries) { - try { - this.app.log.info(`import sql: ${sql}`); - await this.app.db.sequelize.query(sql); - } catch (e) { - if (e.name === 'SequelizeDatabaseError') { - this.app.logger.error(e.message); - } else { - throw e; - } - } - } - } -} diff --git a/packages/plugins/@nocobase/plugin-duplicator/src/server/server.ts b/packages/plugins/@nocobase/plugin-duplicator/src/server/server.ts deleted file mode 100644 index 384c268d8..000000000 --- a/packages/plugins/@nocobase/plugin-duplicator/src/server/server.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { Plugin } from '@nocobase/server'; -import addDumpCommand from './commands/dump-command'; -import addRestoreCommand from './commands/restore-command'; - -import { koaMulter as multer } from '@nocobase/utils'; -import * as os from 'os'; -import dumpAction from './actions/dump-action'; -import dumpableCollections from './actions/dumpable-collections-action'; -import getDictAction from './actions/get-dict-action'; -import { getPackageContent, restoreAction } from './actions/restore-action'; -import zhCN from './locale/zh-CN'; - -export default class Duplicator extends Plugin { - beforeLoad() { - this.app.i18n.addResources('zh-CN', 'duplicator', zhCN); - - addDumpCommand(this.app); - addRestoreCommand(this.app); - } - - async load() { - this.app.resourcer.define({ - name: 'duplicator', - middleware: async (ctx, next) => { - if (ctx.action.actionName !== 'upload') { - return next(); - } - const storage = multer.diskStorage({ - destination: os.tmpdir(), // 获取临时目录 - filename: function (req, file, cb) { - const randomName = Date.now().toString() + Math.random().toString().slice(2); // 随机生成文件名 - cb(null, randomName); - }, - }); - - const upload = multer({ storage }).single('file'); - return upload(ctx, next); - }, - actions: { - restore: restoreAction, - upload: getPackageContent, - dump: dumpAction, - dumpableCollections: dumpableCollections, - getDict: getDictAction, - }, - }); - - this.app.acl.allow('duplicator', 'getDict'); - } -} diff --git a/packages/plugins/@nocobase/plugin-file-manager/src/server/collections/attachments.ts b/packages/plugins/@nocobase/plugin-file-manager/src/server/collections/attachments.ts index 390d0e873..d3887cb7c 100644 --- a/packages/plugins/@nocobase/plugin-file-manager/src/server/collections/attachments.ts +++ b/packages/plugins/@nocobase/plugin-file-manager/src/server/collections/attachments.ts @@ -1,10 +1,11 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'file-manager.attachmentRecords', - duplicator: 'optional', +export default defineCollection({ + dumpRules: { + group: 'user', + }, + shared: true, name: 'attachments', - title: '文件管理器', createdBy: true, updatedBy: true, template: 'file', @@ -62,4 +63,4 @@ export default { // formula: '{{ storage.baseUrl }}{{ path }}/{{ filename }}' }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-file-manager/src/server/collections/storages.ts b/packages/plugins/@nocobase/plugin-file-manager/src/server/collections/storages.ts index 6e6ea7722..d88b5aad3 100644 --- a/packages/plugins/@nocobase/plugin-file-manager/src/server/collections/storages.ts +++ b/packages/plugins/@nocobase/plugin-file-manager/src/server/collections/storages.ts @@ -1,10 +1,9 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'file-manager.storageSetting', - duplicator: 'optional', +export default defineCollection({ + dumpRules: 'required', name: 'storages', - title: '存储引擎', + shared: true, fields: [ { title: '存储引擎名称', @@ -62,4 +61,4 @@ export default { defaultValue: false, }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-graph-collection-manager/src/server/collections/graphPositions.ts b/packages/plugins/@nocobase/plugin-graph-collection-manager/src/server/collections/graphPositions.ts index 9e16f9558..4635d5fb9 100644 --- a/packages/plugins/@nocobase/plugin-graph-collection-manager/src/server/collections/graphPositions.ts +++ b/packages/plugins/@nocobase/plugin-graph-collection-manager/src/server/collections/graphPositions.ts @@ -1,9 +1,9 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'graph-collection-manager.graphCollectionPositions', - duplicator: 'required', + dumpRules: 'required', name: 'graphPositions', + shared: true, fields: [ { type: 'string', diff --git a/packages/plugins/@nocobase/plugin-graph-collection-manager/src/server/index.ts b/packages/plugins/@nocobase/plugin-graph-collection-manager/src/server/index.ts index 5bbf82ba5..c6b2d2649 100644 --- a/packages/plugins/@nocobase/plugin-graph-collection-manager/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-graph-collection-manager/src/server/index.ts @@ -3,9 +3,7 @@ import path from 'path'; export class GraphCollectionManagerPlugin extends Plugin { async load() { - await this.db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); this.app.acl.allow('graphPositions', '*'); } } diff --git a/packages/plugins/@nocobase/plugin-iframe-block/src/server/collections/iframe-html.ts b/packages/plugins/@nocobase/plugin-iframe-block/src/server/collections/iframe-html.ts index 45396bba6..d2ee425c7 100644 --- a/packages/plugins/@nocobase/plugin-iframe-block/src/server/collections/iframe-html.ts +++ b/packages/plugins/@nocobase/plugin-iframe-block/src/server/collections/iframe-html.ts @@ -2,10 +2,11 @@ import { CollectionOptions } from '@nocobase/database'; export default { namespace: 'iframe-block.iframe-html-storage', - duplicator: 'required', + dumpRules: 'required', name: 'iframeHtml', createdBy: true, updatedBy: true, + shared: true, fields: [ { type: 'uid', diff --git a/packages/plugins/@nocobase/plugin-iframe-block/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-iframe-block/src/server/plugin.ts index 2ab1609c9..5b7e19399 100644 --- a/packages/plugins/@nocobase/plugin-iframe-block/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-iframe-block/src/server/plugin.ts @@ -8,9 +8,8 @@ export class IframeBlockPlugin extends Plugin { beforeLoad() {} async load() { - await this.db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); + this.app.actions({ 'iframeHtml:getHtml': getHtml, }); diff --git a/packages/plugins/@nocobase/plugin-localization-management/src/server/collections/localization-texts.ts b/packages/plugins/@nocobase/plugin-localization-management/src/server/collections/localization-texts.ts index 766cb0e9d..7dc016464 100644 --- a/packages/plugins/@nocobase/plugin-localization-management/src/server/collections/localization-texts.ts +++ b/packages/plugins/@nocobase/plugin-localization-management/src/server/collections/localization-texts.ts @@ -1,14 +1,15 @@ -import { CollectionOptions } from '@nocobase/client'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'localization.localization', - duplicator: 'optional', +export default defineCollection({ + dumpRules: { + group: 'required', + }, name: 'localizationTexts', - title: '{{t("Localization Texts")}}', model: 'LocalizationTextModel', createdBy: true, updatedBy: true, logging: true, + shared: true, fields: [ { name: 'id', @@ -61,4 +62,4 @@ export default { fields: ['batch'], }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-localization-management/src/server/collections/localization-translations.ts b/packages/plugins/@nocobase/plugin-localization-management/src/server/collections/localization-translations.ts index 963958f49..9495f6a3c 100644 --- a/packages/plugins/@nocobase/plugin-localization-management/src/server/collections/localization-translations.ts +++ b/packages/plugins/@nocobase/plugin-localization-management/src/server/collections/localization-translations.ts @@ -1,14 +1,16 @@ -import { CollectionOptions } from '@nocobase/client'; +import { defineCollection } from '@nocobase/database'; -export default { +export default defineCollection({ namespace: 'localization.localization', - duplicator: 'optional', + dumpRules: { + group: 'required', + }, name: 'localizationTranslations', - title: '{{t("Localization Translations")}}', model: 'LocalizationTranslationModel', createdBy: true, updatedBy: true, logging: true, + shared: true, fields: [ { name: 'id', @@ -58,4 +60,4 @@ export default { unique: true, }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-localization-management/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-localization-management/src/server/plugin.ts index 265d3969b..333718159 100644 --- a/packages/plugins/@nocobase/plugin-localization-management/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-localization-management/src/server/plugin.ts @@ -47,9 +47,7 @@ export class LocalizationManagementPlugin extends Plugin { beforeLoad() {} async load() { - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); this.db.addMigrations({ namespace: 'localization-management', diff --git a/packages/plugins/@nocobase/plugin-map/src/server/collections/mapConfiguration.ts b/packages/plugins/@nocobase/plugin-map/src/server/collections/mapConfiguration.ts index b96188a27..a9fc64a08 100644 --- a/packages/plugins/@nocobase/plugin-map/src/server/collections/mapConfiguration.ts +++ b/packages/plugins/@nocobase/plugin-map/src/server/collections/mapConfiguration.ts @@ -1,11 +1,12 @@ -import { CollectionOptions } from '@nocobase/client'; +import { defineCollection } from '@nocobase/database'; import { MapConfigurationCollectionName } from '../constants'; -export default { - namespace: 'map.mapConfiguration', - duplicator: 'optional', +export default defineCollection({ + dumpRules: { + group: 'third-party', + }, name: MapConfigurationCollectionName, - title: '{{t("Map Manager")}}', + shared: true, fields: [ { title: 'Access key', @@ -26,4 +27,4 @@ export default { type: 'string', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-map/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-map/src/server/plugin.ts index 0fe68d6d2..e067b3c6e 100644 --- a/packages/plugins/@nocobase/plugin-map/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-map/src/server/plugin.ts @@ -1,5 +1,5 @@ import { InstallOptions, Plugin } from '@nocobase/server'; -import { resolve } from 'path'; +import path from 'path'; import { getConfiguration, setConfiguration } from './actions'; import { CircleField, LineStringField, PointField, PolygonField } from './fields'; import { CircleValueParser, LineStringValueParser, PointValueParser, PolygonValueParser } from './value-parsers'; @@ -24,9 +24,7 @@ export class MapPlugin extends Plugin { } async load() { - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); this.app.resource({ name: 'map-configuration', diff --git a/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/collections/applications.ts b/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/collections/applications.ts index 472be3402..482175df2 100644 --- a/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/collections/applications.ts +++ b/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/collections/applications.ts @@ -1,12 +1,12 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'multi-app-manager.multi-apps', - duplicator: 'optional', + dumpRules: { + group: 'third-party', + }, name: 'applications', model: 'ApplicationModel', autoGenId: false, - title: '{{t("Applications")}}', sortable: 'sort', filterTargetKey: 'name', fields: [ diff --git a/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts b/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts index fc939ce3f..fa61bb0d6 100644 --- a/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts @@ -158,9 +158,7 @@ export class PluginMultiAppManager extends Plugin { } async load() { - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); // after application created this.db.on( diff --git a/packages/plugins/@nocobase/plugin-multi-app-share-collection/src/server/__tests__/collection-sync.test.ts b/packages/plugins/@nocobase/plugin-multi-app-share-collection/src/server/__tests__/collection-sync.test.ts index 0e3067c84..e345a358c 100644 --- a/packages/plugins/@nocobase/plugin-multi-app-share-collection/src/server/__tests__/collection-sync.test.ts +++ b/packages/plugins/@nocobase/plugin-multi-app-share-collection/src/server/__tests__/collection-sync.test.ts @@ -1,9 +1,9 @@ import { BelongsToManyRepository, Database } from '@nocobase/database'; import { AppSupervisor } from '@nocobase/server'; -import { MockServer, mockServer, pgOnly } from '@nocobase/test'; +import { isPg, MockServer, mockServer } from '@nocobase/test'; import * as process from 'process'; -pgOnly()('enable plugin', () => { +describe.runIf(isPg())('enable plugin', () => { let mainDb: Database; let mainApp: MockServer; @@ -41,7 +41,7 @@ pgOnly()('enable plugin', () => { }); }); -pgOnly()('collection sync after main', () => { +describe.runIf(isPg())('collection sync after main', () => { let mainApp: MockServer; beforeEach(async () => { @@ -109,7 +109,7 @@ pgOnly()('collection sync after main', () => { }); }); -pgOnly()('collection sync', () => { +describe.runIf(isPg())('collection sync', () => { let mainDb: Database; let mainApp: MockServer; diff --git a/packages/plugins/@nocobase/plugin-multi-app-share-collection/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-multi-app-share-collection/src/server/plugin.ts index ff64f811e..6ff0b353f 100644 --- a/packages/plugins/@nocobase/plugin-multi-app-share-collection/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-multi-app-share-collection/src/server/plugin.ts @@ -1,3 +1,4 @@ +import Database from '@nocobase/database'; import PluginMultiAppManager from '@nocobase/plugin-multi-app-manager'; import { Application, AppSupervisor, Plugin } from '@nocobase/server'; import lodash from 'lodash'; @@ -11,32 +12,14 @@ class SubAppPlugin extends Plugin { const mainApp = this.options.mainApp; const subApp = this.app; - const sharedCollectionGroups = [ - 'audit-logs', - 'workflow', - 'charts', - 'collection-manager', - 'file-manager', - 'graph-collection-manager', - 'map', - 'sequence-field', - 'snapshot-field', - 'verification', - 'localization-management', - ]; + const sharedCollections = []; - const collectionGroups = mainApp.db.collectionGroupManager.getGroups(); - - const sharedCollectionGroupsCollections = []; - - for (const group of collectionGroups) { - if (sharedCollectionGroups.includes(group.namespace)) { - sharedCollectionGroupsCollections.push(...group.collections); + for (const collection of (mainApp.db as Database).collections.values()) { + if (collection.options.shared) { + sharedCollections.push(collection.name); } } - const sharedCollections = [...sharedCollectionGroupsCollections.flat(), 'users', 'users_jobs']; - subApp.on('beforeLoadPlugin', (plugin) => { if (plugin.name === 'collection-manager') { plugin.setLoadFilter({ @@ -252,9 +235,7 @@ export class MultiAppShareCollectionPlugin extends Plugin { return; } - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); // this.db.addMigrations({ // namespace: 'multi-app-share-collection', diff --git a/packages/plugins/@nocobase/plugin-sample-shop-actions/src/server/index.ts b/packages/plugins/@nocobase/plugin-sample-shop-actions/src/server/index.ts index 930db3527..a014f2eb2 100644 --- a/packages/plugins/@nocobase/plugin-sample-shop-actions/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-sample-shop-actions/src/server/index.ts @@ -8,9 +8,7 @@ export class ShopPlugin extends Plugin { } async load() { - await this.db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); this.app.resource({ name: 'orders', diff --git a/packages/plugins/@nocobase/plugin-sample-shop-events/src/server/index.ts b/packages/plugins/@nocobase/plugin-sample-shop-events/src/server/index.ts index eeb12bcfe..99e9e16cc 100644 --- a/packages/plugins/@nocobase/plugin-sample-shop-events/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-sample-shop-events/src/server/index.ts @@ -45,9 +45,7 @@ export class ShopPlugin extends Plugin { } async load() { - await this.db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); this.db.on('orders.afterCreate', async (order, options) => { const product = await order.getProduct({ diff --git a/packages/plugins/@nocobase/plugin-sample-shop-i18n/src/server/index.ts b/packages/plugins/@nocobase/plugin-sample-shop-i18n/src/server/index.ts index f44105734..7b4dbc1c3 100644 --- a/packages/plugins/@nocobase/plugin-sample-shop-i18n/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-sample-shop-i18n/src/server/index.ts @@ -12,9 +12,7 @@ export class ShopPlugin extends Plugin { } async load() { - await this.db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); this.app.i18n.addResources('zh-CN', ns, zhCN); diff --git a/packages/plugins/@nocobase/plugin-sample-shop-modeling/src/server/index.ts b/packages/plugins/@nocobase/plugin-sample-shop-modeling/src/server/index.ts index e443ec9a1..8325d7287 100644 --- a/packages/plugins/@nocobase/plugin-sample-shop-modeling/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-sample-shop-modeling/src/server/index.ts @@ -16,9 +16,7 @@ export class ShopPlugin extends Plugin { } async load() { - await this.db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); this.app.acl.allow('products', '*'); this.app.acl.allow('categories', '*'); diff --git a/packages/plugins/@nocobase/plugin-sequence-field/src/server/collections/sequences.ts b/packages/plugins/@nocobase/plugin-sequence-field/src/server/collections/sequences.ts index 9cdec0b89..6b2d3695b 100644 --- a/packages/plugins/@nocobase/plugin-sequence-field/src/server/collections/sequences.ts +++ b/packages/plugins/@nocobase/plugin-sequence-field/src/server/collections/sequences.ts @@ -1,24 +1,25 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'sequence-field.sequences', - duplicator: { - dumpable: 'required', + dumpRules: { + group: 'required', async delayRestore(restorer) { const app = restorer.app; const importedCollections = restorer.importedCollections; const sequenceFields = importedCollections - .map((collection) => - [...app.db.getCollection(collection).fields.values()].filter((field) => field.type === 'sequence'), - ) + .map((collection) => { + const collectionInstance = app.db.getCollection(collection); + if (!collectionInstance) throw new Error(`Collection ${collection} not found`); + return [...collectionInstance.fields.values()].filter((field) => field.type === 'sequence'); + }) .flat() .filter(Boolean); // a single sequence field refers to a single row in sequences table const sequencesAttributes = sequenceFields .map((field) => { - const patterns = field.get('patterns').filter((pattern) => pattern.type === 'integer'); + const patterns = field.get('patterns'); return patterns.map((pattern) => { return { @@ -28,7 +29,8 @@ export default defineCollection({ }; }); }) - .flat(); + .flat() + .filter((attr) => attr.collection && attr.field && attr.key); if (sequencesAttributes.length > 0) { await app.db.getRepository('sequences').destroy({ @@ -42,18 +44,17 @@ export default defineCollection({ name: 'sequences', clear: false, rowCondition(row) { - const results = sequencesAttributes.some((attributes) => { + return sequencesAttributes.some((attributes) => { return ( row.collection === attributes.collection && row.field === attributes.field && row.key === attributes.key ); }); - - return results; }, }); }, }, name: 'sequences', + shared: true, fields: [ { name: 'collection', diff --git a/packages/plugins/@nocobase/plugin-snapshot-field/src/server/collections/collectionsHistory.ts b/packages/plugins/@nocobase/plugin-snapshot-field/src/server/collections/collectionsHistory.ts index e1e9e5a17..5ddabde1f 100644 --- a/packages/plugins/@nocobase/plugin-snapshot-field/src/server/collections/collectionsHistory.ts +++ b/packages/plugins/@nocobase/plugin-snapshot-field/src/server/collections/collectionsHistory.ts @@ -1,16 +1,15 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'snapshot-field.snapshot-field', - duplicator: 'required', +export default defineCollection({ + dumpRules: 'required', name: 'collectionsHistory', - title: '数据表历史', sortable: 'sort', autoGenId: false, model: 'CollectionModel', repository: 'CollectionRepository', timestamps: false, filterTargetKey: 'name', + shared: true, fields: [ { type: 'uid', @@ -54,4 +53,4 @@ export default { sortBy: 'sort', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-snapshot-field/src/server/collections/fieldsHistory.ts b/packages/plugins/@nocobase/plugin-snapshot-field/src/server/collections/fieldsHistory.ts index e2c5d3b16..159ead8eb 100644 --- a/packages/plugins/@nocobase/plugin-snapshot-field/src/server/collections/fieldsHistory.ts +++ b/packages/plugins/@nocobase/plugin-snapshot-field/src/server/collections/fieldsHistory.ts @@ -1,13 +1,12 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'snapshot-field.snapshot-field', - duplicator: 'required', +export default defineCollection({ + dumpRules: 'required', name: 'fieldsHistory', - title: '{{t("Fields history")}}', autoGenId: false, model: 'FieldModel', timestamps: false, + shared: true, sortable: { name: 'sort', scopeKey: 'collectionName', @@ -72,4 +71,4 @@ export default { defaultValue: {}, }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-snapshot-field/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-snapshot-field/src/server/plugin.ts index 31c39f127..fb713baa1 100644 --- a/packages/plugins/@nocobase/plugin-snapshot-field/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-snapshot-field/src/server/plugin.ts @@ -87,9 +87,7 @@ export class SnapshotFieldPlugin extends Plugin { async load() { // 导入 collection - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); this.app.db.registerFieldTypes({ snapshot: SnapshotField, diff --git a/packages/plugins/@nocobase/plugin-system-settings/src/server/collections/systemSettings.ts b/packages/plugins/@nocobase/plugin-system-settings/src/server/collections/systemSettings.ts index fd8a3de9a..ccfaa744b 100644 --- a/packages/plugins/@nocobase/plugin-system-settings/src/server/collections/systemSettings.ts +++ b/packages/plugins/@nocobase/plugin-system-settings/src/server/collections/systemSettings.ts @@ -1,8 +1,7 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'system-settings.systemSettings', - duplicator: 'optional', + dumpRules: 'required', name: 'systemSettings', fields: [ { diff --git a/packages/plugins/@nocobase/plugin-system-settings/src/server/server.ts b/packages/plugins/@nocobase/plugin-system-settings/src/server/server.ts index a03e3f1cf..428cce345 100644 --- a/packages/plugins/@nocobase/plugin-system-settings/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-system-settings/src/server/server.ts @@ -36,9 +36,7 @@ export class SystemSettingsPlugin extends Plugin { } async load() { - await this.app.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); this.app.acl.addFixedParams('systemSettings', 'destroy', () => { return { diff --git a/packages/plugins/@nocobase/plugin-theme-editor/src/server/collections/theme-config.ts b/packages/plugins/@nocobase/plugin-theme-editor/src/server/collections/theme-config.ts new file mode 100644 index 000000000..6e229d7be --- /dev/null +++ b/packages/plugins/@nocobase/plugin-theme-editor/src/server/collections/theme-config.ts @@ -0,0 +1,31 @@ +import { defineCollection } from '@nocobase/database'; + +export default defineCollection({ + name: 'themeConfig', + dumpRules: 'required', + fields: [ + // 主题配置内容，一个 JSON 字符串 + { + type: 'json', + name: 'config', + }, + // 主题是否可选 + { + type: 'boolean', + name: 'optional', + }, + { + type: 'boolean', + name: 'isBuiltIn', + }, + { + type: 'uid', + name: 'uid', + }, + { + type: 'radio', + name: 'default', + defaultValue: false, + }, + ], +}); diff --git a/packages/plugins/@nocobase/plugin-theme-editor/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-theme-editor/src/server/plugin.ts index 4b5ab6335..8203f3b48 100644 --- a/packages/plugins/@nocobase/plugin-theme-editor/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-theme-editor/src/server/plugin.ts @@ -1,5 +1,5 @@ import { InstallOptions, Plugin } from '@nocobase/server'; -import { resolve } from 'path'; +import path, { resolve } from 'path'; import { compact, compactDark, dark, defaultTheme } from './builtinThemes'; export class ThemeEditorPlugin extends Plugin { @@ -10,34 +10,7 @@ export class ThemeEditorPlugin extends Plugin { async beforeLoad() {} async load() { - this.db.collection({ - name: 'themeConfig', - fields: [ - // 主题配置内容，一个 JSON 字符串 - { - type: 'json', - name: 'config', - }, - // 主题是否可选 - { - type: 'boolean', - name: 'optional', - }, - { - type: 'boolean', - name: 'isBuiltIn', - }, - { - type: 'uid', - name: 'uid', - }, - { - type: 'radio', - name: 'default', - defaultValue: false, - }, - ], - }); + await this.importCollections(path.resolve(__dirname, './collections')); this.db.addMigrations({ namespace: 'theme-editor', directory: resolve(__dirname, './migrations'), @@ -47,6 +20,7 @@ export class ThemeEditorPlugin extends Plugin { }); this.app.acl.allow('themeConfig', 'list', 'public'); + this.app.acl.registerSnippet({ name: `pm.${this.name}.themeConfig`, actions: ['themeConfig:*'], diff --git a/packages/plugins/@nocobase/plugin-ui-routes-storage/src/server/collections/uiRoutes.ts b/packages/plugins/@nocobase/plugin-ui-routes-storage/src/server/collections/uiRoutes.ts index 03129afd6..67ed42fb7 100644 --- a/packages/plugins/@nocobase/plugin-ui-routes-storage/src/server/collections/uiRoutes.ts +++ b/packages/plugins/@nocobase/plugin-ui-routes-storage/src/server/collections/uiRoutes.ts @@ -1,16 +1,15 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'ui-routes-storage.uiRoutes', - duplicator: 'required', + dumpRules: 'required', name: 'uiRoutes', - title: '前端路由表', model: 'MagicAttributeModel', autoGenId: false, sortable: { name: 'sort', scopeKey: 'parentKey', }, + shared: true, fields: [ { type: 'uid', diff --git a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaServerHooks.ts b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaServerHooks.ts index 80a057dd6..bac2cf370 100644 --- a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaServerHooks.ts +++ b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaServerHooks.ts @@ -1,8 +1,7 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'ui-schema-storage.uiSchemas', - duplicator: 'required', + dumpRules: 'required', name: 'uiSchemaServerHooks', model: 'ServerHookModel', // autoGenId: false, diff --git a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaTemplates.ts b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaTemplates.ts index 113fb212a..45fef2fec 100644 --- a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaTemplates.ts +++ b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaTemplates.ts @@ -1,8 +1,7 @@ import { defineCollection } from '@nocobase/database'; export default defineCollection({ - namespace: 'ui-schema-storage.uiSchemas', - duplicator: 'required', + dumpRules: 'required', name: 'uiSchemaTemplates', autoGenId: false, fields: [ diff --git a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaTreePath.ts b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaTreePath.ts index 1217cb008..9504112fd 100644 --- a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaTreePath.ts +++ b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemaTreePath.ts @@ -1,8 +1,7 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'ui-schema-storage.uiSchemas', - duplicator: 'required', + dumpRules: 'required', name: 'uiSchemaTreePath', autoGenId: false, timestamps: false, diff --git a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemas.ts b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemas.ts index 019fc8cdc..723147a15 100644 --- a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemas.ts +++ b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/collections/uiSchemas.ts @@ -1,10 +1,8 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'ui-schema-storage.uiSchemas', - duplicator: 'required', + dumpRules: 'required', name: 'uiSchemas', - title: '字段配置', autoGenId: false, timestamps: false, repository: 'UiSchemaRepository', diff --git a/packages/plugins/@nocobase/plugin-users/src/server/collections/users.ts b/packages/plugins/@nocobase/plugin-users/src/server/collections/users.ts index bb6db245e..3c14ea0c2 100644 --- a/packages/plugins/@nocobase/plugin-users/src/server/collections/users.ts +++ b/packages/plugins/@nocobase/plugin-users/src/server/collections/users.ts @@ -1,10 +1,8 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'users.users', - duplicator: { - dumpable: 'optional', - with: 'rolesUsers', +export default defineCollection({ + dumpRules: { + group: 'user', }, name: 'users', title: '{{t("Users")}}', @@ -13,6 +11,7 @@ export default { createdBy: true, updatedBy: true, logging: true, + shared: true, fields: [ { name: 'id', @@ -99,4 +98,4 @@ export default { defaultValue: {}, }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-users/src/server/server.ts b/packages/plugins/@nocobase/plugin-users/src/server/server.ts index 427502ad9..42bca96a3 100644 --- a/packages/plugins/@nocobase/plugin-users/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-users/src/server/server.ts @@ -106,9 +106,7 @@ export default class UsersPlugin extends Plugin { } async load() { - await this.db.import({ - directory: resolve(__dirname, 'collections'), - }); + await this.importCollections(resolve(__dirname, 'collections')); this.db.addMigrations({ namespace: 'users', diff --git a/packages/plugins/@nocobase/plugin-verification/src/server/collections/verifications.ts b/packages/plugins/@nocobase/plugin-verification/src/server/collections/verifications.ts index 6b03123b6..c5c14182a 100644 --- a/packages/plugins/@nocobase/plugin-verification/src/server/collections/verifications.ts +++ b/packages/plugins/@nocobase/plugin-verification/src/server/collections/verifications.ts @@ -1,7 +1,11 @@ -export default { - namespace: 'verification.verificationData', - duplicator: 'optional', +import { defineCollection } from '@nocobase/database'; + +export default defineCollection({ + dumpRules: { + group: 'log', + }, name: 'verifications', + shared: true, fields: [ { type: 'uuid', @@ -35,4 +39,4 @@ export default { target: 'verifications_providers', }, ], -}; +}); diff --git a/packages/plugins/@nocobase/plugin-verification/src/server/collections/verifications_providers.ts b/packages/plugins/@nocobase/plugin-verification/src/server/collections/verifications_providers.ts index 6827e1ff3..037503b4c 100644 --- a/packages/plugins/@nocobase/plugin-verification/src/server/collections/verifications_providers.ts +++ b/packages/plugins/@nocobase/plugin-verification/src/server/collections/verifications_providers.ts @@ -1,7 +1,11 @@ -export default { - namespace: 'verification.verificationProviders', - duplicator: 'optional', +import { defineCollection } from '@nocobase/database'; + +export default defineCollection({ + dumpRules: { + group: 'third-party', + }, name: 'verifications_providers', + shared: true, fields: [ { type: 'string', @@ -25,4 +29,4 @@ export default { name: 'default', }, ], -}; +}); diff --git a/packages/plugins/@nocobase/plugin-workflow-manual/src/server/Plugin.ts b/packages/plugins/@nocobase/plugin-workflow-manual/src/server/Plugin.ts index 6ddbed5d6..900b1d188 100644 --- a/packages/plugins/@nocobase/plugin-workflow-manual/src/server/Plugin.ts +++ b/packages/plugins/@nocobase/plugin-workflow-manual/src/server/Plugin.ts @@ -3,18 +3,14 @@ import actions from '@nocobase/actions'; import { HandlerType } from '@nocobase/resourcer'; import WorkflowPlugin, { JOB_STATUS } from '@nocobase/plugin-workflow'; -import jobsCollection from './collections/jobs'; -import usersCollection from './collections/users'; -import usersJobsCollection from './collections/users_jobs'; +import path from 'path'; import { submit } from './actions'; import ManualInstruction from './ManualInstruction'; export default class extends Plugin { async load() { - this.app.db.collection(usersJobsCollection); - this.app.db.extendCollection(usersCollection); - this.app.db.extendCollection(jobsCollection); + await this.importCollections(path.resolve(__dirname, 'collections')); this.app.resource({ name: 'users_jobs', diff --git a/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/users_jobs.ts b/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/1-users_jobs.ts similarity index 81% rename from packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/users_jobs.ts rename to packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/1-users_jobs.ts index 1a57bfb57..be6fbee6c 100644 --- a/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/users_jobs.ts +++ b/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/1-users_jobs.ts @@ -1,9 +1,11 @@ -import { CollectionOptions } from '@nocobase/database'; +import { defineCollection } from '@nocobase/database'; -export default { - namespace: 'workflow.executionLogs', +export default defineCollection({ name: 'users_jobs', - duplicator: 'optional', + dumpRules: { + group: 'log', + }, + shared: true, fields: [ { type: 'bigInt', @@ -47,4 +49,4 @@ export default { name: 'result', }, ], -} as CollectionOptions; +}); diff --git a/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/jobs.ts b/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/2-jobs.ts similarity index 74% rename from packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/jobs.ts rename to packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/2-jobs.ts index 6863d6c34..4f8679316 100644 --- a/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/jobs.ts +++ b/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/2-jobs.ts @@ -1,4 +1,6 @@ -export default { +import { extendCollection } from '@nocobase/database'; + +export default extendCollection({ name: 'jobs', fields: [ { @@ -14,4 +16,4 @@ export default { onDelete: 'CASCADE', }, ], -}; +}); diff --git a/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/users.ts b/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/3-users.ts similarity index 69% rename from packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/users.ts rename to packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/3-users.ts index 87afc0cf3..6e8c0362a 100644 --- a/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/users.ts +++ b/packages/plugins/@nocobase/plugin-workflow-manual/src/server/collections/3-users.ts @@ -1,4 +1,6 @@ -export default { +import { extendCollection } from '@nocobase/database'; + +export default extendCollection({ name: 'users', fields: [ { @@ -12,4 +14,4 @@ export default { target: 'users_jobs', }, ], -}; +}); diff --git a/packages/plugins/@nocobase/plugin-workflow-test/src/server/index.ts b/packages/plugins/@nocobase/plugin-workflow-test/src/server/index.ts index e29b200b1..2a7b189a6 100644 --- a/packages/plugins/@nocobase/plugin-workflow-test/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-workflow-test/src/server/index.ts @@ -71,8 +71,6 @@ export async function getApp({ export default class WorkflowTestPlugin extends Plugin { async load() { - await this.db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); } } diff --git a/packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts b/packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts index 91d338bb9..0fd34ee39 100644 --- a/packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts +++ b/packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts @@ -192,9 +192,7 @@ export default class PluginWorkflowServer extends Plugin { this.app.acl.allow('workflows', ['trigger'], 'loggedIn'); - await db.import({ - directory: path.resolve(__dirname, 'collections'), - }); + await this.importCollections(path.resolve(__dirname, 'collections')); this.db.addMigrations({ namespace: this.name, diff --git a/packages/plugins/@nocobase/plugin-workflow/src/server/collections/executions.ts b/packages/plugins/@nocobase/plugin-workflow/src/server/collections/executions.ts index fe9669df2..33d294d62 100644 --- a/packages/plugins/@nocobase/plugin-workflow/src/server/collections/executions.ts +++ b/packages/plugins/@nocobase/plugin-workflow/src/server/collections/executions.ts @@ -1,9 +1,11 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'workflow.executionLogs', - duplicator: 'optional', + dumpRules: { + group: 'log', + }, name: 'executions', + shared: true, fields: [ { type: 'belongsTo', diff --git a/packages/plugins/@nocobase/plugin-workflow/src/server/collections/flow_nodes.ts b/packages/plugins/@nocobase/plugin-workflow/src/server/collections/flow_nodes.ts index 62dadd794..ffb70cf1d 100644 --- a/packages/plugins/@nocobase/plugin-workflow/src/server/collections/flow_nodes.ts +++ b/packages/plugins/@nocobase/plugin-workflow/src/server/collections/flow_nodes.ts @@ -1,9 +1,9 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'workflow.workflowConfig', - duplicator: 'required', + dumpRules: 'required', name: 'flow_nodes', + shared: true, fields: [ { type: 'uid', diff --git a/packages/plugins/@nocobase/plugin-workflow/src/server/collections/jobs.ts b/packages/plugins/@nocobase/plugin-workflow/src/server/collections/jobs.ts index 3a1d91ba7..5c3f57488 100644 --- a/packages/plugins/@nocobase/plugin-workflow/src/server/collections/jobs.ts +++ b/packages/plugins/@nocobase/plugin-workflow/src/server/collections/jobs.ts @@ -1,9 +1,11 @@ import { CollectionOptions } from '@nocobase/database'; export default { - namespace: 'workflow.executionLogs', - duplicator: 'optional', + dumpRules: { + group: 'log', + }, name: 'jobs', + shared: true, fields: [ { type: 'belongsTo', diff --git a/packages/plugins/@nocobase/plugin-workflow/src/server/collections/workflows.ts b/packages/plugins/@nocobase/plugin-workflow/src/server/collections/workflows.ts index 5c9aff72b..05fe626e0 100644 --- a/packages/plugins/@nocobase/plugin-workflow/src/server/collections/workflows.ts +++ b/packages/plugins/@nocobase/plugin-workflow/src/server/collections/workflows.ts @@ -2,9 +2,9 @@ import { CollectionOptions } from '@nocobase/database'; export default function () { return { - namespace: 'workflow.workflowConfig', - duplicator: 'required', + dumpRules: 'required', name: 'workflows', + shared: true, fields: [ { name: 'key', diff --git a/packages/presets/nocobase/package.json b/packages/presets/nocobase/package.json index cbab2d23d..a7dfd47d4 100644 --- a/packages/presets/nocobase/package.json +++ b/packages/presets/nocobase/package.json @@ -22,7 +22,7 @@ "@nocobase/plugin-collection-manager": "0.18.0-alpha.9", "@nocobase/plugin-custom-request": "0.18.0-alpha.9", "@nocobase/plugin-data-visualization": "0.18.0-alpha.9", - "@nocobase/plugin-duplicator": "0.18.0-alpha.9", + "@nocobase/plugin-backup-restore": "0.18.0-alpha.9", "@nocobase/plugin-error-handler": "0.18.0-alpha.9", "@nocobase/plugin-excel-formula-field": "0.18.0-alpha.9", "@nocobase/plugin-export": "0.18.0-alpha.9", diff --git a/packages/presets/nocobase/src/server/index.ts b/packages/presets/nocobase/src/server/index.ts index 1d50925d7..d80c04de4 100644 --- a/packages/presets/nocobase/src/server/index.ts +++ b/packages/presets/nocobase/src/server/index.ts @@ -29,7 +29,7 @@ export class PresetNocoBase extends Plugin { 'client', 'export', 'import', - 'duplicator', + 'backup-restore', 'iframe-block', 'formula-field', 'data-visualization', diff --git a/yarn.lock b/yarn.lock index 7f4602767..459ecafee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10038,7 +10038,7 @@ content-disposition@0.5.2: resolved "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4" integrity sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA== -content-disposition@~0.5.2: +content-disposition@^0.5.4, content-disposition@~0.5.2: version "0.5.4" resolved "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==