From 505c23b4e1409db109217be1421915025a5ed3db Mon Sep 17 00:00:00 2001 From: ChengLei Shao Date: Mon, 25 Sep 2023 18:17:19 +0800 Subject: [PATCH] feat(database): append tree parent recursively (#2573) * feat(database): append with options * feat: recursively load parent instances * chore: test * fix: load with appends * chore: test * chore: load with belongs to many * chore: test --- packages/core/database/package.json | 3 +- .../adjacency-list-repository.test.ts | 228 ++++++++++++++++++ .../src/__tests__/option-parser.test.ts | 17 ++ .../src/eager-loading/eager-loading-tree.ts | 47 +++- packages/core/database/src/options-parser.ts | 28 ++- .../adjacency-list-repository.ts | 29 +++ .../server/src/errors/plugin-not-exist.ts | 1 + 7 files changed, 350 insertions(+), 3 deletions(-) create mode 100644 packages/core/database/src/__tests__/adjacency-list-repository.test.ts create mode 100644 packages/core/server/src/errors/plugin-not-exist.ts diff --git a/packages/core/database/package.json b/packages/core/database/package.json index 1653a7ee1..b5233b79c 100644 --- a/packages/core/database/package.json +++ b/packages/core/database/package.json @@ -20,7 +20,8 @@ "mathjs": "^10.6.1", "semver": "^7.3.7", "sequelize": "^6.26.0", - "umzug": "^3.1.1" + "umzug": "^3.1.1", + "qs": "^6.11.2" }, "devDependencies": { "@types/glob": "^7.2.0" diff --git a/packages/core/database/src/__tests__/adjacency-list-repository.test.ts b/packages/core/database/src/__tests__/adjacency-list-repository.test.ts new file mode 100644 index 000000000..4f9715f12 --- /dev/null +++ b/packages/core/database/src/__tests__/adjacency-list-repository.test.ts @@ -0,0 +1,228 @@ +import { mockDatabase } from './'; +import Database from '../database'; + +describe('adjacency list repository', () => { + let db: Database; + + beforeEach(async () => { + db = mockDatabase(); + await db.clean({ drop: true }); + }); + + afterEach(async () => { + await db.close(); + }); + + it('should append relation parent recursively with belongs to assoc', async () => { + const Category = db.collection({ + name: 'categories', + tree: 'adjacency-list', + fields: [ + { type: 'string', name: 'name' }, + { + type: 'belongsTo', + name: 'parent', + treeParent: true, + }, + { + type: 'hasMany', + name: 'children', + treeChildren: true, + }, + ], + }); + + const Post = db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { type: 'belongsTo', name: 'category', target: 'categories', foreignKey: 'categoryId' }, + { type: 'belongsTo', name: 'category2', target: 'categories', foreignKey: 'categoryId2' }, + { type: 'belongsTo', name: 'category3', target: 'categories', foreignKey: 'categoryId3' }, + ], + }); + + await db.sync(); + + await Category.repository.create({ + values: [ + { + name: 'c1', + children: [ + { + name: 'c1-1', + children: [ + { + name: 'c1-1-1', + }, + ], + }, + { + name: 'c12', + }, + ], + }, + ], + }); + + const c111 = await Category.repository.findOne({ where: { name: 'c1-1-1' } }); + + const p1 = await Post.repository.create({ + values: [ + { + title: 'p1', + category: { id: c111.id }, + category2: null, + category3: { id: c111.id }, + }, + ], + }); + + const p1WithCategory = await Post.repository.findOne({ + appends: [ + 'category', + 'category.parent(recursively=true)', + 'category2', + 'category2.parent(recursively=true)', + 'category3', + 'category3.parent(recursively=true)', + ], + }); + + expect(p1WithCategory.category.parent.name).toBe('c1-1'); + expect(p1WithCategory.category.parent.parent.name).toBe('c1'); + expect(p1WithCategory.category2).toBeNull(); + expect(p1WithCategory.category3.parent.name).toBe('c1-1'); + expect(p1WithCategory.category3.parent.parent.name).toBe('c1'); + }); + + it('should append relation parent recursively with belongs to many', async () => { + const Category = db.collection({ + name: 'categories', + tree: 'adjacency-list', + fields: [ + { type: 'string', name: 'name' }, + { + type: 'belongsTo', + name: 'parent', + treeParent: true, + }, + { + type: 'hasMany', + name: 'children', + treeChildren: true, + }, + ], + }); + + const Post = db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { type: 'belongsToMany', name: 'categories', target: 'categories', through: 'table1' }, + { type: 'belongsToMany', name: 'categories2', target: 'categories', through: 'table2' }, + { type: 'belongsToMany', name: 'categories3', target: 'categories', through: 'table3' }, + ], + }); + + await db.sync(); + + await Category.repository.create({ + values: [ + { + name: 'c1', + children: [ + { + name: 'c1-1', + children: [ + { + name: 'c1-1-1', + }, + ], + }, + { + name: 'c12', + }, + ], + }, + ], + }); + + const c111 = await Category.repository.findOne({ where: { name: 'c1-1-1' } }); + + const p1 = await Post.repository.create({ + values: [ + { + title: 'p1', + categories: [{ id: c111.id }], + categories2: [{ id: c111.id }], + categories3: [], + }, + ], + }); + + const p1WithCategory = await Post.repository.findOne({ + appends: [ + 'categories', + 'categories.parent(recursively=true)', + 'categories2', + 'categories2.parent(recursively=true)', + 'categories3', + 'categories3.parent(recursively=true)', + ], + }); + + expect(p1WithCategory.categories[0].parent.name).toBe('c1-1'); + expect(p1WithCategory.categories[0].parent.parent.name).toBe('c1'); + + expect(p1WithCategory.categories2[0].parent.name).toBe('c1-1'); + expect(p1WithCategory.categories2[0].parent.parent.name).toBe('c1'); + }); + + it('should append parent recursively', async () => { + const Tree = db.collection({ + name: 'categories', + tree: 'adjacency-list', + fields: [ + { type: 'string', name: 'name' }, + { + type: 'belongsTo', + name: 'parent', + treeParent: true, + }, + { + type: 'hasMany', + name: 'children', + treeChildren: true, + }, + ], + }); + + await db.sync(); + + await Tree.repository.create({ + values: [ + { + name: 'c1', + children: [ + { + name: 'c1-1', + children: [ + { + name: 'c1-1-1', + }, + ], + }, + { + name: 'c12', + }, + ], + }, + ], + }); + + const c111 = await Tree.repository.findOne({ where: { name: 'c1-1-1' }, appends: ['parent(recursively=true)'] }); + expect(c111.parent.name).toBe('c1-1'); + expect(c111.parent.parent.name).toBe('c1'); + }); +}); diff --git a/packages/core/database/src/__tests__/option-parser.test.ts b/packages/core/database/src/__tests__/option-parser.test.ts index 91d0c4fc1..7de220934 100644 --- a/packages/core/database/src/__tests__/option-parser.test.ts +++ b/packages/core/database/src/__tests__/option-parser.test.ts @@ -77,6 +77,7 @@ describe('option parser', () => { { association: 'tags', attributes: ['id', 'name'], + options: {}, }, ], }); @@ -130,6 +131,22 @@ describe('option parser', () => { expect(params['include'][0]['association']).toEqual('posts'); }); + it('should support append with options', () => { + const options = { + appends: ['posts(recursively=true)'], + }; + + const parser = new OptionsParser(options, { + collection: User, + }); + + const { include } = parser.toSequelizeParams(); + + expect(include[0].association).toEqual('posts'); + expect(include[0].association); + expect(include[0].options.recursively).toBeTruthy(); + }); + it('should handle field with association', () => { const options = { appends: ['posts'], diff --git a/packages/core/database/src/eager-loading/eager-loading-tree.ts b/packages/core/database/src/eager-loading/eager-loading-tree.ts index f736f1f76..65ff83da1 100644 --- a/packages/core/database/src/eager-loading/eager-loading-tree.ts +++ b/packages/core/database/src/eager-loading/eager-loading-tree.ts @@ -3,6 +3,7 @@ import { Association, HasOne, Includeable, Model, ModelStatic, Op, Transaction } import Database from '../database'; import { appendChildCollectionNameAfterRepositoryFind } from '../listeners/append-child-collection-name-after-repository-find'; import { OptionsParser } from '../options-parser'; +import { AdjacencyListRepository } from '../repositories/tree-repository/adjacency-list-repository'; interface EagerLoadingNode { model: ModelStatic; @@ -15,6 +16,7 @@ interface EagerLoadingNode { order?: any; where?: any; inspectInheritAttribute?: boolean; + includeOptions?: any; } const pushAttribute = (node, attribute) => { @@ -106,6 +108,7 @@ export class EagerLoadingTree { parent: eagerLoadingTreeParent, where: include.where, children: [], + includeOption: include.options || {}, }); if (associationType == 'HasOne' || associationType == 'HasMany') { @@ -258,9 +261,10 @@ export class EagerLoadingTree { if (associationType == 'BelongsTo') { const foreignKey = association.foreignKey; - const parentInstancesForeignKeyValues = node.parent.instances.map((instance) => instance.get(foreignKey)); + const collection = this.db.modelCollection.get(node.model); + instances = await node.model.findAll({ transaction, where: { @@ -268,6 +272,47 @@ export class EagerLoadingTree { }, attributes: node.attributes, }); + + // load parent instances recursively + if (node.includeOption.recursively && instances.length > 0) { + const targetKey = association.targetKey; + const sql = AdjacencyListRepository.queryParentSQL({ + db: this.db, + collection, + foreignKey, + targetKey, + nodeIds: instances.map((instance) => instance.get(targetKey)), + }); + + const results = await this.db.sequelize.query(sql, { + type: 'SELECT', + transaction, + }); + + const parentInstances = await node.model.findAll({ + transaction, + where: { + [association.targetKey]: results.map((result) => result[targetKey]), + }, + attributes: node.attributes, + }); + + const setInstanceParent = (instance) => { + const parentInstance = parentInstances.find( + (parentInstance) => parentInstance.get(targetKey) == instance.get(foreignKey), + ); + if (!parentInstance) { + return; + } + + setInstanceParent(parentInstance); + instance[association.as] = instance.dataValues[association.as] = parentInstance; + }; + + for (const instance of instances) { + setInstanceParent(instance); + } + } } if (associationType == 'BelongsToMany') { diff --git a/packages/core/database/src/options-parser.ts b/packages/core/database/src/options-parser.ts index 742a04808..38e3c2729 100644 --- a/packages/core/database/src/options-parser.ts +++ b/packages/core/database/src/options-parser.ts @@ -4,6 +4,7 @@ import { Collection } from './collection'; import { Database } from './database'; import FilterParser from './filter-parser'; import { Appends, Except, FindOptions } from './repository'; +import qs from 'qs'; const debug = require('debug')('noco-database'); @@ -237,6 +238,21 @@ export class OptionsParser { return filterParams; } + protected parseAppendWithOptions(append: string) { + const parts = append.split('('); + const obj: { name: string; options?: object; raw?: string } = { + name: parts[0], + }; + + if (parts.length > 1) { + const optionsStr = parts[1].replace(')', ''); + obj.options = qs.parse(optionsStr); + obj.raw = `(${optionsStr})`; + } + + return obj; + } + protected parseAppends(appends: Appends, filterParams: any) { if (!appends) return filterParams; @@ -250,6 +266,10 @@ export class OptionsParser { * @param append */ const setInclude = (model: ModelStatic, queryParams: any, append: string) => { + const appendWithOptions = this.parseAppendWithOptions(append); + + append = appendWithOptions.name; + const appendFields = append.split('.'); const appendAssociation = appendFields[0]; @@ -316,6 +336,7 @@ export class OptionsParser { // association not exists queryParams['include'].push({ association: appendAssociation, + options: appendWithOptions.options || {}, }); existIncludeIndex = queryParams['include'].length - 1; @@ -361,10 +382,15 @@ export class OptionsParser { }; } + let nextAppend = appendFields.filter((_, index) => index !== 0).join('.'); + if (appendWithOptions.raw) { + nextAppend += appendWithOptions.raw; + } + setInclude( model.associations[queryParams['include'][existIncludeIndex].association].target, queryParams['include'][existIncludeIndex], - appendFields.filter((_, index) => index !== 0).join('.'), + nextAppend, ); } }; diff --git a/packages/core/database/src/repositories/tree-repository/adjacency-list-repository.ts b/packages/core/database/src/repositories/tree-repository/adjacency-list-repository.ts index b74a89df7..c26a77a8c 100644 --- a/packages/core/database/src/repositories/tree-repository/adjacency-list-repository.ts +++ b/packages/core/database/src/repositories/tree-repository/adjacency-list-repository.ts @@ -1,7 +1,36 @@ import lodash from 'lodash'; import { FindOptions, Repository } from '../../repository'; +import Database from '../../database'; +import { Collection } from '../../collection'; export class AdjacencyListRepository extends Repository { + static queryParentSQL(options: { + db: Database; + nodeIds: any[]; + collection: Collection; + foreignKey: string; + targetKey: string; + }) { + const { collection, db, nodeIds } = options; + const tableName = collection.quotedTableName(); + const { foreignKey, targetKey } = options; + const foreignKeyField = collection.model.rawAttributes[foreignKey].field; + const targetKeyField = collection.model.rawAttributes[targetKey].field; + + const queryInterface = db.sequelize.getQueryInterface(); + const q = queryInterface.quoteIdentifier.bind(queryInterface); + return `WITH RECURSIVE cte AS ( + SELECT ${q(targetKeyField)}, ${q(foreignKeyField)} + FROM ${tableName} + WHERE ${q(targetKeyField)} IN (${nodeIds.join(',')}) + UNION ALL + SELECT t.${q(targetKeyField)}, t.${q(foreignKeyField)} + FROM ${tableName} AS t + INNER JOIN cte ON t.${q(targetKeyField)} = cte.${q(foreignKeyField)} + ) + SELECT ${q(targetKeyField)} AS ${q(targetKey)}, ${q(foreignKeyField)} AS ${q(foreignKey)} FROM cte`; + } + async update(options): Promise { return super.update({ ...(options || {}), diff --git a/packages/core/server/src/errors/plugin-not-exist.ts b/packages/core/server/src/errors/plugin-not-exist.ts new file mode 100644 index 000000000..32a73708f --- /dev/null +++ b/packages/core/server/src/errors/plugin-not-exist.ts @@ -0,0 +1 @@ +export class PluginNotExist extends Error {}