feat(database): append tree parent recursively (#2573)

* feat(database): append with options

* feat: recursively load parent instances

* chore: test

* fix: load with appends

* chore: test

* chore: load with belongs to many

* chore: test

packages/core/database/package.json

@@ -20,7 +20,8 @@
     "mathjs": "^10.6.1",
     "semver": "^7.3.7",
     "sequelize": "^6.26.0",
-    "umzug": "^3.1.1"
+    "umzug": "^3.1.1",
+    "qs": "^6.11.2"
   },
   "devDependencies": {
     "@types/glob": "^7.2.0"

packages/core/database/src/__tests__/adjacency-list-repository.test.ts

@@ -0,0 +1,228 @@
+import { mockDatabase } from './';
+import Database from '../database';
+
+describe('adjacency list repository', () => {
+  let db: Database;
+
+  beforeEach(async () => {
+    db = mockDatabase();
+    await db.clean({ drop: true });
+  });
+
+  afterEach(async () => {
+    await db.close();
+  });
+
+  it('should append relation parent recursively with belongs to assoc', async () => {
+    const Category = db.collection({
+      name: 'categories',
+      tree: 'adjacency-list',
+      fields: [
+        { type: 'string', name: 'name' },
+        {
+          type: 'belongsTo',
+          name: 'parent',
+          treeParent: true,
+        },
+        {
+          type: 'hasMany',
+          name: 'children',
+          treeChildren: true,
+        },
+      ],
+    });
+
+    const Post = db.collection({
+      name: 'posts',
+      fields: [
+        { type: 'string', name: 'title' },
+        { type: 'belongsTo', name: 'category', target: 'categories', foreignKey: 'categoryId' },
+        { type: 'belongsTo', name: 'category2', target: 'categories', foreignKey: 'categoryId2' },
+        { type: 'belongsTo', name: 'category3', target: 'categories', foreignKey: 'categoryId3' },
+      ],
+    });
+
+    await db.sync();
+
+    await Category.repository.create({
+      values: [
+        {
+          name: 'c1',
+          children: [
+            {
+              name: 'c1-1',
+              children: [
+                {
+                  name: 'c1-1-1',
+                },
+              ],
+            },
+            {
+              name: 'c12',
+            },
+          ],
+        },
+      ],
+    });
+
+    const c111 = await Category.repository.findOne({ where: { name: 'c1-1-1' } });
+
+    const p1 = await Post.repository.create({
+      values: [
+        {
+          title: 'p1',
+          category: { id: c111.id },
+          category2: null,
+          category3: { id: c111.id },
+        },
+      ],
+    });
+
+    const p1WithCategory = await Post.repository.findOne({
+      appends: [
+        'category',
+        'category.parent(recursively=true)',
+        'category2',
+        'category2.parent(recursively=true)',
+        'category3',
+        'category3.parent(recursively=true)',
+      ],
+    });
+
+    expect(p1WithCategory.category.parent.name).toBe('c1-1');
+    expect(p1WithCategory.category.parent.parent.name).toBe('c1');
+    expect(p1WithCategory.category2).toBeNull();
+    expect(p1WithCategory.category3.parent.name).toBe('c1-1');
+    expect(p1WithCategory.category3.parent.parent.name).toBe('c1');
+  });
+
+  it('should append relation parent recursively with belongs to many', async () => {
+    const Category = db.collection({
+      name: 'categories',
+      tree: 'adjacency-list',
+      fields: [
+        { type: 'string', name: 'name' },
+        {
+          type: 'belongsTo',
+          name: 'parent',
+          treeParent: true,
+        },
+        {
+          type: 'hasMany',
+          name: 'children',
+          treeChildren: true,
+        },
+      ],
+    });
+
+    const Post = db.collection({
+      name: 'posts',
+      fields: [
+        { type: 'string', name: 'title' },
+        { type: 'belongsToMany', name: 'categories', target: 'categories', through: 'table1' },
+        { type: 'belongsToMany', name: 'categories2', target: 'categories', through: 'table2' },
+        { type: 'belongsToMany', name: 'categories3', target: 'categories', through: 'table3' },
+      ],
+    });
+
+    await db.sync();
+
+    await Category.repository.create({
+      values: [
+        {
+          name: 'c1',
+          children: [
+            {
+              name: 'c1-1',
+              children: [
+                {
+                  name: 'c1-1-1',
+                },
+              ],
+            },
+            {
+              name: 'c12',
+            },
+          ],
+        },
+      ],
+    });
+
+    const c111 = await Category.repository.findOne({ where: { name: 'c1-1-1' } });
+
+    const p1 = await Post.repository.create({
+      values: [
+        {
+          title: 'p1',
+          categories: [{ id: c111.id }],
+          categories2: [{ id: c111.id }],
+          categories3: [],
+        },
+      ],
+    });
+
+    const p1WithCategory = await Post.repository.findOne({
+      appends: [
+        'categories',
+        'categories.parent(recursively=true)',
+        'categories2',
+        'categories2.parent(recursively=true)',
+        'categories3',
+        'categories3.parent(recursively=true)',
+      ],
+    });
+
+    expect(p1WithCategory.categories[0].parent.name).toBe('c1-1');
+    expect(p1WithCategory.categories[0].parent.parent.name).toBe('c1');
+
+    expect(p1WithCategory.categories2[0].parent.name).toBe('c1-1');
+    expect(p1WithCategory.categories2[0].parent.parent.name).toBe('c1');
+  });
+
+  it('should append parent recursively', async () => {
+    const Tree = db.collection({
+      name: 'categories',
+      tree: 'adjacency-list',
+      fields: [
+        { type: 'string', name: 'name' },
+        {
+          type: 'belongsTo',
+          name: 'parent',
+          treeParent: true,
+        },
+        {
+          type: 'hasMany',
+          name: 'children',
+          treeChildren: true,
+        },
+      ],
+    });
+
+    await db.sync();
+
+    await Tree.repository.create({
+      values: [
+        {
+          name: 'c1',
+          children: [
+            {
+              name: 'c1-1',
+              children: [
+                {
+                  name: 'c1-1-1',
+                },
+              ],
+            },
+            {
+              name: 'c12',
+            },
+          ],
+        },
+      ],
+    });
+
+    const c111 = await Tree.repository.findOne({ where: { name: 'c1-1-1' }, appends: ['parent(recursively=true)'] });
+    expect(c111.parent.name).toBe('c1-1');
+    expect(c111.parent.parent.name).toBe('c1');
+  });
+});

packages/core/database/src/__tests__/option-parser.test.ts

@@ -77,6 +77,7 @@ describe('option parser', () => {
         {
           association: 'tags',
           attributes: ['id', 'name'],
+          options: {},
         },
       ],
     });
@@ -130,6 +131,22 @@ describe('option parser', () => {
       expect(params['include'][0]['association']).toEqual('posts');
     });
 
+    it('should support append with options', () => {
+      const options = {
+        appends: ['posts(recursively=true)'],
+      };
+
+      const parser = new OptionsParser(options, {
+        collection: User,
+      });
+
+      const { include } = parser.toSequelizeParams();
+
+      expect(include[0].association).toEqual('posts');
+      expect(include[0].association);
+      expect(include[0].options.recursively).toBeTruthy();
+    });
+
     it('should handle field with association', () => {
       const options = {
         appends: ['posts'],

packages/core/database/src/eager-loading/eager-loading-tree.ts

@@ -3,6 +3,7 @@ import { Association, HasOne, Includeable, Model, ModelStatic, Op, Transaction }
 import Database from '../database';
 import { appendChildCollectionNameAfterRepositoryFind } from '../listeners/append-child-collection-name-after-repository-find';
 import { OptionsParser } from '../options-parser';
+import { AdjacencyListRepository } from '../repositories/tree-repository/adjacency-list-repository';
 
 interface EagerLoadingNode {
   model: ModelStatic<any>;
@@ -15,6 +16,7 @@ interface EagerLoadingNode {
   order?: any;
   where?: any;
   inspectInheritAttribute?: boolean;
+  includeOptions?: any;
 }
 
 const pushAttribute = (node, attribute) => {
@@ -106,6 +108,7 @@ export class EagerLoadingTree {
           parent: eagerLoadingTreeParent,
           where: include.where,
           children: [],
+          includeOption: include.options || {},
         });
 
         if (associationType == 'HasOne' || associationType == 'HasMany') {
@@ -258,9 +261,10 @@ export class EagerLoadingTree {
 
         if (associationType == 'BelongsTo') {
           const foreignKey = association.foreignKey;
-
           const parentInstancesForeignKeyValues = node.parent.instances.map((instance) => instance.get(foreignKey));
 
+          const collection = this.db.modelCollection.get(node.model);
+
           instances = await node.model.findAll({
             transaction,
             where: {
@@ -268,6 +272,47 @@ export class EagerLoadingTree {
             },
             attributes: node.attributes,
           });
+
+          // load parent instances recursively
+          if (node.includeOption.recursively && instances.length > 0) {
+            const targetKey = association.targetKey;
+            const sql = AdjacencyListRepository.queryParentSQL({
+              db: this.db,
+              collection,
+              foreignKey,
+              targetKey,
+              nodeIds: instances.map((instance) => instance.get(targetKey)),
+            });
+
+            const results = await this.db.sequelize.query(sql, {
+              type: 'SELECT',
+              transaction,
+            });
+
+            const parentInstances = await node.model.findAll({
+              transaction,
+              where: {
+                [association.targetKey]: results.map((result) => result[targetKey]),
+              },
+              attributes: node.attributes,
+            });
+
+            const setInstanceParent = (instance) => {
+              const parentInstance = parentInstances.find(
+                (parentInstance) => parentInstance.get(targetKey) == instance.get(foreignKey),
+              );
+              if (!parentInstance) {
+                return;
+              }
+
+              setInstanceParent(parentInstance);
+              instance[association.as] = instance.dataValues[association.as] = parentInstance;
+            };
+
+            for (const instance of instances) {
+              setInstanceParent(instance);
+            }
+          }
         }
 
         if (associationType == 'BelongsToMany') {

packages/core/database/src/options-parser.ts

@@ -4,6 +4,7 @@ import { Collection } from './collection';
 import { Database } from './database';
 import FilterParser from './filter-parser';
 import { Appends, Except, FindOptions } from './repository';
+import qs from 'qs';
 
 const debug = require('debug')('noco-database');
 
@@ -237,6 +238,21 @@ export class OptionsParser {
     return filterParams;
   }
 
+  protected parseAppendWithOptions(append: string) {
+    const parts = append.split('(');
+    const obj: { name: string; options?: object; raw?: string } = {
+      name: parts[0],
+    };
+
+    if (parts.length > 1) {
+      const optionsStr = parts[1].replace(')', '');
+      obj.options = qs.parse(optionsStr);
+      obj.raw = `(${optionsStr})`;
+    }
+
+    return obj;
+  }
+
   protected parseAppends(appends: Appends, filterParams: any) {
     if (!appends) return filterParams;
 
@@ -250,6 +266,10 @@ export class OptionsParser {
      * @param append
      */
     const setInclude = (model: ModelStatic<any>, queryParams: any, append: string) => {
+      const appendWithOptions = this.parseAppendWithOptions(append);
+
+      append = appendWithOptions.name;
+
       const appendFields = append.split('.');
       const appendAssociation = appendFields[0];
 
@@ -316,6 +336,7 @@ export class OptionsParser {
         // association not exists
         queryParams['include'].push({
           association: appendAssociation,
+          options: appendWithOptions.options || {},
         });
 
         existIncludeIndex = queryParams['include'].length - 1;
@@ -361,10 +382,15 @@ export class OptionsParser {
           };
         }
 
+        let nextAppend = appendFields.filter((_, index) => index !== 0).join('.');
+        if (appendWithOptions.raw) {
+          nextAppend += appendWithOptions.raw;
+        }
+
         setInclude(
           model.associations[queryParams['include'][existIncludeIndex].association].target,
           queryParams['include'][existIncludeIndex],
-          appendFields.filter((_, index) => index !== 0).join('.'),
+          nextAppend,
         );
       }
     };

packages/core/database/src/repositories/tree-repository/adjacency-list-repository.ts

@@ -1,7 +1,36 @@
 import lodash from 'lodash';
 import { FindOptions, Repository } from '../../repository';
+import Database from '../../database';
+import { Collection } from '../../collection';
 
 export class AdjacencyListRepository extends Repository {
+  static queryParentSQL(options: {
+    db: Database;
+    nodeIds: any[];
+    collection: Collection;
+    foreignKey: string;
+    targetKey: string;
+  }) {
+    const { collection, db, nodeIds } = options;
+    const tableName = collection.quotedTableName();
+    const { foreignKey, targetKey } = options;
+    const foreignKeyField = collection.model.rawAttributes[foreignKey].field;
+    const targetKeyField = collection.model.rawAttributes[targetKey].field;
+
+    const queryInterface = db.sequelize.getQueryInterface();
+    const q = queryInterface.quoteIdentifier.bind(queryInterface);
+    return `WITH RECURSIVE cte AS (
+      SELECT ${q(targetKeyField)}, ${q(foreignKeyField)}
+      FROM ${tableName}
+      WHERE ${q(targetKeyField)} IN (${nodeIds.join(',')})
+      UNION ALL
+      SELECT t.${q(targetKeyField)}, t.${q(foreignKeyField)}
+      FROM ${tableName} AS t
+      INNER JOIN cte ON t.${q(targetKeyField)} = cte.${q(foreignKeyField)}
+      )
+      SELECT ${q(targetKeyField)} AS ${q(targetKey)}, ${q(foreignKeyField)} AS ${q(foreignKey)} FROM cte`;
+  }
+
   async update(options): Promise<any> {
     return super.update({
       ...(options || {}),

packages/core/server/src/errors/plugin-not-exist.ts

@@ -0,0 +1 @@
+export class PluginNotExist extends Error {}