fix: tree performance (#1779)

* chore: tree performance

* chore: build index value

* chore: get tree index

* fix: children key

* fix: test

* fix: tree child to json

* fix: tree with fields

packages/core/database/src/__tests__/tree.test.ts

@@ -1,11 +1,15 @@
 import { Database } from '../database';
 import { mockDatabase } from './';
+import { AdjacencyListRepository } from '../tree-repository/adjacency-list-repository';
 
-describe('sort', function () {
+describe('tree test', function () {
   let db: Database;
 
   beforeEach(async () => {
-    db = mockDatabase();
+    db = mockDatabase({
+      tablePrefix: '',
+    });
+    await db.clean({ drop: true });
   });
 
   afterEach(async () => {
@@ -162,7 +166,7 @@ describe('sort', function () {
     expect(instance.toJSON()).toMatchObject(values[0]);
   });
 
-  it('should be tree', async () => {
+  it('should find tree collection', async () => {
     const collection = db.collection({
       name: 'categories',
       tree: 'adjacency-list',
@@ -214,4 +218,100 @@ describe('sort', function () {
 
     expect(instance.toJSON()).toMatchObject(values[0]);
   });
+
+  it('should get adjacency list repository', async () => {
+    const collection = db.collection({
+      name: 'categories',
+      tree: 'adjacency-list',
+      fields: [
+        {
+          type: 'string',
+          name: 'name',
+        },
+        {
+          type: 'belongsTo',
+          name: 'parent',
+          foreignKey: 'parentId',
+          treeParent: true,
+        },
+        {
+          type: 'hasMany',
+          name: 'children',
+          foreignKey: 'parentId',
+          treeChildren: true,
+        },
+      ],
+    });
+
+    const repository = db.getRepository('categories');
+    expect(repository).toBeInstanceOf(AdjacencyListRepository);
+  });
+
+  test('performance', async () => {
+    const collection = db.collection({
+      name: 'categories',
+      tree: 'adjacency-list',
+      fields: [
+        {
+          type: 'string',
+          name: 'name',
+        },
+        {
+          type: 'belongsTo',
+          name: 'parent',
+          foreignKey: 'parentId',
+          treeParent: true,
+        },
+        {
+          type: 'hasMany',
+          name: 'children',
+          foreignKey: 'parentId',
+          treeChildren: true,
+        },
+      ],
+    });
+    await db.sync();
+
+    const values = [];
+    for (let i = 0; i < 10; i++) {
+      const children = [];
+      for (let j = 0; j < 10; j++) {
+        const grandchildren = [];
+        for (let k = 0; k < 10; k++) {
+          grandchildren.push({
+            name: `name-${i}-${j}-${k}`,
+          });
+        }
+        children.push({
+          name: `name-${i}-${j}`,
+          children: grandchildren,
+        });
+      }
+
+      values.push({
+        name: `name-${i}`,
+        description: `description-${i}`,
+        children,
+      });
+    }
+
+    await db.getRepository('categories').create({
+      values,
+    });
+
+    const before = Date.now();
+
+    const instances = await db.getRepository('categories').find({
+      filter: {
+        parentId: null,
+      },
+      tree: true,
+      fields: ['id', 'name'],
+      sort: 'id',
+      limit: 10,
+    });
+
+    const after = Date.now();
+    console.log(after - before);
+  });
 });

packages/core/database/src/collection.ts

@@ -14,6 +14,7 @@ import { BelongsToField, Field, FieldOptions, HasManyField } from './fields';
 import { Model } from './model';
 import { Repository } from './repository';
 import { checkIdentifier, md5, snakeCase } from './utils';
+import { AdjacencyListRepository } from './tree-repository/adjacency-list-repository';
 
 export type RepositoryType = typeof Repository;
 
@@ -223,6 +224,11 @@ export class Collection<
     if (typeof repository === 'string') {
       repo = this.context.database.repositories.get(repository) || Repository;
     }
+
+    if (this.options.tree == 'adjacency-list' || this.options.tree == 'adjacencyList') {
+      repo = AdjacencyListRepository;
+    }
+
     this.repository = new repo(this);
   }
 

packages/core/database/src/listeners/adjacency-list.ts

@@ -17,44 +17,3 @@ export const beforeDefineAdjacencyListCollection = (options: CollectionOptions)
     }
   });
 };
-
-export const afterDefineAdjacencyListCollection = (collection: Collection) => {
-  if (!collection.options.tree) {
-    return;
-  }
-  collection.model.afterFind(async (instances, options: any) => {
-    if (!options.tree) {
-      return;
-    }
-    const foreignKey = collection.treeParentField?.foreignKey ?? 'parentId';
-    const childrenKey = collection.treeChildrenField?.name ?? 'children';
-    const arr: Model[] = Array.isArray(instances) ? instances : [instances];
-    let index = 0;
-    for (const instance of arr) {
-      const opts = {
-        ...lodash.pick(options, ['tree', 'fields', 'appends', 'except', 'sort']),
-      };
-      let __index = `${index++}`;
-      if (options.parentIndex) {
-        __index = `${options.parentIndex}.${__index}`;
-      }
-      instance.setDataValue('__index', __index);
-      const children = await collection.repository.find({
-        filter: {
-          [foreignKey]: instance.id,
-        },
-        transaction: options.transaction,
-        ...opts,
-        // @ts-ignore
-        parentIndex: `${__index}.${childrenKey}`,
-        context: options.context,
-      });
-      if (children?.length > 0) {
-        instance.setDataValue(
-          childrenKey,
-          children.map((r) => r.toJSON()),
-        );
-      }
-    }
-  });
-};

packages/core/database/src/listeners/index.ts

@@ -1,7 +1,6 @@
 import { Database } from '../database';
-import { afterDefineAdjacencyListCollection, beforeDefineAdjacencyListCollection } from './adjacency-list';
+import { beforeDefineAdjacencyListCollection } from './adjacency-list';
 
 export const registerBuiltInListeners = (db: Database) => {
   db.on('beforeDefineCollection', beforeDefineAdjacencyListCollection);
-  db.on('afterDefineCollection', afterDefineAdjacencyListCollection);
 };

packages/core/database/src/tree-repository/adjacency-list-repository.ts

@@ -0,0 +1,138 @@
+import { FindOptions, Repository } from '../repository';
+import lodash from 'lodash';
+
+export class AdjacencyListRepository extends Repository {
+  async find(options?: FindOptions): Promise<any> {
+    const parentNodes = await super.find(lodash.omit(options));
+
+    if (parentNodes.length === 0) {
+      return [];
+    }
+
+    const templateModel = parentNodes[0];
+    const collection = this.database.modelCollection.get(templateModel.constructor);
+    const primaryKey = collection.model.primaryKeyAttribute;
+    const { treeParentField } = collection;
+    const foreignKey = treeParentField.options.foreignKey;
+
+    const childrenKey = collection.treeChildrenField?.name ?? 'children';
+
+    const sql = this.querySQL(
+      parentNodes.map((node) => node.id),
+      collection,
+    );
+
+    const childNodes = await this.database.sequelize.query(sql, {
+      type: 'SELECT',
+      transaction: options.transaction,
+    });
+
+    const childIds = childNodes.map((node) => node[primaryKey]);
+
+    const findChildrenOptions = {
+      ...lodash.omit(options, ['limit', 'offset', 'filterByTk']),
+      filter: {
+        [primaryKey]: childIds,
+      },
+    };
+
+    if (findChildrenOptions.fields) {
+      [primaryKey, foreignKey].forEach((field) => {
+        if (!findChildrenOptions.fields.includes(field)) {
+          findChildrenOptions.fields.push(field);
+        }
+      });
+    }
+
+    const childInstances = (await super.find(findChildrenOptions)).map((r) => {
+      return r.toJSON();
+    });
+
+    const nodeMap = {};
+
+    childInstances.forEach((node) => {
+      if (!nodeMap[`${node[foreignKey]}`]) {
+        nodeMap[`${node[foreignKey]}`] = [];
+      }
+      nodeMap[`${node[foreignKey]}`].push(node);
+    });
+
+    function buildTree(parentId) {
+      const children = nodeMap[parentId];
+
+      if (!children) {
+        return [];
+      }
+
+      return children.map((child) => ({
+        ...child,
+        [childrenKey]: buildTree(child.id),
+      }));
+    }
+
+    for (const parent of parentNodes) {
+      const parentId = parent[primaryKey];
+      const children = buildTree(parentId);
+      parent.setDataValue(childrenKey, children);
+    }
+
+    this.addIndex(parentNodes, childrenKey);
+
+    return parentNodes;
+  }
+
+  private addIndex(treeArray, childrenKey = 'children') {
+    function traverse(node, index) {
+      let children;
+
+      if (lodash.isPlainObject(node)) {
+        node['__index'] = `${index}`;
+        children = node[childrenKey];
+        if (children.length === 0) {
+          delete node[childrenKey];
+        }
+      } else {
+        node.setDataValue('__index', `${index}`);
+        children = node.getDataValue(childrenKey);
+        if (children.length === 0) {
+          node.setDataValue(childrenKey, undefined);
+        }
+      }
+
+      if (children && children.length > 0) {
+        children.forEach((child, i) => {
+          traverse(child, `${index}.${childrenKey}.${i}`);
+        });
+      }
+    }
+
+    treeArray.forEach((tree, i) => {
+      traverse(tree, i);
+    });
+  }
+
+  private querySQL(rootIds, collection) {
+    const { treeChildrenField, treeParentField } = collection;
+    const foreignKey = treeParentField.options.foreignKey;
+    const foreignKeyField = collection.model.rawAttributes[foreignKey].field;
+
+    const primaryKey = collection.model.primaryKeyAttribute;
+
+    const queryInterface = this.database.sequelize.getQueryInterface();
+    const q = queryInterface.quoteIdentifier.bind(queryInterface);
+
+    return `
+      WITH RECURSIVE cte AS (
+        SELECT ${q(primaryKey)}, ${q(foreignKeyField)}, 1 AS level
+        FROM ${collection.quotedTableName()}
+        WHERE ${q(foreignKeyField)} IN (${rootIds.join(',')})
+        UNION ALL
+        SELECT t.${q(primaryKey)}, t.${q(foreignKeyField)}, cte.level + 1 AS level
+        FROM ${collection.quotedTableName()} t
+               JOIN cte ON t.${q(foreignKeyField)} = cte.${q(primaryKey)}
+      )
+      SELECT ${q(primaryKey)}, ${q(foreignKeyField)} as ${q(foreignKey)}, level
+      FROM cte
+    `;
+  }
+}

packages/plugins/acl/src/__tests__/list-action.test.ts

@@ -372,7 +372,7 @@ describe('list association action with acl', () => {
         sortable: false,
         name: 'table_a',
         template: 'tree',
-        tree: 'adjacencyList',
+        tree: 'adjacency-list',
         fields: [
           {
             interface: 'integer',
@@ -391,6 +391,7 @@ describe('list association action with acl', () => {
             interface: 'm2o',
             type: 'belongsTo',
             name: 'parent',
+            treeParent: true,
             foreignKey: 'parentId',
             uiSchema: {
               title: '{{t("Parent")}}',
@@ -409,6 +410,7 @@ describe('list association action with acl', () => {
               'x-component': 'RecordPicker',
               'x-component-props': { multiple: true, fieldNames: { label: 'id', value: 'id' } },
             },
+            treeChildren: true,
             target: 'table_a',
           },
           {