fix: slow join query issued by appends field in find method of repository (#845)
* fix: slow join query issue by appends field in repository.find * feat: handle appending query in multiple relation repository * feat: handle appending query in single relation repository Co-authored-by: chenos <chenlinxh@gmail.com> (cherry picked from commit 9222ff4f0ca92bc554107fccc0cf2a9d6290e56d) # Conflicts: # packages/core/database/src/__tests__/relation-repository/hasone-repository.test.ts # packages/core/database/src/repository.ts
This commit is contained in:
parent
92fda15efd
commit
f490fd358a
@ -1,7 +1,7 @@
|
|||||||
import { mockDatabase } from '../index';
|
|
||||||
import { HasOneRepository } from '../../relation-repository/hasone-repository';
|
|
||||||
import Database from '../../database';
|
|
||||||
import { Collection } from '../../collection';
|
import { Collection } from '../../collection';
|
||||||
|
import Database from '../../database';
|
||||||
|
import { HasOneRepository } from '../../relation-repository/hasone-repository';
|
||||||
|
import { mockDatabase } from '../index';
|
||||||
|
|
||||||
describe('has one repository', () => {
|
describe('has one repository', () => {
|
||||||
let db: Database;
|
let db: Database;
|
||||||
@ -9,6 +9,9 @@ describe('has one repository', () => {
|
|||||||
let User: Collection;
|
let User: Collection;
|
||||||
let Profile: Collection;
|
let Profile: Collection;
|
||||||
|
|
||||||
|
let A1: Collection;
|
||||||
|
let A2: Collection;
|
||||||
|
|
||||||
afterEach(async () => {
|
afterEach(async () => {
|
||||||
await db.close();
|
await db.close();
|
||||||
});
|
});
|
||||||
@ -25,27 +28,73 @@ describe('has one repository', () => {
|
|||||||
|
|
||||||
Profile = db.collection({
|
Profile = db.collection({
|
||||||
name: 'profiles',
|
name: 'profiles',
|
||||||
fields: [{ type: 'string', name: 'avatar' }],
|
fields: [
|
||||||
|
{ type: 'string', name: 'avatar' },
|
||||||
|
{
|
||||||
|
type: 'hasMany',
|
||||||
|
name: 'a1',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'hasMany',
|
||||||
|
name: 'a2',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
A1 = db.collection({
|
||||||
|
name: 'a1',
|
||||||
|
fields: [{ type: 'string', name: 'name' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
A2 = db.collection({
|
||||||
|
name: 'a2',
|
||||||
|
fields: [{ type: 'string', name: 'name' }],
|
||||||
});
|
});
|
||||||
|
|
||||||
await db.sync();
|
await db.sync();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('create', async () => {
|
test('find with appends', async () => {
|
||||||
const user = await User.repository.create({
|
const user = await User.repository.create({
|
||||||
values: { name: 'u1' },
|
|
||||||
});
|
|
||||||
|
|
||||||
const userProfileRepository = new HasOneRepository(User, 'profile', user['id']);
|
|
||||||
let profile = await userProfileRepository.find();
|
|
||||||
|
|
||||||
profile = await userProfileRepository.create({
|
|
||||||
values: {
|
values: {
|
||||||
avatar: 'avatar1',
|
name: 'u1',
|
||||||
|
profile: {
|
||||||
|
avatar: 'avatar',
|
||||||
|
a1: [
|
||||||
|
{
|
||||||
|
name: 'a11',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'a12',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'a13',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
a2: [
|
||||||
|
{
|
||||||
|
name: 'a21',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'a22',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'a23',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(profile.toJSON());
|
const UserProfileRepository = new HasOneRepository(User, 'profile', user['id']);
|
||||||
|
|
||||||
|
const profile = await UserProfileRepository.find({
|
||||||
|
appends: ['a1', 'a2'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const data = profile.toJSON();
|
||||||
|
expect(data['a1']).toBeDefined();
|
||||||
|
expect(data['a2']).toBeDefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('find', async () => {
|
test('find', async () => {
|
||||||
|
@ -10,6 +10,9 @@ describe('repository find', () => {
|
|||||||
let Comment: Collection;
|
let Comment: Collection;
|
||||||
let Tag: Collection;
|
let Tag: Collection;
|
||||||
|
|
||||||
|
let A1: Collection;
|
||||||
|
let A2: Collection;
|
||||||
|
|
||||||
afterEach(async () => {
|
afterEach(async () => {
|
||||||
await db.close();
|
await db.close();
|
||||||
});
|
});
|
||||||
@ -23,9 +26,21 @@ describe('repository find', () => {
|
|||||||
{ type: 'string', name: 'name' },
|
{ type: 'string', name: 'name' },
|
||||||
{ type: 'integer', name: 'age' },
|
{ type: 'integer', name: 'age' },
|
||||||
{ type: 'hasMany', name: 'posts' },
|
{ type: 'hasMany', name: 'posts' },
|
||||||
|
{ type: 'belongsToMany', name: 'a1' },
|
||||||
|
{ type: 'belongsToMany', name: 'a2' },
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
A1 = db.collection({
|
||||||
|
name: 'a1',
|
||||||
|
fields: [{ type: 'string', name: 'name' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
A2 = db.collection({
|
||||||
|
name: 'a2',
|
||||||
|
fields: [{ type: 'string', name: 'name' }],
|
||||||
|
});
|
||||||
|
|
||||||
Post = db.collection({
|
Post = db.collection({
|
||||||
name: 'posts',
|
name: 'posts',
|
||||||
fields: [
|
fields: [
|
||||||
@ -73,11 +88,15 @@ describe('repository find', () => {
|
|||||||
name: 'u1',
|
name: 'u1',
|
||||||
age: 10,
|
age: 10,
|
||||||
posts: [{ title: 'u1t1', comments: [{ content: 'u1t1c1' }], abc1: [{ name: 't1' }] }],
|
posts: [{ title: 'u1t1', comments: [{ content: 'u1t1c1' }], abc1: [{ name: 't1' }] }],
|
||||||
|
a1: [{ name: 'u1a11' }, { name: 'u1a12' }],
|
||||||
|
a2: [{ name: 'u1a21' }, { name: 'u1a22' }],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'u2',
|
name: 'u2',
|
||||||
age: 20,
|
age: 20,
|
||||||
posts: [{ title: 'u2t1', comments: [{ content: 'u2t1c1' }] }],
|
posts: [{ title: 'u2t1', comments: [{ content: 'u2t1c1' }] }],
|
||||||
|
a1: [{ name: 'u2a11' }, { name: 'u2a12' }],
|
||||||
|
a2: [{ name: 'u2a21' }, { name: 'u2a22' }],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'u3',
|
name: 'u3',
|
||||||
@ -161,6 +180,18 @@ describe('repository find', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('find with appends', () => {
|
describe('find with appends', () => {
|
||||||
|
test('toJSON', async () => {
|
||||||
|
const user = await User.repository.findOne({
|
||||||
|
filter: {
|
||||||
|
name: 'u1',
|
||||||
|
},
|
||||||
|
appends: ['a1', 'a2'],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(user['a1']).toBeDefined();
|
||||||
|
expect(user['a2']).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
test('filter attribute', async () => {
|
test('filter attribute', async () => {
|
||||||
const user = await User.repository.findOne({
|
const user = await User.repository.findOne({
|
||||||
filter: {
|
filter: {
|
||||||
|
@ -14,6 +14,7 @@ import {
|
|||||||
import { updateModelByValues } from '../update-associations';
|
import { updateModelByValues } from '../update-associations';
|
||||||
import { UpdateGuard } from '../update-guard';
|
import { UpdateGuard } from '../update-guard';
|
||||||
import { RelationRepository, transaction } from './relation-repository';
|
import { RelationRepository, transaction } from './relation-repository';
|
||||||
|
import { handleAppendsQuery } from '../utils';
|
||||||
|
|
||||||
export interface FindAndCountOptions extends CommonFindOptions {}
|
export interface FindAndCountOptions extends CommonFindOptions {}
|
||||||
|
|
||||||
@ -52,16 +53,30 @@ export abstract class MultipleRelationRepository extends RelationRepository {
|
|||||||
group: `${this.targetModel.name}.${this.targetKey()}`,
|
group: `${this.targetModel.name}.${this.targetKey()}`,
|
||||||
transaction,
|
transaction,
|
||||||
})
|
})
|
||||||
).map((row) => row.get(this.targetKey()));
|
).map((row) => {
|
||||||
|
return { row, pk: row.get(this.targetKey()) };
|
||||||
|
});
|
||||||
|
|
||||||
return await sourceModel[getAccessor]({
|
if (ids.length == 0) {
|
||||||
...omit(findOptions, ['limit', 'offset']),
|
return [];
|
||||||
where: {
|
}
|
||||||
[this.targetKey()]: {
|
|
||||||
[Op.in]: ids,
|
return await handleAppendsQuery({
|
||||||
},
|
templateModel: ids[0].row,
|
||||||
},
|
queryPromises: findOptions.include.map((include) => {
|
||||||
transaction,
|
return sourceModel[getAccessor]({
|
||||||
|
...omit(findOptions, ['limit', 'offset']),
|
||||||
|
include: [include],
|
||||||
|
where: {
|
||||||
|
[this.targetKey()]: {
|
||||||
|
[Op.in]: ids.map((id) => id.pk),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
transaction,
|
||||||
|
}).then((rows) => {
|
||||||
|
return { rows, include };
|
||||||
|
});
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,9 +1,10 @@
|
|||||||
import lodash from 'lodash';
|
import lodash, { omit } from 'lodash';
|
||||||
import { SingleAssociationAccessors, Transactionable } from 'sequelize';
|
import { SingleAssociationAccessors, Transactionable } from 'sequelize';
|
||||||
import { Model } from '../model';
|
import { Model } from '../model';
|
||||||
import { Appends, Except, Fields, Filter, TargetKey, UpdateOptions } from '../repository';
|
import { Appends, Except, Fields, Filter, TargetKey, UpdateOptions } from '../repository';
|
||||||
import { updateModelByValues } from '../update-associations';
|
import { updateModelByValues } from '../update-associations';
|
||||||
import { RelationRepository, transaction } from './relation-repository';
|
import { RelationRepository, transaction } from './relation-repository';
|
||||||
|
import { handleAppendsQuery } from '../utils';
|
||||||
|
|
||||||
export interface SingleRelationFindOption extends Transactionable {
|
export interface SingleRelationFindOption extends Transactionable {
|
||||||
fields?: Fields;
|
fields?: Fields;
|
||||||
@ -52,6 +53,29 @@ export abstract class SingleRelationRepository extends RelationRepository {
|
|||||||
const getAccessor = this.accessors().get;
|
const getAccessor = this.accessors().get;
|
||||||
const sourceModel = await this.getSourceModel(transaction);
|
const sourceModel = await this.getSourceModel(transaction);
|
||||||
|
|
||||||
|
if (findOptions?.include?.length > 0) {
|
||||||
|
const templateModel = await sourceModel[getAccessor]({
|
||||||
|
...findOptions,
|
||||||
|
transaction,
|
||||||
|
attributes: [this.targetKey()],
|
||||||
|
group: `${this.targetModel.name}.${this.targetKey()}`,
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await handleAppendsQuery({
|
||||||
|
templateModel,
|
||||||
|
queryPromises: findOptions.include.map((include) => {
|
||||||
|
return sourceModel[getAccessor]({
|
||||||
|
...findOptions,
|
||||||
|
include: [include],
|
||||||
|
}).then((row) => {
|
||||||
|
return { rows: [row], include };
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
return results[0];
|
||||||
|
}
|
||||||
|
|
||||||
return await sourceModel[getAccessor]({
|
return await sourceModel[getAccessor]({
|
||||||
...findOptions,
|
...findOptions,
|
||||||
transaction,
|
transaction,
|
||||||
|
@ -26,6 +26,7 @@ import { HasOneRepository } from './relation-repository/hasone-repository';
|
|||||||
import { RelationRepository } from './relation-repository/relation-repository';
|
import { RelationRepository } from './relation-repository/relation-repository';
|
||||||
import { updateAssociations, updateModelByValues } from './update-associations';
|
import { updateAssociations, updateModelByValues } from './update-associations';
|
||||||
import { UpdateGuard } from './update-guard';
|
import { UpdateGuard } from './update-guard';
|
||||||
|
import { handleAppendsQuery } from './utils';
|
||||||
|
|
||||||
const debug = require('debug')('noco-database');
|
const debug = require('debug')('noco-database');
|
||||||
|
|
||||||
@ -225,18 +226,34 @@ export class Repository<TModelAttributes extends {} = any, TCreationAttributes e
|
|||||||
group: `${model.name}.${primaryKeyField}`,
|
group: `${model.name}.${primaryKeyField}`,
|
||||||
transaction,
|
transaction,
|
||||||
})
|
})
|
||||||
).map((row) => row.get(primaryKeyField));
|
).map((row) => {
|
||||||
|
return { row, pk: row.get(primaryKeyField) };
|
||||||
|
});
|
||||||
|
|
||||||
|
if (ids.length == 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
const where = {
|
const where = {
|
||||||
[primaryKeyField]: {
|
[primaryKeyField]: {
|
||||||
[Op.in]: ids,
|
[Op.in]: ids.map((id) => id['pk']),
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
return await model.findAll({
|
return await handleAppendsQuery({
|
||||||
...omit(opts, ['limit', 'offset']),
|
queryPromises: opts.include.map((include) => {
|
||||||
where,
|
return model
|
||||||
transaction,
|
.findAll({
|
||||||
|
...omit(opts, ['limit', 'offset']),
|
||||||
|
include: include,
|
||||||
|
where,
|
||||||
|
transaction,
|
||||||
|
})
|
||||||
|
.then((rows) => {
|
||||||
|
return { rows, include };
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
templateModel: ids[0].row,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
45
packages/core/database/src/utils.ts
Normal file
45
packages/core/database/src/utils.ts
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
import { omit } from 'lodash';
|
||||||
|
import { Model } from './model';
|
||||||
|
|
||||||
|
type HandleAppendsQueryOptions = {
|
||||||
|
templateModel: any;
|
||||||
|
queryPromises: Array<any>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function handleAppendsQuery(options: HandleAppendsQueryOptions) {
|
||||||
|
const { templateModel, queryPromises } = options;
|
||||||
|
|
||||||
|
const results = await Promise.all(queryPromises);
|
||||||
|
|
||||||
|
let rows: Array<Model>;
|
||||||
|
|
||||||
|
for (const appendedResult of results) {
|
||||||
|
if (!rows) {
|
||||||
|
rows = appendedResult.rows;
|
||||||
|
|
||||||
|
if (rows.length == 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelOptions = templateModel['_options'];
|
||||||
|
for (const row of rows) {
|
||||||
|
row['_options'] = {
|
||||||
|
...row['_options'],
|
||||||
|
include: modelOptions['include'],
|
||||||
|
includeNames: modelOptions['includeNames'],
|
||||||
|
includeMap: modelOptions['includeMap'],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < appendedResult.rows.length; i++) {
|
||||||
|
const key = appendedResult.include.association;
|
||||||
|
const val = appendedResult.rows[i].get(key);
|
||||||
|
|
||||||
|
rows[i].set(key, val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return rows;
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user