ruiling/tachybase_todo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: sort field init performance (#1675)

Browse Source 
* fix: sort field init performance

* chore: test table prefix

* fix: mysql compability
This commit is contained in:
ChengLei Shao 2023-04-08 22:08:32 +08:00 committed by GitHub
parent e9d5fa7f9c
commit 55c7bb5cf5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 107 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
packages/core/database/src/__tests__/fields/sort-field.test.ts
View File

@ -17,6 +17,50 @@ describe('string field', () => {
await db.close();
});
it('should init sorted value with thousand records', async () => {
const Test = db.collection({
name: 'tests',
fields: [
{
type: 'string',
name: 'name',
},
{
type: 'string',
name: 'group',
},
],
});
await db.sync();
await Test.model.bulkCreate(
(() => {
const values = [];
for (let i = 0; i < 100000; i++) {
values.push({
group: 'a',
name: `r${i}`,
});
values.push({
group: 'b',
name: `r${i}`,
});
}
return values;
})(),
);
Test.setField('sort', { type: 'sort', scopeKey: 'group' });
const begin = Date.now();
await db.sync();
const end = Date.now();
// log time cost as milliseconds
console.log(end - begin);
});
it('should init sorted value with scopeKey', async () => {
const Test = db.collection({
name: 'tests',

98
packages/core/database/src/fields/sort-field.ts
View File

@ -42,7 +42,19 @@ export class SortField extends Field {
};
initRecordsSortValue = async ({ transaction }) => {
const doInit = async (scopeKey = null, scopeValue = null) => {
const orderField = (() => {
const model = this.collection.model;
if (model.primaryKeyAttribute) {
return model.primaryKeyAttribute;
}
if (model.rawAttributes['createdAt']) {
return model.rawAttributes['createdAt'].field;
}
throw new Error(`can not find order key for collection ${this.collection.name}`);
})();
const needInit = async (scopeKey = null, scopeValue = null) => {
const filter = {};
if (scopeKey && scopeValue) {
filter[scopeKey] = scopeValue;
@ -61,44 +73,53 @@ export class SortField extends Field {
transaction,
});
const orderKey = (() => {
const model = this.collection.model;
if (model.primaryKeyAttribute) {
return model.primaryKeyAttribute;
}
if (model.rawAttributes['createdAt']) {
return 'createdAt';
return emptyCount === totalCount && emptyCount > 0;
};
const doInit = async (scopeKey = null, scopeValue = null) => {
const queryInterface = this.collection.db.sequelize.getQueryInterface();
const quotedOrderField = queryInterface.quoteIdentifier(orderField);
const sql = `
WITH ordered_table AS (
SELECT *, ROW_NUMBER() OVER (${
scopeKey ? `PARTITION BY ${queryInterface.quoteIdentifier(scopeKey)}` : ''
} ORDER BY ${quotedOrderField}) AS new_sequence_number
FROM ${this.collection.quotedTableName()}
${
scopeKey
? `WHERE ${queryInterface.quoteIdentifier(scopeKey)} IN (${scopeValue.map((v) => `'${v}'`).join(',')})`
: ''
}
)
${
this.collection.db.inDialect('mysql')
? `
UPDATE ${this.collection.quotedTableName()}, ordered_table
SET ${this.collection.quotedTableName()}.${this.name} = ordered_table.new_sequence_number
WHERE ${this.collection.quotedTableName()}.${quotedOrderField} = ordered_table.${quotedOrderField}
${scopeKey ? `AND ${this.collection.quotedTableName()}.${scopeKey} = ordered_table.${scopeKey}` : ''}
`
: `
UPDATE ${this.collection.quotedTableName()}
SET ${queryInterface.quoteIdentifier(this.name)} = ordered_table.new_sequence_number
FROM ordered_table
WHERE ${this.collection.quotedTableName()}.${quotedOrderField} = ${queryInterface.quoteIdentifier(
'ordered_table',
)}.${quotedOrderField};
`
}
throw new Error(`can not find order key for collection ${this.collection.name}`);
})();
`;
if (emptyCount === totalCount && emptyCount > 0) {
const records = await this.collection.repository.find({
order: [orderKey],
filter,
transaction,
});
let start = 1;
for (const record of records) {
await record.update(
{
sort: start,
},
{
hooks: false,
transaction,
silent: true,
},
);
start += 1;
}
}
await this.collection.db.sequelize.query(sql, {
transaction,
});
};
const scopeKey = this.options.scopeKey;
if (scopeKey) {
const groups = await this.collection.repository.find({
attributes: [scopeKey],
@ -106,10 +127,17 @@ export class SortField extends Field {
raw: true,
});
const needInitGroups = [];
for (const group of groups) {
await doInit(scopeKey, group[scopeKey]);
if (await needInit(scopeKey, group[scopeKey])) {
needInitGroups.push(group['group']);
}
}
} else {
if (needInitGroups.length > 0) {
await doInit(scopeKey, needInitGroups);
}
} else if (await needInit()) {
await doInit();
}
};