feat: using bigint for id field (#1100)
* feat: using bigint for id field * fix: postgres bigint parse * fix: sequelize test * test: update to bigint migrator * chore: updateToBigInt method * fix: mysql update bigint * fix: update to bigint with inherits table * feat: update fields type in fields table * fix: import * fix: bigInt Co-authored-by: chenos <chenlinxh@gmail.com>
This commit is contained in:
parent
903fbfacce
commit
73e2d27e29
@ -10,7 +10,7 @@ export const id: IField = {
|
||||
sortable: true,
|
||||
default: {
|
||||
name: 'id',
|
||||
type: 'integer',
|
||||
type: 'bigInt',
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
allowNull: false,
|
||||
|
48
packages/core/database/src/__tests__/bigint.test.ts
Normal file
48
packages/core/database/src/__tests__/bigint.test.ts
Normal file
@ -0,0 +1,48 @@
|
||||
import { Database } from '../database';
|
||||
import { mockDatabase } from './index';
|
||||
|
||||
const excludeSqlite = () => (process.env.DB_DIALECT != 'sqlite' ? describe : describe.skip);
|
||||
|
||||
excludeSqlite()('collection', () => {
|
||||
let db: Database;
|
||||
|
||||
beforeEach(async () => {
|
||||
db = mockDatabase({
|
||||
logging: console.log,
|
||||
});
|
||||
|
||||
await db.clean({ drop: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await db.close();
|
||||
});
|
||||
|
||||
it('should using bigint for id field', async () => {
|
||||
const collection = db.collection({
|
||||
name: 'users',
|
||||
fields: [{ type: 'hasOne', name: 'profile' }],
|
||||
});
|
||||
|
||||
await db.sync();
|
||||
const tableInfo = await db.sequelize.getQueryInterface().describeTable(collection.model.tableName);
|
||||
|
||||
expect(tableInfo['id'].type).toBe('BIGINT');
|
||||
|
||||
const profile = db.collection({
|
||||
name: 'profiles',
|
||||
fields: [
|
||||
{
|
||||
type: 'belongsTo',
|
||||
name: 'user',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await db.sync();
|
||||
|
||||
const profileTableInfo = await db.sequelize.getQueryInterface().describeTable(profile.model.tableName);
|
||||
|
||||
expect(profileTableInfo['userId'].type).toBe('BIGINT');
|
||||
});
|
||||
});
|
@ -7,7 +7,11 @@ describe('collection', () => {
|
||||
let db: Database;
|
||||
|
||||
beforeEach(async () => {
|
||||
db = mockDatabase();
|
||||
db = mockDatabase({
|
||||
logging: console.log,
|
||||
});
|
||||
|
||||
await db.clean({ drop: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
@ -386,7 +386,7 @@ pgOnly()('collection inherits', () => {
|
||||
name: 'c',
|
||||
inherits: ['a', 'b'],
|
||||
fields: [
|
||||
{ type: 'integer', name: 'id', autoIncrement: true },
|
||||
{ type: 'bigInt', name: 'id', autoIncrement: true },
|
||||
{ type: 'string', name: 'c1' },
|
||||
],
|
||||
});
|
||||
|
@ -12,6 +12,7 @@ describe('belongs to many with target key', function () {
|
||||
beforeEach(async () => {
|
||||
db = mockDatabase();
|
||||
|
||||
await db.clean({ drop: true });
|
||||
Post = db.collection({
|
||||
name: 'posts',
|
||||
filterTargetKey: 'title',
|
||||
@ -121,6 +122,7 @@ describe('belongs to many', () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
db = mockDatabase();
|
||||
await db.clean({ drop: true });
|
||||
PostTag = db.collection({
|
||||
name: 'posts_tags',
|
||||
fields: [{ type: 'string', name: 'tagged_at' }],
|
||||
|
@ -6,6 +6,7 @@ import lodash from 'lodash';
|
||||
import { basename, isAbsolute, resolve } from 'path';
|
||||
import semver from 'semver';
|
||||
import {
|
||||
DataTypes,
|
||||
ModelCtor,
|
||||
Op,
|
||||
Options,
|
||||
@ -73,6 +74,7 @@ interface MapOf<T> {
|
||||
export interface IDatabaseOptions extends Options {
|
||||
tablePrefix?: string;
|
||||
migrator?: any;
|
||||
usingBigIntForId?: boolean;
|
||||
}
|
||||
|
||||
export type DatabaseOptions = IDatabaseOptions;
|
||||
@ -163,8 +165,6 @@ export class Database extends EventEmitter implements AsyncEmitter {
|
||||
constructor(options: DatabaseOptions) {
|
||||
super();
|
||||
|
||||
// this.setMaxListeners(100);
|
||||
|
||||
this.version = new DatabaseVersion(this);
|
||||
|
||||
const opts = {
|
||||
@ -189,6 +189,11 @@ export class Database extends EventEmitter implements AsyncEmitter {
|
||||
opts.timezone = '+00:00';
|
||||
}
|
||||
|
||||
if (options.dialect === 'postgres') {
|
||||
// https://github.com/sequelize/sequelize/issues/1774
|
||||
require('pg').defaults.parseInt8 = true;
|
||||
}
|
||||
|
||||
this.sequelize = new Sequelize(opts);
|
||||
this.options = opts;
|
||||
this.collections = new Map();
|
||||
@ -266,6 +271,16 @@ export class Database extends EventEmitter implements AsyncEmitter {
|
||||
this.on('afterRemoveCollection', (collection) => {
|
||||
this.inheritanceMap.removeNode(collection.name);
|
||||
});
|
||||
|
||||
this.on('afterDefine', (model) => {
|
||||
if (lodash.get(this.options, 'usingBigIntForId', true)) {
|
||||
const idAttribute = model.rawAttributes['id'];
|
||||
if (idAttribute && idAttribute.primaryKey) {
|
||||
model.rawAttributes['id'].type = DataTypes.BIGINT;
|
||||
model.refreshAttributes();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
addMigration(item: MigrationItem) {
|
||||
|
@ -7,7 +7,7 @@ const sortFieldMutex = new Mutex();
|
||||
|
||||
export class SortField extends Field {
|
||||
get dataType() {
|
||||
return DataTypes.INTEGER;
|
||||
return DataTypes.BIGINT;
|
||||
}
|
||||
|
||||
setSortValue = async (instance, options) => {
|
||||
@ -32,14 +32,14 @@ export class SortField extends Field {
|
||||
const newValue = (max || 0) + 1;
|
||||
instance.set(name, newValue);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
onScopeChange = async (instance, options) => {
|
||||
const { scopeKey } = this.options;
|
||||
if (scopeKey && !instance.isNewRecord && instance._previousDataValues[scopeKey] != instance[scopeKey]) {
|
||||
await this.setSortValue(instance, options);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
initRecordsSortValue = async ({ transaction }) => {
|
||||
const totalCount = await this.collection.repository.count({
|
||||
@ -73,7 +73,7 @@ export class SortField extends Field {
|
||||
start += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
bind() {
|
||||
super.bind();
|
||||
|
@ -1,11 +1,13 @@
|
||||
import lodash from 'lodash';
|
||||
import { Model as SequelizeModel, ModelCtor } from 'sequelize';
|
||||
import { DataTypes, Model as SequelizeModel, ModelCtor } from 'sequelize';
|
||||
import { Collection } from './collection';
|
||||
import { Database } from './database';
|
||||
import { Field } from './fields';
|
||||
import type { InheritedCollection } from './inherited-collection';
|
||||
import { SyncRunner } from './sync-runner';
|
||||
|
||||
const _ = lodash;
|
||||
|
||||
interface IModel {
|
||||
[key: string]: any;
|
||||
}
|
||||
|
@ -6,6 +6,7 @@ import Plugin from '../';
|
||||
export async function createApp(options = {}) {
|
||||
const app = mockServer({
|
||||
acl: false,
|
||||
...options,
|
||||
});
|
||||
|
||||
app.plugin(PluginErrorHandler, { name: 'error-handler' });
|
||||
|
@ -0,0 +1,114 @@
|
||||
import { Database, MigrationContext } from '@nocobase/database';
|
||||
import Migrator from '../../migrations/20221117111110-update-id-to-bigint';
|
||||
|
||||
const excludeSqlite = () => (process.env.DB_DIALECT != 'sqlite' ? describe : describe.skip);
|
||||
|
||||
import { createApp } from '../index';
|
||||
import { MockServer } from '@nocobase/test';
|
||||
|
||||
excludeSqlite()('update id to bigint test', () => {
|
||||
let app: MockServer;
|
||||
let db: Database;
|
||||
|
||||
beforeEach(async () => {
|
||||
app = await createApp({
|
||||
database: {
|
||||
usingBigIntForId: false,
|
||||
},
|
||||
});
|
||||
db = app.db;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.destroy();
|
||||
});
|
||||
|
||||
it('should update id to bigint', async () => {
|
||||
db.collection({
|
||||
name: 'groups',
|
||||
});
|
||||
|
||||
const Users = db.collection({
|
||||
name: 'users',
|
||||
fields: [
|
||||
{ type: 'belongsTo', name: 'group', foreignKey: 'groupId' },
|
||||
{
|
||||
type: 'hasOne',
|
||||
name: 'profile',
|
||||
},
|
||||
{
|
||||
type: 'hasMany',
|
||||
name: 'orders',
|
||||
},
|
||||
{
|
||||
type: 'belongsToMany',
|
||||
name: 'tags',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
db.collection({
|
||||
name: 'tags',
|
||||
});
|
||||
|
||||
db.collection({
|
||||
name: 'profiles',
|
||||
fields: [
|
||||
{
|
||||
type: 'belongsTo',
|
||||
name: 'user',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
db.collection({
|
||||
name: 'orders',
|
||||
});
|
||||
await db.sync();
|
||||
|
||||
const assertBigInt = async (collectionName, fieldName) => {
|
||||
const tableInfo = await db.sequelize
|
||||
.getQueryInterface()
|
||||
.describeTable(
|
||||
db.getCollection(collectionName) ? db.getCollection(collectionName).model.tableName : collectionName,
|
||||
);
|
||||
console.log(`${collectionName}, ${fieldName}`, tableInfo[fieldName].type);
|
||||
expect(tableInfo[fieldName].type).toBe('BIGINT');
|
||||
};
|
||||
|
||||
const assertInteger = (val) => {
|
||||
if (db.inDialect('postgres', 'sqlite')) {
|
||||
expect(val).toBe('INTEGER');
|
||||
} else {
|
||||
expect(val).toBe('INT');
|
||||
}
|
||||
};
|
||||
|
||||
let usersTableInfo = await db.sequelize
|
||||
.getQueryInterface()
|
||||
.describeTable(db.getCollection('users').model.tableName);
|
||||
|
||||
assertInteger(usersTableInfo.id.type);
|
||||
|
||||
const migration = new Migrator({ db } as MigrationContext);
|
||||
migration.context.app = app;
|
||||
await migration.up();
|
||||
|
||||
//@ts-ignore
|
||||
const throughTableName = Users.model.associations.tags.through.model.tableName;
|
||||
|
||||
const asserts = [
|
||||
'users#id',
|
||||
'profiles#userId',
|
||||
'users#groupId',
|
||||
'orders#userId',
|
||||
`${throughTableName}#userId`,
|
||||
`${throughTableName}#tagId`,
|
||||
];
|
||||
|
||||
for (const assert of asserts) {
|
||||
const [collectionName, fieldName] = assert.split('#');
|
||||
await assertBigInt(collectionName, fieldName);
|
||||
}
|
||||
});
|
||||
});
|
@ -0,0 +1,123 @@
|
||||
import { Migration } from '@nocobase/server';
|
||||
import { DataTypes } from '@nocobase/database';
|
||||
|
||||
export default class UpdateIdToBigIntMigrator extends Migration {
|
||||
async up() {
|
||||
const db = this.app.db;
|
||||
|
||||
await db.getCollection('fields').repository.update({
|
||||
filter: {
|
||||
name: 'id',
|
||||
type: 'integer',
|
||||
},
|
||||
values: {
|
||||
type: 'bigInt',
|
||||
},
|
||||
});
|
||||
|
||||
if (!db.inDialect('mysql', 'postgres')) {
|
||||
return;
|
||||
}
|
||||
|
||||
const models = [];
|
||||
|
||||
const queryInterface = db.sequelize.getQueryInterface() as any;
|
||||
|
||||
const queryGenerator = queryInterface.queryGenerator as any;
|
||||
|
||||
const updateToBigInt = async (model, fieldName) => {
|
||||
const tableName = model.tableName;
|
||||
if (model.rawAttributes[fieldName].type instanceof DataTypes.INTEGER) {
|
||||
if (db.inDialect('postgres')) {
|
||||
await this.sequelize.query(
|
||||
`ALTER TABLE "${tableName}" ALTER COLUMN "${fieldName}" SET DATA TYPE BIGINT;`,
|
||||
{},
|
||||
);
|
||||
} else if (db.inDialect('mysql')) {
|
||||
const dataTypeOrOptions = model.rawAttributes[fieldName];
|
||||
const attributeName = fieldName;
|
||||
|
||||
const query = queryGenerator.attributesToSQL(
|
||||
{
|
||||
[attributeName]: queryInterface.normalizeAttribute({
|
||||
...dataTypeOrOptions,
|
||||
type: DataTypes.BIGINT,
|
||||
}),
|
||||
},
|
||||
{
|
||||
context: 'changeColumn',
|
||||
table: tableName,
|
||||
},
|
||||
);
|
||||
const sql = queryGenerator.changeColumnQuery(tableName, query);
|
||||
|
||||
await this.sequelize.query(sql.replace(' PRIMARY KEY;', ' ;'), {});
|
||||
}
|
||||
|
||||
this.app.log.info(`updated ${tableName}.${fieldName} to BIGINT`, tableName, fieldName);
|
||||
}
|
||||
};
|
||||
|
||||
//@ts-ignore
|
||||
this.app.db.sequelize.modelManager.forEachModel((model) => {
|
||||
models.push(model);
|
||||
});
|
||||
|
||||
for (const model of models) {
|
||||
try {
|
||||
const primaryKeyField = model.tableAttributes[model.primaryKeyField];
|
||||
|
||||
if (primaryKeyField && primaryKeyField.primaryKey) {
|
||||
await updateToBigInt(model, model.primaryKeyField);
|
||||
}
|
||||
|
||||
if (model.tableAttributes['sort'] && model.tableAttributes['sort'].type instanceof DataTypes.INTEGER) {
|
||||
await updateToBigInt(model, 'sort');
|
||||
}
|
||||
|
||||
const associations = model.associations;
|
||||
for (const associationName of Object.keys(associations)) {
|
||||
const association = associations[associationName];
|
||||
|
||||
const type = association.associationType;
|
||||
let foreignModel;
|
||||
let fieldName;
|
||||
|
||||
if (type === 'BelongsTo') {
|
||||
foreignModel = association.source;
|
||||
fieldName = association.foreignKey;
|
||||
}
|
||||
|
||||
if (type === 'HasMany') {
|
||||
foreignModel = association.target;
|
||||
fieldName = association.foreignKey;
|
||||
}
|
||||
|
||||
if (type === 'HasOne') {
|
||||
foreignModel = association.target;
|
||||
fieldName = association.foreignKey;
|
||||
}
|
||||
|
||||
if (foreignModel && fieldName) {
|
||||
await updateToBigInt(foreignModel, fieldName);
|
||||
}
|
||||
|
||||
if (type === 'BelongsToMany') {
|
||||
const throughModel = association.through.model;
|
||||
const otherKey = association.otherKey;
|
||||
const foreignKey = association.foreignKey;
|
||||
|
||||
await updateToBigInt(throughModel, otherKey);
|
||||
await updateToBigInt(throughModel, foreignKey);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.message.includes('cannot alter inherited column')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -11,7 +11,7 @@ export default {
|
||||
fields: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'integer',
|
||||
type: 'bigInt',
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
allowNull: false,
|
||||
|
@ -2,7 +2,7 @@ import parse from 'json-templates';
|
||||
import { resolve } from 'path';
|
||||
|
||||
import { Collection, Op } from '@nocobase/database';
|
||||
import { HandlerType, Middleware } from '@nocobase/resourcer';
|
||||
import { HandlerType } from '@nocobase/resourcer';
|
||||
import { Plugin } from '@nocobase/server';
|
||||
import { Registry } from '@nocobase/utils';
|
||||
|
||||
@ -56,7 +56,7 @@ export default class UsersPlugin extends Plugin<UserPluginConfig> {
|
||||
if (createdBy === true) {
|
||||
collection.setField('createdById', {
|
||||
type: 'context',
|
||||
dataType: 'integer',
|
||||
dataType: 'bigInt',
|
||||
dataIndex: 'state.currentUser.id',
|
||||
createOnly: true,
|
||||
visible: true,
|
||||
@ -72,7 +72,7 @@ export default class UsersPlugin extends Plugin<UserPluginConfig> {
|
||||
if (updatedBy === true) {
|
||||
collection.setField('updatedById', {
|
||||
type: 'context',
|
||||
dataType: 'integer',
|
||||
dataType: 'bigInt',
|
||||
dataIndex: 'state.currentUser.id',
|
||||
visible: true,
|
||||
index: true,
|
||||
|
@ -4,49 +4,49 @@ export default {
|
||||
name: 'users_jobs',
|
||||
fields: [
|
||||
{
|
||||
type: 'integer',
|
||||
type: 'bigInt',
|
||||
name: 'id',
|
||||
primaryKey: true,
|
||||
autoIncrement: true
|
||||
autoIncrement: true,
|
||||
},
|
||||
{
|
||||
type: 'integer',
|
||||
type: 'bigInt',
|
||||
name: 'userId',
|
||||
primaryKey: false,
|
||||
},
|
||||
{
|
||||
type: 'integer',
|
||||
type: 'bigInt',
|
||||
name: 'jobId',
|
||||
primaryKey: false,
|
||||
},
|
||||
{
|
||||
type: 'belongsTo',
|
||||
name: 'job'
|
||||
name: 'job',
|
||||
},
|
||||
{
|
||||
type: 'belongsTo',
|
||||
name: 'user'
|
||||
name: 'user',
|
||||
},
|
||||
{
|
||||
type: 'belongsTo',
|
||||
name: 'execution'
|
||||
name: 'execution',
|
||||
},
|
||||
{
|
||||
type: 'belongsTo',
|
||||
name: 'node',
|
||||
target: 'flow_nodes'
|
||||
target: 'flow_nodes',
|
||||
},
|
||||
{
|
||||
type: 'belongsTo',
|
||||
name: 'workflow'
|
||||
name: 'workflow',
|
||||
},
|
||||
{
|
||||
type: 'integer',
|
||||
name: 'status'
|
||||
name: 'status',
|
||||
},
|
||||
{
|
||||
type: 'jsonb',
|
||||
name: 'result'
|
||||
}
|
||||
]
|
||||
name: 'result',
|
||||
},
|
||||
],
|
||||
} as CollectionOptions;
|
||||
|
Loading…
Reference in New Issue
Block a user