diff --git a/packages/core/client/src/collection-manager/interfaces/id.ts b/packages/core/client/src/collection-manager/interfaces/id.ts index 19c670f40..4a08ccdb8 100644 --- a/packages/core/client/src/collection-manager/interfaces/id.ts +++ b/packages/core/client/src/collection-manager/interfaces/id.ts @@ -10,7 +10,7 @@ export const id: IField = { sortable: true, default: { name: 'id', - type: 'integer', + type: 'bigInt', autoIncrement: true, primaryKey: true, allowNull: false, diff --git a/packages/core/database/src/__tests__/bigint.test.ts b/packages/core/database/src/__tests__/bigint.test.ts new file mode 100644 index 000000000..0e379e98d --- /dev/null +++ b/packages/core/database/src/__tests__/bigint.test.ts @@ -0,0 +1,48 @@ +import { Database } from '../database'; +import { mockDatabase } from './index'; + +const excludeSqlite = () => (process.env.DB_DIALECT != 'sqlite' ? describe : describe.skip); + +excludeSqlite()('collection', () => { + let db: Database; + + beforeEach(async () => { + db = mockDatabase({ + logging: console.log, + }); + + await db.clean({ drop: true }); + }); + + afterEach(async () => { + await db.close(); + }); + + it('should using bigint for id field', async () => { + const collection = db.collection({ + name: 'users', + fields: [{ type: 'hasOne', name: 'profile' }], + }); + + await db.sync(); + const tableInfo = await db.sequelize.getQueryInterface().describeTable(collection.model.tableName); + + expect(tableInfo['id'].type).toBe('BIGINT'); + + const profile = db.collection({ + name: 'profiles', + fields: [ + { + type: 'belongsTo', + name: 'user', + }, + ], + }); + + await db.sync(); + + const profileTableInfo = await db.sequelize.getQueryInterface().describeTable(profile.model.tableName); + + expect(profileTableInfo['userId'].type).toBe('BIGINT'); + }); +}); diff --git a/packages/core/database/src/__tests__/collection.test.ts b/packages/core/database/src/__tests__/collection.test.ts index a3ac75869..6bf90e1c6 100644 --- a/packages/core/database/src/__tests__/collection.test.ts +++ b/packages/core/database/src/__tests__/collection.test.ts @@ -7,7 +7,11 @@ describe('collection', () => { let db: Database; beforeEach(async () => { - db = mockDatabase(); + db = mockDatabase({ + logging: console.log, + }); + + await db.clean({ drop: true }); }); afterEach(async () => { diff --git a/packages/core/database/src/__tests__/inhertits/collection-inherits.test.ts b/packages/core/database/src/__tests__/inhertits/collection-inherits.test.ts index 35224fe06..e5a2ce81d 100644 --- a/packages/core/database/src/__tests__/inhertits/collection-inherits.test.ts +++ b/packages/core/database/src/__tests__/inhertits/collection-inherits.test.ts @@ -386,7 +386,7 @@ pgOnly()('collection inherits', () => { name: 'c', inherits: ['a', 'b'], fields: [ - { type: 'integer', name: 'id', autoIncrement: true }, + { type: 'bigInt', name: 'id', autoIncrement: true }, { type: 'string', name: 'c1' }, ], }); diff --git a/packages/core/database/src/__tests__/relation-repository/belongs-to-many-repository.test.ts b/packages/core/database/src/__tests__/relation-repository/belongs-to-many-repository.test.ts index c55f09eee..0c2a8fdd1 100644 --- a/packages/core/database/src/__tests__/relation-repository/belongs-to-many-repository.test.ts +++ b/packages/core/database/src/__tests__/relation-repository/belongs-to-many-repository.test.ts @@ -12,6 +12,7 @@ describe('belongs to many with target key', function () { beforeEach(async () => { db = mockDatabase(); + await db.clean({ drop: true }); Post = db.collection({ name: 'posts', filterTargetKey: 'title', @@ -121,6 +122,7 @@ describe('belongs to many', () => { beforeEach(async () => { db = mockDatabase(); + await db.clean({ drop: true }); PostTag = db.collection({ name: 'posts_tags', fields: [{ type: 'string', name: 'tagged_at' }], diff --git a/packages/core/database/src/database.ts b/packages/core/database/src/database.ts index 872db1d11..10c9bd672 100644 --- a/packages/core/database/src/database.ts +++ b/packages/core/database/src/database.ts @@ -6,6 +6,7 @@ import lodash from 'lodash'; import { basename, isAbsolute, resolve } from 'path'; import semver from 'semver'; import { + DataTypes, ModelCtor, Op, Options, @@ -73,6 +74,7 @@ interface MapOf { export interface IDatabaseOptions extends Options { tablePrefix?: string; migrator?: any; + usingBigIntForId?: boolean; } export type DatabaseOptions = IDatabaseOptions; @@ -163,8 +165,6 @@ export class Database extends EventEmitter implements AsyncEmitter { constructor(options: DatabaseOptions) { super(); - // this.setMaxListeners(100); - this.version = new DatabaseVersion(this); const opts = { @@ -189,6 +189,11 @@ export class Database extends EventEmitter implements AsyncEmitter { opts.timezone = '+00:00'; } + if (options.dialect === 'postgres') { + // https://github.com/sequelize/sequelize/issues/1774 + require('pg').defaults.parseInt8 = true; + } + this.sequelize = new Sequelize(opts); this.options = opts; this.collections = new Map(); @@ -266,6 +271,16 @@ export class Database extends EventEmitter implements AsyncEmitter { this.on('afterRemoveCollection', (collection) => { this.inheritanceMap.removeNode(collection.name); }); + + this.on('afterDefine', (model) => { + if (lodash.get(this.options, 'usingBigIntForId', true)) { + const idAttribute = model.rawAttributes['id']; + if (idAttribute && idAttribute.primaryKey) { + model.rawAttributes['id'].type = DataTypes.BIGINT; + model.refreshAttributes(); + } + } + }); } addMigration(item: MigrationItem) { diff --git a/packages/core/database/src/fields/sort-field.ts b/packages/core/database/src/fields/sort-field.ts index 131ba3dc9..d53d82ca5 100644 --- a/packages/core/database/src/fields/sort-field.ts +++ b/packages/core/database/src/fields/sort-field.ts @@ -7,7 +7,7 @@ const sortFieldMutex = new Mutex(); export class SortField extends Field { get dataType() { - return DataTypes.INTEGER; + return DataTypes.BIGINT; } setSortValue = async (instance, options) => { @@ -32,14 +32,14 @@ export class SortField extends Field { const newValue = (max || 0) + 1; instance.set(name, newValue); }); - } + }; onScopeChange = async (instance, options) => { const { scopeKey } = this.options; if (scopeKey && !instance.isNewRecord && instance._previousDataValues[scopeKey] != instance[scopeKey]) { await this.setSortValue(instance, options); } - } + }; initRecordsSortValue = async ({ transaction }) => { const totalCount = await this.collection.repository.count({ @@ -73,7 +73,7 @@ export class SortField extends Field { start += 1; } } - } + }; bind() { super.bind(); diff --git a/packages/core/database/src/model.ts b/packages/core/database/src/model.ts index ca912c5e9..275c02517 100644 --- a/packages/core/database/src/model.ts +++ b/packages/core/database/src/model.ts @@ -1,11 +1,13 @@ import lodash from 'lodash'; -import { Model as SequelizeModel, ModelCtor } from 'sequelize'; +import { DataTypes, Model as SequelizeModel, ModelCtor } from 'sequelize'; import { Collection } from './collection'; import { Database } from './database'; import { Field } from './fields'; import type { InheritedCollection } from './inherited-collection'; import { SyncRunner } from './sync-runner'; +const _ = lodash; + interface IModel { [key: string]: any; } diff --git a/packages/plugins/collection-manager/src/__tests__/index.ts b/packages/plugins/collection-manager/src/__tests__/index.ts index 6aede15ec..6e9dcf4c3 100644 --- a/packages/plugins/collection-manager/src/__tests__/index.ts +++ b/packages/plugins/collection-manager/src/__tests__/index.ts @@ -6,6 +6,7 @@ import Plugin from '../'; export async function createApp(options = {}) { const app = mockServer({ acl: false, + ...options, }); app.plugin(PluginErrorHandler, { name: 'error-handler' }); diff --git a/packages/plugins/collection-manager/src/__tests__/migrations/update-id-to-bigint.test.ts b/packages/plugins/collection-manager/src/__tests__/migrations/update-id-to-bigint.test.ts new file mode 100644 index 000000000..96d5c0075 --- /dev/null +++ b/packages/plugins/collection-manager/src/__tests__/migrations/update-id-to-bigint.test.ts @@ -0,0 +1,114 @@ +import { Database, MigrationContext } from '@nocobase/database'; +import Migrator from '../../migrations/20221117111110-update-id-to-bigint'; + +const excludeSqlite = () => (process.env.DB_DIALECT != 'sqlite' ? describe : describe.skip); + +import { createApp } from '../index'; +import { MockServer } from '@nocobase/test'; + +excludeSqlite()('update id to bigint test', () => { + let app: MockServer; + let db: Database; + + beforeEach(async () => { + app = await createApp({ + database: { + usingBigIntForId: false, + }, + }); + db = app.db; + }); + + afterEach(async () => { + await app.destroy(); + }); + + it('should update id to bigint', async () => { + db.collection({ + name: 'groups', + }); + + const Users = db.collection({ + name: 'users', + fields: [ + { type: 'belongsTo', name: 'group', foreignKey: 'groupId' }, + { + type: 'hasOne', + name: 'profile', + }, + { + type: 'hasMany', + name: 'orders', + }, + { + type: 'belongsToMany', + name: 'tags', + }, + ], + }); + + db.collection({ + name: 'tags', + }); + + db.collection({ + name: 'profiles', + fields: [ + { + type: 'belongsTo', + name: 'user', + }, + ], + }); + + db.collection({ + name: 'orders', + }); + await db.sync(); + + const assertBigInt = async (collectionName, fieldName) => { + const tableInfo = await db.sequelize + .getQueryInterface() + .describeTable( + db.getCollection(collectionName) ? db.getCollection(collectionName).model.tableName : collectionName, + ); + console.log(`${collectionName}, ${fieldName}`, tableInfo[fieldName].type); + expect(tableInfo[fieldName].type).toBe('BIGINT'); + }; + + const assertInteger = (val) => { + if (db.inDialect('postgres', 'sqlite')) { + expect(val).toBe('INTEGER'); + } else { + expect(val).toBe('INT'); + } + }; + + let usersTableInfo = await db.sequelize + .getQueryInterface() + .describeTable(db.getCollection('users').model.tableName); + + assertInteger(usersTableInfo.id.type); + + const migration = new Migrator({ db } as MigrationContext); + migration.context.app = app; + await migration.up(); + + //@ts-ignore + const throughTableName = Users.model.associations.tags.through.model.tableName; + + const asserts = [ + 'users#id', + 'profiles#userId', + 'users#groupId', + 'orders#userId', + `${throughTableName}#userId`, + `${throughTableName}#tagId`, + ]; + + for (const assert of asserts) { + const [collectionName, fieldName] = assert.split('#'); + await assertBigInt(collectionName, fieldName); + } + }); +}); diff --git a/packages/plugins/collection-manager/src/migrations/20221117111110-update-id-to-bigint.ts b/packages/plugins/collection-manager/src/migrations/20221117111110-update-id-to-bigint.ts new file mode 100644 index 000000000..2abd03585 --- /dev/null +++ b/packages/plugins/collection-manager/src/migrations/20221117111110-update-id-to-bigint.ts @@ -0,0 +1,123 @@ +import { Migration } from '@nocobase/server'; +import { DataTypes } from '@nocobase/database'; + +export default class UpdateIdToBigIntMigrator extends Migration { + async up() { + const db = this.app.db; + + await db.getCollection('fields').repository.update({ + filter: { + name: 'id', + type: 'integer', + }, + values: { + type: 'bigInt', + }, + }); + + if (!db.inDialect('mysql', 'postgres')) { + return; + } + + const models = []; + + const queryInterface = db.sequelize.getQueryInterface() as any; + + const queryGenerator = queryInterface.queryGenerator as any; + + const updateToBigInt = async (model, fieldName) => { + const tableName = model.tableName; + if (model.rawAttributes[fieldName].type instanceof DataTypes.INTEGER) { + if (db.inDialect('postgres')) { + await this.sequelize.query( + `ALTER TABLE "${tableName}" ALTER COLUMN "${fieldName}" SET DATA TYPE BIGINT;`, + {}, + ); + } else if (db.inDialect('mysql')) { + const dataTypeOrOptions = model.rawAttributes[fieldName]; + const attributeName = fieldName; + + const query = queryGenerator.attributesToSQL( + { + [attributeName]: queryInterface.normalizeAttribute({ + ...dataTypeOrOptions, + type: DataTypes.BIGINT, + }), + }, + { + context: 'changeColumn', + table: tableName, + }, + ); + const sql = queryGenerator.changeColumnQuery(tableName, query); + + await this.sequelize.query(sql.replace(' PRIMARY KEY;', ' ;'), {}); + } + + this.app.log.info(`updated ${tableName}.${fieldName} to BIGINT`, tableName, fieldName); + } + }; + + //@ts-ignore + this.app.db.sequelize.modelManager.forEachModel((model) => { + models.push(model); + }); + + for (const model of models) { + try { + const primaryKeyField = model.tableAttributes[model.primaryKeyField]; + + if (primaryKeyField && primaryKeyField.primaryKey) { + await updateToBigInt(model, model.primaryKeyField); + } + + if (model.tableAttributes['sort'] && model.tableAttributes['sort'].type instanceof DataTypes.INTEGER) { + await updateToBigInt(model, 'sort'); + } + + const associations = model.associations; + for (const associationName of Object.keys(associations)) { + const association = associations[associationName]; + + const type = association.associationType; + let foreignModel; + let fieldName; + + if (type === 'BelongsTo') { + foreignModel = association.source; + fieldName = association.foreignKey; + } + + if (type === 'HasMany') { + foreignModel = association.target; + fieldName = association.foreignKey; + } + + if (type === 'HasOne') { + foreignModel = association.target; + fieldName = association.foreignKey; + } + + if (foreignModel && fieldName) { + await updateToBigInt(foreignModel, fieldName); + } + + if (type === 'BelongsToMany') { + const throughModel = association.through.model; + const otherKey = association.otherKey; + const foreignKey = association.foreignKey; + + await updateToBigInt(throughModel, otherKey); + await updateToBigInt(throughModel, foreignKey); + } + } + } catch (error) { + if (error.message.includes('cannot alter inherited column')) { + continue; + } + + throw error; + } + } + } +} diff --git a/packages/plugins/users/src/collections/users.ts b/packages/plugins/users/src/collections/users.ts index 2e276eb58..6d4828261 100644 --- a/packages/plugins/users/src/collections/users.ts +++ b/packages/plugins/users/src/collections/users.ts @@ -11,7 +11,7 @@ export default { fields: [ { name: 'id', - type: 'integer', + type: 'bigInt', autoIncrement: true, primaryKey: true, allowNull: false, diff --git a/packages/plugins/users/src/server.ts b/packages/plugins/users/src/server.ts index aa1c07422..96f66567d 100644 --- a/packages/plugins/users/src/server.ts +++ b/packages/plugins/users/src/server.ts @@ -2,7 +2,7 @@ import parse from 'json-templates'; import { resolve } from 'path'; import { Collection, Op } from '@nocobase/database'; -import { HandlerType, Middleware } from '@nocobase/resourcer'; +import { HandlerType } from '@nocobase/resourcer'; import { Plugin } from '@nocobase/server'; import { Registry } from '@nocobase/utils'; @@ -56,7 +56,7 @@ export default class UsersPlugin extends Plugin { if (createdBy === true) { collection.setField('createdById', { type: 'context', - dataType: 'integer', + dataType: 'bigInt', dataIndex: 'state.currentUser.id', createOnly: true, visible: true, @@ -72,7 +72,7 @@ export default class UsersPlugin extends Plugin { if (updatedBy === true) { collection.setField('updatedById', { type: 'context', - dataType: 'integer', + dataType: 'bigInt', dataIndex: 'state.currentUser.id', visible: true, index: true, diff --git a/packages/plugins/workflow/src/server/extensions/assignees/collections/users_jobs.ts b/packages/plugins/workflow/src/server/extensions/assignees/collections/users_jobs.ts index 26b0811c3..5839dda6b 100644 --- a/packages/plugins/workflow/src/server/extensions/assignees/collections/users_jobs.ts +++ b/packages/plugins/workflow/src/server/extensions/assignees/collections/users_jobs.ts @@ -4,49 +4,49 @@ export default { name: 'users_jobs', fields: [ { - type: 'integer', + type: 'bigInt', name: 'id', primaryKey: true, - autoIncrement: true + autoIncrement: true, }, { - type: 'integer', + type: 'bigInt', name: 'userId', primaryKey: false, }, { - type: 'integer', + type: 'bigInt', name: 'jobId', primaryKey: false, }, { type: 'belongsTo', - name: 'job' + name: 'job', }, { type: 'belongsTo', - name: 'user' + name: 'user', }, { type: 'belongsTo', - name: 'execution' + name: 'execution', }, { type: 'belongsTo', name: 'node', - target: 'flow_nodes' + target: 'flow_nodes', }, { type: 'belongsTo', - name: 'workflow' + name: 'workflow', }, { type: 'integer', - name: 'status' + name: 'status', }, { type: 'jsonb', - name: 'result' - } - ] + name: 'result', + }, + ], } as CollectionOptions;