feat: improve migrations (#510)

* feat: improve upgrade

* feat: addMigrations

* fix: get version

* feat: retry

* feat: migration context

* feat: get the version number from the server
This commit is contained in:
chenos 2022-06-17 10:25:59 +08:00 committed by GitHub
parent 42a45d1fb6
commit 34e17004c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 274 additions and 46 deletions

View File

@ -267,7 +267,7 @@ $ yarn nocobase db:auth -h
Usage: nocobase db:auth [options]
Options:
-r, --repeat [repeat] Number of reconnections
-r, --retry [retry] retry times
-h, --help
```

View File

@ -267,7 +267,7 @@ $ yarn nocobase db:auth -h
Usage: nocobase db:auth [options]
Options:
-r, --repeat [repeat] 重连次数
-r, --retry [retry] 重试次数
-h, --help
```

View File

@ -3,11 +3,26 @@ import React, { createContext, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useHistory } from 'react-router-dom';
import { useAPIClient, useCurrentUserContext } from '..';
import { useRequest } from '../api-client';
import { ChangePassword } from './ChangePassword';
import { EditProfile } from './EditProfile';
import { LanguageSettings } from './LanguageSettings';
import { SwitchRole } from './SwitchRole';
const ApplicationVersion = () => {
const { data, loading } = useRequest({
url: 'app:getInfo',
});
if (loading) {
return null;
}
return (
<Menu.Item key="version" disabled>
Version {data?.data?.version}
</Menu.Item>
);
};
export const DropdownVisibleContext = createContext(null);
export const CurrentUser = () => {
@ -26,7 +41,7 @@ export const CurrentUser = () => {
}}
overlay={
<Menu>
<Menu.Item key="version" disabled>Version {process.env.VERSION}</Menu.Item>
<ApplicationVersion />
<Menu.Divider />
<EditProfile />
<ChangePassword />

View File

@ -21,6 +21,9 @@
"umzug": "^3.1.1",
"mathjs": "^10.6.1"
},
"devDependencies": {
"@types/glob": "^7.2.0"
},
"repository": {
"type": "git",
"url": "git+https://github.com/nocobase/nocobase.git",

View File

@ -0,0 +1,7 @@
import { Migration } from '@nocobase/database';
export default class extends Migration {
async up() {}
async down() {}
}

View File

@ -0,0 +1,7 @@
import { Migration } from '@nocobase/database';
export default class extends Migration {
async up() {}
async down() {}
}

View File

@ -1,4 +1,5 @@
import { Database, Migration, mockDatabase } from '@nocobase/database';
import { resolve } from 'path';
const names = (migrations: Array<{ name: string }>) => migrations.map(m => m.name);
@ -18,6 +19,23 @@ describe('migrator', () => {
await db.close();
});
test('addMigrations', async () => {
db.addMigrations({
directory: resolve(__dirname, './fixtures/migrations'),
});
await db.migrator.up();
expect(names(await db.migrator.executed())).toEqual(['m1', 'm2']);
});
test('addMigrations', async () => {
db.addMigrations({
namespace: 'test',
directory: resolve(__dirname, './fixtures/migrations'),
});
await db.migrator.up();
expect(names(await db.migrator.executed())).toEqual(['test/m1', 'test/m2']);
});
test('up and down', async () => {
const spy = jest.fn();
db.addMigration({

View File

@ -1,8 +1,9 @@
import { applyMixins, AsyncEmitter } from '@nocobase/utils';
import merge from 'deepmerge';
import { EventEmitter } from 'events';
import glob from 'glob';
import lodash from 'lodash';
import { isAbsolute, resolve } from 'path';
import { basename, isAbsolute, resolve } from 'path';
import {
ModelCtor,
Op,
@ -25,6 +26,7 @@ import extendOperators from './operators';
import { RelationRepository } from './relation-repository/relation-repository';
import { Repository } from './repository';
export interface MergeOptions extends merge.Options {}
export interface PendingOptions {
@ -54,6 +56,13 @@ export interface CleanOptions extends QueryInterfaceDropAllTablesOptions {
drop?: boolean;
}
export type AddMigrationsOptions = {
context?: any;
namespace?: string;
extensions?: string[];
directory: string;
};
type OperatorFunc = (value: any, ctx?: RegisterOperatorsContext) => any;
export class Database extends EventEmitter implements AsyncEmitter {
@ -135,13 +144,12 @@ export class Database extends EventEmitter implements AsyncEmitter {
};
this.migrations = new Migrations(context);
this.migrator = new Umzug({
logger: migratorOptions.logger || console,
migrations: this.migrations.callback(),
context,
storage: new SequelizeStorage({
modelName: `${this.options.tablePrefix||''}migrations`,
modelName: `${this.options.tablePrefix || ''}migrations`,
...migratorOptions.storage,
sequelize: this.sequelize,
}),
@ -152,6 +160,33 @@ export class Database extends EventEmitter implements AsyncEmitter {
return this.migrations.add(item);
}
addMigrations(options: AddMigrationsOptions) {
const { namespace, context, extensions = ['js', 'ts'], directory } = options;
const patten = `${directory}/*.{${extensions.join(',')}}`;
const files = glob.sync(patten, {
ignore: ['**/*.d.ts'],
});
for (const file of files) {
let filename = basename(file);
filename = filename.substring(0, filename.lastIndexOf('.')) || filename;
this.migrations.add({
name: namespace ? `${namespace}/${filename}` : filename,
migration: this.requireModule(file),
context,
});
}
}
private requireModule(module: any) {
if (typeof module === 'string') {
module = require(module);
}
if (typeof module !== 'object') {
return module;
}
return module.__esModule ? module.default : module;
}
/**
* Add collection to database
* @param options
@ -304,12 +339,17 @@ export class Database extends EventEmitter implements AsyncEmitter {
}
}
async doesCollectionExistInDb(name) {
const tables = await this.sequelize.getQueryInterface().showAllTables();
return tables.find((table) => table === `${this.getTablePrefix()}${name}`);
}
public isSqliteMemory() {
return this.sequelize.getDialect() === 'sqlite' && lodash.get(this.options, 'storage') == ':memory:';
}
async auth(options: QueryOptions & { repeat?: number } = {}) {
const { repeat = 10, ...others } = options;
async auth(options: QueryOptions & { retry?: number } = {}) {
const { retry = 10, ...others } = options;
const delay = (ms) => new Promise((yea) => setTimeout(yea, ms));
let count = 1;
const authenticate = async () => {
@ -318,7 +358,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
console.log('Connection has been established successfully.');
return true;
} catch (error) {
if (count >= repeat) {
if (count >= retry) {
throw error;
}
console.log('reconnecting...', count);
@ -327,7 +367,6 @@ export class Database extends EventEmitter implements AsyncEmitter {
return await authenticate();
}
};
return await authenticate();
}

View File

@ -10,7 +10,7 @@ export interface MigrationContext {
export class Migration {
public name: string;
public context: { db: Database };
public context: { db: Database; [key: string]: any };
constructor(context: MigrationContext) {
this.context = context;
@ -40,6 +40,7 @@ export class Migration {
export interface MigrationItem {
name: string;
migration?: typeof Migration;
context?: any;
up?: any;
down?: any;
}
@ -59,7 +60,7 @@ export class Migrations {
add(item: MigrationItem) {
const Migration = item.migration;
if (Migration) {
const migration = new Migration(this.context);
const migration = new Migration({ ...this.context, ...item.context });
migration.name = item.name;
this.items.push(migration);
} else {
@ -68,7 +69,7 @@ export class Migrations {
}
callback() {
return (ctx) => {
return async (ctx) => {
return this.items;
};
}

View File

@ -24,7 +24,11 @@
"koa": "^2.13.4",
"koa-bodyparser": "^4.3.0",
"koa-static": "^5.0.0",
"lodash": "^4.17.21"
"lodash": "^4.17.21",
"semver": "^7.3.7"
},
"devDependencies": {
"@types/semver": "^7.3.9"
},
"gitHead": "a00b45a2686695c5f4824d074ac5e1aff210793a"
}

View File

@ -1,6 +1,6 @@
import { ACL } from '@nocobase/acl';
import { registerActions } from '@nocobase/actions';
import Database, { CollectionOptions, IDatabaseOptions } from '@nocobase/database';
import Database, { Collection, CollectionOptions, IDatabaseOptions } from '@nocobase/database';
import Resourcer, { ResourceOptions } from '@nocobase/resourcer';
import { applyMixins, AsyncEmitter } from '@nocobase/utils';
import { Command, CommandOptions } from 'commander';
@ -8,10 +8,11 @@ import { Server } from 'http';
import { i18n, InitOptions } from 'i18next';
import Koa from 'koa';
import { isBoolean } from 'lodash';
import semver from 'semver';
import { createACL } from './acl';
import { AppManager } from './app-manager';
import { registerCli } from './commands';
import { createDatabase, createI18n, createResourcer, registerMiddlewares } from './helper';
import { createI18n, createResourcer, registerMiddlewares } from './helper';
import { Plugin } from './plugin';
import { InstallOptions, PluginManager } from './plugin-manager';
@ -79,6 +80,50 @@ interface StartOptions {
listen?: ListenOptions;
}
export class ApplicationVersion {
protected app: Application;
protected collection: Collection;
constructor(app: Application) {
this.app = app;
if (!app.db.hasCollection('applicationVersion')) {
app.db.collection({
name: 'applicationVersion',
timestamps: false,
fields: [{ name: 'value', type: 'string' }],
});
}
this.collection = this.app.db.getCollection('applicationVersion');
}
async get() {
if (await this.app.db.doesCollectionExistInDb('applicationVersion')) {
const model = await this.collection.model.findOne();
return model.get('value') as any;
}
return null;
}
async update() {
await this.collection.sync();
await this.collection.model.destroy({
truncate: true,
});
await this.collection.model.create({
value: this.app.getVersion(),
});
}
async satisfies(range: string) {
if (await this.app.db.doesCollectionExistInDb('applicationVersion')) {
const model = await this.collection.model.findOne();
const version = model.get('value') as any;
return semver.satisfies(version, range);
}
return true;
}
}
export class Application<StateT = DefaultState, ContextT = DefaultContext> extends Koa implements AsyncEmitter {
public readonly db: Database;
@ -94,6 +139,8 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
public readonly appManager: AppManager;
public readonly version: ApplicationVersion;
protected plugins = new Map<string, Plugin>();
public listenServer: Server;
@ -102,7 +149,7 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
super();
this.acl = createACL();
this.db = createDatabase(options);
this.db = this.createDatabase(options);
this.resourcer = createResourcer(options);
this.cli = new Command('nocobase').usage('[command] [options]');
this.i18n = createI18n(options);
@ -122,6 +169,21 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
this.loadPluginConfig(options.plugins || []);
registerCli(this);
this.version = new ApplicationVersion(this);
}
private createDatabase(options: ApplicationOptions) {
if (options.database instanceof Database) {
return options.database;
} else {
return new Database({
...options.database,
migrator: {
context: { app: this },
},
});
}
}
getVersion() {
@ -267,9 +329,23 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
await this.db.sync(options?.sync);
await this.pm.install(options);
await this.version.update();
await this.emitAsync('afterInstall', this, options);
}
async upgrade(options: any = {}) {
const force = false;
await this.db.migrator.up();
await this.db.sync({
force,
alter: {
drop: force,
},
});
await this.version.update();
}
declare emitAsync: (event: string | symbol, ...args: any[]) => Promise<boolean>;
}

View File

@ -3,8 +3,8 @@ import Application from '../application';
export default (app: Application) => {
app
.command('db:auth')
.option('-r, --repeat [repeat]')
.option('-r, --retry [retry]')
.action(async (opts) => {
await app.db.auth({ repeat: opts.repeat || 10 });
await app.db.auth({ retry: opts.retry || 10 });
});
};

View File

@ -3,11 +3,10 @@ import Application from '../application';
export default (app: Application) => {
app
.command('db:sync')
.option('-f, --force')
.action(async (...cliArgs) => {
const [opts] = cliArgs;
console.log('db sync...');
const force = !!opts.force;
const force = false;
await app.db.sync({
force,
alter: {

View File

@ -7,21 +7,20 @@ export default (app: Application) => {
.option('-f, --force')
.option('-c, --clean')
.option('-s, --silent')
.option('-r, --repeat [repeat]')
.option('-r, --retry [retry]')
.action(async (...cliArgs) => {
let installed = false;
const [opts] = cliArgs;
try {
await app.db.auth({ repeat: opts.repeat || 1 });
await app.db.auth({ retry: opts.retry || 1 });
} catch (error) {
console.log(chalk.red('Unable to connect to the database. Please check the database environment variables in the .env file.'));
return;
}
if (!opts?.clean && !opts?.force) {
const tables = await app.db.sequelize.getQueryInterface().showAllTables();
if (tables.includes('collections')) {
if (app.db.doesCollectionExistInDb('applicationVersion')) {
installed = true;
if (!opts.silent) {
console.log('NocoBase is already installed. To reinstall, please execute:');

View File

@ -10,13 +10,7 @@ export default (app: Application) => {
.action(async (...cliArgs) => {
const [opts] = cliArgs;
console.log('upgrading...');
const force = false;
await app.db.sync({
force,
alter: {
drop: force,
},
});
await app.upgrade();
await app.stop({
cliArgs,
});

View File

@ -2,6 +2,7 @@ export { AppManager } from './app-manager';
export * from './application';
export { Application as default } from './application';
export * as middlewares from './middlewares';
export * from './migration';
export * from './plugin';
export * from './plugin-manager';
export * from './read-config';

View File

@ -0,0 +1,13 @@
import { Migration as DbMigration } from '@nocobase/database';
import Application from './application';
import Plugin from './plugin';
export class Migration extends DbMigration {
get app() {
return this.context.app as Application;
}
get plugin() {
return this.context.plugin as Plugin;
}
}

View File

@ -1,9 +1,9 @@
import { Database } from '@nocobase/database';
import { Application } from './application';
import finder from 'find-package-json';
import { Application } from './application';
import { InstallOptions } from './plugin-manager';
export interface PluginInterface {
beforeLoad?: () => void;
load();
@ -63,3 +63,5 @@ export abstract class Plugin<O = any> implements PluginInterface {
return packageObj['name'];
}
}
export default Plugin;

View File

@ -13,6 +13,13 @@ export default class PluginActionLogs extends Plugin {
await this.db.import({
directory: path.resolve(__dirname, 'collections'),
});
this.db.addMigrations({
namespace: 'audit-logs',
directory: path.resolve(__dirname, './migrations'),
context: {
plugin: this,
},
});
}
getName(): string {

View File

@ -0,0 +1,18 @@
import { Migration } from '@nocobase/server';
export default class LoggingMigration extends Migration {
async up() {
const result = await this.app.version.satisfies('<=0.7.0-alpha.83');
if (!result) {
return;
}
const repository = this.context.db.getRepository('collections');
const collections = await repository.find();
for (const collection of collections) {
if (!collection.get('logging')) {
collection.set('logging', true);
await collection.save();
}
}
}
}

View File

@ -19,6 +19,7 @@ export class ClientPlugin extends Plugin {
async load() {
this.app.acl.allow('app', 'getLang');
this.app.acl.allow('app', 'getInfo');
this.app.acl.allow('plugins', 'getPinned', 'loggedIn');
this.app.resource({
name: 'app',
@ -33,7 +34,7 @@ export class ClientPlugin extends Plugin {
lang = currentUser?.appLang;
}
ctx.body = {
version: this.app.getVersion(),
version: await ctx.app.version.get(),
lang,
};
await next();

View File

@ -1,11 +1,13 @@
import { Migration } from '@nocobase/database';
import { Migration } from '@nocobase/server';
export default class AlertSubTableMigration extends Migration {
versionRange = '<=0.7.0-alpha.83';
async up() {
const repository = this.context.db.getRepository('fields');
const fields = await repository.find();
const result = await this.app.version.satisfies('<=0.7.0-alpha.83');
if (!result) {
return;
}
const Field = this.context.db.getRepository('fields');
const fields = await Field.find();
for (const field of fields) {
if (field.get('interface') === 'subTable') {
field.set('interface', 'o2m');

View File

@ -9,7 +9,6 @@ import {
beforeCreateForReverseField,
beforeInitOptions
} from './hooks';
import AlertSubTableMigration from './migrations/20220613103214-alert-sub-table';
import { CollectionModel, FieldModel } from './models';
export class CollectionManagerPlugin extends Plugin {
@ -19,9 +18,12 @@ export class CollectionManagerPlugin extends Plugin {
FieldModel,
});
this.db.addMigration({
name: 'collection-manager/20220613103214-alert-sub-table',
migration: AlertSubTableMigration,
this.db.addMigrations({
namespace: 'collection-manager',
directory: path.resolve(__dirname, './migrations'),
context: {
plugin: this,
},
});
this.app.db.registerRepositories({

View File

@ -4972,6 +4972,14 @@
"@types/qs" "*"
"@types/serve-static" "*"
"@types/glob@^7.2.0":
version "7.2.0"
resolved "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb"
integrity sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==
dependencies:
"@types/minimatch" "*"
"@types/node" "*"
"@types/graceful-fs@^4.1.2":
version "4.1.5"
resolved "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15"
@ -5153,7 +5161,7 @@
resolved "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==
"@types/minimatch@^3.0.3":
"@types/minimatch@*", "@types/minimatch@^3.0.3":
version "3.0.5"
resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40"
integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==
@ -5327,6 +5335,11 @@
resolved "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39"
integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==
"@types/semver@^7.3.9":
version "7.3.9"
resolved "https://registry.npmjs.org/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc"
integrity sha512-L/TMpyURfBkf+o/526Zb6kd/tchUP3iBDEPjqjb+U2MAJhVRxxrmr2fwpe08E7QsV7YLcpq0tUaQ9O9x97ZIxQ==
"@types/serve-static@*":
version "1.13.10"
resolved "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz#f5e0ce8797d2d7cc5ebeda48a52c96c4fa47a8d9"
@ -20112,6 +20125,13 @@ semver@7.3.5, semver@7.x, semver@^7.1.1, semver@^7.1.3, semver@^7.2, semver@^7.2
dependencies:
lru-cache "^6.0.0"
semver@^7.3.7:
version "7.3.7"
resolved "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f"
integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==
dependencies:
lru-cache "^6.0.0"
semver@~7.2.0:
version "7.2.3"
resolved "https://registry.npmjs.org/semver/-/semver-7.2.3.tgz#3641217233c6382173c76bf2c7ecd1e1c16b0d8a"