From 02559b61ba67ae85046ec8277fb5d1db04fc46dc Mon Sep 17 00:00:00 2001 From: ChengLei Shao Date: Tue, 17 Oct 2023 22:22:45 +0800 Subject: [PATCH] feat(ci): run backend tests concurrently (#2815) * chore: run backend tests concurrently * fix: mock app server create database * chore: using @swc/jest instead of ts-jest * chore: create test database * chore: database distributor * fix: test collection manager * fix: test * chore: to use Jest.mock, change the formatter to the commonjs module * chore: db acquire * chore: github action * fix: pg test action * chore: formatter * chore: jest workaround * fix: mysql test * chore: mysql github action * chore: clean storage * chore: gitignore * chore: max workers --- .github/workflows/nocobase-test-backend.yml | 19 +- .gitignore | 1 + jest.config.js | 27 +- jest.setupAfterEnv.ts | 14 +- package.json | 7 +- packages/core/actions/src/__tests__/index.ts | 34 +-- packages/core/database/package.json | 6 +- .../src/__tests__/repository/find.test.ts | 1 - .../src/__tests__/view/view-inference.test.ts | 4 +- packages/core/database/src/database.ts | 50 +++- packages/core/database/src/mock-database.ts | 49 +++- packages/core/server/src/app-supervisor.ts | 2 - packages/core/server/src/application.ts | 2 +- packages/core/test/src/mockServer.ts | 19 +- .../core/test/src/scripts/test-db-creator.ts | 150 ++++++++++ .../test/src/scripts/test-db-distributor.ts | 259 ++++++++++++++++++ .../src/server/__tests__/through.test.ts | 3 + .../repositories/collection-repository.ts | 4 - .../src/server/__tests__/query.test.ts | 3 +- .../src/server/server.ts | 5 +- .../__tests__/actions/workflows.test.ts | 3 - .../__tests__/triggers/collection.test.ts | 2 + yarn.lock | 101 ++++++- 23 files changed, 667 insertions(+), 98 deletions(-) create mode 100644 packages/core/test/src/scripts/test-db-creator.ts create mode 100644 packages/core/test/src/scripts/test-db-distributor.ts diff --git a/.github/workflows/nocobase-test-backend.yml b/.github/workflows/nocobase-test-backend.yml index 7cc50bdc0..46f16809e 100644 --- a/.github/workflows/nocobase-test-backend.yml +++ b/.github/workflows/nocobase-test-backend.yml @@ -38,11 +38,12 @@ jobs: cache: 'yarn' - run: yarn install - name: Test with Sqlite - run: yarn nocobase install -f && node --max_old_space_size=4096 ./node_modules/.bin/jest --maxWorkers=1 --workerIdleMemoryLimit=3000MB + run: yarn nocobase install -f && node --max_old_space_size=4096 ./node_modules/.bin/jest --maxWorkers=100% --workerIdleMemoryLimit=3000MB env: LOGGER_LEVEL: error DB_DIALECT: sqlite DB_STORAGE: /tmp/db.sqlite + DB_TEST_PREFIX: test_ DB_UNDERSCORED: ${{ matrix.underscored }} timeout-minutes: 40 @@ -80,7 +81,11 @@ jobs: - run: yarn install # - run: yarn build - name: Test with postgres - run: yarn nocobase install -f && node --max_old_space_size=4096 ./node_modules/.bin/jest --maxWorkers=1 --workerIdleMemoryLimit=3000MB + run: | + yarn nocobase install -f + ./node_modules/.bin/tsx packages/core/test/src/scripts/test-db-creator.ts & + sleep 1 + node --max_old_space_size=4096 ./node_modules/.bin/jest --maxWorkers=100% --workerIdleMemoryLimit=3000MB env: LOGGER_LEVEL: error DB_DIALECT: postgres @@ -92,6 +97,8 @@ jobs: DB_UNDERSCORED: ${{ matrix.underscored }} DB_SCHEMA: ${{ matrix.schema }} COLLECTION_MANAGER_SCHEMA: ${{ matrix.collection_schema }} + DB_TEST_DISTRIBUTOR_PORT: 23450 + DB_TEST_PREFIX: test_ timeout-minutes: 40 mysql-test: @@ -118,7 +125,11 @@ jobs: - run: yarn install # - run: yarn build - name: Test with MySQL - run: yarn nocobase install -f && node --max_old_space_size=4096 ./node_modules/.bin/jest --maxWorkers=1 --workerIdleMemoryLimit=3000MB + run: | + yarn nocobase install -f + ./node_modules/.bin/tsx packages/core/test/src/scripts/test-db-creator.ts & + sleep 1 + node --max_old_space_size=4096 ./node_modules/.bin/jest --maxWorkers=100% --workerIdleMemoryLimit=3000MB env: LOGGER_LEVEL: error DB_DIALECT: mysql @@ -128,4 +139,6 @@ jobs: DB_PASSWORD: password DB_DATABASE: nocobase DB_UNDERSCORED: ${{ matrix.underscored }} + DB_TEST_DISTRIBUTOR_PORT: 23450 + DB_TEST_PREFIX: test_ timeout-minutes: 40 diff --git a/.gitignore b/.gitignore index a67d4e23d..8e3c831e3 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ storage/logs-e2e storage/uploads-e2e tsconfig.paths.json playwright +.swc \ No newline at end of file diff --git a/jest.config.js b/jest.config.js index 71c4f2140..76cb7a338 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,8 +1,20 @@ const { pathsToModuleNameMapper } = require('ts-jest'); const { compilerOptions } = require('./tsconfig.paths.json'); -const { defaults } = require('jest-config'); -module.exports = { +const swcrc = { + jsc: { + parser: { + syntax: 'typescript', + tsx: false, + decorators: true, + dynamicImport: false, + }, + }, +}; + +((swcrc.jsc ??= {}).experimental ??= {}).plugins = [['jest_workaround', {}]]; + +const config = { rootDir: process.cwd(), collectCoverage: false, verbose: true, @@ -16,14 +28,7 @@ module.exports = { }), }, transform: { - '^.+\\.{ts|tsx}?$': [ - 'ts-jest', - { - babelConfig: false, - tsconfig: './tsconfig.jest.json', - diagnostics: false, - }, - ], + '^.+\\.(t|j)sx?$': ['@swc/jest', swcrc], }, modulePathIgnorePatterns: ['/esm/', '/es/', '/dist/', '/lib/', '/client/', '/sdk/', '\\.test\\.tsx$'], coveragePathIgnorePatterns: [ @@ -37,3 +42,5 @@ module.exports = { '/storage/', ], }; + +module.exports = config; diff --git a/jest.setupAfterEnv.ts b/jest.setupAfterEnv.ts index 2c95f5081..ba00d5522 100644 --- a/jest.setupAfterEnv.ts +++ b/jest.setupAfterEnv.ts @@ -1,13 +1,3 @@ -import prettyFormat from 'pretty-format'; +import { jest } from '@jest/globals'; -global['prettyFormat'] = prettyFormat; - -jest.setTimeout(300000); - -// 把 console.error 转换成 error，方便断言 -(() => { - const spy = jest.spyOn(console, 'error'); - afterAll(() => { - spy.mockRestore(); - }); -})(); +jest.setTimeout(100000); diff --git a/package.json b/package.json index 8216745fc..e826d2aee 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,8 @@ "@commitlint/prompt-cli": "^16.1.0", "@faker-js/faker": "8.1.0", "@playwright/test": "^1.37.1", + "@swc/core": "^1.3.92", + "@swc/jest": "^0.2.29", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^14.0.0", "@testing-library/user-event": "^14.4.3", @@ -79,6 +81,7 @@ "@types/react-dom": "^17.0.0", "@vitejs/plugin-react": "^4.0.0", "auto-changelog": "^2.4.0", + "axios": "^0.26.1", "commander": "^9.2.0", "dumi": "^2.2.0", "dumi-theme-nocobase": "^0.2.14", @@ -88,6 +91,7 @@ "ghooks": "^2.0.4", "jest": "^29.6.2", "jest-cli": "^29.6.2", + "jest_workaround": "^0.79.19", "jsdom-worker": "^0.3.0", "lint-staged": "^13.2.3", "pretty-format": "^24.0.0", @@ -97,8 +101,7 @@ "ts-jest": "^29.1.1", "typescript": "5.1.3", "vite": "^4.4.9", - "vitest": "^0.34.3", - "axios": "^0.26.1" + "vitest": "^0.34.3" }, "volta": { "node": "18.14.2", diff --git a/packages/core/actions/src/__tests__/index.ts b/packages/core/actions/src/__tests__/index.ts index aafb435b4..3c41afd97 100644 --- a/packages/core/actions/src/__tests__/index.ts +++ b/packages/core/actions/src/__tests__/index.ts @@ -1,4 +1,4 @@ -import Database, { CollectionOptions, DatabaseOptions } from '@nocobase/database'; +import Database, { CollectionOptions, DatabaseOptions, mockDatabase } from '@nocobase/database'; import { Handlers, ResourceOptions, Resourcer } from '@nocobase/resourcer'; import merge from 'deepmerge'; import Koa from 'koa'; @@ -8,38 +8,6 @@ import supertest, { SuperAgentTest } from 'supertest'; import db2resource from '../../../server/src/middlewares/db2resource'; import { uid } from '@nocobase/utils'; -export function generatePrefixByPath() { - return `mock_${uid(6)}`; -} - -export function getConfig(config = {}, options?: any): DatabaseOptions { - return merge( - { - username: process.env.DB_USER, - password: process.env.DB_PASSWORD, - database: process.env.DB_DATABASE, - host: process.env.DB_HOST, - port: process.env.DB_PORT, - dialect: process.env.DB_DIALECT, - storage: process.env.DB_STORAGE, - logging: process.env.DB_LOGGING === 'on', - sync: { - force: true, - }, - hooks: { - beforeDefine(model, options) { - options.tableName = `${generatePrefixByPath()}_${options.tableName || options.name.plural}`; - }, - }, - }, - config || {}, - options, - ) as any; -} - -export function mockDatabase(options?: DatabaseOptions): Database { - return new Database(getConfig(options)); -} interface ActionParams { fields?: string[]; diff --git a/packages/core/database/package.json b/packages/core/database/package.json index 61569e533..62d6640ee 100644 --- a/packages/core/database/package.json +++ b/packages/core/database/package.json @@ -21,10 +21,12 @@ "qs": "^6.11.2", "semver": "^7.3.7", "sequelize": "^6.26.0", - "umzug": "^3.1.1" + "umzug": "^3.1.1", + "nanoid": "^3.3.6" }, "devDependencies": { - "@types/glob": "^7.2.0" + "@types/glob": "^7.2.0", + "node-fetch": "^2.6.7" }, "repository": { "type": "git", diff --git a/packages/core/database/src/__tests__/repository/find.test.ts b/packages/core/database/src/__tests__/repository/find.test.ts index 000020c4d..9f9bc80f5 100644 --- a/packages/core/database/src/__tests__/repository/find.test.ts +++ b/packages/core/database/src/__tests__/repository/find.test.ts @@ -336,7 +336,6 @@ describe('repository find', () => { beforeEach(async () => { db = mockDatabase(); - User = db.collection<{ id: number; name: string }, { name: string }>({ name: 'users', fields: [ diff --git a/packages/core/database/src/__tests__/view/view-inference.test.ts b/packages/core/database/src/__tests__/view/view-inference.test.ts index d8de7ac24..8294bfba4 100644 --- a/packages/core/database/src/__tests__/view/view-inference.test.ts +++ b/packages/core/database/src/__tests__/view/view-inference.test.ts @@ -5,9 +5,7 @@ describe('view inference', function () { let db: Database; beforeEach(async () => { - db = mockDatabase({ - tablePrefix: '', - }); + db = mockDatabase(); await db.clean({ drop: true }); }); diff --git a/packages/core/database/src/database.ts b/packages/core/database/src/database.ts index 76169c7d5..ac1c98e55 100644 --- a/packages/core/database/src/database.ts +++ b/packages/core/database/src/database.ts @@ -71,6 +71,7 @@ import { patchSequelizeQueryInterface, snakeCase } from './utils'; import { BaseValueParser, registerFieldValueParsers } from './value-parsers'; import { ViewCollection } from './view-collection'; import { SqlCollection } from './sql-collection/sql-collection'; +import { nanoid } from 'nanoid'; export type MergeOptions = merge.Options; @@ -88,6 +89,8 @@ export interface IDatabaseOptions extends Options { migrator?: any; usingBigIntForId?: boolean; underscored?: boolean; + customHooks?: any; + instanceId?: string; } export type DatabaseOptions = IDatabaseOptions; @@ -172,9 +175,11 @@ export class Database extends EventEmitter implements AsyncEmitter { pendingFields = new Map(); modelCollection = new Map, Collection>(); tableNameCollectionMap = new Map(); - + context: any = {}; queryInterface: QueryInterface; + _instanceId: string; + utils = new DatabaseUtils(this); referenceMap = new ReferencesMap(); inheritanceMap = new InheritanceMap(); @@ -206,6 +211,12 @@ export class Database extends EventEmitter implements AsyncEmitter { ...lodash.clone(options), }; + if (!options.instanceId) { + this._instanceId = nanoid(); + } else { + this._instanceId = options.instanceId; + } + if (options.storage && options.storage !== ':memory:') { if (!isAbsolute(options.storage)) { opts.storage = resolve(process.cwd(), options.storage); @@ -224,7 +235,8 @@ export class Database extends EventEmitter implements AsyncEmitter { } this.options = opts; - this.sequelize = new Sequelize(this.sequelizeOptions(this.options)); + const sequelizeOptions = this.sequelizeOptions(this.options); + this.sequelize = new Sequelize(sequelizeOptions); this.queryInterface = buildQueryInterface(this); @@ -297,17 +309,31 @@ export class Database extends EventEmitter implements AsyncEmitter { patchSequelizeQueryInterface(this); } + get instanceId() { + return this._instanceId; + } + + setContext(context: any) { + this.context = context; + } + setLogger(logger: Logger) { this.logger = logger; } sequelizeOptions(options) { if (options.dialect === 'postgres') { - options.hooks = { - afterConnect: async (connection) => { - await connection.query('SET search_path TO public;'); - }, - }; + if (!options.hooks) { + options.hooks = {}; + } + + if (!options.hooks['afterConnect']) { + options.hooks['afterConnect'] = []; + } + + options.hooks['afterConnect'].push(async (connection) => { + await connection.query('SET search_path TO public;'); + }); } return options; } @@ -769,7 +795,15 @@ export class Database extends EventEmitter implements AsyncEmitter { return; } - return this.sequelize.close(); + await this.emitAsync('beforeClose', this); + + const closeResult = this.sequelize.close(); + + if (this.options?.customHooks?.['afterClose']) { + await this.options.customHooks['afterClose'](this); + } + + return closeResult; } on(event: EventType, listener: any): this; diff --git a/packages/core/database/src/mock-database.ts b/packages/core/database/src/mock-database.ts index cc064f09f..50ceea3b4 100644 --- a/packages/core/database/src/mock-database.ts +++ b/packages/core/database/src/mock-database.ts @@ -1,12 +1,13 @@ import { merge, uid } from '@nocobase/utils'; import { resolve } from 'path'; import { Database, IDatabaseOptions } from './database'; - +import fetch from 'node-fetch'; +import path from 'path'; +import { customAlphabet } from 'nanoid'; export class MockDatabase extends Database { constructor(options: IDatabaseOptions) { super({ storage: ':memory:', - tablePrefix: `mock_${uid(6)}_`, dialect: 'sqlite', ...options, }); @@ -52,5 +53,47 @@ function customLogger(queryString, queryObject) { export function mockDatabase(options: IDatabaseOptions = {}): MockDatabase { const dbOptions = merge(getConfigByEnv(), options) as any; - return new MockDatabase(dbOptions); + + if (process.env['DB_TEST_PREFIX']) { + let configKey = 'database'; + if (dbOptions.dialect === 'sqlite') { + configKey = 'storage'; + } else { + configKey = 'database'; + } + + const shouldChange = () => { + if (dbOptions.dialect === 'sqlite') { + return !dbOptions[configKey].includes(process.env['DB_TEST_PREFIX']); + } + + return !dbOptions[configKey].startsWith(process.env['DB_TEST_PREFIX']); + }; + + if (dbOptions[configKey] && shouldChange()) { + const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 10); + + const instanceId = `d_${nanoid()}`; + const databaseName = `${process.env['DB_TEST_PREFIX']}_${instanceId}`; + + if (dbOptions.dialect === 'sqlite') { + dbOptions.storage = path.resolve(path.dirname(dbOptions.storage), databaseName); + } else { + dbOptions.database = databaseName; + } + } + + if (process.env['DB_TEST_DISTRIBUTOR_PORT']) { + dbOptions.hooks = dbOptions.hooks || {}; + + dbOptions.hooks.beforeConnect = async (config) => { + const url = `http://127.0.0.1:${process.env['DB_TEST_DISTRIBUTOR_PORT']}/acquire?via=${db.instanceId}&name=${config.database}`; + await fetch(url); + }; + } + } + + const db = new MockDatabase(dbOptions); + + return db; } diff --git a/packages/core/server/src/app-supervisor.ts b/packages/core/server/src/app-supervisor.ts index aa63763e3..cefbf1235 100644 --- a/packages/core/server/src/app-supervisor.ts +++ b/packages/core/server/src/app-supervisor.ts @@ -178,8 +178,6 @@ export class AppSupervisor extends EventEmitter implements AsyncEmitter { throw new Error(`app ${app.name} already exists`); } - console.log(`add app ${app.name} into supervisor`); - this.bindAppEvents(app); this.apps[app.name] = app; diff --git a/packages/core/server/src/application.ts b/packages/core/server/src/application.ts index e7c721e7c..73fd06250 100644 --- a/packages/core/server/src/application.ts +++ b/packages/core/server/src/application.ts @@ -749,7 +749,7 @@ export class Application exten this._version = new ApplicationVersion(this); } - private createDatabase(options: ApplicationOptions) { + protected createDatabase(options: ApplicationOptions) { const db = new Database({ ...(options.database instanceof Database ? options.database.options : options.database), migrator: { diff --git a/packages/core/test/src/mockServer.ts b/packages/core/test/src/mockServer.ts index 835091f2a..bf5ab8928 100644 --- a/packages/core/test/src/mockServer.ts +++ b/packages/core/test/src/mockServer.ts @@ -81,6 +81,17 @@ export class MockServer extends Application { await this.db.clean({ drop: true }); } + protected createDatabase(options: ApplicationOptions) { + const oldDatabase = this._db; + + const databaseOptions = oldDatabase ? oldDatabase.options : options?.database || {}; + const database = mockDatabase(databaseOptions); + database.setLogger(this._logger); + database.setContext({ app: this }); + + return database; + } + async destroy(options: any = {}): Promise { await super.destroy(options); @@ -195,17 +206,9 @@ export function mockServer(options: ApplicationOptions = {}) { PluginManager.findPackagePatched = true; } - let database; - if (options?.database instanceof Database) { - database = options.database; - } else { - database = mockDatabase(options?.database || {}); - } - const app = new MockServer({ acl: false, ...options, - database, }); return app; diff --git a/packages/core/test/src/scripts/test-db-creator.ts b/packages/core/test/src/scripts/test-db-creator.ts new file mode 100644 index 000000000..6925b7df7 --- /dev/null +++ b/packages/core/test/src/scripts/test-db-creator.ts @@ -0,0 +1,150 @@ +import http from 'http'; +import url from 'url'; +import pg from 'pg'; +import dotenv from 'dotenv'; +import path from 'path'; +import mysql from 'mysql2/promise'; + +dotenv.config({ path: path.resolve(process.cwd(), '.env.test') }); + +abstract class BaseClient { + private createdDBs: Set = new Set(); + protected _client: Client | null = null; + + abstract _createDB(name: string): Promise; + abstract _createConnection(): Promise; + abstract _removeDB(name: string): Promise; + + async createDB(name: string) { + if (this.createdDBs.has(name)) { + return; + } + + if (!this._client) { + this._client = await this._createConnection(); + } + + console.log(`Creating database: ${name}`); + await this._createDB(name); + this.createdDBs.add(name); + } + + async releaseAll() { + if (!this._client) { + return; + } + + const dbNames = Array.from(this.createdDBs); + + for (const name of dbNames) { + console.log(`Removing database: ${name}`); + await this._removeDB(name); + this.createdDBs.delete(name); + } + } +} + +class PostgresClient extends BaseClient { + async _removeDB(name: string): Promise { + await this._client.query(`DROP DATABASE IF EXISTS ${name}`); + } + + async _createDB(name: string): Promise { + await this._client.query(`DROP DATABASE IF EXISTS ${name}`); + await this._client.query(`CREATE DATABASE ${name};`); + } + + async _createConnection(): Promise { + const client = new pg.Client({ + host: process.env['DB_HOST'], + port: Number(process.env['DB_PORT']), + user: process.env['DB_USER'], + password: process.env['DB_PASSWORD'], + database: process.env['DB_DATABASE'], + }); + + await client.connect(); + return client; + } +} + +class MySQLClient extends BaseClient { + async _removeDB(name: string): Promise { + await this._client.query(`DROP DATABASE IF EXISTS ${name}`); + } + + async _createDB(name: string): Promise { + await this._client.query(`CREATE DATABASE IF NOT EXISTS ${name}`); + } + + async _createConnection(): Promise { + const connection = await mysql.createConnection({ + host: process.env['DB_HOST'], + port: Number(process.env['DB_PORT']), + user: process.env['DB_USER'], + password: process.env['DB_PASSWORD'], + database: process.env['DB_DATABASE'], + }); + + return connection; + } +} + +const client = { + postgres: () => { + return new PostgresClient(); + }, + mysql: () => { + return new MySQLClient(); + }, +}; + +const dialect = process.env['DB_DIALECT']; + +if (!client[dialect]) { + throw new Error(`Unknown dialect: ${dialect}`); +} + +const dbClient = client[dialect](); + +const server = http.createServer((req, res) => { + const parsedUrl = url.parse(req.url, true); + const path = parsedUrl.pathname; + const trimmedPath = path.replace(/^\/+|\/+$/g, ''); + + if (trimmedPath === 'acquire') { + const via = parsedUrl.query.via as string; + const name = parsedUrl.query.name as string | undefined; + + dbClient + .createDB(name) + .then(() => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(); + }) + .catch((error) => { + console.error(error); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + }); + } else if (trimmedPath === 'release') { + dbClient + .releaseAll() + .then(() => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(); + }) + .catch((error) => { + console.error(error); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + }); + } else { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Not Found\n'); + } +}); + +server.listen(23450, '127.0.0.1', () => { + console.log('Server is running at http://127.0.0.1:23450/'); +}); diff --git a/packages/core/test/src/scripts/test-db-distributor.ts b/packages/core/test/src/scripts/test-db-distributor.ts new file mode 100644 index 000000000..b82f84773 --- /dev/null +++ b/packages/core/test/src/scripts/test-db-distributor.ts @@ -0,0 +1,259 @@ +import http from 'http'; +import url from 'url'; +import pg from 'pg'; +import dotenv from 'dotenv'; +import path from 'path'; +import fs from 'fs'; + +dotenv.config({ path: path.resolve(process.cwd(), '.env.test') }); +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +class DBManager { + private acquiredDBs: Map> = new Map(); + + acquire(name: string, via: string) { + console.log('acquire', name, 'via', via); + if (this.acquiredDBs.has(name)) { + // If DB is already acquired, add the via to the set + this.acquiredDBs.get(name)!.add(via); + } else { + // If DB is not acquired yet, set the set with the via + this.acquiredDBs.set(name, new Set([via])); + } + } + + async release(name: string, via: string, relaseDb?: () => Promise) { + console.log('release', name, 'via', via); + const vias = this.acquiredDBs.get(name); + if (!vias || !vias.has(via)) { + console.log(`Cannot release ${name}, it is not acquired via ${via}`); + return; + } + + // Remove the via from the set + vias.delete(via); + + // If no more vias, remove the DB from the map + if (vias.size === 0) { + console.log('DB', name, 'is not used anymore, release it'); + // delay 1000ms to make sure the DB is not used anymore + await delay(1000); + + if (this.acquiredDBs.get(name)?.size === 0) { + console.log('start to release DB', name); + await relaseDb?.(); + this.acquiredDBs.delete(name); + console.log('DB', name, 'is released, current usesd db count:', this.acquiredDBs.size); + } + } + + return null; + } + + isAcquired(name: string): boolean { + return this.acquiredDBs.has(name); + } +} + +const getDBNames = (size: number, name: string) => { + const names = []; + for (let i = 0; i < size; i++) { + names.push(`auto_named_${name}_${i}`); + } + return names; +}; + +abstract class BasePool { + dbManager: DBManager = new DBManager(); + constructor(protected size: number) {} + + abstract createDatabase(name: string, options?: any): Promise; + abstract cleanDatabase(name: string): Promise; + + abstract getConfiguredDatabaseName(): string; + abstract getDatabaseConfiguration(): any; + + async init() { + const promises = []; + for (const name of getDBNames(this.size, this.getConfiguredDatabaseName())) { + promises.push( + (async () => { + console.log('create database', name); + await this.createDatabase(name); + })(), + ); + } + + await Promise.all(promises); + } + + async acquire(name: string | undefined, via: string) { + if (!name) { + name = getDBNames(this.size, this.getConfiguredDatabaseName()).find((name) => !this.dbManager.isAcquired(name)); + } + + if (!name) { + throw new Error('No available database'); + } + + this.dbManager.acquire(name, via); + return name; + } + + async release(name: string, via: string) { + await this.dbManager.release(name, via, async () => { + await this.cleanDatabase(name); + }); + } +} + +class PostgresPool extends BasePool { + private async _createConnection(options, callback) { + const config = this.getDatabaseConfiguration(); + const databaseName = this.getConfiguredDatabaseName(); + + const client = new pg.Client({ + host: config['host'], + port: config['port'], + user: config['username'], + password: config['password'], + database: databaseName, + ...options, + }); + + await client.connect(); + + await callback(client); + + await client.end(); + } + + async cleanDatabase(name: string): Promise { + await this._createConnection({ database: name }, async (client) => { + await client.query(`DROP SCHEMA public CASCADE;CREATE SCHEMA public;`); + }); + } + + async createDatabase(name: string, options?: any): Promise { + const { log } = options || {}; + + await this._createConnection({}, async (client) => { + if (log) { + console.log(`DROP DATABASE IF EXISTS ${name}`); + } + await client.query(`DROP DATABASE IF EXISTS ${name}`); + if (log) { + console.log(`CREATE DATABASE ${name}`); + } + await client.query(`CREATE DATABASE ${name}`); + + if (log) { + console.log(`end`); + } + }); + } + + getDatabaseConfiguration() { + return { + host: process.env.DB_HOST, + port: process.env.DB_PORT, + username: process.env.DB_USERNAME, + password: process.env.DB_PASSWORD, + }; + } + + getConfiguredDatabaseName() { + return process.env.DB_DATABASE; + } +} + +class SqlitePool extends BasePool { + async createDatabase(name: string, options?: any): Promise { + return fs.promises.writeFile(path.resolve(this.getStoragePath(), name), ''); + } + + async cleanDatabase(name: string): Promise { + return fs.promises.unlink(path.resolve(this.getStoragePath(), name)); + } + + getDatabaseConfiguration(): any { + return { + storage: process.env.DB_STORAGE, + }; + } + + getConfiguredDatabaseName() { + const storagePath = process.env.DB_STORAGE; + if (storagePath && storagePath !== ':memory:') { + return path.basename(storagePath); + } + } + + getStoragePath() { + const storagePath = process.env.DB_STORAGE; + if (storagePath && storagePath !== ':memory:') { + // return path without file name + return path.dirname(storagePath); + } + } +} + +const pools = { + postgres: PostgresPool, + sqlite: SqlitePool, +}; + +(async () => { + const poolSize = process.env.TEST_DB_POOL_SIZE || 100; + const poolClass = pools[process.env.DB_DIALECT]; + + if (!poolClass) { + throw new Error(`Unknown pool class ${process.env.DB_DIALECT}`); + } + + const pool = new poolClass(poolSize); + await pool.init(); + + return pool; +})() + .then((pool: BasePool) => { + const server = http.createServer((req, res) => { + const parsedUrl = url.parse(req.url, true); + const path = parsedUrl.pathname; + const trimmedPath = path.replace(/^\/+|\/+$/g, ''); + + if (trimmedPath === 'acquire') { + const via = parsedUrl.query.via as string; + const name = parsedUrl.query.name as string | undefined; + pool + .acquire(name, via) + .then((name) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ name })); + }) + .catch((err) => { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: err.message })); + }); + } else if (trimmedPath === 'release') { + const via = parsedUrl.query.via as string; + const name = parsedUrl.query.name as string; + + pool.release(name, via); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(); + } else { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Not Found\n'); + } + }); + + server.listen(23450, '127.0.0.1', () => { + console.log('Server is running at http://127.0.0.1:23450/'); + }); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/packages/plugins/@nocobase/plugin-collection-manager/src/server/__tests__/through.test.ts b/packages/plugins/@nocobase/plugin-collection-manager/src/server/__tests__/through.test.ts index df7bd16de..9018f9578 100644 --- a/packages/plugins/@nocobase/plugin-collection-manager/src/server/__tests__/through.test.ts +++ b/packages/plugins/@nocobase/plugin-collection-manager/src/server/__tests__/through.test.ts @@ -119,8 +119,11 @@ describe('collections repository', () => { const app2 = mockServer({ database: { tablePrefix: 'through_', + database: app1.db.options.database, + storage: app1.db.options.storage, }, }); + app2.plugin(PluginErrorHandler, { name: 'error-handler' }); app2.plugin(Plugin, { name: 'collection-manager' }); await app2.load(); diff --git a/packages/plugins/@nocobase/plugin-collection-manager/src/server/repositories/collection-repository.ts b/packages/plugins/@nocobase/plugin-collection-manager/src/server/repositories/collection-repository.ts index ac4521076..10a33a5f7 100644 --- a/packages/plugins/@nocobase/plugin-collection-manager/src/server/repositories/collection-repository.ts +++ b/packages/plugins/@nocobase/plugin-collection-manager/src/server/repositories/collection-repository.ts @@ -18,9 +18,7 @@ export class CollectionRepository extends Repository { async load(options: LoadOptions = {}) { const { filter, skipExist } = options; - console.log('start load collections'); const instances = (await this.find({ filter, appends: ['fields'] })) as CollectionModel[]; - console.log('end load collections'); const graphlib = CollectionsGraph.graphlib(); @@ -111,8 +109,6 @@ export class CollectionRepository extends Repository { this.app.setMaintainingMessage(`load ${collectionName} collection fields`); await nameMap[collectionName].loadFields({ includeFields: skipField }); } - - console.log('finished load collection'); } async db2cm(collectionName: string) { diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts index ef3962311..dbccb95a4 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts @@ -1,5 +1,5 @@ import { MockServer, mockServer } from '@nocobase/test'; -import * as formatter from '../actions/formatter'; +const formatter = require('../actions/formatter'); import { cacheMiddleware, parseBuilder, parseFieldAndAssociations } from '../actions/query'; import compose from 'koa-compose'; @@ -128,6 +128,7 @@ describe('query', () => { it('should parse dimensions', async () => { jest.spyOn(formatter, 'formatter').mockReturnValue('formatted-field'); + const dimensions = [ { field: ['createdAt'], diff --git a/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts b/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts index e1976329d..c8445a79c 100644 --- a/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-multi-app-manager/src/server/server.ts @@ -123,11 +123,14 @@ export class PluginMultiAppManager extends Plugin { private beforeGetApplicationMutex = new Mutex(); static getDatabaseConfig(app: Application): IDatabaseOptions { - const oldConfig = + let oldConfig = app.options.database instanceof Database ? (app.options.database as Database).options : (app.options.database as IDatabaseOptions); + if (!oldConfig && app.db) { + oldConfig = app.db.options; + } return lodash.cloneDeep(lodash.omit(oldConfig, ['migrator'])); } diff --git a/packages/plugins/@nocobase/plugin-workflow/src/server/__tests__/actions/workflows.test.ts b/packages/plugins/@nocobase/plugin-workflow/src/server/__tests__/actions/workflows.test.ts index 5ff428ccd..d24f221fe 100644 --- a/packages/plugins/@nocobase/plugin-workflow/src/server/__tests__/actions/workflows.test.ts +++ b/packages/plugins/@nocobase/plugin-workflow/src/server/__tests__/actions/workflows.test.ts @@ -21,9 +21,6 @@ describe('workflow > actions > workflows', () => { afterEach(() => app.destroy()); - // describe('create', () => { - // }); - describe('update', () => { it('update unexecuted workflow should be ok', async () => { const workflow = await WorkflowModel.create({ diff --git a/packages/plugins/@nocobase/plugin-workflow/src/server/__tests__/triggers/collection.test.ts b/packages/plugins/@nocobase/plugin-workflow/src/server/__tests__/triggers/collection.test.ts index a853ec8cd..71a00f03c 100644 --- a/packages/plugins/@nocobase/plugin-workflow/src/server/__tests__/triggers/collection.test.ts +++ b/packages/plugins/@nocobase/plugin-workflow/src/server/__tests__/triggers/collection.test.ts @@ -85,6 +85,8 @@ describe('workflow > triggers > collection', () => { plugins: ['error-handler', 'collection-manager'], database: { tablePrefix: db.options.tablePrefix, + storage: db.options.storage, + database: db.options.database, }, }); diff --git a/yarn.lock b/yarn.lock index d8b22f177..ed32a29de 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3268,6 +3268,13 @@ slash "^3.0.0" strip-ansi "^6.0.0" +"@jest/create-cache-key-function@^27.4.2": + version "27.5.1" + resolved "https://registry.yarnpkg.com/@jest/create-cache-key-function/-/create-cache-key-function-27.5.1.tgz#7448fae15602ea95c828f5eceed35c202a820b31" + integrity sha512-dmH1yW+makpTSURTy8VzdUwFnfQh1G8R+DxO2Ho2FFmBbKFEVm+3jWdvFhE2VqB/LATCTokkP0dotjyQyw5/AQ== + dependencies: + "@jest/types" "^27.5.1" + "@jest/environment@^29.6.1": version "29.6.1" resolved "https://registry.npmmirror.com/@jest/environment/-/environment-29.6.1.tgz#ee358fff2f68168394b4a50f18c68278a21fe82f" @@ -3532,7 +3539,7 @@ slash "^3.0.0" write-file-atomic "^4.0.2" -"@jest/types@27.5.1": +"@jest/types@27.5.1", "@jest/types@^27.5.1": version "27.5.1" resolved "https://registry.npmmirror.com/@jest/types/-/types-27.5.1.tgz#3c79ec4a8ba61c170bf937bcf9e98a9df175ec80" dependencies: @@ -5255,42 +5262,92 @@ version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-darwin-arm64/-/core-darwin-arm64-1.3.57.tgz#6af39458bf4ae58c8e4d12be2f2d69563b5a932a" +"@swc/core-darwin-arm64@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-darwin-arm64/-/core-darwin-arm64-1.3.92.tgz#0498d3584cf877e39107c94705c38fa4a8c04789" + integrity sha512-v7PqZUBtIF6Q5Cp48gqUiG8zQQnEICpnfNdoiY3xjQAglCGIQCjJIDjreZBoeZQZspB27lQN4eZ43CX18+2SnA== + "@swc/core-darwin-x64@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-darwin-x64/-/core-darwin-x64-1.3.57.tgz#e7133f88e1aa166ad7e5966302df495551c3505c" +"@swc/core-darwin-x64@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-darwin-x64/-/core-darwin-x64-1.3.92.tgz#1728e7ebbfe37b56c07d99e29dde78bfa90cf8d1" + integrity sha512-Q3XIgQfXyxxxms3bPN+xGgvwk0TtG9l89IomApu+yTKzaIIlf051mS+lGngjnh9L0aUiCp6ICyjDLtutWP54fw== + "@swc/core-linux-arm-gnueabihf@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.3.57.tgz#f8d95c8a9e03d1cccbb85d214fe24ca95940852a" +"@swc/core-linux-arm-gnueabihf@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.3.92.tgz#6f7c20833b739f8911c936c9783976ded2c449dc" + integrity sha512-tnOCoCpNVXC+0FCfG84PBZJyLlz0Vfj9MQhyhCvlJz9hQmvpf8nTdKH7RHrOn8VfxtUBLdVi80dXgIFgbvl7qA== + "@swc/core-linux-arm64-gnu@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.3.57.tgz#66fb53f081f43a652de79374d8b7c78b10469ef8" +"@swc/core-linux-arm64-gnu@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.3.92.tgz#bb01dd9b922b0c076c38924013bd10036ce39c7c" + integrity sha512-lFfGhX32w8h1j74Iyz0Wv7JByXIwX11OE9UxG+oT7lG0RyXkF4zKyxP8EoxfLrDXse4Oop434p95e3UNC3IfCw== + "@swc/core-linux-arm64-musl@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.3.57.tgz#28709d982d2eeb55b173c79a7d5feed1a2cc065d" +"@swc/core-linux-arm64-musl@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.3.92.tgz#0070165eed2805475c98eb732bab8bdca955932e" + integrity sha512-rOZtRcLj57MSAbiecMsqjzBcZDuaCZ8F6l6JDwGkQ7u1NYR57cqF0QDyU7RKS1Jq27Z/Vg21z5cwqoH5fLN+Sg== + "@swc/core-linux-x64-gnu@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.3.57.tgz#3bea8b7d115fc2bf65a7b4b6930dd878f434034f" +"@swc/core-linux-x64-gnu@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.3.92.tgz#d9785f93b9121eeef0f54e8d845dd216698e0115" + integrity sha512-qptoMGnBL6v89x/Qpn+l1TH1Y0ed+v0qhNfAEVzZvCvzEMTFXphhlhYbDdpxbzRmCjH6GOGq7Y+xrWt9T1/ARg== + "@swc/core-linux-x64-musl@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.3.57.tgz#40bf395ac8971d561e38ef2af54b72d3705c199e" +"@swc/core-linux-x64-musl@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.3.92.tgz#8fe5cf244695bf4f0bc7dc7df450a9bd1bfccc2b" + integrity sha512-g2KrJ43bZkCZHH4zsIV5ErojuV1OIpUHaEyW1gf7JWKaFBpWYVyubzFPvPkjcxHGLbMsEzO7w/NVfxtGMlFH/Q== + "@swc/core-win32-arm64-msvc@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.3.57.tgz#5edf457fb86c455b6cecb89e4fa29a8dfa47910b" +"@swc/core-win32-arm64-msvc@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.3.92.tgz#d6150785455c813a8e62f4e4b0a22773baf398eb" + integrity sha512-3MCRGPAYDoQ8Yyd3WsCMc8eFSyKXY5kQLyg/R5zEqA0uthomo0m0F5/fxAJMZGaSdYkU1DgF73ctOWOf+Z/EzQ== + "@swc/core-win32-ia32-msvc@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.3.57.tgz#0d9d4b851f5380dce04094412ecae8ac538f5dc8" +"@swc/core-win32-ia32-msvc@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.3.92.tgz#8142166bceafbaa209d440b36fdc8cd4b4f82768" + integrity sha512-zqTBKQhgfWm73SVGS8FKhFYDovyRl1f5dTX1IwSKynO0qHkRCqJwauFJv/yevkpJWsI2pFh03xsRs9HncTQKSA== + "@swc/core-win32-x64-msvc@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.3.57.tgz#4923a71e9bd3a1bae3ab3a1c6e07a31973d45f61" +"@swc/core-win32-x64-msvc@1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.3.92.tgz#4ba542875fc690b579232721ccec7873e139646a" + integrity sha512-41bE66ddr9o/Fi1FBh0sHdaKdENPTuDpv1IFHxSg0dJyM/jX8LbkjnpdInYXHBxhcLVAPraVRrNsC4SaoPw2Pg== + "@swc/core@1.3.57": version "1.3.57" resolved "https://registry.npmmirror.com/@swc/core/-/core-1.3.57.tgz#a6c2c04ad72668a3144661fee27508556fff3fed" @@ -5306,6 +5363,43 @@ "@swc/core-win32-ia32-msvc" "1.3.57" "@swc/core-win32-x64-msvc" "1.3.57" +"@swc/core@^1.3.92": + version "1.3.92" + resolved "https://registry.yarnpkg.com/@swc/core/-/core-1.3.92.tgz#f51808cdb6cbb90b0877b9a51806eea9a70eafca" + integrity sha512-vx0vUrf4YTEw59njOJ46Ha5i0cZTMYdRHQ7KXU29efN1MxcmJH2RajWLPlvQarOP1ab9iv9cApD7SMchDyx2vA== + dependencies: + "@swc/counter" "^0.1.1" + "@swc/types" "^0.1.5" + optionalDependencies: + "@swc/core-darwin-arm64" "1.3.92" + "@swc/core-darwin-x64" "1.3.92" + "@swc/core-linux-arm-gnueabihf" "1.3.92" + "@swc/core-linux-arm64-gnu" "1.3.92" + "@swc/core-linux-arm64-musl" "1.3.92" + "@swc/core-linux-x64-gnu" "1.3.92" + "@swc/core-linux-x64-musl" "1.3.92" + "@swc/core-win32-arm64-msvc" "1.3.92" + "@swc/core-win32-ia32-msvc" "1.3.92" + "@swc/core-win32-x64-msvc" "1.3.92" + +"@swc/counter@^0.1.1": + version "0.1.2" + resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.2.tgz#bf06d0770e47c6f1102270b744e17b934586985e" + integrity sha512-9F4ys4C74eSTEUNndnER3VJ15oru2NumfQxS8geE+f3eB5xvfxpWyqE5XlVnxb/R14uoXi6SLbBwwiDSkv+XEw== + +"@swc/jest@^0.2.29": + version "0.2.29" + resolved "https://registry.yarnpkg.com/@swc/jest/-/jest-0.2.29.tgz#b27d647ec430c909f9bb567d1df2a47eaa3841f4" + integrity sha512-8reh5RvHBsSikDC3WGCd5ZTd2BXKkyOdK7QwynrCH58jk2cQFhhHhFBg/jvnWZehUQe/EoOImLENc9/DwbBFow== + dependencies: + "@jest/create-cache-key-function" "^27.4.2" + jsonc-parser "^3.2.0" + +"@swc/types@^0.1.5": + version "0.1.5" + resolved "https://registry.yarnpkg.com/@swc/types/-/types-0.1.5.tgz#043b731d4f56a79b4897a3de1af35e75d56bc63a" + integrity sha512-myfUej5naTBWnqOCc/MdVOLVjXUXtIA+NpDrDBKJtLLg2shUjBu3cZmB/85RyitKc55+lUUyl7oRfLOvkr2hsw== + "@szmarczak/http-timer@^1.1.2": version "1.1.2" resolved "https://registry.npmmirror.com/@szmarczak/http-timer/-/http-timer-1.1.2.tgz#b1665e2c461a2cd92f4c1bbf50d5454de0d4b421" @@ -15015,6 +15109,11 @@ jest@^29.6.2: import-local "^3.0.2" jest-cli "^29.6.2" +jest_workaround@^0.79.19: + version "0.79.19" + resolved "https://registry.yarnpkg.com/jest_workaround/-/jest_workaround-0.79.19.tgz#f5bc569163c33c4533e1c4053c533a8811152a91" + integrity sha512-g/MtKSwyb4Ohnd5GHeJaduTgznkyst81x+eUBGOSGK7f8doWuRMPpt6XM/13sM2jLB2QNzT/7Djj7o2PhsozIA== + jose@^4.14.1: version "4.14.4" resolved "https://registry.npmmirror.com/jose/-/jose-4.14.4.tgz#59e09204e2670c3164ee24cbfe7115c6f8bff9ca"