perf(server): optimize performance of APIs (#3079)

* perf: add perf_hooks

* perf: add cache

* fix: test

* feat: support bloom filter

* feat: caching token black list

* perf: caching i18n instance

* fix: test

* fix: test

* chore: remove prePerfHooks on app

* chore: improve i18n instances cache

* chore: remove performance measure

* fix: package.json

* perf: optimize cache strategy

* fix: test

* fix: bug

* test: storer of auth-manager

* fix: afterDestroy hook when updating null value

* fix: version

* chore: fix bug and add test

* fix: test

* fix: test
This commit is contained in:
YANG QIA 2023-12-12 23:02:09 +08:00 committed by GitHub
parent 2c37910894
commit 474b09c7f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 805 additions and 46 deletions

View File

@ -76,7 +76,10 @@ const app = new Koa();
app.use(async (ctx, next) => {
const repository = db.getRepository('users');
ctx.body = await repository.find();
ctx.body = await repository.findAndCount({
limit: 20,
offset: 0,
});
await next();
});

View File

@ -35,7 +35,10 @@ const User = sequelize.define(
const app = new Koa();
app.use(async (ctx, next) => {
ctx.body = await User.findAll();
ctx.body = await User.findAndCountAll({
offset: 0,
limit: 20,
});
await next();
});

View File

@ -1,5 +1,6 @@
const { Application } = require('@nocobase/server');
const dotenv = require('dotenv');
const { PerformanceObserver, createHistogram } = require('perf_hooks');
dotenv.config();
@ -21,8 +22,14 @@ const app = new Application({
resourcer: {
prefix: '/api',
},
logger: {
// skip: () => true,
// transports: ['console'],
// level: 'error',
},
acl: false,
plugins: [],
perfHooks: true,
});
app.db.collection({
@ -78,6 +85,13 @@ app.db.collection({
],
});
// const obs = new PerformanceObserver((items) => {
// items.getEntries().forEach((item) => {
// console.log(item);
// });
// });
// obs.observe({ entryTypes: ['measure'] });
app.listen(13030, (err) => {
console.log('nocobase-server: http://localhost:13030/api/users');
});

View File

@ -11,5 +11,6 @@ export async function getConfig() {
plugins,
cacheManager,
logger,
perfHooks: process.env.ENABLE_PERF_HOOKS ? true : false,
};
}

View File

@ -6,6 +6,7 @@
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"dependencies": {
"@nocobase/cache": "0.17.0-alpha.4",
"@nocobase/actions": "0.17.0-alpha.4",
"@nocobase/database": "0.17.0-alpha.4",
"@nocobase/resourcer": "0.17.0-alpha.4",

View File

@ -5,9 +5,9 @@ import { Auth, AuthExtend } from './auth';
import { JwtOptions, JwtService } from './base/jwt-service';
import { ITokenBlacklistService } from './base/token-blacklist-service';
type Storer = {
export interface Storer {
get: (name: string) => Promise<Model>;
};
}
export type AuthManagerOptions = {
authKey: string;
@ -45,8 +45,8 @@ export class AuthManager {
* @description Add a new authenticate type and the corresponding authenticator.
* The types will show in the authenticators list of the admin panel.
*
* @param {string} authType - The type of the authenticator. It is required to be unique.
* @param {AuthConfig} authConfig - Configurations of the kind of authenticator.
* @param authType - The type of the authenticator. It is required to be unique.
* @param authConfig - Configurations of the kind of authenticator.
*/
registerTypes(authType: string, authConfig: AuthConfig) {
this.authTypes.register(authType, authConfig);
@ -66,8 +66,8 @@ export class AuthManager {
/**
* get
* @description Get authenticator instance by name.
* @param {string} name - The name of the authenticator.
* @return {Promise<Auth>} authenticator instance.
* @param name - The name of the authenticator.
* @return authenticator instance.
*/
async get(name: string, ctx: Context) {
if (!this.storer) {

View File

@ -1,6 +1,7 @@
import { Collection, Model } from '@nocobase/database';
import { Auth, AuthConfig } from '../auth';
import { JwtService } from './jwt-service';
import { Cache } from '@nocobase/cache';
/**
* BaseAuth
@ -35,6 +36,10 @@ export class BaseAuth extends Auth {
return this.ctx.state.currentUser;
}
getCacheKey(userId: number) {
return `auth:${userId}`;
}
validateUsername(username: string) {
return /^[^@.<>"'/]{2,16}$/.test(username);
}
@ -51,11 +56,15 @@ export class BaseAuth extends Auth {
this.ctx.headers['x-role'] = roleName;
}
return await this.userRepository.findOne({
filter: {
id: userId,
},
});
const cache = this.ctx.cache as Cache;
return await cache.wrap(this.getCacheKey(userId), () =>
this.userRepository.findOne({
filter: {
id: userId,
},
raw: true,
}),
);
} catch (err) {
this.ctx.logger.error(err);
return null;
@ -71,7 +80,6 @@ export class BaseAuth extends Auth {
try {
user = await this.validate();
} catch (err) {
console.log(err);
this.ctx.throw(401, err.message);
}
if (!user) {
@ -91,6 +99,9 @@ export class BaseAuth extends Auth {
if (!token) {
return;
}
const { userId } = await this.jwt.decode(token);
await this.ctx.app.emitAsync('beforeSignOut', { userId });
await this.ctx.cache.del(this.getCacheKey(userId));
return await this.jwt.block(token);
}
}

View File

@ -6,6 +6,7 @@
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"dependencies": {
"bloom-filters": "^3.0.1",
"cache-manager": "^5.2.4",
"cache-manager-redis-yet": "^4.1.2"
},

View File

@ -0,0 +1,24 @@
import { BloomFilter } from '../bloom-filter';
import { CacheManager } from '../cache-manager';
describe('bloomFilter', () => {
let bloomFilter: BloomFilter;
let cacheManager: CacheManager;
beforeEach(async () => {
cacheManager = new CacheManager();
cacheManager.registerStore({ name: 'memory', store: 'memory' });
bloomFilter = await cacheManager.createBloomFilter({ store: 'memory' });
await bloomFilter.reserve('bloom-test', 0.01, 1000);
});
afterEach(async () => {
await cacheManager.flushAll();
});
it('should add and check', async () => {
await bloomFilter.add('bloom-test', 'hello');
expect(await bloomFilter.exists('bloom-test', 'hello')).toBeTruthy();
expect(await bloomFilter.exists('bloom-test', 'world')).toBeFalsy();
});
});

View File

@ -0,0 +1,6 @@
export interface BloomFilter {
reserve(key: string, errorRate: number, capacity: number): Promise<void>;
add(key: string, val: string): Promise<void>;
mAdd(key: string, vals: string[]): Promise<void>;
exists(key: string, val: string): Promise<boolean>;
}

View File

@ -0,0 +1,39 @@
import { BloomFilter as IBloomFilter } from '.';
import { Cache } from '../cache';
import { BloomFilter } from 'bloom-filters';
export class MemoryBloomFilter implements IBloomFilter {
cache: Cache;
constructor(cache: Cache) {
this.cache = cache;
}
async reserve(key: string, errorRate: number, capacity: number) {
const filter = BloomFilter.create(capacity, errorRate);
await this.cache.set(key, filter);
}
async add(key: string, value: string) {
const filter = (await this.cache.get(key)) as BloomFilter;
if (!filter) {
return;
}
filter.add(value);
}
async mAdd(key: string, values: string[]) {
const filter = (await this.cache.get(key)) as BloomFilter;
if (!filter) {
return;
}
values.forEach((value) => filter.add(value));
}
async exists(key: string, value: string) {
const filter = (await this.cache.get(key)) as BloomFilter;
if (!filter) {
return false;
}
return filter.has(value);
}
}

View File

@ -0,0 +1,34 @@
import { RedisStore } from 'cache-manager-redis-yet';
import { BloomFilter } from '.';
import { Cache } from '../cache';
export class RedisBloomFilter implements BloomFilter {
cache: Cache;
constructor(cache: Cache) {
this.cache = cache;
}
getStore() {
return this.cache.store.store as RedisStore;
}
async reserve(key: string, errorRate: number, capacity: number) {
const store = this.getStore();
await store.client.bf.reserve(key, errorRate, capacity);
}
async add(key: string, value: string) {
const store = this.getStore();
await store.client.bf.add(key, value);
}
async mAdd(key: string, values: string[]) {
const store = this.getStore();
await store.client.bf.mAdd(key, values);
}
async exists(key: string, value: string) {
const store = this.getStore();
return await store.client.bf.exists(key, value);
}
}

View File

@ -3,6 +3,9 @@ import { Cache } from './cache';
import lodash from 'lodash';
import { RedisStore, redisStore } from 'cache-manager-redis-yet';
import deepmerge from 'deepmerge';
import { MemoryBloomFilter } from './bloom-filter/memory-bloom-filter';
import { BloomFilter } from './bloom-filter';
import { RedisBloomFilter } from './bloom-filter/redis-bloom-filter';
type StoreOptions = {
store?: 'memory' | FactoryStore<Store, any>;
@ -118,4 +121,23 @@ export class CacheManager {
}
await Promise.all(promises);
}
async createBloomFilter(options?: { store?: string }): Promise<BloomFilter> {
const name = 'bloom-filter';
const { store = this.defaultStore } = options || {};
let cache: Cache;
try {
cache = this.getCache(name);
} catch (error) {
cache = await this.createCache({ name, store });
}
switch (store) {
case 'memory':
return new MemoryBloomFilter(cache);
case 'redis':
return new RedisBloomFilter(cache);
default:
throw new Error(`BloomFilter store [${store}] is not supported`);
}
}
}

View File

@ -97,4 +97,10 @@ export class Cache {
const object = (await this.get(key)) || {};
return object[objectKey];
}
async delValueInObject(key: string, objectKey: string) {
const object = (await this.get(key)) || {};
delete object[objectKey];
await this.set(key, object);
}
}

View File

@ -1,2 +1,3 @@
export * from './cache-manager';
export * from './cache';
export * from './bloom-filter';

View File

@ -208,7 +208,7 @@ export class EagerLoadingTree {
include: includeForFilter,
} as any)
).map((row) => {
return { row, pk: row.get(primaryKeyField) };
return { row, pk: row[primaryKeyField] };
});
const findOptions = {

View File

@ -385,7 +385,7 @@ export async function updateMultipleAssociation(
const createAccessor = association.accessors.create;
if (isUndefinedOrNull(value)) {
await model[setAccessor](null, { transaction, context });
await model[setAccessor](null, { transaction, context, individualHooks: true });
model.setDataValue(key, null);
return;
}

View File

@ -67,6 +67,7 @@ export function createAppLogger(options: AppLoggerOptions = {}) {
}
info['req'] = pick(info['req'], requestWhitelist);
info['res'] = pick(info['res'], responseWhitelist);
ctx.logger.log(info);
}

View File

@ -1,9 +1,10 @@
import { assign, MergeStrategies, requireModule } from '@nocobase/utils';
import { assign, MergeStrategies, prePerfHooksWrap, requireModule } from '@nocobase/utils';
import compose from 'koa-compose';
import _ from 'lodash';
import Middleware, { MiddlewareType } from './middleware';
import Resource from './resource';
import { HandlerType } from './resourcer';
import { RecordableHistogram, performance } from 'perf_hooks';
export type ActionType = string | HandlerType | ActionOptions;
@ -309,6 +310,8 @@ export class Action {
this.getHandler(),
].filter(Boolean);
// handlers = handlers.map((handler) => prePerfHooksWrap(handler));
return handlers;
}

View File

@ -20,11 +20,19 @@ import { createCacheManager } from './cache';
import { registerCli } from './commands';
import { CronJobManager } from './cron/cron-job-manager';
import { ApplicationNotInstall } from './errors/application-not-install';
import { createAppProxy, createI18n, createResourcer, getCommandFullName, registerMiddlewares } from './helper';
import {
createAppProxy,
createI18n,
createResourcer,
getCommandFullName,
registerMiddlewares,
enablePerfHooks,
} from './helper';
import { ApplicationVersion } from './helpers/application-version';
import { Locale } from './locale';
import { Plugin } from './plugin';
import { InstallOptions, PluginManager } from './plugin-manager';
import { RecordableHistogram, performance } from 'node:perf_hooks';
const packageJson = require('../package.json');
@ -50,6 +58,7 @@ export interface ApplicationOptions {
pmSock?: string;
name?: string;
authManager?: AuthManagerOptions;
perfHooks?: boolean;
}
export interface DefaultState extends KoaDefaultState {
@ -115,6 +124,7 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
name: string;
} = null;
public running = false;
public perfHistograms = new Map<string, RecordableHistogram>();
protected plugins = new Map<string, Plugin>();
protected _appSupervisor: AppSupervisor = AppSupervisor.getInstance();
protected _started: boolean;
@ -560,6 +570,10 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
this.log.error(e);
}
if (this._cacheManager) {
await this._cacheManager.close();
}
await this.emitAsync('afterStop', this, options);
this.stopped = true;
@ -729,6 +743,10 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
this._locales = new Locale(createAppProxy(this));
if (options.perfHooks) {
enablePerfHooks(this);
}
registerMiddlewares(this, options);
if (options.registerActions !== false) {

View File

@ -1,7 +1,7 @@
import cors from '@koa/cors';
import Database from '@nocobase/database';
import { Resourcer } from '@nocobase/resourcer';
import { uid } from '@nocobase/utils';
import { postPerfHooksWrap, prePerfHooksWrap, uid } from '@nocobase/utils';
import { Command } from 'commander';
import fs from 'fs';
import i18next from 'i18next';
@ -13,6 +13,7 @@ import { dateTemplate } from './middlewares/data-template';
import { dataWrapping } from './middlewares/data-wrapping';
import { db2resource } from './middlewares/db2resource';
import { i18n } from './middlewares/i18n';
import { createHistogram, RecordableHistogram } from 'perf_hooks';
export function createI18n(options: ApplicationOptions) {
const instance = i18next.createInstance();
@ -80,7 +81,10 @@ export function registerMiddlewares(app: Application, options: ApplicationOption
app.use(dataWrapping(), { tag: 'dataWrapping', after: 'i18n' });
}
app.resourcer.use(parseVariables, { tag: 'parseVariables', after: 'acl' });
app.resourcer.use(parseVariables, {
tag: 'parseVariables',
after: 'acl',
});
app.resourcer.use(dateTemplate, { tag: 'dateTemplate', after: 'acl' });
app.use(db2resource, { tag: 'db2resource', after: 'dataWrapping' });
@ -119,3 +123,35 @@ export const tsxRerunning = async () => {
const file = resolve(process.cwd(), 'storage/app.watch.ts');
await fs.promises.writeFile(file, `export const watchId = '${uid()}';`, 'utf-8');
};
export const enablePerfHooks = (app: Application) => {
app.context.getPerfHistogram = (name: string) => {
if (!app.perfHistograms.has(name)) {
app.perfHistograms.set(name, createHistogram());
}
return app.perfHistograms.get(name);
};
app.resourcer.define({
name: 'perf',
actions: {
view: async (ctx, next) => {
const result = {};
const histograms = ctx.app.perfHistograms as Map<string, RecordableHistogram>;
const sortedHistograms = [...histograms.entries()].sort(([i, a], [j, b]) => b.mean - a.mean);
sortedHistograms.forEach(([name, histogram]) => {
result[name] = histogram;
});
ctx.body = result;
await next();
},
reset: async (ctx, next) => {
const histograms = ctx.app.perfHistograms as Map<string, RecordableHistogram>;
histograms.forEach((histogram: RecordableHistogram) => histogram.reset());
await next();
},
},
});
app.acl.allow('perf', '*', 'public');
};

View File

@ -9,6 +9,7 @@ export class Locale {
defaultLang = 'en-US';
localeFn = new Map();
resourceCached = new Map();
i18nInstances = new Map();
constructor(app: Application) {
this.app = app;
@ -97,4 +98,16 @@ export class Locale {
});
return resources;
}
async getI18nInstance(lang: string) {
if (lang === '*' || !lang) {
return this.app.i18n.cloneInstance({ initImmediate: false });
}
let instance = this.i18nInstances.get(lang);
if (!instance) {
instance = this.app.i18n.cloneInstance({ initImmediate: false });
this.i18nInstances.set(lang, instance);
}
return instance;
}
}

View File

@ -1,7 +1,6 @@
import { Locale } from '../locale';
export async function i18n(ctx, next) {
const i18n = ctx.app.i18n.cloneInstance({ initImmediate: false });
ctx.i18n = i18n;
ctx.t = i18n.t.bind(i18n);
ctx.getCurrentLocale = () => {
const lng =
ctx.get('X-Locale') ||
@ -12,9 +11,13 @@ export async function i18n(ctx, next) {
return lng;
};
const lng = ctx.getCurrentLocale();
const localeManager = ctx.app.localeManager as Locale;
const i18n = await localeManager.getI18nInstance(lng);
ctx.i18n = i18n;
ctx.t = i18n.t.bind(i18n);
if (lng !== '*' && lng) {
i18n.changeLanguage(lng);
await ctx.app.localeManager.loadResourcesByLang(lng);
await localeManager.loadResourcesByLang(lng);
}
await next();
}

View File

@ -21,5 +21,6 @@ export * from './toposort';
export * from './uid';
export * from './url';
export * from './measure-execution-time';
export * from './perf-hooks';
export { dayjs, lodash };

View File

@ -0,0 +1,32 @@
import { RecordableHistogram, performance } from 'perf_hooks';
export const prePerfHooksWrap = (handler: any, options?: { name?: string }) => {
const { name } = options || {};
return async (ctx: any, next: any) => {
if (!ctx.getPerfHistogram) {
return await handler(ctx, next);
}
const histogram = ctx.getPerfHistogram(name || handler) as RecordableHistogram;
const start = performance.now();
await handler(ctx, async () => {
const duration = performance.now() - start;
histogram.record(Math.ceil(duration * 1e6));
await next();
});
};
};
export const postPerfHooksWrap = (handler: any, options: { name?: string }) => {
const { name } = options || {};
return async (ctx: any, next: any) => {
if (!ctx.getPerfHistogram) {
return await handler(ctx, next);
}
await next();
const histogram = ctx.getPerfHistogram(name || handler) as RecordableHistogram;
const start = performance.now();
await handler(ctx, async () => {});
const duration = performance.now() - start;
histogram.record(Math.ceil(duration * 1e6));
};
};

View File

@ -21,7 +21,8 @@
"@nocobase/database": "0.x",
"@nocobase/server": "0.x",
"@nocobase/test": "0.x",
"@nocobase/utils": "0.x"
"@nocobase/utils": "0.x",
"@nocobase/cache": "0.x"
},
"repository": {
"type": "git",

View File

@ -19,6 +19,7 @@ describe('role', () => {
ctx = {
db,
cache: api.cache,
state: {
currentRole: '',
},
@ -83,4 +84,153 @@ describe('role', () => {
await setCurrentRole(ctx, () => {});
expect(ctx.state.currentRole).toBe('anonymous');
});
it('should set role in cache', async () => {
ctx.state.currentUser = await db.getRepository('users').findOne({
appends: ['roles'],
});
ctx.get = function (name) {
if (name === 'X-Role') {
return 'admin';
}
};
await setCurrentRole(ctx, () => {});
const roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeDefined();
});
it('should update cache when role added', async () => {
ctx.get = function (name) {
if (name === 'X-Role') {
return 'admin';
}
};
await db.getRepository('roles').create({
values: {
name: 'test',
title: 'Test',
},
});
ctx.state.currentUser = await db.getRepository('users').findOne({
appends: ['roles'],
});
await setCurrentRole(ctx, () => {});
let roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeDefined();
let testRole = roles.find((role) => role.name === 'test');
expect(testRole).toBeUndefined();
await db.getRepository('users').update({
values: {
roles: [
...ctx.state.currentUser.roles,
{
name: 'test',
},
],
},
filterByTk: ctx.state.currentUser.id,
});
roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeUndefined();
await setCurrentRole(ctx, () => {});
roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeDefined();
testRole = roles.find((role) => role.name === 'test');
expect(testRole).toBeDefined();
});
it('should update cache when one role removed', async () => {
ctx.get = function (name) {
if (name === 'X-Role') {
return 'admin';
}
};
ctx.state.currentUser = await db.getRepository('users').findOne({
appends: ['roles'],
});
await setCurrentRole(ctx, () => {});
let roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeDefined();
let testRole = roles.find((role) => role.name === 'member');
expect(testRole).toBeDefined();
await db.getRepository('users').update({
values: {
roles: [
{
name: 'root',
},
{
name: 'admin',
},
],
},
filterByTk: ctx.state.currentUser.id,
});
roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeUndefined();
await setCurrentRole(ctx, () => {});
roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeDefined();
testRole = roles.find((role) => role.name === 'member');
expect(testRole).toBeUndefined();
});
it('should update cache when all roles removed', async () => {
ctx.get = function (name) {
if (name === 'X-Role') {
return 'admin';
}
};
ctx.state.currentUser = await db.getRepository('users').findOne({
appends: ['roles'],
});
await setCurrentRole(ctx, () => {});
let roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeDefined();
await db.getRepository('users').update({
values: {
roles: null,
},
filterByTk: ctx.state.currentUser.id,
});
roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeUndefined();
const throwFn = jest.fn();
ctx.throw = throwFn;
await setCurrentRole(ctx, () => {});
expect(throwFn).lastCalledWith(401, { code: 'ROLE_NOT_FOUND_ERR', message: 'The user role does not exist.' });
expect(ctx.state.currentRole).not.toBeDefined();
});
it('should update cache when role deleted', async () => {
ctx.get = function (name) {
if (name === 'X-Role') {
return 'admin';
}
};
ctx.state.currentUser = await db.getRepository('users').findOne({
appends: ['roles'],
});
await setCurrentRole(ctx, () => {});
let roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeDefined();
let testRole = roles.find((role) => role.name === 'member');
expect(testRole).toBeDefined();
await db.getRepository('roles').destroy({
filter: {
name: 'member',
},
});
roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeUndefined();
await setCurrentRole(ctx, () => {});
roles = await ctx.cache.get(`roles:${ctx.state.currentUser.id}`);
expect(roles).toBeDefined();
testRole = roles.find((role) => role.name === 'member');
expect(testRole).toBeUndefined();
});
});

View File

@ -83,6 +83,7 @@ describe('actions', () => {
await db.getRepository('roles').destroy({
filterByTk: 'test',
});
await app.cache.reset();
response = await agent.post('/auth:signIn').send({
email: 'test2@nocobase.com',

View File

@ -22,7 +22,7 @@ export async function setDefaultRole(ctx: Context, next: Next) {
await db.sequelize.transaction(async (transaction) => {
await repository.update({
filter: {
userId: currentUser.get('id'),
userId: currentUser.id,
},
values: {
default: false,
@ -31,7 +31,7 @@ export async function setDefaultRole(ctx: Context, next: Next) {
});
await repository.update({
filter: {
userId: currentUser.get('id'),
userId: currentUser.id,
roleName,
},
values: {

View File

@ -1,5 +1,6 @@
import { Context } from '@nocobase/actions';
import { Repository } from '@nocobase/database';
import { Cache } from '@nocobase/cache';
import { Model, Repository } from '@nocobase/database';
export async function setCurrentRole(ctx: Context, next) {
const currentRole = ctx.get('X-Role');
@ -13,9 +14,14 @@ export async function setCurrentRole(ctx: Context, next) {
return next();
}
const cache = ctx.cache as Cache;
const repository = ctx.db.getRepository('users.roles', ctx.state.currentUser.id) as unknown as Repository;
const roles = await repository.find();
ctx.state.currentUser.setDataValue('roles', roles);
const roles = (await cache.wrap(`roles:${ctx.state.currentUser.id}`, () =>
repository.find({
raw: true,
}),
)) as Model[];
ctx.state.currentUser.roles = roles;
// 1. If the X-Role is set, use the specified role
if (currentRole) {

View File

@ -13,6 +13,7 @@ import { setCurrentRole } from './middlewares/setCurrentRole';
import { RoleModel } from './model/RoleModel';
import { RoleResourceActionModel } from './model/RoleResourceActionModel';
import { RoleResourceModel } from './model/RoleResourceModel';
import { Cache } from '@nocobase/cache';
export interface AssociationFieldAction {
associationActions: string[];
@ -347,6 +348,16 @@ export class PluginACL extends Plugin {
});
});
// Delete cache when the roles of a user changed
this.app.db.on('rolesUsers.afterSave', async (model) => {
const cache = this.app.cache as Cache;
await cache.del(`roles:${model.get('userId')}`);
});
this.app.db.on('rolesUsers.afterDestroy', async (model) => {
const cache = this.app.cache as Cache;
await cache.del(`roles:${model.get('userId')}`);
});
const writeRolesToACL = async (app, options) => {
const exists = await this.app.db.collectionExistsInDb('roles');
if (exists) {
@ -433,6 +444,9 @@ export class PluginACL extends Plugin {
});
});
this.app.on('beforeSignOut', ({ userId }) => {
this.app.cache.del(`roles:${userId}`);
});
this.app.resourcer.use(setCurrentRole, { tag: 'setCurrentRole', before: 'acl', after: 'auth' });
this.app.acl.allow('users', 'setDefaultRole', 'loggedIn');

View File

@ -0,0 +1,78 @@
import { Database, Model } from '@nocobase/database';
import { MockServer, mockServer } from '@nocobase/test';
import { BaseAuth } from '@nocobase/auth';
describe('auth', () => {
let auth: BaseAuth;
let app: MockServer;
let db: Database;
let user: Model;
beforeEach(async () => {
app = mockServer({
plugins: ['users', 'auth'],
});
await app.quickstart({ clean: true });
db = app.db;
user = await db.getRepository('users').create({
values: {
username: 'admin',
},
});
const jwt = app.authManager.jwt;
auth = new BaseAuth({
userCollection: db.getCollection('users'),
ctx: {
app,
getBearerToken() {
return jwt.sign({ userId: user.id });
},
cache: app.cache,
} as any,
} as any);
await app.cache.reset();
});
afterEach(async () => {
await app.cache.reset();
await app.destroy();
});
it('should get user from cache', async () => {
expect(await app.cache.get(auth.getCacheKey(user.id))).toBeUndefined();
let userData = await auth.check();
expect(userData).not.toBeNull();
expect(await app.cache.get(auth.getCacheKey(user.id))).toBeDefined();
userData = await auth.check();
expect(userData).not.toBeNull();
});
it('should update cache when user changed', async () => {
await auth.check();
let cacheData = await app.cache.get(auth.getCacheKey(user.id));
expect(cacheData['nickname']).toBeNull();
await db.getRepository('users').update({
values: {
nickname: 'admin',
},
filterByTk: user.id,
});
cacheData = await app.cache.get(auth.getCacheKey(user.id));
console.log(cacheData);
expect(cacheData['nickname']).toBe('admin');
});
it('should delete cache when user deleted', async () => {
await auth.check();
let cacheData = await app.cache.get(auth.getCacheKey(user.id));
expect(cacheData['nickname']).toBeNull();
await db.getRepository('users').destroy({
filterByTk: user.id,
});
cacheData = await app.cache.get(auth.getCacheKey(user.id));
expect(cacheData).toBeUndefined();
});
});

View File

@ -0,0 +1,89 @@
import { Cache, CacheManager } from '@nocobase/cache';
import { Storer } from '../storer';
class MockDB {
data: any;
hooks = {};
constructor(data: any) {
this.data = data;
}
on(name: string, func: (...args: any[]) => Promise<void>) {
this.hooks[name] = func;
}
async emitAsync(name: string, ...args: any[]) {
const func = this.hooks[name];
if (func) {
await func(...args);
}
}
getRepository() {
return {
find: async () => {
return this.data;
},
};
}
}
describe('storer', () => {
let db: any;
let storer: Storer;
let cache: Cache;
const data = [
{
name: 'test1',
enabled: true,
},
{
name: 'test2',
enabled: true,
},
];
beforeEach(async () => {
const cacheManager = new CacheManager();
cache = await cacheManager.createCache({ name: 'test' });
db = new MockDB(data);
storer = new Storer({ db, cache });
});
afterEach(() => {
db = undefined;
storer = undefined;
});
it('should get authenticator from cache', async () => {
expect(await cache.get('authenticators')).toBeUndefined();
let authenticator = await storer.get('test1');
expect(authenticator).toBeDefined();
expect(await cache.get('authenticators')).toBeDefined();
authenticator = await storer.get('test1');
expect(authenticator).toBeDefined();
});
it('should delete from cache on afterDestory', async () => {
expect(await storer.get('test1')).toBeDefined();
await db.emitAsync('authenticators.afterDestroy', data[0]);
const authenticators = await cache.get('authenticators');
expect(authenticators['test1']).toBeUndefined();
});
it('should delete from cache on afterSave as disabled', async () => {
expect(await storer.get('test1')).toBeDefined();
await db.emitAsync('authenticators.afterSave', { ...data[0], enabled: false });
const authenticators = await cache.get('authenticators');
expect(authenticators['test1']).toBeUndefined();
});
it('should set cache on afterSave as enabled', async () => {
expect(await storer.get('test1')).toBeDefined();
await db.emitAsync('authenticators.afterSave', { name: 'test3', enabled: true });
const authenticators = await cache.get('authenticators');
expect(authenticators['test3']).toBeDefined();
});
});

View File

@ -1,12 +1,12 @@
import Database, { Repository } from '@nocobase/database';
import { MockServer, mockServer } from '@nocobase/test';
import { TokenBlacklistService } from '../token-blacklist';
import { ITokenBlacklistService } from '@nocobase/auth';
describe('token-blacklist', () => {
let app: MockServer;
let db: Database;
let repo: Repository;
let tokenBlacklist: TokenBlacklistService;
let tokenBlacklist: ITokenBlacklistService;
beforeAll(async () => {
app = mockServer({
@ -15,7 +15,7 @@ describe('token-blacklist', () => {
await app.loadAndInstall({ clean: true });
db = app.db;
repo = db.getRepository('tokenBlacklist');
tokenBlacklist = new TokenBlacklistService(app.getPlugin('auth'));
tokenBlacklist = app.authManager.jwt.blacklist;
});
afterAll(async () => {

View File

@ -8,8 +8,12 @@ import { BasicAuth } from './basic-auth';
import { enUS, zhCN } from './locale';
import { AuthModel } from './model/authenticator';
import { TokenBlacklistService } from './token-blacklist';
import { Cache } from '@nocobase/cache';
import { Storer } from './storer';
export class AuthPlugin extends Plugin {
cache: Cache;
afterAdd() {}
async beforeLoad() {
this.app.i18n.addResources('zh-CN', namespace, zhCN);
@ -30,16 +34,19 @@ export class AuthPlugin extends Plugin {
plugin: this,
},
});
// Set up auth manager and register preset auth type
this.app.authManager.setStorer({
get: async (name: string) => {
const repo = this.db.getRepository('authenticators');
const authenticators = await repo.find({ filter: { enabled: true } });
const authenticator = authenticators.find((authenticator: Model) => authenticator.name === name);
return authenticator || authenticators[0];
},
this.cache = await this.app.cacheManager.createCache({
name: 'auth',
prefix: 'auth',
store: 'memory',
});
// Set up auth manager and register preset auth type
const storer = new Storer({
db: this.db,
cache: this.cache,
});
this.app.authManager.setStorer(storer);
if (!this.app.authManager.jwt.blacklist) {
// If blacklist service is not set, should configure default blacklist service
this.app.authManager.setTokenBlacklistService(new TokenBlacklistService(this));
@ -64,6 +71,16 @@ export class AuthPlugin extends Plugin {
name: `pm.${this.name}.authenticators`,
actions: ['authenticators:*'],
});
// Change cache when user changed
this.app.db.on('users.afterSave', async (user: Model) => {
const cache = this.app.cache as Cache;
await cache.set(`auth:${user.id}`, user.toJSON());
});
this.app.db.on('users.afterDestroy', async (user: Model) => {
const cache = this.app.cache as Cache;
await cache.del(`auth:${user.id}`);
});
}
async install(options?: InstallOptions) {

View File

@ -0,0 +1,53 @@
import { Storer as IStorer } from '@nocobase/auth';
import { Cache } from '@nocobase/cache';
import { Database, Model } from '@nocobase/database';
import { AuthModel } from './model/authenticator';
export class Storer implements IStorer {
db: Database;
cache: Cache;
key = 'authenticators';
constructor({ db, cache }: { db: Database; cache: Cache }) {
this.db = db;
this.cache = cache;
this.db.on('authenticators.afterSave', async (model: AuthModel) => {
if (!model.enabled) {
await this.cache.delValueInObject(this.key, model.name);
return;
}
await this.cache.setValueInObject(this.key, model.name, model);
});
this.db.on('authenticators.afterDestroy', async (model: AuthModel) => {
await this.cache.delValueInObject(this.key, model.name);
});
}
async getCache(): Promise<AuthModel[]> {
const authenticators = (await this.cache.get(this.key)) as Record<string, AuthModel>;
if (!authenticators) {
return [];
}
return Object.values(authenticators);
}
async setCache(authenticators: AuthModel[]) {
const obj = authenticators.reduce((obj, authenticator) => {
obj[authenticator.name] = authenticator;
return obj;
}, {});
await this.cache.set(this.key, obj);
}
async get(name: string) {
let authenticators = await this.getCache();
if (!authenticators.length) {
const repo = this.db.getRepository('authenticators');
authenticators = await repo.find({ filter: { enabled: true } });
await this.setCache(authenticators);
}
const authenticator = authenticators.find((authenticator: Model) => authenticator.name === name);
return authenticator || authenticators[0];
}
}

View File

@ -2,13 +2,33 @@ import { ITokenBlacklistService } from '@nocobase/auth';
import { Repository } from '@nocobase/database';
import { CronJob } from 'cron';
import AuthPlugin from './plugin';
import { BloomFilter } from '@nocobase/cache';
export class TokenBlacklistService implements ITokenBlacklistService {
repo: Repository;
cronJob: CronJob;
bloomFilter: BloomFilter;
cacheKey = 'token-black-list';
constructor(protected plugin: AuthPlugin) {
this.repo = plugin.db.getRepository('tokenBlacklist');
// Try to create a bloom filter and cache blocked tokens in it
plugin.app.on('beforeStart', async () => {
try {
this.bloomFilter = await plugin.app.cacheManager.createBloomFilter();
// https://redis.io/docs/data-types/probabilistic/bloom-filter/#reserving-bloom-filters
// 0.1% error rate requires 14.4 bits per item
// 14.4*1000000/8/1024/1024 = 1.72MB
await this.bloomFilter.reserve(this.cacheKey, 0.001, 1000000);
const data = await this.repo.find({ fields: ['token'], raw: true });
const tokens = data.map((item: any) => item.token);
await this.bloomFilter.mAdd(this.cacheKey, tokens);
} catch (error) {
plugin.app.logger.error('token-blacklist: create bloom filter failed', error);
this.bloomFilter = null;
}
});
}
get app() {
@ -16,6 +36,12 @@ export class TokenBlacklistService implements ITokenBlacklistService {
}
async has(token: string) {
if (this.bloomFilter) {
const exists = await this.bloomFilter.exists(this.cacheKey, token);
if (!exists) {
return false;
}
}
return !!(await this.repo.findOne({
filter: {
token,
@ -25,10 +51,14 @@ export class TokenBlacklistService implements ITokenBlacklistService {
async add(values) {
await this.deleteExpiredTokens();
const { token } = values;
if (this.bloomFilter) {
await this.bloomFilter.add(this.cacheKey, token);
}
return this.repo.model.findOrCreate({
defaults: values,
where: {
token: values.token,
token,
},
});
}

View File

@ -50,8 +50,8 @@ describe('createdBy/updatedBy', () => {
});
const data = p2.toJSON();
expect(data.createdBy.id).toBe(currentUser.get('id'));
expect(data.updatedBy.id).toBe(currentUser.get('id'));
expect(data.createdBy.id).toBe(currentUser.id);
expect(data.updatedBy.id).toBe(currentUser.id);
});
it('case 3', async () => {

View File

@ -56,12 +56,12 @@ export async function submit(context: Context, next) {
}
const presetValues = processor.getParsedValue(actionItem.values ?? {}, userJob.nodeId, {
// @deprecated
currentUser: currentUser.toJSON(),
currentUser: currentUser,
// @deprecated
currentRecord: values.result[formKey],
// @deprecated
currentTime: new Date(),
$user: currentUser.toJSON(),
$user: currentUser,
$nForm: values.result[formKey],
$nDate: {
now: new Date(),

View File

@ -8124,6 +8124,11 @@ balanced-match@^1.0.0:
version "1.0.2"
resolved "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
base64-arraybuffer@^1.0.2:
version "1.0.2"
resolved "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc"
integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==
base64-js@^1.0.2, base64-js@^1.3.1:
version "1.5.1"
resolved "https://registry.npmmirror.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
@ -8199,6 +8204,21 @@ blessed@0.1.81:
version "0.1.81"
resolved "https://registry.npmmirror.com/blessed/-/blessed-0.1.81.tgz#f962d687ec2c369570ae71af843256e6d0ca1129"
bloom-filters@^3.0.1:
version "3.0.1"
resolved "https://registry.npmjs.org/bloom-filters/-/bloom-filters-3.0.1.tgz#13e28ed22febe2489cd00ba5bd98fdc90e820180"
integrity sha512-rU9IU6bgZ1jmqcLWhlKSidrFjbIGjB89CJBsQqUj1+3/11tAJDwn+f7iRu4bbQ2srTjGgNeoWNwcnelumqdi0g==
dependencies:
base64-arraybuffer "^1.0.2"
is-buffer "^2.0.5"
lodash "^4.17.15"
lodash.eq "^4.0.0"
lodash.indexof "^4.0.5"
long "^5.2.0"
reflect-metadata "^0.1.13"
seedrandom "^3.0.5"
xxhashjs "^0.2.2"
bluebird@^3.5.0, bluebird@^3.5.1:
version "3.7.2"
resolved "https://registry.npmmirror.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
@ -9966,6 +9986,11 @@ csstype@^3.0.10, csstype@^3.0.2, csstype@^3.0.8:
version "3.1.2"
resolved "https://registry.npmmirror.com/csstype/-/csstype-3.1.2.tgz#1d4bf9d572f11c14031f0436e1c10bc1f571f50b"
cuint@^0.2.2:
version "0.2.2"
resolved "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz#408086d409550c2631155619e9fa7bcadc3b991b"
integrity sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==
culvert@^0.1.2:
version "0.1.2"
resolved "https://registry.npmmirror.com/culvert/-/culvert-0.1.2.tgz#9502f5f0154a2d5a22a023e79f71cc936fa6ef6f"
@ -16090,6 +16115,11 @@ lodash.difference@^4.5.0:
version "4.5.0"
resolved "https://registry.npmmirror.com/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c"
lodash.eq@^4.0.0:
version "4.0.0"
resolved "https://registry.npmjs.org/lodash.eq/-/lodash.eq-4.0.0.tgz#a39f06779e72f9c0d1f310c90cd292c1661d5035"
integrity sha512-vbrJpXL6kQNG6TkInxX12DZRfuYVllSxhwYqjYB78g2zF3UI15nFO/0AgmZnZRnaQ38sZtjCiVjGr2rnKt4v0g==
lodash.flatten@^4.4.0:
version "4.4.0"
resolved "https://registry.npmmirror.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
@ -16102,6 +16132,11 @@ lodash.includes@^4.3.0:
version "4.3.0"
resolved "https://registry.npmmirror.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
lodash.indexof@^4.0.5:
version "4.0.5"
resolved "https://registry.npmjs.org/lodash.indexof/-/lodash.indexof-4.0.5.tgz#53714adc2cddd6ed87638f893aa9b6c24e31ef3c"
integrity sha512-t9wLWMQsawdVmf6/IcAgVGqAJkNzYVcn4BHYZKTPW//l7N5Oq7Bq138BaVk19agcsPZePcidSgTTw4NqS1nUAw==
lodash.isboolean@^3.0.3:
version "3.0.3"
resolved "https://registry.npmmirror.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
@ -20628,6 +20663,11 @@ redux@^4.0.0, redux@^4.0.4:
dependencies:
"@babel/runtime" "^7.9.2"
reflect-metadata@^0.1.13:
version "0.1.13"
resolved "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"
integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==
reflect.getprototypeof@^1.0.2:
version "1.0.3"
resolved "https://registry.npmmirror.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.3.tgz#2738fd896fcc3477ffbd4190b40c2458026b6928"
@ -24346,6 +24386,13 @@ xtend@~2.1.1:
dependencies:
object-keys "~0.4.0"
xxhashjs@^0.2.2:
version "0.2.2"
resolved "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz#8a6251567621a1c46a5ae204da0249c7f8caa9d8"
integrity sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==
dependencies:
cuint "^0.2.2"
y18n@^3.2.1:
version "3.2.2"
resolved "https://registry.npmmirror.com/y18n/-/y18n-3.2.2.tgz#85c901bd6470ce71fc4bb723ad209b70f7f28696"