From daac2ae0db7e5d0fa41338e0144c920c34fab404 Mon Sep 17 00:00:00 2001 From: YANG QIA <2013xile@gmail.com> Date: Mon, 20 Nov 2023 17:14:20 +0800 Subject: [PATCH] refactor(cache): improve cache (#3004) * feat: improve cache * fix: bug * fix: test * fix: test * fix: test * chore: add cache test * feat: add wrapWithCondition * fix: test * refactor: improve api * fix: test * fix: test * fix: test * fix: improve code * fix: test * feat: register redis store * fix: tst * fix: test * fix: bug * chore: update * fix: ttl unit * chore: cachemanager constructor * chore: remove code * feat: support close connection * chore: add close options for redis store --- .env.example | 6 +- packages/core/app/src/config/cache.ts | 21 ++- packages/core/app/src/config/index.ts | 4 +- packages/core/cache/package.json | 6 +- .../cache/src/__tests__/cache-manager.test.ts | 42 ++++++ .../core/cache/src/__tests__/cache.test.ts | 64 +++++++++ .../core/cache/src/__tests__/index.test.ts | 72 ----------- packages/core/cache/src/cache-manager.ts | 121 ++++++++++++++++++ packages/core/cache/src/cache.ts | 100 +++++++++++++++ packages/core/cache/src/index.ts | 72 +---------- packages/core/server/src/application.ts | 22 +++- packages/core/server/src/cache/index.ts | 10 ++ packages/core/server/src/locale/locale.ts | 27 ++-- .../src/server/__tests__/query.test.ts | 10 +- .../src/server/actions/query.ts | 6 +- .../src/server/plugin.ts | 9 +- .../src/server/__tests__/resources.test.ts | 56 ++++---- .../src/server/plugin.ts | 7 +- .../src/server/resources.ts | 15 +-- .../ui-schema-repository-with-cache.test.ts | 2 +- yarn.lock | 111 ++++++++++++---- 21 files changed, 538 insertions(+), 245 deletions(-) create mode 100644 packages/core/cache/src/__tests__/cache-manager.test.ts create mode 100644 packages/core/cache/src/__tests__/cache.test.ts delete mode 100644 packages/core/cache/src/__tests__/index.test.ts create mode 100644 packages/core/cache/src/cache-manager.ts create mode 100644 packages/core/cache/src/cache.ts create mode 100644 packages/core/server/src/cache/index.ts diff --git a/.env.example b/.env.example index c2545bd57..b70e98443 100644 --- a/.env.example +++ b/.env.example @@ -43,8 +43,10 @@ DB_TABLE_PREFIX= # DB_DIALECT_OPTIONS_SSL_REJECT_UNAUTHORIZED=true ################# CACHE ################# -# default is memory cache, when develop mode,code's change will be clear memory cache, so can use 'cache-manager-fs-hash' -# CACHE_CONFIG={"storePackage":"cache-manager-fs-hash","ttl":86400,"max":1000} +CACHE_DEFAULT_STORE=memory +# max number of items in memory cache +CACHE_MEMORY_MAX=2000 +# CACHE_REDIS_URL= ################# STORAGE (Initialization only) ################# diff --git a/packages/core/app/src/config/cache.ts b/packages/core/app/src/config/cache.ts index 251fbd097..09287bac8 100644 --- a/packages/core/app/src/config/cache.ts +++ b/packages/core/app/src/config/cache.ts @@ -1,5 +1,18 @@ -import { createDefaultCacheConfig } from '@nocobase/cache'; +import { CacheManagerOptions } from '@nocobase/cache'; -const cacheConfig = process.env.CACHE_CONFIG ? JSON.parse(process.env.CACHE_CONFIG) : createDefaultCacheConfig(); - -export default cacheConfig; +export const cacheManager = { + defaultStore: process.env.CACHE_DEFAULT_STORE || 'memory', + stores: { + memory: { + store: 'memory', + max: parseInt(process.env.CACHE_MEMORY_MAX) || 2000, + }, + ...(process.env.CACHE_REDIS_URL + ? { + redis: { + url: process.env.CACHE_REDIS_URL, + }, + } + : {}), + }, +} as CacheManagerOptions; diff --git a/packages/core/app/src/config/index.ts b/packages/core/app/src/config/index.ts index 4167a2c4e..e68dd0226 100644 --- a/packages/core/app/src/config/index.ts +++ b/packages/core/app/src/config/index.ts @@ -1,4 +1,4 @@ -import cache from './cache'; +import { cacheManager } from './cache'; import { parseDatabaseOptions } from './database'; import logger from './logger'; import plugins from './plugins'; @@ -9,7 +9,7 @@ export async function getConfig() { database: await parseDatabaseOptions(), resourcer, plugins, - cache, + cacheManager, logger, }; } diff --git a/packages/core/cache/package.json b/packages/core/cache/package.json index a9e37fc83..8ef8fe386 100644 --- a/packages/core/cache/package.json +++ b/packages/core/cache/package.json @@ -6,11 +6,11 @@ "main": "./lib/index.js", "types": "./lib/index.d.ts", "dependencies": { - "cache-manager": "^4.1.0" + "cache-manager": "^5.2.4", + "cache-manager-redis-yet": "^4.1.2" }, "devDependencies": { - "@types/cache-manager": "^4.0.2", - "cache-manager-fs-hash": "^1.0.0" + "redis": "^4.6.10" }, "repository": { "type": "git", diff --git a/packages/core/cache/src/__tests__/cache-manager.test.ts b/packages/core/cache/src/__tests__/cache-manager.test.ts new file mode 100644 index 000000000..77c10872c --- /dev/null +++ b/packages/core/cache/src/__tests__/cache-manager.test.ts @@ -0,0 +1,42 @@ +import { Cache } from '../cache'; +import { CacheManager } from '../cache-manager'; + +describe('cache-manager', () => { + let cacheManager: CacheManager; + + beforeEach(() => { + cacheManager = new CacheManager(); + }); + + afterEach(() => { + cacheManager = null; + }); + + it('create with default config', async () => { + cacheManager.registerStore({ name: 'memory', store: 'memory' }); + const cache = await cacheManager.createCache({ name: 'test', store: 'memory' }); + expect(cache).toBeDefined(); + expect(cache.name).toBe('test'); + expect(cacheManager.caches.has('test')).toBeTruthy(); + }); + + it('create with custom config', async () => { + cacheManager.registerStore({ name: 'memory', store: 'memory' }); + const cache = (await cacheManager.createCache({ name: 'test', store: 'memory', ttl: 100 })) as Cache; + expect(cache).toBeDefined(); + expect(cache.name).toBe('test'); + expect(cacheManager.caches.has('test')).toBeTruthy(); + }); + + it('should close store', async () => { + const close = jest.fn(); + cacheManager.registerStore({ + name: 'memory', + store: 'memory', + close, + }); + await cacheManager.createCache({ name: 'test', store: 'memory' }); + await cacheManager.close(); + expect(close).toBeCalled(); + }); +}); diff --git a/packages/core/cache/src/__tests__/cache.test.ts b/packages/core/cache/src/__tests__/cache.test.ts new file mode 100644 index 000000000..fb20f6d8b --- /dev/null +++ b/packages/core/cache/src/__tests__/cache.test.ts @@ -0,0 +1,64 @@ +import { Cache } from '../cache'; +import { CacheManager } from '../cache-manager'; +import lodash from 'lodash'; + +describe('cache', () => { + let cache: Cache; + + beforeEach(async () => { + const cacheManager = new CacheManager(); + cacheManager.registerStore({ name: 'memory', store: 'memory' }); + cache = await cacheManager.createCache({ name: 'test', store: 'memory' }); + }); + + afterEach(async () => { + await cache.reset(); + }); + + it('should set and get value', async () => { + await cache.set('key', 'value'); + const value = await cache.get('key'); + expect(value).toBe('value'); + }); + + it('set and get value in object', async () => { + const value = { a: 1 }; + await cache.set('key', value); + const cacheA = await cache.getValueInObject('key', 'a'); + expect(cacheA).toEqual(1); + + await cache.setValueInObject('key', 'a', 2); + const cacheVal2 = await cache.getValueInObject('key', 'a'); + expect(cacheVal2).toEqual(2); + }); + + it('wrap with condition, useCache', async () => { + const obj = {}; + const get = () => obj; + const val = await cache.wrapWithCondition('key', get, { + useCache: false, + }); + expect(val).toBe(obj); + expect(await cache.get('key')).toBeUndefined(); + const val2 = await cache.wrapWithCondition('key', get); + expect(val2).toBe(obj); + expect(await cache.get('key')).toMatchObject(obj); + }); + + it('wrap with condition, isCacheable', async () => { + let obj = {}; + const get = () => obj; + const isCacheable = (val: any) => !lodash.isEmpty(val); + const val = await cache.wrapWithCondition('key', get, { + isCacheable, + }); + expect(val).toBe(obj); + expect(await cache.get('key')).toBeUndefined(); + obj = { a: 1 }; + const val2 = await cache.wrapWithCondition('key', get, { + isCacheable, + }); + expect(val2).toBe(obj); + expect(await cache.get('key')).toMatchObject(obj); + }); +}); diff --git a/packages/core/cache/src/__tests__/index.test.ts b/packages/core/cache/src/__tests__/index.test.ts deleted file mode 100644 index 28a9d5f15..000000000 --- a/packages/core/cache/src/__tests__/index.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { createCache, createDefaultCacheConfig } from '@nocobase/cache'; - -export function sleep(ms?: number) { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -describe('cache', () => { - it('createCache-with-mem', async () => { - const cacheConfig = createDefaultCacheConfig(); - cacheConfig.ttl = 1; - const cache = createCache(cacheConfig); - await cache.set('name', 'Emma'); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(100); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(1005); - expect(await cache.get('name')).toBeUndefined(); - }); - - it('createCache-with-single-config', async () => { - let cacheConfigStr = '{"store":"memory","ttl":1,"max":10}'; - let cacheConfig = JSON.parse(cacheConfigStr); - let cache = createCache(cacheConfig); - await cache.set('name', 'Emma'); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(100); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(1005); - expect(await cache.get('name')).toBeUndefined(); - - cacheConfigStr = '[{"store":"memory","ttl":1,"max":10}]'; - cacheConfig = JSON.parse(cacheConfigStr); - cache = createCache(cacheConfig); - await cache.set('name', 'Emma'); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(100); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(1005); - expect(await cache.get('name')).toBeUndefined(); - }); - - it('createCache-with-default-cache-manager-fs-hash', async () => { - const cacheConfig = createDefaultCacheConfig(); - cacheConfig.ttl = 1; - cacheConfig.storePackage = 'cache-manager-fs-hash'; - const cache = createCache(cacheConfig); - await cache.set('name', 'Emma'); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(100); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(1005); - expect(await cache.get('name')).toBeUndefined(); - }); - - it('createCache-multi-cache', async () => { - const cacheConfigStr = - '[{"store":"memory","ttl":1,"max":10},{"storePackage":"cache-manager-fs-hash","ttl":10,"max":100}]'; - const cacheConfig = JSON.parse(cacheConfigStr); - const cache = createCache(cacheConfig); - - await cache.set('name', 'Emma'); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(1005); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(5000); - expect(await cache.get('name')).toEqual('Emma'); - await sleep(5000); - expect(await cache.get('name')).toBeUndefined(); - }); -}); diff --git a/packages/core/cache/src/cache-manager.ts b/packages/core/cache/src/cache-manager.ts new file mode 100644 index 000000000..d681f0010 --- /dev/null +++ b/packages/core/cache/src/cache-manager.ts @@ -0,0 +1,121 @@ +import { FactoryStore, Store, caching, Cache as BasicCache } from 'cache-manager'; +import { Cache } from './cache'; +import lodash from 'lodash'; +import { RedisStore, redisStore } from 'cache-manager-redis-yet'; +import deepmerge from 'deepmerge'; + +type StoreOptions = { + store?: 'memory' | FactoryStore; + close?: (store: Store) => Promise; + // global config + [key: string]: any; +}; + +export type CacheManagerOptions = Partial<{ + defaultStore: string; + stores: { + [storeType: string]: StoreOptions; + }; +}>; + +export class CacheManager { + defaultStore: string; + private stores = new Map< + string, + { + store: BasicCache; + close?: (store: Store) => Promise; + } + >(); + storeTypes = new Map(); + caches = new Map(); + + constructor(options?: CacheManagerOptions) { + const defaultOptions: CacheManagerOptions = { + defaultStore: 'memory', + stores: { + memory: { + store: 'memory', + // global config + max: 2000, + }, + redis: { + store: redisStore, + close: async (redis: RedisStore) => { + await redis.client.quit(); + }, + }, + }, + }; + const cacheOptions = deepmerge(defaultOptions, options || {}); + const { defaultStore = 'memory', stores } = cacheOptions; + this.defaultStore = defaultStore; + for (const [name, store] of Object.entries(stores)) { + const { store: s, ...globalConfig } = store; + this.registerStore({ name, store: s, ...globalConfig }); + } + } + + private async createStore(options: { name: string; storeType: string; [key: string]: any }) { + const { name, storeType: type, ...config } = options; + const storeType = this.storeTypes.get(type) as any; + if (!storeType) { + throw new Error(`Create cache failed, store type [${type}] is unavailable or not registered`); + } + const { store: s, close, ...globalConfig } = storeType; + const store = await caching(s, { ...globalConfig, ...config }); + this.stores.set(name, { close, store }); + return store; + } + + registerStore(options: { name: string } & StoreOptions) { + const { name, ...rest } = options; + this.storeTypes.set(name, rest); + } + + private newCache(options: { name: string; prefix?: string; store: BasicCache }) { + const { name, prefix, store } = options; + const cache = new Cache({ name, prefix, store }); + this.caches.set(name, cache); + return cache; + } + + async createCache(options: { name: string; prefix?: string; store?: string; [key: string]: any }) { + const { name, prefix, store = this.defaultStore, ...config } = options; + if (!lodash.isEmpty(config)) { + const newStore = await this.createStore({ name, storeType: store, ...config }); + return this.newCache({ name, prefix, store: newStore }); + } + const s = this.stores.get(store); + if (!s) { + const defaultStore = await this.createStore({ name: store, storeType: store }); + return this.newCache({ name, prefix, store: defaultStore }); + } + return this.newCache({ name, prefix, store: s.store }); + } + + getCache(name: string): Cache { + const cache = this.caches.get(name); + if (!cache) { + throw new Error(`Get cache failed, ${name} is not found`); + } + return cache; + } + + async flushAll() { + const promises = []; + for (const cache of this.caches.values()) { + promises.push(cache.reset()); + } + await Promise.all(promises); + } + + async close() { + const promises = []; + for (const s of this.stores.values()) { + const { close, store } = s; + close && promises.push(close(store.store)); + } + await Promise.all(promises); + } +} diff --git a/packages/core/cache/src/cache.ts b/packages/core/cache/src/cache.ts new file mode 100644 index 000000000..ce8ce6af0 --- /dev/null +++ b/packages/core/cache/src/cache.ts @@ -0,0 +1,100 @@ +import { Cache as BasicCache, Milliseconds } from 'cache-manager'; + +export class Cache { + name: string; + prefix?: string; + store: BasicCache; + + constructor({ name, prefix, store }: { name: string; store: BasicCache; prefix?: string }) { + this.name = name; + this.prefix = prefix; + this.store = store; + } + + key(key: string): string { + return this.prefix ? `${this.prefix}:${key}` : key; + } + + async set(key: string, value: unknown, ttl?: Milliseconds): Promise { + await this.store.set(this.key(key), value, ttl); + } + + async get(key: string): Promise { + return await this.store.get(this.key(key)); + } + + async del(key: string): Promise { + await this.store.del(this.key(key)); + } + + async reset(): Promise { + await this.store.reset(); + } + + async wrap(key: string, fn: () => Promise, ttl?: Milliseconds): Promise { + return await this.store.wrap(this.key(key), fn, ttl); + } + + async wrapWithCondition( + key: string, + fn: () => T | Promise, + options?: { + useCache?: boolean; + isCacheable?: (val: unknown) => boolean | Promise; + ttl?: Milliseconds; + }, + ): Promise { + const { useCache, isCacheable, ttl } = options || {}; + if (useCache === false) { + return await fn(); + } + const value = await this.get(key); + if (value) { + return value; + } + const result = await fn(); + const cacheable = isCacheable ? await isCacheable(result) : result; + if (!cacheable) { + return result; + } + await this.set(key, result, ttl); + return result; + } + + async mset(args: [string, unknown][], ttl?: Milliseconds): Promise { + await this.store.store.mset( + args.map(([key, value]) => [this.key(key), value]), + ttl, + ); + } + + async mget(...args: string[]): Promise { + args = args.map((key) => this.key(key)); + return await this.store.store.mget(...args); + } + + async mdel(...args: string[]): Promise { + args = args.map((key) => this.key(key)); + await this.store.store.mdel(...args); + } + + async keys(pattern?: string): Promise { + const keys = await this.store.store.keys(pattern); + return keys.map((key) => key.replace(`${this.name}:`, '')); + } + + async ttl(key: string): Promise { + return await this.store.store.ttl(this.key(key)); + } + + async setValueInObject(key: string, objectKey: string, value: unknown) { + const object = (await this.get(key)) || {}; + object[objectKey] = value; + await this.set(key, object); + } + + async getValueInObject(key: string, objectKey: string) { + const object = (await this.get(key)) || {}; + return object[objectKey]; + } +} diff --git a/packages/core/cache/src/index.ts b/packages/core/cache/src/index.ts index ef794fd47..adb5e4a26 100644 --- a/packages/core/cache/src/index.ts +++ b/packages/core/cache/src/index.ts @@ -1,70 +1,2 @@ -import { CacheOptions, caching, CachingConfig, multiCaching, StoreConfig, WrapArgsType } from 'cache-manager'; - -/** - * be used for create cache {@link createCache} - */ -export type ICacheConfig = StoreConfig & - CacheOptions & { - // every storeConfig init a store instance - storePackage?: string; - }; - -/** - * create a default cache config object - * @returns {ICacheConfig} - */ -export function createDefaultCacheConfig(): ICacheConfig { - return { - ttl: 86400, // seconds - max: 1000, - store: 'memory', - }; -} - -/** - * cache and multi cache common method and only keep promise method - */ -export interface Cache { - set(key: string, value: T, options?: CachingConfig): Promise; - - set(key: string, value: T, ttl: number): Promise; - - wrap(...args: WrapArgsType[]): Promise; - - get(key: string): Promise; - - del(key: string): Promise; - - reset(): Promise; -} - -/** - * create cache - *
if cacheConfig is array and length gt 1 then will be return multi cache, else will be return cache - * @param {ICacheConfig | ICacheConfig[]} cacheConfig - * @returns {Cache} - */ -export function createCache(cacheConfig: ICacheConfig | ICacheConfig[] = createDefaultCacheConfig()): Cache { - if (Array.isArray(cacheConfig)) { - // multi cache - if (cacheConfig.length === 1) { - return createCacheByICacheConfig(cacheConfig[0]); - } else { - const caches = []; - for (const cacheConfigEle of cacheConfig) { - caches.push(createCacheByICacheConfig(cacheConfigEle)); - } - return multiCaching(caches) as Cache; - } - } else { - return createCacheByICacheConfig(cacheConfig); - } -} - -function createCacheByICacheConfig(cacheConfig: ICacheConfig): Cache { - // if storePackage exist then load storePackage and instead store - if (cacheConfig.storePackage) { - cacheConfig.store = require(cacheConfig.storePackage); - } - return caching(cacheConfig); -} +export * from './cache-manager'; +export * from './cache'; diff --git a/packages/core/server/src/application.ts b/packages/core/server/src/application.ts index e28edc9c0..3f773d404 100644 --- a/packages/core/server/src/application.ts +++ b/packages/core/server/src/application.ts @@ -1,7 +1,7 @@ import { ACL } from '@nocobase/acl'; import { registerActions } from '@nocobase/actions'; import { actions as authActions, AuthManager, AuthManagerOptions } from '@nocobase/auth'; -import { Cache, createCache, ICacheConfig } from '@nocobase/cache'; +import { CacheManagerOptions, Cache, CacheManager } from '@nocobase/cache'; import Database, { CollectionOptions, IDatabaseOptions } from '@nocobase/database'; import { AppLoggerOptions, createAppLogger, Logger } from '@nocobase/logger'; import { ResourceOptions, Resourcer } from '@nocobase/resourcer'; @@ -24,6 +24,7 @@ import { Locale } from './locale'; import { Plugin } from './plugin'; import { InstallOptions, PluginManager } from './plugin-manager'; import { CronJobManager } from './cron/cron-job-manager'; +import { createCacheManager } from './cache'; const packageJson = require('../package.json'); @@ -36,7 +37,7 @@ export interface ResourcerOptions { export interface ApplicationOptions { database?: IDatabaseOptions | Database; - cache?: ICacheConfig | ICacheConfig[]; + cacheManager?: CacheManagerOptions; resourcer?: ResourcerOptions; bodyParser?: any; cors?: any; @@ -167,8 +168,18 @@ export class Application exten return this._resourcer; } + protected _cacheManager: CacheManager; + + get cacheManager() { + return this._cacheManager; + } + protected _cache: Cache; + set cache(cache: Cache) { + this._cache = cache; + } + get cache() { return this._cache; } @@ -317,8 +328,13 @@ export class Application exten if (!oldDb.closed()) { await oldDb.close(); } + if (this._cacheManager) { + await this._cacheManager.close(); + } } + this._cacheManager = await createCacheManager(this, this.options.cacheManager); + this.setMaintainingMessage('init plugins'); await this.pm.initPlugins(); @@ -677,10 +693,10 @@ export class Application exten this._resourcer = createResourcer(options); this._cli = this.createCli(); this._i18n = createI18n(options); - this._cache = createCache(options.cache); this.context.db = this._db; this.context.logger = this._logger; this.context.resourcer = this._resourcer; + this.context.cacheManager = this._cacheManager; this.context.cache = this._cache; const plugins = this._pm ? this._pm.options.plugins : options.plugins; diff --git a/packages/core/server/src/cache/index.ts b/packages/core/server/src/cache/index.ts new file mode 100644 index 000000000..ddd032951 --- /dev/null +++ b/packages/core/server/src/cache/index.ts @@ -0,0 +1,10 @@ +import { CacheManagerOptions, CacheManager } from '@nocobase/cache'; +import Application from '../application'; + +export const createCacheManager = async (app: Application, options: CacheManagerOptions) => { + const cacheManager = new CacheManager(options); + const defaultCache = await cacheManager.createCache({ name: app.name }); + app.cache = defaultCache; + app.context.cache = defaultCache; + return cacheManager; +}; diff --git a/packages/core/server/src/locale/locale.ts b/packages/core/server/src/locale/locale.ts index c7cc2d836..0eabf4df5 100644 --- a/packages/core/server/src/locale/locale.ts +++ b/packages/core/server/src/locale/locale.ts @@ -1,4 +1,4 @@ -import { Cache, createCache } from '@nocobase/cache'; +import { Cache } from '@nocobase/cache'; import { lodash } from '@nocobase/utils'; import Application from '../application'; import { getResource } from './resource'; @@ -11,8 +11,6 @@ export class Locale { constructor(app: Application) { this.app = app; - this.cache = createCache(); - this.app.on('afterLoad', async () => { this.app.log.debug('load locale resource'); this.app.setMaintainingMessage('load locale resource'); @@ -23,6 +21,12 @@ export class Locale { } async load() { + this.cache = await this.app.cacheManager.createCache({ + name: 'locale', + prefix: 'locale', + store: 'memory', + }); + await this.get(this.defaultLang); } @@ -36,7 +40,7 @@ export class Locale { }; for (const [name, fn] of this.localeFn) { // this.app.log.debug(`load [${name}] locale resource `); - const result = await this.wrapCache(`locale:${name}:${lang}`, async () => await fn(lang)); + const result = await this.wrapCache(`${name}:${lang}`, async () => await fn(lang)); if (result) { defaults[name] = result; } @@ -45,20 +49,13 @@ export class Locale { } async wrapCache(key: string, fn: () => any) { - const result = await this.cache.get(key); - if (result) { - return result; - } - const value = await fn(); - if (lodash.isEmpty(value)) { - return value; - } - await this.cache.set(key, value); - return value; + return await this.cache.wrapWithCondition(key, fn, { + isCacheable: (val: any) => !lodash.isEmpty(val), + }); } async getCacheResources(lang: string) { - return await this.wrapCache(`locale:resources:${lang}`, () => this.getResources(lang)); + return await this.wrapCache(`resources:${lang}`, () => this.getResources(lang)); } getResources(lang: string) { diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts index dbccb95a4..71cce1118 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts @@ -206,7 +206,9 @@ describe('query', () => { const cache = new MockCache(); ctx = { app: { - getPlugin: () => ({ cache }), + cacheManager: { + getCache: () => cache, + }, }, }; }); @@ -220,7 +222,7 @@ describe('query', () => { }, }, }; - const cache = context.app.getPlugin().cache; + const cache = context.app.cacheManager.getCache(); expect(cache.get(key)).toBeUndefined(); await compose([cacheMiddleware, query])(context, async () => {}); expect(query).toBeCalled(); @@ -242,7 +244,7 @@ describe('query', () => { }, }, }; - const cache = context.app.getPlugin().cache; + const cache = context.app.cacheManager.getCache(); cache.set(key, value); expect(cache.get(key)).toBeDefined(); await compose([cacheMiddleware, query])(context, async () => {}); @@ -259,7 +261,7 @@ describe('query', () => { }, }, }; - const cache = context.app.getPlugin().cache; + const cache = context.app.cacheManager.getCache(); expect(cache.get(key)).toBeUndefined(); await compose([cacheMiddleware, query])(context, async () => {}); expect(query).toBeCalled(); diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/actions/query.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/actions/query.ts index 2d71df86a..2cb7d1ece 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/actions/query.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/actions/query.ts @@ -4,6 +4,7 @@ import ChartsV2Plugin from '../plugin'; import { formatter } from './formatter'; import compose from 'koa-compose'; import { parseFilter, getDateVars } from '@nocobase/utils'; +import { Cache } from '@nocobase/cache'; type MeasureProps = { field: string | string[]; @@ -291,8 +292,7 @@ export const parseVariables = async (ctx: Context, next: Next) => { export const cacheMiddleware = async (ctx: Context, next: Next) => { const { uid, cache: cacheConfig, refresh } = ctx.action.params.values as QueryParams; - const plugin = ctx.app.getPlugin('data-visualization') as ChartsV2Plugin; - const cache = plugin.cache; + const cache = ctx.app.cacheManager.getCache('data-visualization') as Cache; const useCache = cacheConfig?.enabled && uid; if (useCache && !refresh) { @@ -304,7 +304,7 @@ export const cacheMiddleware = async (ctx: Context, next: Next) => { } await next(); if (useCache) { - await cache.set(uid, ctx.body, cacheConfig?.ttl || 30); + await cache.set(uid, ctx.body, cacheConfig?.ttl * 1000); } }; diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/plugin.ts index 18cda06e9..4c4bd08b4 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/plugin.ts @@ -1,4 +1,4 @@ -import { Cache, createCache } from '@nocobase/cache'; +import { Cache } from '@nocobase/cache'; import { InstallOptions, Plugin } from '@nocobase/server'; import { query } from './actions/query'; import { resolve } from 'path'; @@ -27,10 +27,11 @@ export class DataVisualizationPlugin extends Plugin { }, }); - this.cache = createCache({ - ttl: 30, // seconds - max: 1000, + this.cache = await this.app.cacheManager.createCache({ + name: 'data-visualization', store: 'memory', + ttl: 30 * 1000, // millseconds + max: 1000, }); } diff --git a/packages/plugins/@nocobase/plugin-localization-management/src/server/__tests__/resources.test.ts b/packages/plugins/@nocobase/plugin-localization-management/src/server/__tests__/resources.test.ts index 3c6203831..a1a450adb 100644 --- a/packages/plugins/@nocobase/plugin-localization-management/src/server/__tests__/resources.test.ts +++ b/packages/plugins/@nocobase/plugin-localization-management/src/server/__tests__/resources.test.ts @@ -1,43 +1,49 @@ +import { CacheManager } from '@nocobase/cache'; import Resources from '../resources'; describe('resources', () => { let resources: Resources; - beforeAll(() => { - resources = new Resources({ - getRepository: (name: string) => { - if (name === 'localizationTexts') { - return { - find: () => [ - { id: 1, module: 'resources.client', text: 'Edit' }, - { id: 2, module: 'resources.client', text: 'Add new' }, - { id: 3, module: 'resources.acl', text: 'Admin' }, - ], - }; - } - if (name === 'localizationTranslations') { - return { - find: () => [ - { textId: 1, translation: '编辑' }, - { textId: 3, translation: '管理员' }, - ], - }; - } - }, - } as any); + beforeAll(async () => { + const cacheManager = new CacheManager(); + const cache = await cacheManager.createCache({ name: 'locale', store: 'memory' }); + resources = new Resources( + { + getRepository: (name: string) => { + if (name === 'localizationTexts') { + return { + find: () => [ + { id: 1, module: 'resources.client', text: 'Edit' }, + { id: 2, module: 'resources.client', text: 'Add new' }, + { id: 3, module: 'resources.acl', text: 'Admin' }, + ], + }; + } + if (name === 'localizationTranslations') { + return { + find: () => [ + { textId: 1, translation: '编辑' }, + { textId: 3, translation: '管理员' }, + ], + }; + } + }, + } as any, + cache, + ); }); test('getTexts', async () => { const texts = await resources.getTexts(); expect(texts).toBeDefined(); - const cache = await resources.cache.get('localization:texts'); + const cache = await resources.cache.get('texts'); expect(cache).toBeDefined(); }); test('getTranslations', async () => { const translations = await resources.getTranslations('zh-CN'); expect(translations).toBeDefined(); - const cache = await resources.cache.get('localization:translations:zh-CN'); + const cache = await resources.cache.get('translations:zh-CN'); expect(cache).toBeDefined(); }); @@ -61,7 +67,7 @@ describe('resources', () => { test('updateCacheTexts', async () => { const texts = [{ id: 4, module: 'resources.acl', text: 'Test' }]; await resources.updateCacheTexts(texts); - const cache = await resources.cache.get('localization:texts'); + const cache = await resources.cache.get('texts'); expect(cache).toBeDefined(); expect((cache as any[]).length).toBe(4); }); diff --git a/packages/plugins/@nocobase/plugin-localization-management/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-localization-management/src/server/plugin.ts index a23cd6c78..29f69b675 100644 --- a/packages/plugins/@nocobase/plugin-localization-management/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-localization-management/src/server/plugin.ts @@ -97,7 +97,12 @@ export class LocalizationManagementPlugin extends Plugin { .catch((err) => {}); }); - this.resources = new Resources(this.db); + const cache = await this.app.cacheManager.createCache({ + name: 'localization', + prefix: 'localization', + store: 'memory', + }); + this.resources = new Resources(this.db, cache); this.registerUISchemahook(); diff --git a/packages/plugins/@nocobase/plugin-localization-management/src/server/resources.ts b/packages/plugins/@nocobase/plugin-localization-management/src/server/resources.ts index 085ceb415..30f7b40c2 100644 --- a/packages/plugins/@nocobase/plugin-localization-management/src/server/resources.ts +++ b/packages/plugins/@nocobase/plugin-localization-management/src/server/resources.ts @@ -1,18 +1,17 @@ -import { Cache, createCache } from '@nocobase/cache'; +import { Cache } from '@nocobase/cache'; import { Database } from '@nocobase/database'; export default class Resources { cache: Cache; db: Database; - CACHE_KEY_PREFIX = 'localization:'; - constructor(db: Database) { - this.cache = createCache(); + constructor(db: Database, cache: Cache) { + this.cache = cache; this.db = db; } async getTexts() { - return await this.cache.wrap(`${this.CACHE_KEY_PREFIX}texts`, async () => { + return await this.cache.wrap(`texts`, async () => { return await this.db.getRepository('localizationTexts').find({ fields: ['id', 'module', 'text'], raw: true, @@ -21,7 +20,7 @@ export default class Resources { } async getTranslations(locale: string) { - return await this.cache.wrap(`${this.CACHE_KEY_PREFIX}translations:${locale}`, async () => { + return await this.cache.wrap(`translations:${locale}`, async () => { return await this.db.getRepository('localizationTranslations').find({ fields: ['textId', 'translation'], filter: { locale }, @@ -69,10 +68,10 @@ export default class Resources { text: text.text, })); const existTexts = await this.getTexts(); - await this.cache.set(`${this.CACHE_KEY_PREFIX}texts`, [...existTexts, ...newTexts]); + await this.cache.set(`texts`, [...existTexts, ...newTexts]); } async resetCache(locale: string) { - await this.cache.del(`${this.CACHE_KEY_PREFIX}translations:${locale}`); + await this.cache.del(`translations:${locale}`); } } diff --git a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/__tests__/ui-schema-repository-with-cache.test.ts b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/__tests__/ui-schema-repository-with-cache.test.ts index f56335867..f13d2bbd5 100644 --- a/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/__tests__/ui-schema-repository-with-cache.test.ts +++ b/packages/plugins/@nocobase/plugin-ui-schema-storage/src/server/__tests__/ui-schema-repository-with-cache.test.ts @@ -21,7 +21,6 @@ describe('ui_schema repository with cache', () => { }); db = app.db; - cache = app.cache; await db.clean({ drop: true }); @@ -35,6 +34,7 @@ describe('ui_schema repository with cache', () => { }, }); repository = db.getCollection('uiSchemas').repository as UiSchemaRepository; + cache = app.cache; repository.setCache(cache); schema = { diff --git a/yarn.lock b/yarn.lock index 76740cdab..2813ad69d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4752,6 +4752,40 @@ classcat "^5.0.3" zustand "^4.3.1" +"@redis/bloom@1.2.0", "@redis/bloom@^1.2.0": + version "1.2.0" + resolved "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + +"@redis/client@1.5.11", "@redis/client@^1.5.8": + version "1.5.11" + resolved "https://registry.npmjs.org/@redis/client/-/client-1.5.11.tgz#5ee8620fea56c67cb427228c35d8403518efe622" + integrity sha512-cV7yHcOAtNQ5x/yQl7Yw1xf53kO0FNDTdDU6bFIMbW6ljB7U7ns0YRM+QIkpoqTAt6zK5k9Fq0QWlUbLcq9AvA== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + +"@redis/graph@1.1.0", "@redis/graph@^1.1.0": + version "1.1.0" + resolved "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" + integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== + +"@redis/json@1.0.6", "@redis/json@^1.0.4": + version "1.0.6" + resolved "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz#b7a7725bbb907765d84c99d55eac3fcf772e180e" + integrity sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw== + +"@redis/search@1.1.5", "@redis/search@^1.1.3": + version "1.1.5" + resolved "https://registry.npmjs.org/@redis/search/-/search-1.1.5.tgz#682b68114049ff28fdf2d82c580044dfb74199fe" + integrity sha512-hPP8w7GfGsbtYEJdn4n7nXa6xt6hVZnnDktKW4ArMaFQ/m/aR7eFvsLQmG/mn1Upq99btPJk+F27IQ2dYpCoUg== + +"@redis/time-series@1.0.5", "@redis/time-series@^1.0.4": + version "1.0.5" + resolved "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad" + integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg== + "@remix-run/router@1.7.1": version "1.7.1" resolved "https://registry.npmjs.org/@remix-run/router/-/router-1.7.1.tgz#fea7ac35ae4014637c130011f59428f618730498" @@ -5508,10 +5542,6 @@ "@types/connect" "*" "@types/node" "*" -"@types/cache-manager@^4.0.2": - version "4.0.2" - resolved "https://registry.npmmirror.com/@types/cache-manager/-/cache-manager-4.0.2.tgz#5e76dd9e7881c23f332c2f48e5f326bd05ba9ac9" - "@types/chai-subset@^1.3.3": version "1.3.3" resolved "https://registry.npmmirror.com/@types/chai-subset/-/chai-subset-1.3.3.tgz#97893814e92abd2c534de422cb377e0e0bdaac94" @@ -7685,10 +7715,6 @@ async-validator@^4.1.0: version "4.2.5" resolved "https://registry.npmmirror.com/async-validator/-/async-validator-4.2.5.tgz#c96ea3332a521699d0afaaceed510a54656c6339" -async@3.2.3: - version "3.2.3" - resolved "https://registry.npmmirror.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" - async@^2.6.3, async@^2.6.4, async@~2.6.1: version "2.6.4" resolved "https://registry.npmmirror.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221" @@ -8417,19 +8443,27 @@ cache-content-type@^1.0.0: mime-types "^2.1.18" ylru "^1.2.0" -cache-manager-fs-hash@^1.0.0: - version "1.0.0" - resolved "https://registry.npmmirror.com/cache-manager-fs-hash/-/cache-manager-fs-hash-1.0.0.tgz#9a3f3fa239c48c54fc6b00575032b72c07dcad99" +cache-manager-redis-yet@^4.1.2: + version "4.1.2" + resolved "https://registry.npmjs.org/cache-manager-redis-yet/-/cache-manager-redis-yet-4.1.2.tgz#fa04df1a979a42585393a7a9918168978a7211ec" + integrity sha512-pM2K1ZlOv8gQpE1Z5mcDrfLj5CsNKVRiYua/SZ12j7LEDgfDeFVntI6JSgIw0siFSR/9P/FpG30scI3frHwibA== dependencies: - lockfile "^1.0.4" + "@redis/bloom" "^1.2.0" + "@redis/client" "^1.5.8" + "@redis/graph" "^1.1.0" + "@redis/json" "^1.0.4" + "@redis/search" "^1.1.3" + "@redis/time-series" "^1.0.4" + cache-manager "^5.2.2" + redis "^4.6.7" -cache-manager@^4.1.0: - version "4.1.0" - resolved "https://registry.npmmirror.com/cache-manager/-/cache-manager-4.1.0.tgz#aa986421f1c975a862d6de88edb9ab1d30f4bd39" +cache-manager@^5.2.2, cache-manager@^5.2.4: + version "5.2.4" + resolved "https://registry.npmjs.org/cache-manager/-/cache-manager-5.2.4.tgz#01bebe2cc6bef993e3e959d59d3a25a3f2658df1" + integrity sha512-gkuCjug16NdGvKm/sydxGVx17uffrSWcEe2xraBtwRCgdYcFxwJAla4OYpASAZT2yhSoxgDiWL9XH6IAChcZJA== dependencies: - async "3.2.3" lodash.clonedeep "^4.5.0" - lru-cache "^7.10.1" + lru-cache "^10.0.1" cacheable-request@^6.0.0: version "6.1.0" @@ -8881,6 +8915,11 @@ clsx@^1.2.1: version "1.2.1" resolved "https://registry.npmmirror.com/clsx/-/clsx-1.2.1.tgz#0ddc4a20a549b59c93a4116bb26f5294ca17dc12" +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + cmd-shim@^4.1.0: version "4.1.0" resolved "https://registry.npmmirror.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" @@ -12414,6 +12453,11 @@ generate-function@^2.3.1: dependencies: is-property "^1.0.2" +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + genfun@^4.0.1: version "4.0.1" resolved "https://registry.npmmirror.com/genfun/-/genfun-4.0.1.tgz#ed10041f2e4a7f1b0a38466d17a5c3e27df1dfc1" @@ -15788,12 +15832,6 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" -lockfile@^1.0.4: - version "1.0.4" - resolved "https://registry.npmmirror.com/lockfile/-/lockfile-1.0.4.tgz#07f819d25ae48f87e538e6578b6964a4981a5609" - dependencies: - signal-exit "^3.0.2" - lodash-es@^4.17.15, lodash-es@^4.17.21: version "4.17.21" resolved "https://registry.npmmirror.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee" @@ -15993,6 +16031,11 @@ lru-cache@8.0.5: version "8.0.5" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-8.0.5.tgz#983fe337f3e176667f8e567cfcce7cb064ea214e" +lru-cache@^10.0.1: + version "10.0.1" + resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.1.tgz#0a3be479df549cca0e5d693ac402ff19537a6b7a" + integrity sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g== + lru-cache@^4.0.1, lru-cache@^4.1.1: version "4.1.5" resolved "https://registry.npmmirror.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd" @@ -16012,7 +16055,7 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -lru-cache@^7.10.1, lru-cache@^7.14.1: +lru-cache@^7.14.1: version "7.18.3" resolved "https://registry.npmmirror.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89" @@ -20347,6 +20390,18 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" +redis@^4.6.10, redis@^4.6.7: + version "4.6.10" + resolved "https://registry.npmjs.org/redis/-/redis-4.6.10.tgz#07f6ea2b2c5455b098e76d1e8c9b3376114e9458" + integrity sha512-mmbyhuKgDiJ5TWUhiKhBssz+mjsuSI/lSZNPI9QvZOYzWvYGejtb+W3RlDDf8LD6Bdl5/mZeG8O1feUGhXTxEg== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.5.11" + "@redis/graph" "1.1.0" + "@redis/json" "1.0.6" + "@redis/search" "1.1.5" + "@redis/time-series" "1.0.5" + redux@^4.0.0, redux@^4.0.4: version "4.2.1" resolved "https://registry.npmmirror.com/redux/-/redux-4.2.1.tgz#c08f4306826c49b5e9dc901dee0452ea8fce6197" @@ -24020,6 +24075,10 @@ y18n@^5.0.5: version "5.0.8" resolved "https://registry.npmmirror.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" +yallist@4.0.0, yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + yallist@^2.1.2: version "2.1.2" resolved "https://registry.npmmirror.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" @@ -24028,10 +24087,6 @@ yallist@^3.0.0, yallist@^3.0.2, yallist@^3.1.1: version "3.1.1" resolved "https://registry.npmmirror.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - yaml@^1.10.0: version "1.10.2" resolved "https://registry.npmmirror.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"