From 92fda15efdd5c7d65c3bb1b51d46686115a6aa55 Mon Sep 17 00:00:00 2001 From: lyf-coder <58352715+lyf-coder@users.noreply.github.com> Date: Mon, 10 Oct 2022 15:37:47 +0800 Subject: [PATCH] feat(core/cache): support cache (#876) * feat(core/cache): support cache * build(create-nocobase-app): remove --cache-store-package cli option * perf(core/cache): modify default cache config and remove unnecessary logic code (cherry picked from commit 6e6086de7a908f38fe516e5340cb2154229c3843) # Conflicts: # packages/core/server/src/application.ts # packages/core/server/src/helper.ts --- .env.example | 4 + .gitignore | 3 +- packages/app/server/src/config/cache.ts | 5 ++ packages/app/server/src/config/index.ts | 2 + packages/core/actions/package.json | 3 +- packages/core/actions/src/index.ts | 3 + packages/core/cache/.npmignore | 8 ++ packages/core/cache/package.json | 26 +++++++ .../core/cache/src/__tests__/index.test.ts | 74 +++++++++++++++++++ packages/core/cache/src/index.ts | 70 ++++++++++++++++++ packages/core/server/src/application.ts | 15 +++- yarn.lock | 48 ++++++++++-- 12 files changed, 253 insertions(+), 8 deletions(-) create mode 100644 packages/app/server/src/config/cache.ts create mode 100644 packages/core/cache/.npmignore create mode 100644 packages/core/cache/package.json create mode 100644 packages/core/cache/src/__tests__/index.test.ts create mode 100644 packages/core/cache/src/index.ts diff --git a/.env.example b/.env.example index 2e559a3ce..8bce59791 100644 --- a/.env.example +++ b/.env.example @@ -27,6 +27,10 @@ DB_STORAGE=storage/db/nocobase.sqlite # DB_PASSWORD=nocobase # DB_LOGGING=on +################# CACHE ################# +# default is memory cache, when develop mode,code's change will be clear memory cache, so can use 'cache-manager-fs-hash' +# CACHE_CONFIG={"storePackage":"cache-manager-fs-hash","ttl":86400,"max":1000} + ################# STORAGE (Initialization only) ################# INIT_ROOT_EMAIL=admin@nocobase.com diff --git a/.gitignore b/.gitignore index 2be23bcba..2a1fdc02a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ coverage docs-dist/ .npmrc dist/ -docker/**/storage \ No newline at end of file +docker/**/storage +cache/diskstore-* diff --git a/packages/app/server/src/config/cache.ts b/packages/app/server/src/config/cache.ts new file mode 100644 index 000000000..8fbb26e1a --- /dev/null +++ b/packages/app/server/src/config/cache.ts @@ -0,0 +1,5 @@ +import { createDefaultCacheConfig } from '@nocobase/cache'; + +const cacheConfig = !!process.env.CACHE_CONFIG ? JSON.parse(process.env.CACHE_CONFIG) : createDefaultCacheConfig(); + +export default cacheConfig; diff --git a/packages/app/server/src/config/index.ts b/packages/app/server/src/config/index.ts index 05aa3c881..ae26a7c7a 100644 --- a/packages/app/server/src/config/index.ts +++ b/packages/app/server/src/config/index.ts @@ -1,9 +1,11 @@ import database from './database'; import plugins from './plugins'; import resourcer from './resourcer'; +import cache from './cache'; export default { database, resourcer, plugins, + cache, }; diff --git a/packages/core/actions/package.json b/packages/core/actions/package.json index 0f59412c5..1e7273a6d 100644 --- a/packages/core/actions/package.json +++ b/packages/core/actions/package.json @@ -13,7 +13,8 @@ "types": "./lib/index.d.ts", "dependencies": { "@nocobase/database": "0.7.4-alpha.7", - "@nocobase/resourcer": "0.7.4-alpha.7" + "@nocobase/resourcer": "0.7.4-alpha.7", + "@nocobase/cache": "0.7.4-alpha.7" }, "repository": { "type": "git", diff --git a/packages/core/actions/src/index.ts b/packages/core/actions/src/index.ts index 50d111bb3..cce1598f1 100644 --- a/packages/core/actions/src/index.ts +++ b/packages/core/actions/src/index.ts @@ -1,6 +1,7 @@ import Koa from 'koa'; import { Database } from '@nocobase/database'; import { Action } from '@nocobase/resourcer'; +import { Cache } from '@nocobase/cache'; import lodash from 'lodash'; import * as actions from './actions'; @@ -10,9 +11,11 @@ export type Next = () => Promise; export interface Context extends Koa.Context { db: Database; + cache: Cache; action: Action; body: any; app: any; + [key: string]: any; } diff --git a/packages/core/cache/.npmignore b/packages/core/cache/.npmignore new file mode 100644 index 000000000..d44041bb5 --- /dev/null +++ b/packages/core/cache/.npmignore @@ -0,0 +1,8 @@ +node_modules +*.log +docs +__tests__ +jest.config.js +tsconfig.json +src +.fatherrc.ts \ No newline at end of file diff --git a/packages/core/cache/package.json b/packages/core/cache/package.json new file mode 100644 index 000000000..b9044e626 --- /dev/null +++ b/packages/core/cache/package.json @@ -0,0 +1,26 @@ +{ + "name": "@nocobase/cache", + "version": "0.7.4-alpha.7", + "description": "", + "license": "Apache-2.0", + "licenses": [ + { + "type": "Apache-2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ], + "main": "./lib/index.js", + "types": "./lib/index.d.ts", + "dependencies": { + "cache-manager": "^4.1.0" + }, + "devDependencies": { + "@types/cache-manager": "^4.0.2", + "cache-manager-fs-hash": "^1.0.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/nocobase/nocobase.git", + "directory": "packages/cache" + } +} diff --git a/packages/core/cache/src/__tests__/index.test.ts b/packages/core/cache/src/__tests__/index.test.ts new file mode 100644 index 000000000..9f2ea2389 --- /dev/null +++ b/packages/core/cache/src/__tests__/index.test.ts @@ -0,0 +1,74 @@ +import { createCache, createDefaultCacheConfig } from '@nocobase/cache'; + +export function sleep(ms: number) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +describe('cache', () => { + it('createCache-with-mem', async () => { + const cacheConfig = createDefaultCacheConfig(); + cacheConfig.ttl = 1; + const cache = createCache(cacheConfig); + await cache.set('name', 'Emma'); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(100); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(1005); + expect(await cache.get('name')).toBeUndefined(); + }); + + it('createCache-with-single-config', async () => { + let cacheConfigStr = + '{"store":"memory","ttl":1,"max":10}'; + let cacheConfig = JSON.parse(cacheConfigStr); + let cache = createCache(cacheConfig); + await cache.set('name', 'Emma'); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(100); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(1005); + expect(await cache.get('name')).toBeUndefined(); + + cacheConfigStr = + '[{"store":"memory","ttl":1,"max":10}]'; + cacheConfig = JSON.parse(cacheConfigStr); + cache = createCache(cacheConfig); + await cache.set('name', 'Emma'); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(100); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(1005); + expect(await cache.get('name')).toBeUndefined(); + }); + + it('createCache-with-default-cache-manager-fs-hash', async () => { + const cacheConfig = createDefaultCacheConfig(); + cacheConfig.ttl = 1; + cacheConfig.storePackage = 'cache-manager-fs-hash'; + const cache = createCache(cacheConfig); + await cache.set('name', 'Emma'); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(100); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(1005); + expect(await cache.get('name')).toBeUndefined(); + }); + + it('createCache-multi-cache', async () => { + const cacheConfigStr = + '[{"store":"memory","ttl":1,"max":10},{"storePackage":"cache-manager-fs-hash","ttl":10,"max":100}]'; + const cacheConfig = JSON.parse(cacheConfigStr); + const cache = createCache(cacheConfig); + + await cache.set('name', 'Emma'); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(1005); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(5000); + expect(await cache.get('name')).toEqual('Emma'); + await sleep(5000); + expect(await cache.get('name')).toBeUndefined(); + }); +}); diff --git a/packages/core/cache/src/index.ts b/packages/core/cache/src/index.ts new file mode 100644 index 000000000..ee8b4dc31 --- /dev/null +++ b/packages/core/cache/src/index.ts @@ -0,0 +1,70 @@ +import { CacheOptions, caching, CachingConfig, multiCaching, StoreConfig, WrapArgsType } from 'cache-manager'; + +/** + * be used for create cache {@link createCache} + */ +export type ICacheConfig = StoreConfig & + CacheOptions & { + // every storeConfig init a store instance + storePackage?: string; + }; + +/** + * create a default cache config object + * @returns {ICacheConfig} + */ +export function createDefaultCacheConfig(): ICacheConfig { + return { + ttl: 86400, // seconds + max: 1000, + store: 'memory', + }; +} + +/** + * cache and multi cache common method and only keep promise method + */ +export interface Cache { + set(key: string, value: T, options?: CachingConfig): Promise; + + set(key: string, value: T, ttl: number): Promise; + + wrap(...args: WrapArgsType[]): Promise; + + get(key: string): Promise; + + del(key: string): Promise; + + reset(): Promise; +} + +/** + * create cache + *
if cacheConfig is array and length gt 1 then will be return multi cache, else will be return cache + * @param {ICacheConfig | ICacheConfig[]} cacheConfig + * @returns {Cache} + */ +export function createCache(cacheConfig: ICacheConfig | ICacheConfig[] = createDefaultCacheConfig()): Cache { + if (Array.isArray(cacheConfig)) { + // multi cache + if (cacheConfig.length === 1) { + return createCacheByICacheConfig(cacheConfig[0]); + } else { + let caches = []; + for (const cacheConfigEle of cacheConfig) { + caches.push(createCacheByICacheConfig(cacheConfigEle)); + } + return multiCaching(caches) as Cache; + } + } else { + return createCacheByICacheConfig(cacheConfig); + } +} + +function createCacheByICacheConfig(cacheConfig: ICacheConfig): Cache { + // if storePackage exist then load storePackage and instead store + if (cacheConfig.storePackage) { + cacheConfig.store = require(cacheConfig.storePackage); + } + return caching(cacheConfig); +} diff --git a/packages/core/server/src/application.ts b/packages/core/server/src/application.ts index c41efe1d7..1f18d834a 100644 --- a/packages/core/server/src/application.ts +++ b/packages/core/server/src/application.ts @@ -1,6 +1,6 @@ -import { promisify } from 'util'; import { ACL } from '@nocobase/acl'; import { registerActions } from '@nocobase/actions'; +import { Cache, createCache, ICacheConfig } from '@nocobase/cache'; import Database, { Collection, CollectionOptions, IDatabaseOptions } from '@nocobase/database'; import Resourcer, { ResourceOptions } from '@nocobase/resourcer'; import { applyMixins, AsyncEmitter, Toposort, ToposortOptions } from '@nocobase/utils'; @@ -11,12 +11,14 @@ import Koa, { DefaultContext as KoaDefaultContext, DefaultState as KoaDefaultSta import compose from 'koa-compose'; import { isBoolean } from 'lodash'; import semver from 'semver'; +import { promisify } from 'util'; import { createACL } from './acl'; import { AppManager } from './app-manager'; import { registerCli } from './commands'; import { createI18n, createResourcer, registerMiddlewares } from './helper'; import { Plugin } from './plugin'; import { InstallOptions, PluginManager } from './plugin-manager'; + const packageJson = require('../package.json'); export type PluginConfiguration = string | [string, any]; @@ -27,6 +29,7 @@ export interface ResourcerOptions { export interface ApplicationOptions { database?: IDatabaseOptions | Database; + cache?: ICacheConfig | ICacheConfig[]; resourcer?: ResourcerOptions; bodyParser?: any; cors?: any; @@ -39,11 +42,13 @@ export interface ApplicationOptions { export interface DefaultState extends KoaDefaultState { currentUser?: any; + [key: string]: any; } export interface DefaultContext extends KoaDefaultContext { db: Database; + cache: Cache; resourcer: Resourcer; i18n: any; [key: string]: any; @@ -134,6 +139,8 @@ export class Application exten protected _resourcer: Resourcer; + protected _cache: Cache; + protected _cli: Command; protected _i18n: i18n; @@ -161,6 +168,10 @@ export class Application exten return this._db; } + get cache() { + return this._cache; + } + get resourcer() { return this._resourcer; } @@ -203,8 +214,10 @@ export class Application exten this._resourcer = createResourcer(options); this._cli = new Command('nocobase').usage('[command] [options]'); this._i18n = createI18n(options); + this._cache = createCache(options.cache); this.context.db = this._db; this.context.resourcer = this._resourcer; + this.context.cache = this._cache; this._pm = new PluginManager({ app: this, diff --git a/yarn.lock b/yarn.lock index 1781eb391..c6c56fbb0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4990,6 +4990,11 @@ "@types/connect" "*" "@types/node" "*" +"@types/cache-manager@^4.0.2": + version "4.0.2" + resolved "https://registry.npmmirror.com/@types/cache-manager/-/cache-manager-4.0.2.tgz#5e76dd9e7881c23f332c2f48e5f326bd05ba9ac9" + integrity sha512-fT5FMdzsiSX0AbgnS5gDvHl2Nco0h5zYyjwDQy4yPC7Ww6DeGMVKPRqIZtg9HOXDV2kkc18SL1B0N8f0BecrCA== + "@types/connect@*": version "3.4.35" resolved "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" @@ -6697,6 +6702,11 @@ async-validator@^4.0.2: resolved "https://registry.npmjs.org/async-validator/-/async-validator-4.0.7.tgz#034a0fd2103a6b2ebf010da75183bec299247afe" integrity sha512-Pj2IR7u8hmUEDOwB++su6baaRi+QvsgajuFB9j95foM1N2gy5HM4z60hfusIO0fBPG5uLAEl6yCJr1jNSVugEQ== +async@3.2.3, async@^3.2.0, async@~3.2.0: + version "3.2.3" + resolved "https://registry.npmjs.org/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" + integrity sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g== + async@^2.6.2: version "2.6.4" resolved "https://registry.npmmirror.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221" @@ -6711,11 +6721,6 @@ async@^2.6.3, async@~2.6.1: dependencies: lodash "^4.17.14" -async@^3.2.0, async@~3.2.0: - version "3.2.3" - resolved "https://registry.npmjs.org/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" - integrity sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g== - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -7457,6 +7462,22 @@ cache-content-type@^1.0.0: mime-types "^2.1.18" ylru "^1.2.0" +cache-manager-fs-hash@^1.0.0: + version "1.0.0" + resolved "https://registry.npmmirror.com/cache-manager-fs-hash/-/cache-manager-fs-hash-1.0.0.tgz#9a3f3fa239c48c54fc6b00575032b72c07dcad99" + integrity sha512-fK2vEAhWh7IAzBP+JwUEJkng0OPOxw3ji86SjQS5SWdLx2Cytku6xFNoQK32pedtuae/wBl/jA/X4abiAYHS/Q== + dependencies: + lockfile "^1.0.4" + +cache-manager@^4.1.0: + version "4.1.0" + resolved "https://registry.npmmirror.com/cache-manager/-/cache-manager-4.1.0.tgz#aa986421f1c975a862d6de88edb9ab1d30f4bd39" + integrity sha512-ZGM6dLxrP65bfOZmcviWMadUOCICqpLs92+P/S5tj8onz+k+tB7Gr+SAgOUHCQtfm2gYEQDHiKeul4+tYPOJ8A== + dependencies: + async "3.2.3" + lodash.clonedeep "^4.5.0" + lru-cache "^7.10.1" + cacheable-request@^6.0.0: version "6.1.0" resolved "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" @@ -14504,6 +14525,13 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" +lockfile@^1.0.4: + version "1.0.4" + resolved "https://registry.npmmirror.com/lockfile/-/lockfile-1.0.4.tgz#07f819d25ae48f87e538e6578b6964a4981a5609" + integrity sha512-cvbTwETRfsFh4nHsL1eGWapU1XFi5Ot9E85sWAwia7Y7EgB7vfqcZhTKZ+l7hCGxSPoushMv5GKhT5PdLv03WA== + dependencies: + signal-exit "^3.0.2" + lodash-es@^4.17.11: version "4.17.21" resolved "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee" @@ -14536,6 +14564,11 @@ lodash.clone@4.5.0, lodash.clone@^4.3.2: resolved "https://registry.npmjs.org/lodash.clone/-/lodash.clone-4.5.0.tgz#195870450f5a13192478df4bc3d23d2dea1907b6" integrity sha1-GVhwRQ9aExkkeN9Lw9I9LeoZB7Y= +lodash.clonedeep@^4.5.0: + version "4.5.0" + resolved "https://registry.npmmirror.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" + integrity sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ== + lodash.debounce@^4.0.8: version "4.0.8" resolved "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" @@ -14735,6 +14768,11 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" +lru-cache@^7.10.1: + version "7.14.0" + resolved "https://registry.npmmirror.com/lru-cache/-/lru-cache-7.14.0.tgz#21be64954a4680e303a09e9468f880b98a0b3c7f" + integrity sha512-EIRtP1GrSJny0dqb50QXRUNBxHJhcpxHC++M5tD7RYbvLLn5KVWKsbyswSSqDuU15UFi3bgTQIY8nhDMeF6aDQ== + luxon@^1.28.0: version "1.28.0" resolved "https://registry.yarnpkg.com/luxon/-/luxon-1.28.0.tgz#e7f96daad3938c06a62de0fb027115d251251fbf"