feat(bi): allow to use variables in query filter (#2609)

* chore: rearrange hooks

* chore: allow parse filter from params.values

* feat: support use variables when using chart filter

* refactor: improve query code by composing process middlwares

* chore: `$date` -> `$nDate`
This commit is contained in:
YANG QIA 2023-09-07 23:27:59 +08:00 committed by GitHub
parent b655517a74
commit f82b466aaa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 800 additions and 381 deletions

View File

@ -120,9 +120,8 @@ const ConstantTypes = {
function getTypedConstantOption(type: string, types: true | string[], fieldNames) {
const allTypes = Object.values(ConstantTypes);
const children = (types
? allTypes.filter((item) => (Array.isArray(types) && types.includes(item.value)) || types === true)
: allTypes
const children = (
types ? allTypes.filter((item) => (Array.isArray(types) && types.includes(item.value)) || types === true) : allTypes
).map((item) =>
Object.keys(item).reduce(
(result, key) =>
@ -182,29 +181,27 @@ export function Input(props) {
fieldNames ?? {},
);
const {
component: ConstantComponent,
...constantOption
}: DefaultOptionType & { component?: React.FC<any> } = useMemo(() => {
if (children) {
const { component: ConstantComponent, ...constantOption }: DefaultOptionType & { component?: React.FC<any> } =
useMemo(() => {
if (children) {
return {
value: '',
label: t('Constant'),
[names.value]: '',
[names.label]: t('Constant'),
};
}
if (useTypedConstant) {
return getTypedConstantOption(type, useTypedConstant, names);
}
return {
value: '',
label: t('Constant'),
label: t('Null'),
[names.value]: '',
[names.label]: t('Constant'),
[names.label]: t('Null'),
component: ConstantTypes.null.component,
};
}
if (useTypedConstant) {
return getTypedConstantOption(type, useTypedConstant, names);
}
return {
value: '',
label: t('Null'),
[names.value]: '',
[names.label]: t('Null'),
component: ConstantTypes.null.component,
};
}, [type, useTypedConstant]);
}, [type, useTypedConstant]);
useEffect(() => {
setOptions([compile(constantOption), ...(scope ? cloneDeep(scope) : [])]);
@ -298,6 +295,7 @@ export function Input(props) {
margin: 0;
padding: 2px 7px;
border-radius: 10px;
white-space: nowrap;
}
}
`,

View File

@ -1,4 +1,5 @@
import { getDateVars, parseFilter } from '@nocobase/utils';
import lodash from 'lodash';
function getUser(ctx) {
return async ({ fields }) => {

View File

@ -17,6 +17,7 @@
"@testing-library/react": "^14.0.0",
"antd": "5.x",
"classnames": "^2.3.1",
"koa-compose": "^4.1.0",
"lodash": "^4.17.21",
"react": "^18.2.0",
"react-error-boundary": "^4.0.10",

View File

@ -1,7 +1,7 @@
import { SchemaInitializerButtonContext, useDesignable } from '@nocobase/client';
import React, { useState } from 'react';
import { ChartRendererProvider } from '../renderer';
import { ChartConfigContext, ChartConfigCurrent, ChartConfigure } from './ChartConfigure';
import { ChartConfigContext, ChartConfigCurrent, ChartConfigure } from '../configure/ChartConfigure';
export const ChartV2Block: React.FC = (props) => {
const { insertAdjacent } = useDesignable();

View File

@ -4,7 +4,7 @@ import { uid } from '@formily/shared';
import { SchemaInitializer, useACLRoleContext, useCollectionDataSourceItems } from '@nocobase/client';
import React, { useContext } from 'react';
import { useChartsTranslation } from '../locale';
import { ChartConfigContext } from './ChartConfigure';
import { ChartConfigContext } from '../configure/ChartConfigure';
const itemWrap = SchemaInitializer.itemWrap;
const ConfigureButton = itemWrap((props) => {

View File

@ -1,4 +1,3 @@
export * from './ChartBlock';
export * from './ChartBlockDesigner';
export * from './ChartBlockInitializer';
export * from './ChartConfigure';

View File

@ -30,6 +30,7 @@ import { ChartRenderer, ChartRendererContext } from '../renderer';
import { createRendererSchema, getField, getSelectedFields } from '../utils';
import { getConfigSchema, querySchema, transformSchema } from './schemas/configure';
import { useChartTypes, useCharts, useDefaultChartType } from '../chart/library';
import { FilterDynamicComponent } from './FilterDynamicComponent';
const { Paragraph, Text } = Typography;
export type ChartConfigCurrent = {
@ -353,7 +354,7 @@ ChartConfigure.Query = function Query() {
collection: current?.collection,
useOrderReaction: useOrderReaction(compiledFieldOptions, fields),
}}
components={{ ArrayItems, Editable, FormCollapse, FormItem, Space, Switch, FromSql }}
components={{ ArrayItems, Editable, FormCollapse, FormItem, Space, Switch, FromSql, FilterDynamicComponent }}
/>
);
};

View File

@ -0,0 +1,14 @@
import { Variable } from '@nocobase/client';
import React from 'react';
import { useVariableOptions } from '../hooks';
export function FilterDynamicComponent(props) {
const { value, onChange, renderSchemaComponent } = props;
const options = useVariableOptions();
return (
<Variable.Input value={value} onChange={onChange} scope={options}>
{renderSchemaComponent()}
</Variable.Input>
);
}

View File

@ -315,7 +315,7 @@ export const querySchema: ISchema = {
'x-component': 'Filter',
'x-component-props': {
options: '{{ filterOptions }}',
// dynamicComponent: 'Input',
dynamicComponent: 'FilterDynamicComponent',
},
},
},

View File

@ -0,0 +1,3 @@
export * from './query';
export * from './transformer';
export * from './useVariableOptions';

View File

@ -3,12 +3,11 @@ import { ISchema, Schema, useForm } from '@formily/react';
import { CollectionFieldOptions, useACLRoleContext, useCollectionManager } from '@nocobase/client';
import { useContext } from 'react';
import { useTranslation } from 'react-i18next';
import { ChartConfigContext } from './block/ChartConfigure';
import formatters from './block/formatters';
import transformers from './block/transformers';
import { lang, useChartsTranslation } from './locale';
import { ChartRendererContext, ChartRendererProps } from './renderer';
import { getField, getSelectedFields, parseField, processData } from './utils';
import { ChartConfigContext } from '../configure/ChartConfigure';
import formatters from '../block/formatters';
import { useChartsTranslation } from '../locale';
import { ChartRendererContext } from '../renderer';
import { getField, getSelectedFields, parseField, processData } from '../utils';
export type FieldOption = {
value: string;
@ -143,71 +142,6 @@ export const useCollectionOptions = () => {
return Schema.compile(options, { t });
};
/**
* useFieldTypes
* Get field types for using transformers
* Only supported types will be displayed
* Some interfaces and types will be mapped to supported types
*/
export const useFieldTypes = (fields: FieldOption[]) => (field: any) => {
const selectedField = field.query('.field').get('value');
const query = field.query('query').get('value') || {};
const selectedFields = getSelectedFields(fields, query);
const fieldProps = selectedFields.find((field) => field.value === selectedField);
const supports = Object.keys(transformers);
field.dataSource = supports.map((key) => ({
label: lang(key),
value: key,
}));
const map = {
createdAt: 'datetime',
updatedAt: 'datetime',
double: 'number',
integer: 'number',
percent: 'number',
};
const fieldInterface = fieldProps?.interface;
const fieldType = fieldProps?.type;
const key = map[fieldInterface] || map[fieldType] || fieldType;
if (supports.includes(key)) {
field.setState({
value: key,
disabled: true,
});
return;
}
field.setState({
value: null,
disabled: false,
});
};
export const useTransformers = (field: any) => {
const selectedType = field.query('.type').get('value');
if (!selectedType) {
field.dataSource = [];
return;
}
const options = Object.keys(transformers[selectedType] || {}).map((key) => ({
label: lang(key),
value: key,
}));
field.dataSource = options;
};
export const useFieldTransformer = (transform: ChartRendererProps['transform'], locale = 'en-US') => {
return (transform || [])
.filter((item) => item.field && item.type && item.format)
.reduce((mp, item) => {
const transformer = transformers[item.type][item.format];
if (!transformer) {
return mp;
}
mp[item.field] = (val: any) => transformer(val, locale);
return mp;
}, {});
};
export const useOrderFieldsOptions = (defaultOptions: any[], fields: FieldOption[]) => (field: any) => {
const query = field.query('query').get('value') || {};
const { measures = [] } = query;

View File

@ -0,0 +1,70 @@
import transformers from '../block/transformers';
import { lang } from '../locale';
import { ChartRendererProps } from '../renderer';
import { getSelectedFields } from '../utils';
import { FieldOption } from './query';
/**
* useFieldTypes
* Get field types for using transformers
* Only supported types will be displayed
* Some interfaces and types will be mapped to supported types
*/
export const useFieldTypes = (fields: FieldOption[]) => (field: any) => {
const selectedField = field.query('.field').get('value');
const query = field.query('query').get('value') || {};
const selectedFields = getSelectedFields(fields, query);
const fieldProps = selectedFields.find((field) => field.value === selectedField);
const supports = Object.keys(transformers);
field.dataSource = supports.map((key) => ({
label: lang(key),
value: key,
}));
const map = {
createdAt: 'datetime',
updatedAt: 'datetime',
double: 'number',
integer: 'number',
percent: 'number',
};
const fieldInterface = fieldProps?.interface;
const fieldType = fieldProps?.type;
const key = map[fieldInterface] || map[fieldType] || fieldType;
if (supports.includes(key)) {
field.setState({
value: key,
disabled: true,
});
return;
}
field.setState({
value: null,
disabled: false,
});
};
export const useTransformers = (field: any) => {
const selectedType = field.query('.type').get('value');
if (!selectedType) {
field.dataSource = [];
return;
}
const options = Object.keys(transformers[selectedType] || {}).map((key) => ({
label: lang(key),
value: key,
}));
field.dataSource = options;
};
export const useFieldTransformer = (transform: ChartRendererProps['transform'], locale = 'en-US') => {
return (transform || [])
.filter((item) => item.field && item.type && item.format)
.reduce((mp, item) => {
const transformer = transformers[item.type][item.format];
if (!transformer) {
return mp;
}
mp[item.field] = (val: any) => transformer(val, locale);
return mp;
}, {});
};

View File

@ -0,0 +1,163 @@
import { useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useMemoizedFn } from 'ahooks';
export const useDateVariable = ({ operator, schema }) => {
const { t: trans } = useTranslation();
const t = useMemoizedFn(trans);
const operatorValue = operator?.value || '';
const component = schema?.['x-component'];
const disabled = !['DatePicker', 'DatePicker.RangePicker'].includes(component);
const dateOptions = useMemo(
() => [
{
key: 'now',
value: 'now',
label: t('Now'),
disabled: component !== 'DatePicker' || operatorValue === '$dateBetween',
},
{
key: 'yesterday',
value: 'yesterday',
label: t('Yesterday'),
disabled,
},
{
key: 'today',
value: 'today',
label: t('Today'),
disabled,
},
{
key: 'tomorrow',
value: 'tomorrow',
label: t('Tomorrow'),
disabled,
},
{
key: 'lastIsoWeek',
value: 'lastIsoWeek',
label: t('Last week'),
disabled,
},
{
key: 'thisIsoWeek',
value: 'thisIsoWeek',
label: t('This week'),
disabled,
},
{
key: 'nextIsoWeek',
value: 'nextIsoWeek',
label: t('Next week'),
disabled,
},
{
key: 'lastMonth',
value: 'lastMonth',
label: t('Last month'),
disabled,
},
{
key: 'thisMonth',
value: 'thisMonth',
label: t('This month'),
disabled,
},
{
key: 'nextMonth',
value: 'nextMonth',
label: t('Next month'),
disabled,
},
{
key: 'lastQuarter',
value: 'lastQuarter',
label: t('Last quarter'),
disabled,
},
{
key: 'thisQuarter',
value: 'thisQuarter',
label: t('This quarter'),
disabled,
},
{
key: 'nextQuarter',
value: 'nextQuarter',
label: t('Next quarter'),
disabled,
},
{
key: 'lastYear',
value: 'lastYear',
label: t('Last year'),
disabled,
},
{
key: 'thisYear',
value: 'thisYear',
label: t('This year'),
disabled,
},
{
key: 'nextYear',
value: 'nextYear',
label: t('Next year'),
disabled,
},
{
key: 'last7Days',
value: 'last7Days',
label: t('Last 7 days'),
disabled,
},
{
key: 'next7Days',
value: 'next7Days',
label: t('Next 7 days'),
disabled,
},
{
key: 'last30Days',
value: 'last30Days',
label: t('Last 30 days'),
disabled,
},
{
key: 'next30Days',
value: 'next30Days',
label: t('Next 30 days'),
disabled,
},
{
key: 'last90Days',
value: 'last90Days',
label: t('Last 90 days'),
disabled,
},
{
key: 'next90Days',
value: 'next90Days',
label: t('Next 90 days'),
disabled,
},
],
[disabled, operatorValue, t, component],
);
const result = useMemo(
() => ({
label: t('Date variables'),
value: '$nDate',
key: '$nDate',
disabled: dateOptions.every((option) => option.disabled),
children: dateOptions,
}),
[dateOptions, t],
);
if (!operator || !schema) return null;
return result;
};

View File

@ -0,0 +1,40 @@
import { useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { Schema } from '@formily/react';
import { useCollectionFieldsOptions } from '@nocobase/client';
import { useMemoizedFn } from 'ahooks';
export const useUserVariable = ({ schema }: { schema: any }) => {
const { t: trans } = useTranslation();
const t = useMemoizedFn(trans);
const options = useCollectionFieldsOptions('users');
const component = schema?.['x-component'];
const getOptions = useCallback(
(options: any) =>
options.map((option: any) => {
const result = {
key: option.name,
value: option.name,
disabled: !option.children && component !== option.schema?.['x-component'],
label: Schema.compile(option.title, { t }),
};
if (option.children) {
result['children'] = getOptions(option.children);
}
return result;
}),
[t, component],
);
const children = useMemo(() => getOptions(options), [getOptions, options]);
const result = useMemo(
() => ({
label: t('Current user'),
value: '$user',
key: '$user',
children,
}),
[children, t],
);
return result;
};

View File

@ -0,0 +1,17 @@
import { useField } from '@formily/react';
import { useMemo } from 'react';
import { useDateVariable } from './useDateVariable';
import { useUserVariable } from './useUserVariable';
export const useVariableOptions = () => {
const field = useField<any>();
const { operator, schema } = field.data || {};
const userVariable = useUserVariable({ schema });
const dateVariable = useDateVariable({ operator, schema });
const result = useMemo(() => [userVariable, dateVariable].filter(Boolean), [dateVariable, userVariable]);
if (!operator || !schema) return [];
return result;
};

View File

@ -10,7 +10,7 @@ import {
import { Empty, Result, Spin, Typography } from 'antd';
import React, { useContext } from 'react';
import { ErrorBoundary } from 'react-error-boundary';
import { ChartConfigContext } from '../block';
import { ChartConfigContext } from '../configure/ChartConfigure';
import { useData, useFieldTransformer, useFieldsWithAssociation } from '../hooks';
import { useChartsTranslation } from '../locale';
import { createRendererSchema, getField } from '../utils';

View File

@ -1,6 +1,6 @@
import { Schema } from '@formily/react';
import { uid } from '@formily/shared';
import { SelectedField } from './block';
import { SelectedField } from './configure/ChartConfigure';
import { FieldOption } from './hooks';
import { QueryProps } from './renderer';

View File

@ -1,7 +1,8 @@
import { Database } from '@nocobase/database';
import { MockServer, mockServer } from '@nocobase/test';
import { queryData } from '../actions/query';
import { parseBuilder, parseFieldAndAssociations, queryData } from '../actions/query';
import ChartsV2Plugin from '../plugin';
import compose from 'koa-compose';
describe('api', () => {
let app: MockServer;
@ -52,51 +53,67 @@ describe('api', () => {
});
test('query', async () => {
const result = await queryData({ db } as any, {
collection: 'chart_test',
measures: [
{
field: ['price'],
alias: 'Price',
const ctx = {
db,
action: {
params: {
values: {
collection: 'chart_test',
measures: [
{
field: ['price'],
alias: 'Price',
},
{
field: ['count'],
alias: 'Count',
},
],
dimensions: [
{
field: ['title'],
alias: 'Title',
},
],
},
},
{
field: ['count'],
alias: 'Count',
},
],
dimensions: [
{
field: ['title'],
alias: 'Title',
},
],
});
expect(result).toBeDefined();
},
} as any;
await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {});
expect(ctx.action.params.values.data).toBeDefined();
});
test('query with sort', async () => {
const result = await queryData({ db } as any, {
collection: 'chart_test',
measures: [
{
field: ['price'],
aggregation: 'sum',
alias: 'Price',
const ctx = {
db,
action: {
params: {
values: {
collection: 'chart_test',
measures: [
{
field: ['price'],
aggregation: 'sum',
alias: 'Price',
},
],
dimensions: [
{
field: ['title'],
alias: 'Title',
},
{
field: ['createdAt'],
format: 'YYYY-MM',
},
],
orders: [{ field: 'createdAt', order: 'asc' }],
},
},
],
dimensions: [
{
field: ['title'],
alias: 'Title',
},
{
field: ['createdAt'],
format: 'YYYY-MM',
},
],
orders: [{ field: 'createdAt', order: 'asc' }],
});
expect(result).toBeDefined();
expect(result).toMatchObject([{ createdAt: '2023-01' }, { createdAt: '2023-02' }]);
},
} as any;
await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {});
expect(ctx.action.params.values.data).toBeDefined();
expect(ctx.action.params.values.data).toMatchObject([{ createdAt: '2023-01' }, { createdAt: '2023-02' }]);
});
});

View File

@ -1,6 +1,7 @@
import { MockServer, mockServer } from '@nocobase/test';
import * as formatter from '../actions/formatter';
import { cacheWrap, parseBuilder, parseFieldAndAssociations } from '../actions/query';
import { cacheMiddleware, parseBuilder, parseFieldAndAssociations } from '../actions/query';
import compose from 'koa-compose';
describe('query', () => {
describe('parseBuilder', () => {
@ -51,6 +52,7 @@ describe('query', () => {
],
});
ctx = {
app,
db: {
sequelize,
getRepository: (name: string) => app.db.getRepository(name),
@ -63,13 +65,21 @@ describe('query', () => {
};
});
it('should parse field and associations', () => {
const associations = parseFieldAndAssociations(ctx, {
collection: 'orders',
measures: [{ field: ['price'], aggregation: 'sum', alias: 'price' }],
dimensions: [{ field: ['createdAt'] }, { field: ['user', 'name'] }],
});
expect(associations).toMatchObject({
it('should parse field and associations', async () => {
const context = {
...ctx,
action: {
params: {
values: {
collection: 'orders',
measures: [{ field: ['price'], aggregation: 'sum', alias: 'price' }],
dimensions: [{ field: ['createdAt'] }, { field: ['user', 'name'] }],
},
},
},
};
await parseFieldAndAssociations(context, async () => {});
expect(context.action.params.values).toMatchObject({
measures: [{ field: 'orders.price', aggregation: 'sum', alias: 'price', type: 'double' }],
dimensions: [
{ field: 'orders.created_at', alias: 'createdAt', type: 'date' },
@ -79,14 +89,22 @@ describe('query', () => {
});
});
it('should parse measures', () => {
it('should parse measures', async () => {
const measures1 = [
{
field: ['price'],
},
];
const { queryParams: result1 } = parseBuilder(ctx, { collection: 'orders', measures: measures1 });
expect(result1.attributes).toEqual([['orders.price', 'price']]);
const context = {
...ctx,
action: {
params: {
values: { collection: 'orders', measures: measures1 },
},
},
};
await compose([parseFieldAndAssociations, parseBuilder])(context, async () => {});
expect(context.action.params.values.queryParams.attributes).toEqual([['orders.price', 'price']]);
const measures2 = [
{
@ -95,11 +113,19 @@ describe('query', () => {
alias: 'price-alias',
},
];
const { queryParams: result2 } = parseBuilder(ctx, { collection: 'orders', measures: measures2 });
expect(result2.attributes).toEqual([[['sum', 'orders.price'], 'price-alias']]);
const context2 = {
...ctx,
action: {
params: {
values: { collection: 'orders', measures: measures2 },
},
},
};
await compose([parseFieldAndAssociations, parseBuilder])(context2, async () => {});
expect(context2.action.params.values.queryParams.attributes).toEqual([[['sum', 'orders.price'], 'price-alias']]);
});
it('should parse dimensions', () => {
it('should parse dimensions', async () => {
jest.spyOn(formatter, 'formatter').mockReturnValue('formatted-field');
const dimensions = [
{
@ -108,9 +134,17 @@ describe('query', () => {
alias: 'Created at',
},
];
const { queryParams: result } = parseBuilder(ctx, { collection: 'orders', dimensions });
expect(result.attributes).toEqual([['formatted-field', 'Created at']]);
expect(result.group).toEqual([]);
const context = {
...ctx,
action: {
params: {
values: { collection: 'orders', dimensions },
},
},
};
await compose([parseFieldAndAssociations, parseBuilder])(context, async () => {});
expect(context.action.params.values.queryParams.attributes).toEqual([['formatted-field', 'Created at']]);
expect(context.action.params.values.queryParams.group).toEqual([]);
const measures = [
{
@ -118,30 +152,46 @@ describe('query', () => {
aggregation: 'sum',
},
];
const { queryParams: result2 } = parseBuilder(ctx, { collection: 'orders', measures, dimensions });
expect(result2.group).toEqual(['formatted-field']);
const context2 = {
...ctx,
action: {
params: {
values: { collection: 'orders', measures, dimensions },
},
},
};
await compose([parseFieldAndAssociations, parseBuilder])(context2, async () => {});
expect(context2.action.params.values.queryParams.group).toEqual(['formatted-field']);
});
it('should parse filter', () => {
it('should parse filter', async () => {
const filter = {
createdAt: {
$gt: '2020-01-01',
},
};
const { queryParams: result } = parseBuilder(ctx, { collection: 'orders', filter });
expect(result.where.createdAt).toBeDefined();
const context = {
...ctx,
action: {
params: {
values: { collection: 'orders', filter },
},
},
};
await compose([parseFieldAndAssociations, parseBuilder])(context, async () => {});
expect(context.action.params.values.queryParams.where.createdAt).toBeDefined();
});
});
describe('cacheWrap', () => {
describe('cacheMiddleware', () => {
const key = 'test-key';
const value = 'test-val';
const query = jest.fn().mockImplementation(async (ctx, next) => {
ctx.body = value;
await next();
});
class MockCache {
map: Map<string, any> = new Map();
async func() {
return value;
}
get(key: string) {
return this.map.get(key);
}
@ -149,72 +199,74 @@ describe('query', () => {
this.map.set(key, value);
}
}
let cache: any;
let query: () => Promise<any>;
let ctx: any;
beforeEach(() => {
cache = new MockCache();
const cache = new MockCache();
ctx = {
app: {
getPlugin: () => ({ cache }),
},
};
});
it('should use cache', async () => {
query = async () =>
await cacheWrap(cache, {
key,
func: cache.func,
useCache: true,
refresh: false,
});
const spy = jest.spyOn(cache, 'func');
const context = {
...ctx,
action: {
params: {
values: { cache: { enabled: true }, refresh: false, uid: key },
},
},
};
const cache = context.app.getPlugin().cache;
expect(cache.get(key)).toBeUndefined();
const result = await query();
expect(cache.func).toBeCalled();
expect(result).toEqual(value);
await compose([cacheMiddleware, query])(context, async () => {});
expect(query).toBeCalled();
expect(context.body).toEqual(value);
expect(cache.get(key)).toEqual(value);
spy.mockReset();
const result2 = await query();
expect(result2).toEqual(value);
expect(cache.func).not.toBeCalled();
jest.clearAllMocks();
await compose([cacheMiddleware, query])(context, async () => {});
expect(context.body).toEqual(value);
expect(query).not.toBeCalled();
});
it('should not use cache', async () => {
query = async () =>
await cacheWrap(cache, {
key,
func: cache.func,
useCache: false,
refresh: false,
});
const context = {
...ctx,
action: {
params: {
values: { uid: key },
},
},
};
const cache = context.app.getPlugin().cache;
cache.set(key, value);
expect(cache.get(key)).toBeDefined();
jest.spyOn(cache, 'func');
const result = await query();
expect(cache.func).toBeCalled();
expect(result).toEqual(value);
await compose([cacheMiddleware, query])(context, async () => {});
expect(query).toBeCalled();
expect(context.body).toEqual(value);
});
it('should refresh', async () => {
query = async () =>
await cacheWrap(cache, {
key,
func: cache.func,
useCache: true,
refresh: true,
});
const spy = jest.spyOn(cache, 'func');
const context = {
...ctx,
action: {
params: {
values: { cache: { enabled: true }, refresh: true, uid: key },
},
},
};
const cache = context.app.getPlugin().cache;
expect(cache.get(key)).toBeUndefined();
const result = await query();
expect(cache.func).toBeCalled();
expect(result).toEqual(value);
await compose([cacheMiddleware, query])(context, async () => {});
expect(query).toBeCalled();
expect(context.body).toEqual(value);
expect(cache.get(key)).toEqual(value);
spy.mockClear();
const result2 = await query();
expect(cache.func).toBeCalled();
expect(result2).toEqual(value);
await compose([cacheMiddleware, query])(context, async () => {});
expect(query).toBeCalled();
expect(context.body).toEqual(value);
});
});
});

View File

@ -1,8 +1,9 @@
import { Context, Next } from '@nocobase/actions';
import { Cache } from '@nocobase/cache';
import { Field, FilterParser, snakeCase } from '@nocobase/database';
import ChartsV2Plugin from '../plugin';
import { formatter } from './formatter';
import compose from 'koa-compose';
import { parseFilter, getDateVars } from '@nocobase/utils';
type MeasureProps = {
field: string | string[];
@ -44,8 +45,126 @@ type QueryParams = Partial<{
refresh: boolean;
}>;
export const parseFieldAndAssociations = (ctx: Context, params: QueryParams) => {
const { collection: collectionName, measures, dimensions, orders, filter } = params;
export const postProcess = async (ctx: Context, next: Next) => {
const { sequelize } = ctx.db;
const dialect = sequelize.getDialect();
const { data, fieldMap } = ctx.action.params.values as {
data: any[];
fieldMap: { [source: string]: { type?: string } };
};
switch (dialect) {
case 'postgres':
// https://github.com/sequelize/sequelize/issues/4550
ctx.body = data.map((record) => {
const result = {};
Object.entries(record).forEach(([key, value]) => {
const { type } = fieldMap[key] || {};
switch (type) {
case 'bigInt':
case 'integer':
case 'float':
case 'double':
value = Number(value);
break;
}
result[key] = value;
});
return result;
});
break;
default:
ctx.body = data;
}
await next();
};
export const queryData = async (ctx: Context, next: Next) => {
const { collection, queryParams, fieldMap } = ctx.action.params.values;
const model = ctx.db.getModel(collection);
const data = await model.findAll(queryParams);
ctx.action.params.values = {
data,
fieldMap,
};
await next();
// if (!sql) {
// return await repository.find(parseBuilder(ctx, { collection, measures, dimensions, orders, filter, limit }));
// }
// const statement = `SELECT ${sql.fields} FROM ${collection} ${sql.clauses}`;
// const [data] = await ctx.db.sequelize.query(statement);
// return data;
};
export const parseBuilder = async (ctx: Context, next: Next) => {
const { sequelize } = ctx.db;
const { measures, dimensions, orders, include, where, limit } = ctx.action.params.values;
const attributes = [];
const group = [];
const order = [];
const fieldMap = {};
let hasAgg = false;
measures.forEach((measure: MeasureProps & { field: string }) => {
const { field, aggregation, alias } = measure;
const attribute = [];
const col = sequelize.col(field);
if (aggregation) {
hasAgg = true;
attribute.push(sequelize.fn(aggregation, col));
} else {
attribute.push(col);
}
if (alias) {
attribute.push(alias);
}
attributes.push(attribute.length > 1 ? attribute : attribute[0]);
fieldMap[alias || field] = measure;
});
dimensions.forEach((dimension: DimensionProps & { field: string }) => {
const { field, format, alias, type } = dimension;
const attribute = [];
const col = sequelize.col(field);
if (format) {
attribute.push(formatter(sequelize, type, field, format));
} else {
attribute.push(col);
}
if (alias) {
attribute.push(alias);
}
attributes.push(attribute.length > 1 ? attribute : attribute[0]);
if (hasAgg) {
group.push(attribute[0]);
}
fieldMap[alias || field] = dimension;
});
orders.forEach((item: OrderProps) => {
const alias = sequelize.getQueryInterface().quoteIdentifier(item.alias);
const name = hasAgg ? sequelize.literal(alias) : sequelize.col(item.field as string);
order.push([name, item.order || 'ASC']);
});
ctx.action.params.values = {
...ctx.action.params.values,
queryParams: {
where,
attributes,
include,
group,
order,
limit: limit > 2000 ? 2000 : limit,
raw: true,
},
fieldMap,
};
await next();
};
export const parseFieldAndAssociations = async (ctx: Context, next: Next) => {
const { collection: collectionName, measures, dimensions, orders, filter } = ctx.action.params.values as QueryParams;
const collection = ctx.db.getCollection(collectionName);
const fields = collection.fields;
const underscored = ctx.db.options.underscored;
@ -109,182 +228,111 @@ export const parseFieldAndAssociations = (ctx: Context, params: QueryParams) =>
return item;
});
return {
ctx.action.params.values = {
...ctx.action.params.values,
where,
measures: parsedMeasures,
dimensions: parsedDimensions,
orders: parsedOrders,
include: [...include, ...(parsedFilterInclude || [])],
};
await next();
};
export const parseBuilder = (ctx: Context, builder: QueryParams) => {
const { sequelize } = ctx.db;
const { limit } = builder;
const { measures, dimensions, orders, include, where } = parseFieldAndAssociations(ctx, builder);
const attributes = [];
const group = [];
const order = [];
const fieldMap = {};
let hasAgg = false;
measures.forEach((measure: MeasureProps & { field: string }) => {
const { field, aggregation, alias } = measure;
const attribute = [];
const col = sequelize.col(field);
if (aggregation) {
hasAgg = true;
attribute.push(sequelize.fn(aggregation, col));
} else {
attribute.push(col);
}
if (alias) {
attribute.push(alias);
}
attributes.push(attribute.length > 1 ? attribute : attribute[0]);
fieldMap[alias || field] = measure;
});
dimensions.forEach((dimension: DimensionProps & { field: string }) => {
const { field, format, alias, type } = dimension;
const attribute = [];
const col = sequelize.col(field);
if (format) {
attribute.push(formatter(sequelize, type, field, format));
} else {
attribute.push(col);
}
if (alias) {
attribute.push(alias);
}
attributes.push(attribute.length > 1 ? attribute : attribute[0]);
if (hasAgg) {
group.push(attribute[0]);
}
fieldMap[alias || field] = dimension;
});
orders.forEach((item: OrderProps) => {
const alias = sequelize.getQueryInterface().quoteIdentifier(item.alias);
const name = hasAgg ? sequelize.literal(alias) : sequelize.col(item.field as string);
order.push([name, item.order || 'ASC']);
});
return {
queryParams: {
where,
attributes,
include,
group,
order,
limit: limit > 2000 ? 2000 : limit,
raw: true,
},
fieldMap,
export const parseVariables = async (ctx: Context, next: Next) => {
const { filter } = ctx.action.params.values;
if (!filter) {
return next();
}
const isNumeric = (str: any) => {
if (typeof str === 'number') return true;
if (typeof str != 'string') return false;
return !isNaN(str as any) && !isNaN(parseFloat(str));
};
};
export const processData = (ctx: Context, data: any[], fieldMap: { [source: string]: { type?: string } }) => {
const { sequelize } = ctx.db;
const dialect = sequelize.getDialect();
switch (dialect) {
case 'postgres':
// https://github.com/sequelize/sequelize/issues/4550
return data.map((record) => {
const result = {};
Object.entries(record).forEach(([key, value]) => {
const { type } = fieldMap[key] || {};
switch (type) {
case 'bigInt':
case 'integer':
case 'float':
case 'double':
value = Number(value);
break;
}
result[key] = value;
});
return result;
const getUser = () => {
return async ({ fields }) => {
const userFields = fields.filter((f) => f && ctx.db.getFieldByPath('users.' + f));
ctx.logger?.info('filter-parse: ', { userFields });
if (!ctx.state.currentUser) {
return;
}
if (!userFields.length) {
return;
}
const user = await ctx.db.getRepository('users').findOne({
filterByTk: ctx.state.currentUser.id,
fields: userFields,
});
default:
return data;
}
ctx.logger?.info('filter-parse: ', {
$user: user?.toJSON(),
});
return user;
};
};
ctx.action.params.values.filter = await parseFilter(filter, {
timezone: ctx.get('x-timezone'),
now: new Date().toISOString(),
getField: (path: string) => {
const fieldPath = path
.split('.')
.filter((p) => !p.startsWith('$') && !isNumeric(p))
.join('.');
const { resourceName } = ctx.action;
return ctx.db.getFieldByPath(`${resourceName}.${fieldPath}`);
},
vars: {
$nDate: getDateVars(),
$user: getUser(),
},
});
await next();
};
export const queryData = async (ctx: Context, builder: QueryParams) => {
const { collection, measures, dimensions, orders, filter, limit, sql } = builder;
const model = ctx.db.getModel(collection);
const { queryParams, fieldMap } = parseBuilder(ctx, { collection, measures, dimensions, orders, filter, limit });
const data = await model.findAll(queryParams);
return processData(ctx, data, fieldMap);
// if (!sql) {
// return await repository.find(parseBuilder(ctx, { collection, measures, dimensions, orders, filter, limit }));
// }
export const cacheMiddleware = async (ctx: Context, next: Next) => {
const { uid, cache: cacheConfig, refresh } = ctx.action.params.values as QueryParams;
const plugin = ctx.app.getPlugin('data-visualization') as ChartsV2Plugin;
const cache = plugin.cache;
const useCache = cacheConfig?.enabled && uid;
// const statement = `SELECT ${sql.fields} FROM ${collection} ${sql.clauses}`;
// const [data] = await ctx.db.sequelize.query(statement);
// return data;
};
export const cacheWrap = async (
cache: Cache,
options: {
func: () => Promise<any>;
key: string;
ttl?: number;
useCache?: boolean;
refresh?: boolean;
},
) => {
const { func, key, ttl, useCache, refresh } = options;
if (useCache && !refresh) {
const data = await cache.get(key);
const data = await cache.get(uid);
if (data) {
return data;
ctx.body = data;
return;
}
}
const data = await func();
await next();
if (useCache) {
await cache.set(key, data, ttl);
console.log(uid, ctx.body);
await cache.set(uid, ctx.body, cacheConfig?.ttl || 30);
console.log(cache.get(uid));
}
return data;
};
export const query = async (ctx: Context, next: Next) => {
const {
uid,
collection,
measures,
dimensions,
orders,
filter,
limit,
sql,
cache: cacheConfig,
refresh,
} = ctx.action.params.values as QueryParams;
const checkPermission = (ctx: Context, next: Next) => {
const { collection } = ctx.action.params.values as QueryParams;
const roleName = ctx.state.currentRole || 'anonymous';
const can = ctx.app.acl.can({ role: roleName, resource: collection, action: 'list' });
if (!can && roleName !== 'root') {
ctx.throw(403, 'No permissions');
}
return next();
};
const plugin = ctx.app.getPlugin('data-visualization') as ChartsV2Plugin;
const cache = plugin.cache;
const useCache = cacheConfig?.enabled && uid;
export const query = async (ctx: Context, next: Next) => {
try {
ctx.body = await cacheWrap(cache, {
func: async () => await queryData(ctx, { collection, measures, dimensions, orders, filter, limit, sql }),
key: uid,
ttl: cacheConfig?.ttl || 30,
useCache: useCache ? true : false,
refresh,
});
await compose([
checkPermission,
cacheMiddleware,
parseVariables,
parseFieldAndAssociations,
parseBuilder,
queryData,
postProcess,
])(ctx, async () => {});
} catch (err) {
ctx.app.logger.error('charts query: ', err);
ctx.throw(500, err);
}
await next();
};

View File

@ -4751,10 +4751,6 @@
version "1.7.1"
resolved "https://registry.npmjs.org/@remix-run/router/-/router-1.7.1.tgz#fea7ac35ae4014637c130011f59428f618730498"
"@remix-run/router@1.7.2":
version "1.7.2"
resolved "https://registry.npmmirror.com/@remix-run/router/-/router-1.7.2.tgz#cba1cf0a04bc04cb66027c51fa600e9cbc388bc8"
"@restart/hooks@^0.4.7":
version "0.4.9"
resolved "https://registry.npmmirror.com/@restart/hooks/-/hooks-0.4.9.tgz#ad858fb39d99e252cccce19416adc18fc3f18fcb"
@ -6037,12 +6033,19 @@
"@types/prop-types" "*"
"@types/react" "*"
"@types/react-dom@17.x", "@types/react-dom@^17.0.0", "@types/react-dom@^18.0.0":
"@types/react-dom@17.x", "@types/react-dom@^17.0.0":
version "17.0.20"
resolved "https://registry.npmmirror.com/@types/react-dom/-/react-dom-17.0.20.tgz#e0c8901469d732b36d8473b40b679ad899da1b53"
dependencies:
"@types/react" "^17"
"@types/react-dom@^18.0.0":
version "18.2.7"
resolved "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.7.tgz#67222a08c0a6ae0a0da33c3532348277c70abb63"
integrity sha512-GRaAEriuT4zp9N4p1i8BDBYmEyfo+xQ3yHjJU4eiK5NDa1RmUZG+unZABUTK4/Ox/M+GaHwb6Ow8rUITrtjszA==
dependencies:
"@types/react" "*"
"@types/react-redux@^7.1.20":
version "7.1.25"
resolved "https://registry.npmmirror.com/@types/react-redux/-/react-redux-7.1.25.tgz#de841631205b24f9dfb4967dd4a7901e048f9a88"
@ -6052,7 +6055,7 @@
hoist-non-react-statics "^3.3.0"
redux "^4.0.0"
"@types/react@*", "@types/react@16 || 17 || 18", "@types/react@17.x", "@types/react@>=16.9.11", "@types/react@^17", "@types/react@^17.0.0", "@types/react@^18.0.0":
"@types/react@*", "@types/react@16 || 17 || 18", "@types/react@17.x", "@types/react@>=16.9.11", "@types/react@^17", "@types/react@^17.0.0":
version "17.0.62"
resolved "https://registry.npmmirror.com/@types/react/-/react-17.0.62.tgz#2efe8ddf8533500ec44b1334dd1a97caa2f860e3"
dependencies:
@ -6060,6 +6063,15 @@
"@types/scheduler" "*"
csstype "^3.0.2"
"@types/react@^18.0.0":
version "18.2.21"
resolved "https://registry.npmjs.org/@types/react/-/react-18.2.21.tgz#774c37fd01b522d0b91aed04811b58e4e0514ed9"
integrity sha512-neFKG/sBAwGxHgXiIxnbm3/AAVQ/cMRS93hvBpg8xYRbeQSPVABp9U2bRnPf0iI4+Ucdv3plSxKK+3CW2ENJxA==
dependencies:
"@types/prop-types" "*"
"@types/scheduler" "*"
csstype "^3.0.2"
"@types/readdir-glob@*":
version "1.1.1"
resolved "https://registry.npmmirror.com/@types/readdir-glob/-/readdir-glob-1.1.1.tgz#27ac2db283e6aa3d110b14ff9da44fcd1a5c38b1"
@ -6263,7 +6275,17 @@
semver "^7.5.4"
ts-api-utils "^1.0.1"
"@typescript-eslint/parser@5.48.1", "@typescript-eslint/parser@^6.2.0":
"@typescript-eslint/parser@5.48.1":
version "5.48.1"
resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.48.1.tgz#d0125792dab7e232035434ab8ef0658154db2f10"
integrity sha512-4yg+FJR/V1M9Xoq56SF9Iygqm+r5LMXvheo6DQ7/yUWynQ4YfCRnsKuRgqH4EQ5Ya76rVwlEpw4Xu+TgWQUcdA==
dependencies:
"@typescript-eslint/scope-manager" "5.48.1"
"@typescript-eslint/types" "5.48.1"
"@typescript-eslint/typescript-estree" "5.48.1"
debug "^4.3.4"
"@typescript-eslint/parser@^6.2.0":
version "6.2.0"
resolved "https://registry.npmmirror.com/@typescript-eslint/parser/-/parser-6.2.0.tgz#d37c30b0f459c6f39455335d8f4f085919a1c644"
dependencies:
@ -13010,7 +13032,7 @@ highlight.js@^10.1.0, highlight.js@^10.2.0:
version "10.7.3"
resolved "https://registry.npmmirror.com/highlight.js/-/highlight.js-10.7.3.tgz#697272e3991356e40c3cac566a74eef681756531"
history@5.3.0:
history@5.3.0, history@^5.2.0:
version "5.3.0"
resolved "https://registry.npmmirror.com/history/-/history-5.3.0.tgz#1548abaa245ba47992f063a0783db91ef201c73b"
dependencies:
@ -15233,7 +15255,8 @@ koa-bodyparser@^4.3.0:
koa-compose@^4.1.0:
version "4.1.0"
resolved "https://registry.npmmirror.com/koa-compose/-/koa-compose-4.1.0.tgz#507306b9371901db41121c812e923d0d67d3e877"
resolved "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz#507306b9371901db41121c812e923d0d67d3e877"
integrity sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==
koa-convert@^2.0.0:
version "2.0.0"
@ -19732,7 +19755,15 @@ react-copy-to-clipboard@^5.1.0:
copy-to-clipboard "^3.3.1"
prop-types "^15.8.1"
react-dom@18.1.0, react-dom@18.x, react-dom@^18.0.0, react-dom@^18.2.0:
react-dom@18.1.0:
version "18.1.0"
resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.1.0.tgz#7f6dd84b706408adde05e1df575b3a024d7e8a2f"
integrity sha512-fU1Txz7Budmvamp7bshe4Zi32d0ll7ect+ccxNu9FlObT605GOEB8BfO4tmRJ39R5Zj831VCpvQ05QPBW5yb+w==
dependencies:
loose-envify "^1.1.0"
scheduler "^0.22.0"
react-dom@18.x, react-dom@^18.0.0, react-dom@^18.2.0:
version "18.2.0"
resolved "https://registry.npmmirror.com/react-dom/-/react-dom-18.2.0.tgz#22aaf38708db2674ed9ada224ca4aa708d821e3d"
dependencies:
@ -19895,18 +19926,34 @@ react-refresh@0.14.0, react-refresh@^0.14.0:
version "0.14.0"
resolved "https://registry.npmmirror.com/react-refresh/-/react-refresh-0.14.0.tgz#4e02825378a5f227079554d4284889354e5f553e"
react-router-dom@6.3.0, react-router-dom@6.x, react-router-dom@^6.11.2:
react-router-dom@6.3.0:
version "6.3.0"
resolved "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d"
integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw==
dependencies:
history "^5.2.0"
react-router "6.3.0"
react-router-dom@6.x, react-router-dom@^6.11.2:
version "6.14.1"
resolved "https://registry.npmmirror.com/react-router-dom/-/react-router-dom-6.14.1.tgz#0ad7ba7abdf75baa61169d49f096f0494907a36f"
dependencies:
"@remix-run/router" "1.7.1"
react-router "6.14.1"
react-router@6.14.1, react-router@6.3.0, react-router@^6.11.2:
version "6.14.2"
resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.14.2.tgz#1f60994d8c369de7b8ba7a78d8f7ec23df76b300"
react-router@6.14.1:
version "6.14.1"
resolved "https://registry.npmjs.org/react-router/-/react-router-6.14.1.tgz#5e82bcdabf21add859dc04b1859f91066b3a5810"
integrity sha512-U4PfgvG55LdvbQjg5Y9QRWyVxIdO1LlpYT7x+tMAxd9/vmiPuJhIwdxZuIQLN/9e3O4KFDHYfR9gzGeYMasW8g==
dependencies:
"@remix-run/router" "1.7.2"
"@remix-run/router" "1.7.1"
react-router@6.3.0:
version "6.3.0"
resolved "https://registry.npmjs.org/react-router/-/react-router-6.3.0.tgz#3970cc64b4cb4eae0c1ea5203a80334fdd175557"
integrity sha512-7Wh1DzVQ+tlFjkeo+ujvjSqSJmkt1+8JO+T5xklPlgrh70y7ogx75ODRW0ThWhY7S+6yEDks8TYrtQe/aoboBQ==
dependencies:
history "^5.2.0"
react-side-effect@^2.1.0:
version "2.1.2"
@ -19922,7 +19969,14 @@ react-to-print@^2.14.7:
version "2.14.13"
resolved "https://registry.npmmirror.com/react-to-print/-/react-to-print-2.14.13.tgz#cd0349f7ef93c8af5120fac0ef6c4f3d100df490"
react@18.1.0, react@18.x, react@^18.0.0, react@^18.2.0:
react@18.1.0:
version "18.1.0"
resolved "https://registry.npmjs.org/react/-/react-18.1.0.tgz#6f8620382decb17fdc5cc223a115e2adbf104890"
integrity sha512-4oL8ivCz5ZEPyclFQXaNksK3adutVS8l2xzZU0cqEFrE9Sb7fC0EFK5uEk74wIreL1DERyjvsU915j1pcT2uEQ==
dependencies:
loose-envify "^1.1.0"
react@18.x, react@^18.0.0, react@^18.2.0:
version "18.2.0"
resolved "https://registry.npmmirror.com/react/-/react-18.2.0.tgz#555bd98592883255fa00de14f1151a917b5d77d5"
dependencies:
@ -20760,6 +20814,13 @@ saxes@^5.0.1:
dependencies:
xmlchars "^2.2.0"
scheduler@^0.22.0:
version "0.22.0"
resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.22.0.tgz#83a5d63594edf074add9a7198b1bae76c3db01b8"
integrity sha512-6QAm1BgQI88NPYymgGQLCZgvep4FyePDWFpXVK+zNSUgHwlqpJy8VEh8Et0KxTACS4VWwMousBElAZOH9nkkoQ==
dependencies:
loose-envify "^1.1.0"
scheduler@^0.23.0:
version "0.23.0"
resolved "https://registry.npmmirror.com/scheduler/-/scheduler-0.23.0.tgz#ba8041afc3d30eb206a487b6b384002e4e61fdfe"