From 3367a2f85d5ec9d4bc60f20462c32edbdad7f005 Mon Sep 17 00:00:00 2001 From: "bai.zixv" Date: Thu, 11 Jul 2024 20:42:28 +0800 Subject: [PATCH] feat: data mapping, finish (#1312) Reviewed-on: https://git.daoyoucloud.com/daoyoucloud/tachybase/pulls/1312 Reviewed-by: sealday Co-authored-by: bai.zixv Co-committed-by: bai.zixv --- .../plugin-workflow/src/client/Plugin.tsx | 2 + .../data-mapping/DataMapping.instruction.tsx | 227 ++++++++++++++++++ .../client/features/data-mapping/index.tsx | 14 ++ .../plugin-workflow/src/common/constants.ts | 8 +- .../plugin-workflow/src/server/Plugin.ts | 2 + .../data-mapping/DataMapping.instruction.ts | 118 +++++++++ .../src/server/features/data-mapping/index.ts | 1 + .../server/features/data-mapping/plugin.ts | 14 ++ 8 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/DataMapping.instruction.tsx create mode 100644 packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/index.tsx create mode 100644 packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/DataMapping.instruction.ts create mode 100644 packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/index.ts create mode 100644 packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/plugin.ts diff --git a/packages/plugins/@tachybase/plugin-workflow/src/client/Plugin.tsx b/packages/plugins/@tachybase/plugin-workflow/src/client/Plugin.tsx index 5e9d5038d..c558b1413 100644 --- a/packages/plugins/@tachybase/plugin-workflow/src/client/Plugin.tsx +++ b/packages/plugins/@tachybase/plugin-workflow/src/client/Plugin.tsx @@ -8,6 +8,7 @@ import { PluginActionTrigger } from './features/action-trigger'; import { PluginAggregate } from './features/aggregate'; import { PluginAPIRegularClient } from './features/api-regular'; import PluginApproval from './features/approval'; +import PluginWorkflowDataMappingClient from './features/data-mapping'; import { PluginDelay } from './features/delay'; import { PluginDaynamicCalculation } from './features/dynamic-calculation'; import { PluginWorkflowInterceptor } from './features/interceptor'; @@ -89,6 +90,7 @@ export class PluginWorkflow extends Plugin { await this.pm.add(PluginActionTrigger); await this.pm.add(PluginWorkflowJsonParseClient); await this.pm.add(PluginWorkflowJSParseClient); + await this.pm.add(PluginWorkflowDataMappingClient); await this.pm.add(PluginAPIRegularClient); await this.pm.add(PluginWorkflowInterceptor); await this.pm.add(PluginVariables); diff --git a/packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/DataMapping.instruction.tsx b/packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/DataMapping.instruction.tsx new file mode 100644 index 000000000..5e4609451 --- /dev/null +++ b/packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/DataMapping.instruction.tsx @@ -0,0 +1,227 @@ +import { css } from '@tachybase/client'; +import { ArrayTable } from '@tachybase/components'; + +import { VariableOption, WorkflowVariableInput } from '../..'; +import { NAMESPACE_INSTRUCTION_DATA_MAPPING } from '../../../common/constants'; +import { tval } from '../../locale'; +import { Instruction } from '../../nodes'; + +export class DataMappingInstruction extends Instruction { + title = tval('Data Mapping'); + type = NAMESPACE_INSTRUCTION_DATA_MAPPING; + group = 'extended'; + description = tval('Get specific data from JSON result of any node BY js code or json code;'); + fieldset = { + sourceArray: { + type: 'array', + title: tval('Data source array'), + description: tval('Data source array'), + 'x-decorator': 'FormItem', + 'x-component': 'ArrayTable', + items: { + type: 'object', + properties: { + keyName: { + type: 'void', + 'x-component': 'ArrayTable.Column', + 'x-component-props': { + title: tval('keyName'), + }, + properties: { + keyName: { + type: 'string', + name: 'keyName', + 'x-decorator': 'FormItem', + 'x-component': 'Input', + }, + }, + }, + sourcePath: { + type: 'void', + 'x-component': 'ArrayTable.Column', + 'x-component-props': { + title: tval('Property path'), + }, + properties: { + sourcePath: { + type: 'string', + name: 'sourcePath', + required: true, + 'x-decorator': 'FormItem', + 'x-component': 'WorkflowVariableInput', + 'x-component-props': { + changeOnSelect: true, + }, + }, + }, + }, + + operations: { + type: 'void', + 'x-component': 'ArrayTable.Column', + 'x-component-props': { + dataIndex: 'operations', + fixed: 'right', + className: css` + > *:not(:last-child) { + margin-right: 0.5em; + } + button { + padding: 0; + } + `, + }, + properties: { + remove: { + type: 'void', + 'x-component': 'ArrayTable.Remove', + }, + }, + }, + }, + }, + properties: { + add: { + type: 'void', + title: tval('Add property'), + 'x-component': 'ArrayTable.Addition', + 'x-component-props': { + defaultValue: {}, + }, + }, + }, + }, + type: { + type: 'string', + title: tval('type'), + 'x-decorator': 'FormItem', + 'x-component': 'Radio.Group', + enum: [ + { label: 'js', value: 'js' }, + { label: 'JSONata', value: 'jsonata' }, + ], + default: 'js', + }, + code: { + type: 'string', + title: tval('expression'), + 'x-decorator': 'FormItem', + 'x-decorator-props': { + tooltip: 'jscode: ctx.data\nctx.body\n__ctx\nlib.JSON\nlib.qrcode\ncanvas\nlib.dayjs\nlib.log', + }, + 'x-component': 'CodeMirror', + }, + model: { + type: 'array', + title: tval('Properties mapping'), + description: tval( + 'If the type of query result is object or array of object, could map the properties which to be accessed in subsequent nodes.', + ), + 'x-decorator': 'FormItem', + 'x-component': 'ArrayTable', + items: { + type: 'object', + properties: { + path: { + type: 'void', + 'x-component': 'ArrayTable.Column', + 'x-component-props': { + title: tval('Property path'), + }, + properties: { + path: { + type: 'string', + name: 'path', + required: true, + 'x-decorator': 'FormItem', + 'x-component': 'Input', + }, + }, + }, + alias: { + type: 'void', + 'x-component': 'ArrayTable.Column', + 'x-component-props': { + title: tval('Alias'), + }, + properties: { + alias: { + type: 'string', + name: 'alias', + 'x-decorator': 'FormItem', + 'x-component': 'Input', + }, + }, + }, + label: { + type: 'void', + 'x-component': 'ArrayTable.Column', + 'x-component-props': { + title: tval('Label'), + }, + properties: { + label: { + type: 'string', + name: 'label', + required: true, + 'x-decorator': 'FormItem', + 'x-component': 'Input', + }, + }, + }, + operations: { + type: 'void', + 'x-component': 'ArrayTable.Column', + 'x-component-props': { + dataIndex: 'operations', + fixed: 'right', + className: css` + > *:not(:last-child) { + margin-right: 0.5em; + } + button { + padding: 0; + } + `, + }, + properties: { + remove: { + type: 'void', + 'x-component': 'ArrayTable.Remove', + }, + }, + }, + }, + }, + properties: { + add: { + type: 'void', + title: tval('Add property'), + 'x-component': 'ArrayTable.Addition', + 'x-component-props': { + defaultValue: {}, + }, + }, + }, + }, + }; + components = { + ArrayTable, + WorkflowVariableInput, + }; + + useVariables(node, options): VariableOption { + const { key, title, config } = node; + const { types, fieldNames } = options; + const model = config.model || []; + const result = { + [fieldNames.label]: title, + [fieldNames.value]: key, + [fieldNames.children]: model.map((item) => ({ + [fieldNames.label]: item.label, + [fieldNames.value]: item.alias || item.path, + })), + }; + return result; + } +} diff --git a/packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/index.tsx b/packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/index.tsx new file mode 100644 index 000000000..ac95e87f5 --- /dev/null +++ b/packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/index.tsx @@ -0,0 +1,14 @@ +import { Plugin } from '@tachybase/client'; + +import PluginWorkflow from '../..'; +import { NAMESPACE_INSTRUCTION_DATA_MAPPING } from '../../../common/constants'; +import { DataMappingInstruction } from './DataMapping.instruction'; + +export class PluginWorkflowDataMappingClient extends Plugin { + async load() { + const pluginWorkflow = this.app.pm.get(PluginWorkflow); + pluginWorkflow.registerInstruction(NAMESPACE_INSTRUCTION_DATA_MAPPING, DataMappingInstruction); + } +} + +export default PluginWorkflowDataMappingClient; diff --git a/packages/plugins/@tachybase/plugin-workflow/src/common/constants.ts b/packages/plugins/@tachybase/plugin-workflow/src/common/constants.ts index 172074d6a..b7371a57e 100644 --- a/packages/plugins/@tachybase/plugin-workflow/src/common/constants.ts +++ b/packages/plugins/@tachybase/plugin-workflow/src/common/constants.ts @@ -1,5 +1,7 @@ +// triggers +export const NAMESPACE_TRIGGER_API_REGULAR = 'api-regular'; + +// instructions export const NAMESPACE_INSTRUCTION_JSON_PARSE = 'json-parse'; export const NAMESPACE_INSTRUCTION_JS_PARSE = 'js-parse'; - -// api-fetch -export const NAMESPACE_TRIGGER_API_REGULAR = 'api-regular'; +export const NAMESPACE_INSTRUCTION_DATA_MAPPING = 'data-mapping'; diff --git a/packages/plugins/@tachybase/plugin-workflow/src/server/Plugin.ts b/packages/plugins/@tachybase/plugin-workflow/src/server/Plugin.ts index aaaabf11f..240a9cf56 100644 --- a/packages/plugins/@tachybase/plugin-workflow/src/server/Plugin.ts +++ b/packages/plugins/@tachybase/plugin-workflow/src/server/Plugin.ts @@ -12,6 +12,7 @@ import { PluginActionTrigger } from './features/action-trigger/Plugin'; import { PluginAggregate } from './features/aggregate/Plugin'; import PluginWorkflowAPIRegularServer from './features/api-regular/plugin'; import PluginWorkflowApproval from './features/approval'; +import PluginWorkflowDataMappingServer from './features/data-mapping/plugin'; import { PluginDelay } from './features/delay/Plugin'; import { PluginDynamicCalculation } from './features/dynamic-calculation/Plugin'; import { PluginInterception } from './features/interception'; @@ -77,6 +78,7 @@ export default class PluginWorkflowServer extends Plugin { this.addFeature(PluginActionTrigger); this.addFeature(PluginWorkflowJSONParseServer); this.addFeature(PluginWorkflowJSParseServer); + this.addFeature(PluginWorkflowDataMappingServer); this.addFeature(PluginWorkflowAPIRegularServer); this.addFeature(PluginInterception); this.addFeature(PluginVariables); diff --git a/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/DataMapping.instruction.ts b/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/DataMapping.instruction.ts new file mode 100644 index 000000000..8afe0e20b --- /dev/null +++ b/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/DataMapping.instruction.ts @@ -0,0 +1,118 @@ +import crypto from 'crypto'; + +import * as canvas from 'canvas'; +import jsonata from 'jsonata'; +import _ from 'lodash'; +import qrcode from 'qrcode'; + +import { FlowNodeModel, Instruction, JOB_STATUS, Processor } from '../..'; + +export class DataMappingInstruction extends Instruction { + async run(node: FlowNodeModel, input: any, processor: Processor) { + const { sourceArray, type, code = '', model } = node.config; + // 1. 获取数据源 + let data = {}; + if (sourceArray.length > 1) { + // 多个数据源, 进行合并 + data = sourceArray.reduce( + (cookedData, { keyName, sourcePath }) => ({ + ...cookedData, + [keyName]: processor.getParsedValue(sourcePath, node.id), + }), + {}, + ); + } else { + // 单数据源, 平铺为单对象 + const source = sourceArray[0]['sourcePath']; + data = processor.getParsedValue(source, node.id); + } + + try { + // 2. 根据 type 类型, 对源数据进行复杂数据映射 + let result = {}; + switch (type) { + case 'jsonata': + result = await convertByJSONata(code, data); + break; + case 'js': + default: + result = await convertByJsCode(code, data); + } + + // 3. 将结果集, 进行简单数据映射 + if (typeof result === 'object' && result && model?.length) { + if (Array.isArray(result)) { + result = result.map((item) => mapModel(item, model)); + } else { + result = mapModel(result, model); + } + } + + // 4. 返回结果集, 和节点执行状态 + return { + result, + status: JOB_STATUS.RESOLVED, + }; + } catch (err) { + return { + result: err.toString(), + status: JOB_STATUS.ERROR, + }; + } + } +} + +// utils-JSONata +async function convertByJSONata(code, data) { + const engine = (expression, data) => jsonata(expression).evaluate(data); + const result = await engine(code, data); + return result; +} + +// utils-jsCode +async function convertByJsCode(code, data) { + const ctx = { + data, + body: {}, + }; + await evalSimulate(code, { + ctx, + lib: { + log: console.log, + JSON, + canvas, + qrcode, + crypto, + jsonata, + }, + }); + + return ctx.body; +} + +// utils +function mapModel(data, model) { + if (typeof data !== 'object' || data === null) { + throw new Error('Invalid data: data should be a non-null object'); + } + + const result = model.reduce((acc, { path, alias }) => { + const key = alias ?? path.replace(/\./g, '_'); + const value = _.get(data, path); + acc[key] = value; + + return acc; + }, {}); + + return result; +} + +async function evalSimulate(jsCode, { ctx, lib }) { + const AsyncFunction: any = async function () {}.constructor; + return await new AsyncFunction('$root', `with($root) { ${jsCode}; }`)({ + ctx, + // 允许用户覆盖，这个时候可以使用 _ctx + __ctx: ctx, + lib, + }); +} diff --git a/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/index.ts b/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/index.ts new file mode 100644 index 000000000..b68aea57f --- /dev/null +++ b/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/index.ts @@ -0,0 +1 @@ +export { default } from './plugin'; diff --git a/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/plugin.ts b/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/plugin.ts new file mode 100644 index 000000000..df82b2f6a --- /dev/null +++ b/packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/plugin.ts @@ -0,0 +1,14 @@ +import { Plugin } from '@tachybase/server'; + +import WorkflowPlugin from '../..'; +import { NAMESPACE_INSTRUCTION_DATA_MAPPING } from '../../../common/constants'; +import { DataMappingInstruction } from './DataMapping.instruction'; + +export class PluginWorkflowDataMappingServer extends Plugin { + async load() { + const pluginWorkflow: any = this.app.pm.get(WorkflowPlugin); + pluginWorkflow.registerInstruction(NAMESPACE_INSTRUCTION_DATA_MAPPING, DataMappingInstruction); + } +} + +export default PluginWorkflowDataMappingServer;