feat: data mapping, finish (#1312)

Reviewed-on: daoyoucloud/tachybase#1312
Reviewed-by: sealday <zhanglin@daoyoucloud.com>
Co-authored-by: bai.zixv <bai.zixv@foxmail.com>
Co-committed-by: bai.zixv <bai.zixv@foxmail.com>

packages/plugins/@tachybase/plugin-workflow/src/client/Plugin.tsx

@@ -8,6 +8,7 @@ import { PluginActionTrigger } from './features/action-trigger';
 import { PluginAggregate } from './features/aggregate';
 import { PluginAPIRegularClient } from './features/api-regular';
 import PluginApproval from './features/approval';
+import PluginWorkflowDataMappingClient from './features/data-mapping';
 import { PluginDelay } from './features/delay';
 import { PluginDaynamicCalculation } from './features/dynamic-calculation';
 import { PluginWorkflowInterceptor } from './features/interceptor';
@@ -89,6 +90,7 @@ export class PluginWorkflow extends Plugin {
     await this.pm.add(PluginActionTrigger);
     await this.pm.add(PluginWorkflowJsonParseClient);
     await this.pm.add(PluginWorkflowJSParseClient);
+    await this.pm.add(PluginWorkflowDataMappingClient);
     await this.pm.add(PluginAPIRegularClient);
     await this.pm.add(PluginWorkflowInterceptor);
     await this.pm.add(PluginVariables);

packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/DataMapping.instruction.tsx

@@ -0,0 +1,227 @@
+import { css } from '@tachybase/client';
+import { ArrayTable } from '@tachybase/components';
+
+import { VariableOption, WorkflowVariableInput } from '../..';
+import { NAMESPACE_INSTRUCTION_DATA_MAPPING } from '../../../common/constants';
+import { tval } from '../../locale';
+import { Instruction } from '../../nodes';
+
+export class DataMappingInstruction extends Instruction {
+  title = tval('Data Mapping');
+  type = NAMESPACE_INSTRUCTION_DATA_MAPPING;
+  group = 'extended';
+  description = tval('Get specific data from JSON result of any node BY js code or json code;');
+  fieldset = {
+    sourceArray: {
+      type: 'array',
+      title: tval('Data source array'),
+      description: tval('Data source array'),
+      'x-decorator': 'FormItem',
+      'x-component': 'ArrayTable',
+      items: {
+        type: 'object',
+        properties: {
+          keyName: {
+            type: 'void',
+            'x-component': 'ArrayTable.Column',
+            'x-component-props': {
+              title: tval('keyName'),
+            },
+            properties: {
+              keyName: {
+                type: 'string',
+                name: 'keyName',
+                'x-decorator': 'FormItem',
+                'x-component': 'Input',
+              },
+            },
+          },
+          sourcePath: {
+            type: 'void',
+            'x-component': 'ArrayTable.Column',
+            'x-component-props': {
+              title: tval('Property path'),
+            },
+            properties: {
+              sourcePath: {
+                type: 'string',
+                name: 'sourcePath',
+                required: true,
+                'x-decorator': 'FormItem',
+                'x-component': 'WorkflowVariableInput',
+                'x-component-props': {
+                  changeOnSelect: true,
+                },
+              },
+            },
+          },
+
+          operations: {
+            type: 'void',
+            'x-component': 'ArrayTable.Column',
+            'x-component-props': {
+              dataIndex: 'operations',
+              fixed: 'right',
+              className: css`
+                > *:not(:last-child) {
+                  margin-right: 0.5em;
+                }
+                button {
+                  padding: 0;
+                }
+              `,
+            },
+            properties: {
+              remove: {
+                type: 'void',
+                'x-component': 'ArrayTable.Remove',
+              },
+            },
+          },
+        },
+      },
+      properties: {
+        add: {
+          type: 'void',
+          title: tval('Add property'),
+          'x-component': 'ArrayTable.Addition',
+          'x-component-props': {
+            defaultValue: {},
+          },
+        },
+      },
+    },
+    type: {
+      type: 'string',
+      title: tval('type'),
+      'x-decorator': 'FormItem',
+      'x-component': 'Radio.Group',
+      enum: [
+        { label: 'js', value: 'js' },
+        { label: 'JSONata', value: 'jsonata' },
+      ],
+      default: 'js',
+    },
+    code: {
+      type: 'string',
+      title: tval('expression'),
+      'x-decorator': 'FormItem',
+      'x-decorator-props': {
+        tooltip: 'jscode: ctx.data\nctx.body\n__ctx\nlib.JSON\nlib.qrcode\ncanvas\nlib.dayjs\nlib.log',
+      },
+      'x-component': 'CodeMirror',
+    },
+    model: {
+      type: 'array',
+      title: tval('Properties mapping'),
+      description: tval(
+        'If the type of query result is object or array of object, could map the properties which to be accessed in subsequent nodes.',
+      ),
+      'x-decorator': 'FormItem',
+      'x-component': 'ArrayTable',
+      items: {
+        type: 'object',
+        properties: {
+          path: {
+            type: 'void',
+            'x-component': 'ArrayTable.Column',
+            'x-component-props': {
+              title: tval('Property path'),
+            },
+            properties: {
+              path: {
+                type: 'string',
+                name: 'path',
+                required: true,
+                'x-decorator': 'FormItem',
+                'x-component': 'Input',
+              },
+            },
+          },
+          alias: {
+            type: 'void',
+            'x-component': 'ArrayTable.Column',
+            'x-component-props': {
+              title: tval('Alias'),
+            },
+            properties: {
+              alias: {
+                type: 'string',
+                name: 'alias',
+                'x-decorator': 'FormItem',
+                'x-component': 'Input',
+              },
+            },
+          },
+          label: {
+            type: 'void',
+            'x-component': 'ArrayTable.Column',
+            'x-component-props': {
+              title: tval('Label'),
+            },
+            properties: {
+              label: {
+                type: 'string',
+                name: 'label',
+                required: true,
+                'x-decorator': 'FormItem',
+                'x-component': 'Input',
+              },
+            },
+          },
+          operations: {
+            type: 'void',
+            'x-component': 'ArrayTable.Column',
+            'x-component-props': {
+              dataIndex: 'operations',
+              fixed: 'right',
+              className: css`
+                > *:not(:last-child) {
+                  margin-right: 0.5em;
+                }
+                button {
+                  padding: 0;
+                }
+              `,
+            },
+            properties: {
+              remove: {
+                type: 'void',
+                'x-component': 'ArrayTable.Remove',
+              },
+            },
+          },
+        },
+      },
+      properties: {
+        add: {
+          type: 'void',
+          title: tval('Add property'),
+          'x-component': 'ArrayTable.Addition',
+          'x-component-props': {
+            defaultValue: {},
+          },
+        },
+      },
+    },
+  };
+  components = {
+    ArrayTable,
+    WorkflowVariableInput,
+  };
+
+  useVariables(node, options): VariableOption {
+    const { key, title, config } = node;
+    const { types, fieldNames } = options;
+    const model = config.model || [];
+    const result = {
+      [fieldNames.label]: title,
+      [fieldNames.value]: key,
+      [fieldNames.children]: model.map((item) => ({
+        [fieldNames.label]: item.label,
+        [fieldNames.value]: item.alias || item.path,
+      })),
+    };
+    return result;
+  }
+}

packages/plugins/@tachybase/plugin-workflow/src/client/features/data-mapping/index.tsx

@@ -0,0 +1,14 @@
+import { Plugin } from '@tachybase/client';
+
+import PluginWorkflow from '../..';
+import { NAMESPACE_INSTRUCTION_DATA_MAPPING } from '../../../common/constants';
+import { DataMappingInstruction } from './DataMapping.instruction';
+
+export class PluginWorkflowDataMappingClient extends Plugin {
+  async load() {
+    const pluginWorkflow = this.app.pm.get(PluginWorkflow);
+    pluginWorkflow.registerInstruction(NAMESPACE_INSTRUCTION_DATA_MAPPING, DataMappingInstruction);
+  }
+}
+
+export default PluginWorkflowDataMappingClient;

packages/plugins/@tachybase/plugin-workflow/src/common/constants.ts

@@ -1,5 +1,7 @@
+// triggers
+export const NAMESPACE_TRIGGER_API_REGULAR = 'api-regular';
+
+// instructions
 export const NAMESPACE_INSTRUCTION_JSON_PARSE = 'json-parse';
 export const NAMESPACE_INSTRUCTION_JS_PARSE = 'js-parse';
-
+export const NAMESPACE_INSTRUCTION_DATA_MAPPING = 'data-mapping';
-// api-fetch
-export const NAMESPACE_TRIGGER_API_REGULAR = 'api-regular';

packages/plugins/@tachybase/plugin-workflow/src/server/Plugin.ts

@@ -12,6 +12,7 @@ import { PluginActionTrigger } from './features/action-trigger/Plugin';
 import { PluginAggregate } from './features/aggregate/Plugin';
 import PluginWorkflowAPIRegularServer from './features/api-regular/plugin';
 import PluginWorkflowApproval from './features/approval';
+import PluginWorkflowDataMappingServer from './features/data-mapping/plugin';
 import { PluginDelay } from './features/delay/Plugin';
 import { PluginDynamicCalculation } from './features/dynamic-calculation/Plugin';
 import { PluginInterception } from './features/interception';
@@ -77,6 +78,7 @@ export default class PluginWorkflowServer extends Plugin {
     this.addFeature(PluginActionTrigger);
     this.addFeature(PluginWorkflowJSONParseServer);
     this.addFeature(PluginWorkflowJSParseServer);
+    this.addFeature(PluginWorkflowDataMappingServer);
     this.addFeature(PluginWorkflowAPIRegularServer);
     this.addFeature(PluginInterception);
     this.addFeature(PluginVariables);

packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/DataMapping.instruction.ts

@@ -0,0 +1,118 @@
+import crypto from 'crypto';
+
+import * as canvas from 'canvas';
+import jsonata from 'jsonata';
+import _ from 'lodash';
+import qrcode from 'qrcode';
+
+import { FlowNodeModel, Instruction, JOB_STATUS, Processor } from '../..';
+
+export class DataMappingInstruction extends Instruction {
+  async run(node: FlowNodeModel, input: any, processor: Processor) {
+    const { sourceArray, type, code = '', model } = node.config;
+    // 1. 获取数据源
+    let data = {};
+    if (sourceArray.length > 1) {
+      // 多个数据源, 进行合并
+      data = sourceArray.reduce(
+        (cookedData, { keyName, sourcePath }) => ({
+          ...cookedData,
+          [keyName]: processor.getParsedValue(sourcePath, node.id),
+        }),
+        {},
+      );
+    } else {
+      // 单数据源, 平铺为单对象
+      const source = sourceArray[0]['sourcePath'];
+      data = processor.getParsedValue(source, node.id);
+    }
+
+    try {
+      // 2. 根据 type 类型, 对源数据进行复杂数据映射
+      let result = {};
+      switch (type) {
+        case 'jsonata':
+          result = await convertByJSONata(code, data);
+          break;
+        case 'js':
+        default:
+          result = await convertByJsCode(code, data);
+      }
+
+      // 3. 将结果集, 进行简单数据映射
+      if (typeof result === 'object' && result && model?.length) {
+        if (Array.isArray(result)) {
+          result = result.map((item) => mapModel(item, model));
+        } else {
+          result = mapModel(result, model);
+        }
+      }
+
+      // 4. 返回结果集, 和节点执行状态
+      return {
+        result,
+        status: JOB_STATUS.RESOLVED,
+      };
+    } catch (err) {
+      return {
+        result: err.toString(),
+        status: JOB_STATUS.ERROR,
+      };
+    }
+  }
+}
+
+// utils-JSONata
+async function convertByJSONata(code, data) {
+  const engine = (expression, data) => jsonata(expression).evaluate(data);
+  const result = await engine(code, data);
+  return result;
+}
+
+// utils-jsCode
+async function convertByJsCode(code, data) {
+  const ctx = {
+    data,
+    body: {},
+  };
+  await evalSimulate(code, {
+    ctx,
+    lib: {
+      log: console.log,
+      JSON,
+      canvas,
+      qrcode,
+      crypto,
+      jsonata,
+    },
+  });
+
+  return ctx.body;
+}
+
+// utils
+function mapModel(data, model) {
+  if (typeof data !== 'object' || data === null) {
+    throw new Error('Invalid data: data should be a non-null object');
+  }
+
+  const result = model.reduce((acc, { path, alias }) => {
+    const key = alias ?? path.replace(/\./g, '_');
+    const value = _.get(data, path);
+    acc[key] = value;
+
+    return acc;
+  }, {});
+
+  return result;
+}
+
+async function evalSimulate(jsCode, { ctx, lib }) {
+  const AsyncFunction: any = async function () {}.constructor;
+  return await new AsyncFunction('$root', `with($root) { ${jsCode}; }`)({
+    ctx,
+    // 允许用户覆盖，这个时候可以使用 _ctx
+    __ctx: ctx,
+    lib,
+  });
+}

packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/index.ts

@@ -0,0 +1 @@
+export { default } from './plugin';

packages/plugins/@tachybase/plugin-workflow/src/server/features/data-mapping/plugin.ts

@@ -0,0 +1,14 @@
+import { Plugin } from '@tachybase/server';
+
+import WorkflowPlugin from '../..';
+import { NAMESPACE_INSTRUCTION_DATA_MAPPING } from '../../../common/constants';
+import { DataMappingInstruction } from './DataMapping.instruction';
+
+export class PluginWorkflowDataMappingServer extends Plugin {
+  async load() {
+    const pluginWorkflow: any = this.app.pm.get(WorkflowPlugin);
+    pluginWorkflow.registerInstruction(NAMESPACE_INSTRUCTION_DATA_MAPPING, DataMappingInstruction);
+  }
+}
+
+export default PluginWorkflowDataMappingServer;