fix(plugin-workflow): use dual pipes to process triggers (#1187)
* fix(plugin-workflow): use dual pipes to process triggers * refactor(plugin-workflow): refactor dual pipe structure
This commit is contained in:
		
							parent
							
								
									e5a9721674
								
							
						
					
					
						commit
						c19ca1f30b
					
				@ -48,8 +48,8 @@ export interface Trigger {
 | 
				
			|||||||
  view?: ISchema;
 | 
					  view?: ISchema;
 | 
				
			||||||
  scope?: { [key: string]: any };
 | 
					  scope?: { [key: string]: any };
 | 
				
			||||||
  components?: { [key: string]: any };
 | 
					  components?: { [key: string]: any };
 | 
				
			||||||
  render?(props): React.ReactElement;
 | 
					  render?(props): React.ReactNode;
 | 
				
			||||||
  getter?(node: any): React.ReactElement;
 | 
					  getter?(node: any): React.ReactNode;
 | 
				
			||||||
};
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
export const triggers = new Registry<Trigger>();
 | 
					export const triggers = new Registry<Trigger>();
 | 
				
			||||||
@ -70,9 +70,7 @@ function TriggerExecution() {
 | 
				
			|||||||
    <SchemaComponent
 | 
					    <SchemaComponent
 | 
				
			||||||
      schema={{
 | 
					      schema={{
 | 
				
			||||||
        type: 'void',
 | 
					        type: 'void',
 | 
				
			||||||
        properties: {
 | 
					        name: execution.id,
 | 
				
			||||||
          trigger: {
 | 
					 | 
				
			||||||
            type: 'void',
 | 
					 | 
				
			||||||
        'x-component': 'Action',
 | 
					        'x-component': 'Action',
 | 
				
			||||||
        'x-component-props': {
 | 
					        'x-component-props': {
 | 
				
			||||||
          title: <InfoOutlined />,
 | 
					          title: <InfoOutlined />,
 | 
				
			||||||
@ -122,8 +120,6 @@ function TriggerExecution() {
 | 
				
			|||||||
            }
 | 
					            }
 | 
				
			||||||
          }
 | 
					          }
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
          }
 | 
					 | 
				
			||||||
        }
 | 
					 | 
				
			||||||
      }}
 | 
					      }}
 | 
				
			||||||
    />
 | 
					    />
 | 
				
			||||||
  );
 | 
					  );
 | 
				
			||||||
 | 
				
			|||||||
@ -16,14 +16,15 @@ import initTriggers, { Trigger } from './triggers';
 | 
				
			|||||||
import JobModel from './models/Job';
 | 
					import JobModel from './models/Job';
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					type Pending = [ExecutionModel, JobModel?];
 | 
				
			||||||
export default class WorkflowPlugin extends Plugin {
 | 
					export default class WorkflowPlugin extends Plugin {
 | 
				
			||||||
  instructions: Registry<Instruction> = new Registry();
 | 
					  instructions: Registry<Instruction> = new Registry();
 | 
				
			||||||
  triggers: Registry<Trigger> = new Registry();
 | 
					  triggers: Registry<Trigger> = new Registry();
 | 
				
			||||||
  calculators = calculators;
 | 
					  calculators = calculators;
 | 
				
			||||||
  extensions = extensions;
 | 
					  extensions = extensions;
 | 
				
			||||||
  executing: Promise<any> = null;
 | 
					  executing: ExecutionModel = null;
 | 
				
			||||||
  pending: [ExecutionModel, JobModel][] = [];
 | 
					  pending: Pending[] = [];
 | 
				
			||||||
 | 
					  events: [WorkflowModel, any, { context?: any }][] = [];
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  onBeforeSave = async (instance: WorkflowModel, options) => {
 | 
					  onBeforeSave = async (instance: WorkflowModel, options) => {
 | 
				
			||||||
    const Model = <typeof WorkflowModel>instance.constructor;
 | 
					    const Model = <typeof WorkflowModel>instance.constructor;
 | 
				
			||||||
@ -102,7 +103,7 @@ export default class WorkflowPlugin extends Plugin {
 | 
				
			|||||||
      });
 | 
					      });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      // check for not started executions
 | 
					      // check for not started executions
 | 
				
			||||||
      await this.dispatch();
 | 
					      this.dispatch();
 | 
				
			||||||
    });
 | 
					    });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    this.app.on('beforeStop', async () => {
 | 
					    this.app.on('beforeStop', async () => {
 | 
				
			||||||
@ -132,12 +133,25 @@ export default class WorkflowPlugin extends Plugin {
 | 
				
			|||||||
    }
 | 
					    }
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  public async trigger(workflow: WorkflowModel, context: Object, options: Transactionable & { context?: any } = {}): Promise<void> {
 | 
					  public trigger(workflow: WorkflowModel, context: Object, options: { context?: any } = {}): Promise<void> {
 | 
				
			||||||
    // `null` means not to trigger
 | 
					    // `null` means not to trigger
 | 
				
			||||||
    if (context == null) {
 | 
					    if (context == null) {
 | 
				
			||||||
      return null;
 | 
					      return;
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    this.events.push([workflow, context, options]);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    if (this.events.length > 1) {
 | 
				
			||||||
 | 
					      return;
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    // NOTE: no await for quick return
 | 
				
			||||||
 | 
					    setTimeout(this.prepare);
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  private prepare = async () => {
 | 
				
			||||||
 | 
					    const [workflow, context, options] = this.events[0];
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    if (options.context?.executionId) {
 | 
					    if (options.context?.executionId) {
 | 
				
			||||||
      // NOTE: no transaction here for read-uncommitted execution
 | 
					      // NOTE: no transaction here for read-uncommitted execution
 | 
				
			||||||
      const existed = await workflow.countExecutions({
 | 
					      const existed = await workflow.countExecutions({
 | 
				
			||||||
@ -152,8 +166,7 @@ export default class WorkflowPlugin extends Plugin {
 | 
				
			|||||||
      }
 | 
					      }
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    const transaction = await (<typeof WorkflowModel>workflow.constructor).database.sequelize.transaction();
 | 
					    await this.db.sequelize.transaction(async transaction => {
 | 
				
			||||||
 | 
					 | 
				
			||||||
      const execution = await workflow.createExecution({
 | 
					      const execution = await workflow.createExecution({
 | 
				
			||||||
        context,
 | 
					        context,
 | 
				
			||||||
        key: workflow.key,
 | 
					        key: workflow.key,
 | 
				
			||||||
@ -161,8 +174,6 @@ export default class WorkflowPlugin extends Plugin {
 | 
				
			|||||||
        useTransaction: workflow.useTransaction,
 | 
					        useTransaction: workflow.useTransaction,
 | 
				
			||||||
      }, { transaction });
 | 
					      }, { transaction });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    console.log('workflow triggered:', new Date(), workflow.id, execution.id);
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
      const executed = await workflow.countExecutions({ transaction });
 | 
					      const executed = await workflow.countExecutions({ transaction });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      // NOTE: not to trigger afterUpdate hook here
 | 
					      // NOTE: not to trigger afterUpdate hook here
 | 
				
			||||||
@ -186,9 +197,16 @@ export default class WorkflowPlugin extends Plugin {
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
      execution.workflow = workflow;
 | 
					      execution.workflow = workflow;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    await transaction.commit();
 | 
					      return execution;
 | 
				
			||||||
 | 
					    });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    setTimeout(() => this.dispatch(execution));
 | 
					    this.events.shift();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    if (this.events.length) {
 | 
				
			||||||
 | 
					      await this.prepare();
 | 
				
			||||||
 | 
					    } else {
 | 
				
			||||||
 | 
					      this.dispatch();
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  public async resume(job) {
 | 
					  public async resume(job) {
 | 
				
			||||||
@ -196,45 +214,51 @@ export default class WorkflowPlugin extends Plugin {
 | 
				
			|||||||
      job.execution = await job.getExecution();
 | 
					      job.execution = await job.getExecution();
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    setTimeout(() => this.dispatch(job.execution, job));
 | 
					    this.pending.push([job.execution, job]);
 | 
				
			||||||
 | 
					    this.dispatch();
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private async dispatch(execution?: ExecutionModel, job?: JobModel) {
 | 
					  private async dispatch() {
 | 
				
			||||||
    if (this.executing) {
 | 
					    if (this.executing) {
 | 
				
			||||||
      if (job) {
 | 
					 | 
				
			||||||
        this.pending.push([execution, job]);
 | 
					 | 
				
			||||||
      }
 | 
					 | 
				
			||||||
      return;
 | 
					      return;
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    if (!execution) {
 | 
					    let next: Pending;
 | 
				
			||||||
      execution = await this.db.getRepository('executions').findOne({
 | 
					    // resuming has high priority
 | 
				
			||||||
 | 
					    if (this.pending.length) {
 | 
				
			||||||
 | 
					      next = this.pending.shift();
 | 
				
			||||||
 | 
					    } else {
 | 
				
			||||||
 | 
					      const execution = await this.db.getRepository('executions').findOne({
 | 
				
			||||||
        filter: {
 | 
					        filter: {
 | 
				
			||||||
          status: EXECUTION_STATUS.CREATED
 | 
					          status: EXECUTION_STATUS.CREATED
 | 
				
			||||||
        },
 | 
					        },
 | 
				
			||||||
        sort: 'createdAt'
 | 
					        sort: 'createdAt'
 | 
				
			||||||
      }) as ExecutionModel;
 | 
					      }) as ExecutionModel;
 | 
				
			||||||
      if (!execution) {
 | 
					      if (execution) {
 | 
				
			||||||
        return;
 | 
					        next = [execution];
 | 
				
			||||||
 | 
					      }
 | 
				
			||||||
 | 
					    };
 | 
				
			||||||
 | 
					    if (next) {
 | 
				
			||||||
 | 
					      this.process(...next);
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  private async process(execution: ExecutionModel, job?: JobModel) {
 | 
				
			||||||
 | 
					    this.executing = execution;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    if (execution.status === EXECUTION_STATUS.CREATED) {
 | 
					    if (execution.status === EXECUTION_STATUS.CREATED) {
 | 
				
			||||||
      await execution.update({ status: EXECUTION_STATUS.STARTED });
 | 
					      await execution.update({ status: EXECUTION_STATUS.STARTED });
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    const processor = this.createProcessor(execution);
 | 
					    const processor = this.createProcessor(execution);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    this.executing = job ? processor.resume(job) : processor.start();
 | 
					    console.log('workflow processing:', new Date(), execution.workflowId, execution.id);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    await this.executing;
 | 
					    await (job ? processor.resume(job) : processor.start());
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    this.executing = null;
 | 
					    this.executing = null;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    setTimeout(() => {
 | 
					    this.dispatch();
 | 
				
			||||||
      const args = this.pending.length ? this.pending.shift() : [];
 | 
					 | 
				
			||||||
      this.dispatch(...args);
 | 
					 | 
				
			||||||
    });
 | 
					 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private createProcessor(execution: ExecutionModel, options = {}): Processor {
 | 
					  private createProcessor(execution: ExecutionModel, options = {}): Processor {
 | 
				
			||||||
 | 
				
			|||||||
@ -70,15 +70,13 @@ export default class Processor {
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
    const { options } = this;
 | 
					    const { options } = this;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    const { sequelize } = (<typeof ExecutionModel>this.execution.constructor).database;
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
    // @ts-ignore
 | 
					    // @ts-ignore
 | 
				
			||||||
    return options.transaction && !options.transaction.finished
 | 
					    return options.transaction && !options.transaction.finished
 | 
				
			||||||
      ? options.transaction
 | 
					      ? options.transaction
 | 
				
			||||||
      : await sequelize.transaction();
 | 
					      : await options.plugin.db.sequelize.transaction();
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  async prepare(commit?: boolean) {
 | 
					  private async prepare() {
 | 
				
			||||||
    const transaction = await this.getTransaction();
 | 
					    const transaction = await this.getTransaction();
 | 
				
			||||||
    this.transaction = transaction;
 | 
					    this.transaction = transaction;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -97,10 +95,6 @@ export default class Processor {
 | 
				
			|||||||
    });
 | 
					    });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    this.makeJobs(jobs);
 | 
					    this.makeJobs(jobs);
 | 
				
			||||||
 | 
					 | 
				
			||||||
    if (commit) {
 | 
					 | 
				
			||||||
      await this.commit();
 | 
					 | 
				
			||||||
    }
 | 
					 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  public async start() {
 | 
					  public async start() {
 | 
				
			||||||
 | 
				
			|||||||
@ -183,6 +183,41 @@ describe('workflow > Plugin', () => {
 | 
				
			|||||||
    });
 | 
					    });
 | 
				
			||||||
  });
 | 
					  });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  describe('cycling trigger', () => {
 | 
				
			||||||
 | 
					    it('trigger should not be triggered more than once in same execution', async () => {
 | 
				
			||||||
 | 
					      const workflow = await WorkflowModel.create({
 | 
				
			||||||
 | 
					        enabled: true,
 | 
				
			||||||
 | 
					        type: 'collection',
 | 
				
			||||||
 | 
					        config: {
 | 
				
			||||||
 | 
					          mode: 1,
 | 
				
			||||||
 | 
					          collection: 'posts'
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					      });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      const n1 = await workflow.createNode({
 | 
				
			||||||
 | 
					        type: 'create',
 | 
				
			||||||
 | 
					        config: {
 | 
				
			||||||
 | 
					          collection: 'posts',
 | 
				
			||||||
 | 
					          params: {
 | 
				
			||||||
 | 
					            values: {
 | 
				
			||||||
 | 
					              title: 't2'
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					          }
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					      });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      const post = await PostRepo.create({ values: { title: 't1' } });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      await sleep(500);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      const posts = await PostRepo.find();
 | 
				
			||||||
 | 
					      expect(posts.length).toBe(2);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      const [execution] = await workflow.getExecutions();
 | 
				
			||||||
 | 
					      expect(execution.status).toBe(EXECUTION_STATUS.RESOLVED);
 | 
				
			||||||
 | 
					    });
 | 
				
			||||||
 | 
					  });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  describe('dispatcher', () => {
 | 
					  describe('dispatcher', () => {
 | 
				
			||||||
    it.skip('multiple triggers in same event', async () => {
 | 
					    it.skip('multiple triggers in same event', async () => {
 | 
				
			||||||
      const w1 = await WorkflowModel.create({
 | 
					      const w1 = await WorkflowModel.create({
 | 
				
			||||||
@ -203,15 +238,27 @@ describe('workflow > Plugin', () => {
 | 
				
			|||||||
        }
 | 
					        }
 | 
				
			||||||
      });
 | 
					      });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      const w3 = await WorkflowModel.create({
 | 
				
			||||||
 | 
					        enabled: true,
 | 
				
			||||||
 | 
					        type: 'collection',
 | 
				
			||||||
 | 
					        config: {
 | 
				
			||||||
 | 
					          mode: 1,
 | 
				
			||||||
 | 
					          collection: 'posts'
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					      });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      const p1 = await PostRepo.create({ values: { title: 't1' } });
 | 
					      const p1 = await PostRepo.create({ values: { title: 't1' } });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      await sleep(500);
 | 
					      await sleep(1000);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      const [e1] = await w1.getExecutions();
 | 
					      const [e1] = await w1.getExecutions();
 | 
				
			||||||
      expect(e1.status).toBe(EXECUTION_STATUS.RESOLVED);
 | 
					      expect(e1.status).toBe(EXECUTION_STATUS.RESOLVED);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      const [e2] = await w2.getExecutions();
 | 
					      const [e2] = await w2.getExecutions();
 | 
				
			||||||
      expect(e2.status).toBe(EXECUTION_STATUS.RESOLVED);
 | 
					      expect(e2.status).toBe(EXECUTION_STATUS.RESOLVED);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      const [e3] = await w3.getExecutions();
 | 
				
			||||||
 | 
					      expect(e3.status).toBe(EXECUTION_STATUS.RESOLVED);
 | 
				
			||||||
    });
 | 
					    });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    it('when server starts, process all created executions', async () => {
 | 
					    it('when server starts, process all created executions', async () => {
 | 
				
			||||||
 | 
				
			|||||||
@ -425,30 +425,4 @@ describe('workflow > Processor', () => {
 | 
				
			|||||||
      expect(jobs.length).toEqual(5);
 | 
					      expect(jobs.length).toEqual(5);
 | 
				
			||||||
    });
 | 
					    });
 | 
				
			||||||
  });
 | 
					  });
 | 
				
			||||||
 | 
					 | 
				
			||||||
  describe('cycling trigger', () => {
 | 
					 | 
				
			||||||
    it('trigger should not be triggered more than once in same execution', async () => {
 | 
					 | 
				
			||||||
      const n1 = await workflow.createNode({
 | 
					 | 
				
			||||||
        type: 'create',
 | 
					 | 
				
			||||||
        config: {
 | 
					 | 
				
			||||||
          collection: 'posts',
 | 
					 | 
				
			||||||
          params: {
 | 
					 | 
				
			||||||
            values: {
 | 
					 | 
				
			||||||
              title: 't2'
 | 
					 | 
				
			||||||
            }
 | 
					 | 
				
			||||||
          }
 | 
					 | 
				
			||||||
        }
 | 
					 | 
				
			||||||
      });
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
      const post = await PostRepo.create({ values: { title: 't1' } });
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
      await sleep(500);
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
      const posts = await PostRepo.find();
 | 
					 | 
				
			||||||
      expect(posts.length).toBe(2);
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
      const [execution] = await workflow.getExecutions();
 | 
					 | 
				
			||||||
      expect(execution.status).toBe(EXECUTION_STATUS.RESOLVED);
 | 
					 | 
				
			||||||
    });
 | 
					 | 
				
			||||||
  });
 | 
					 | 
				
			||||||
});
 | 
					});
 | 
				
			||||||
 | 
				
			|||||||
@ -70,11 +70,9 @@ async function handler(this: CollectionTrigger, workflow: WorkflowModel, data: M
 | 
				
			|||||||
    }
 | 
					    }
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  setTimeout(() => {
 | 
					 | 
				
			||||||
  this.plugin.trigger(workflow, { data: data.get() }, {
 | 
					  this.plugin.trigger(workflow, { data: data.get() }, {
 | 
				
			||||||
    context
 | 
					    context
 | 
				
			||||||
  });
 | 
					  });
 | 
				
			||||||
  });
 | 
					 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
export default class CollectionTrigger extends Trigger {
 | 
					export default class CollectionTrigger extends Trigger {
 | 
				
			||||||
 | 
				
			|||||||
		Loading…
	
		Reference in New Issue
	
	Block a user