From d7c9febb8b5b2519462a6d4885f75c054703037b Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Mon, 22 Jul 2024 17:18:04 +0200 Subject: [PATCH 01/54] --wip-- --wip-- --wip-- --wip-- clean up use spaces in package.json --- jest.config.js | 4 +- packages/cli/new-partial-execution | 0 packages/cli/package.json | 2 +- packages/cli/src/FeatureFlags.ts | 10 + packages/cli/src/license.ts | 2 + packages/cli/src/workflow-runner.ts | 54 +- packages/core/src/WorkflowExecute.ts | 134 ++++ packages/core/src/__tests__/helpers.ts | 75 +++ packages/core/src/__tests__/utils-2.test.ts | 300 +++++++++ packages/core/src/__tests__/utils.test.ts | 638 ++++++++++++++++++++ packages/core/src/utils-2.ts | 168 ++++++ packages/core/src/utils.ts | 626 +++++++++++++++++++ packages/workflow/src/Interfaces.ts | 11 +- 13 files changed, 2006 insertions(+), 18 deletions(-) create mode 100644 packages/cli/new-partial-execution create mode 100644 packages/cli/src/FeatureFlags.ts create mode 100644 packages/core/src/__tests__/helpers.ts create mode 100644 packages/core/src/__tests__/utils-2.test.ts create mode 100644 packages/core/src/__tests__/utils.test.ts create mode 100644 packages/core/src/utils-2.ts create mode 100644 packages/core/src/utils.ts diff --git a/jest.config.js b/jest.config.js index 3caac38ef9dbf..1a6d7c31a22bc 100644 --- a/jest.config.js +++ b/jest.config.js @@ -32,8 +32,8 @@ const config = { return acc; }, {}), setupFilesAfterEnv: ['jest-expect-message'], - collectCoverage: process.env.COVERAGE_ENABLED === 'true', - coverageReporters: ['text-summary'], + collectCoverage: true, + coverageReporters: ['html'], collectCoverageFrom: ['src/**/*.ts'], }; diff --git a/packages/cli/new-partial-execution b/packages/cli/new-partial-execution new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/cli/package.json b/packages/cli/package.json index 1b33555a962f3..9b9b2afa00790 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -22,7 +22,7 @@ "lint": "eslint . --quiet", "lintfix": "eslint . --fix", "start": "run-script-os", - "start:default": "cd bin && ./n8n", + "start:default": "node --inspect bin/n8n", "start:windows": "cd bin && n8n", "test": "pnpm test:sqlite", "test:sqlite": "N8N_LOG_LEVEL=silent DB_TYPE=sqlite jest", diff --git a/packages/cli/src/FeatureFlags.ts b/packages/cli/src/FeatureFlags.ts new file mode 100644 index 0000000000000..49b95c1088400 --- /dev/null +++ b/packages/cli/src/FeatureFlags.ts @@ -0,0 +1,10 @@ +import fs from 'fs/promises'; + +export async function isPartialExecutionEnabled() { + try { + await fs.access('new-partial-execution'); + return true; + } catch (error) { + return false; + } +} diff --git a/packages/cli/src/license.ts b/packages/cli/src/license.ts index 75a57efd2ca33..400a2c7623f24 100644 --- a/packages/cli/src/license.ts +++ b/packages/cli/src/license.ts @@ -230,6 +230,7 @@ export class License { } isFeatureEnabled(feature: BooleanLicenseFeature) { + return true; return this.manager?.hasFeatureEnabled(feature) ?? false; } @@ -370,6 +371,7 @@ export class License { } getTeamProjectLimit() { + return UNLIMITED_LICENSE_QUOTA; return this.getFeatureValue(LICENSE_QUOTAS.TEAM_PROJECT_LIMIT) ?? 0; } diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index aea8bfc15e520..71b577d21d459 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -39,6 +39,8 @@ import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.serv import { EventService } from './events/event.service'; +import { isPartialExecutionEnabled } from './FeatureFlags'; + @Service() export class WorkflowRunner { private scalingService: ScalingService; @@ -113,6 +115,7 @@ export class WorkflowRunner { async run( data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, + // TODO: Figure out what this is for realtime?: boolean, restartExecutionId?: string, responsePromise?: IDeferredPromise, @@ -138,6 +141,7 @@ export class WorkflowRunner { this.activeExecutions.attachResponsePromise(executionId, responsePromise); } + // NOTE: queue mode if (this.executionsMode === 'queue' && data.executionMode !== 'manual') { // Do not run "manual" executions in bull because sending events to the // frontend would not be possible @@ -198,6 +202,9 @@ export class WorkflowRunner { ): Promise { const workflowId = data.workflowData.id; if (loadStaticData === true && workflowId) { + // TODO: Can we assign static data to a variable instead of mutating `data`? + // NOTE: This is the workflow and node specific data that can be saved + // and retrieved with the code node. data.workflowData.staticData = await this.workflowStaticDataService.getStaticDataById(workflowId); } @@ -215,6 +222,8 @@ export class WorkflowRunner { let pinData: IPinData | undefined; if (data.executionMode === 'manual') { + // TODO: Find out why pin data exists on both objects and if we need both + // or if one can be cleaned up. pinData = data.pinData ?? data.workflowData.pinData; } @@ -229,6 +238,8 @@ export class WorkflowRunner { settings: workflowSettings, pinData, }); + // NOTE: This seems like a catchall so we can pass anything deep into the + // workflow execution engine. const additionalData = await WorkflowExecuteAdditionalData.getBase( data.userId, undefined, @@ -244,6 +255,8 @@ export class WorkflowRunner { { executionId }, ); let workflowExecution: PCancelable; + // NOTE: This is were we update the status of the execution in the + // database. And this is where the race condition happens. await this.executionRepository.updateStatus(executionId, 'running'); try { @@ -255,6 +268,8 @@ export class WorkflowRunner { }, ]; + // TODO: Why the detour through the WorkflowExecuteAdditionalData to call + // ActiveExecutions? additionalData.setExecutionStatus = WorkflowExecuteAdditionalData.setExecutionStatus.bind({ executionId, }); @@ -264,7 +279,12 @@ export class WorkflowRunner { }); if (data.executionData !== undefined) { - this.logger.debug(`Execution ID ${executionId} had Execution data. Running with payload.`, { + // TODO: What's the difference between `data.executionData` and `data.runData`? + // I think this is the data coming from a webhook or a trigger, e.g. the + // body of a POST request or the message of a queue message. + console.trace('data.executionData', JSON.stringify(data.executionData, null, 2)); + + console.debug(`Execution ID ${executionId} had Execution data. Running with payload.`, { executionId, }); const workflowExecute = new WorkflowExecute( @@ -278,7 +298,8 @@ export class WorkflowRunner { data.startNodes === undefined || data.startNodes.length === 0 ) { - this.logger.debug(`Execution ID ${executionId} will run executing all nodes.`, { + // Full Execution + console.debug(`Execution ID ${executionId} will run executing all nodes.`, { executionId, }); // Execute all nodes @@ -294,16 +315,29 @@ export class WorkflowRunner { data.pinData, ); } else { - this.logger.debug(`Execution ID ${executionId} is a partial execution.`, { executionId }); + // Partial Execution + console.debug(`Execution ID ${executionId} is a partial execution.`, { executionId }); // Execute only the nodes between start and destination nodes const workflowExecute = new WorkflowExecute(additionalData, data.executionMode); - workflowExecution = workflowExecute.runPartialWorkflow( - workflow, - data.runData, - data.startNodes, - data.destinationNode, - data.pinData, - ); + + if (await isPartialExecutionEnabled()) { + console.debug('Partial execution is enabled'); + workflowExecution = workflowExecute.runPartialWorkflow2( + workflow, + data.runData, + data.startNodes, + data.destinationNode, + data.pinData, + ); + } else { + workflowExecution = workflowExecute.runPartialWorkflow( + workflow, + data.runData, + data.startNodes, + data.destinationNode, + data.pinData, + ); + } } this.activeExecutions.attachWorkflowExecution(executionId, workflowExecution); diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index fe4d60587b7e9..9e3eb3e18437a 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-use-before-define */ /* eslint-disable @typescript-eslint/prefer-optional-chain */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ @@ -49,6 +50,16 @@ import { import get from 'lodash/get'; import * as NodeExecuteFunctions from './NodeExecuteFunctions'; +import * as a from 'assert'; +import { + DirectedGraph, + findCycles, + findStartNodes, + findSubgraph2, + findTriggerForPartialExecution, +} from './utils'; +import { recreateNodeExecutionStack } from './utils-2'; + export class WorkflowExecute { private status: ExecutionStatus = 'new'; @@ -284,6 +295,8 @@ export class WorkflowExecute { } } + console.log(JSON.stringify(nodeExecutionStack, null, 2)); + this.runExecutionData = { startData: { destinationNode, @@ -305,6 +318,127 @@ export class WorkflowExecute { return this.processRunExecutionData(workflow); } + // eslint-disable-next-line @typescript-eslint/promise-function-async, complexity + runPartialWorkflow2( + workflow: Workflow, + runData: IRunData, + _startNodes: StartNodeData[], + destinationNodeName?: string, + pinData?: IPinData, + ): PCancelable { + debugger; + try { + const graph = DirectedGraph.fromWorkflow(workflow); + + if (destinationNodeName === undefined) { + throw new ApplicationError('destinationNodeName is undefined'); + } + + const destinationNode = workflow.getNode(destinationNodeName); + + if (destinationNode === null) { + throw new ApplicationError( + `Could not find a node with the name ${destinationNodeName} in the workflow.`, + ); + } + + // 1. Find the Trigger + + const trigger = findTriggerForPartialExecution(workflow, destinationNodeName); + + if (trigger === undefined) { + throw new ApplicationError( + 'The destination node is not connected to any trigger. Partial executions need a trigger.', + ); + } + + // 2. Find the Subgraph + + // TODO: filter out the branches that connect to other triggers than the one + // selected above. + const subgraph = findSubgraph2(graph, destinationNode, trigger); + //const filteredNodes = findSubgraph(workflow, destinationNode); + const filteredNodes = subgraph.getNodes(); + console.log('filteredNodes', filteredNodes); + + // 3. Find the Start Nodes + + const startNodes = findStartNodes(subgraph, trigger, destinationNode, runData); + console.log('startNodes', JSON.stringify(startNodes, null, 2)); + + // 4. Detect Cycles + + const cycles = findCycles(workflow); + + // 5. Handle Cycles + + if (cycles.length) { + // TODO: handle + } + + // 6. Clean Run Data + + // 7. Recreate Execution Stack + + this.status = 'running'; + + //_startNodes = startNodes.map((sn) => ({ + // name: sn.node.name, + // sourceData: sn.sourceData + // ? { + // ...sn.sourceData, + // previousNode: sn.sourceData.previousNode.name, + // } + // : null, + //})); + + //console.log('_startNodes', _startNodes); + + // Initialize the nodeExecutionStack and waitingExecution with + // the data from runData + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack( + subgraph.toWorkflow({ ...workflow }), + startNodes, + destinationNode.name, + runData, + pinData ?? {}, + ); + + //console.log(JSON.stringify(nodeExecutionStack, null, 2)); + + // 8. Execute + + this.runExecutionData = { + startData: { + destinationNode: destinationNodeName, + runNodeFilter: Array.from(filteredNodes.values()).map((node) => node.name), + }, + resultData: { + runData, + pinData, + }, + executionData: { + contextData: {}, + nodeExecutionStack, + metadata: {}, + waitingExecution, + waitingExecutionSource, + }, + }; + + return this.processRunExecutionData( + //workflow, + subgraph.toWorkflow({ + ...workflow, + }), + ); + } catch (error) { + console.error(error); + throw error; + } + } + /** * Executes the hook with the given name * diff --git a/packages/core/src/__tests__/helpers.ts b/packages/core/src/__tests__/helpers.ts new file mode 100644 index 0000000000000..fa9039b5a8e61 --- /dev/null +++ b/packages/core/src/__tests__/helpers.ts @@ -0,0 +1,75 @@ +import { NodeConnectionType } from 'n8n-workflow'; +import type { INodeParameters, INode, ITaskData, IDataObject } from 'n8n-workflow'; + +interface StubNode { + name: string; + parameters?: INodeParameters; + disabled?: boolean; +} + +export function createNodeData(stubData: StubNode): INode { + return { + name: stubData.name, + parameters: stubData.parameters ?? {}, + type: 'test.set', + typeVersion: 1, + id: 'uuid-1234', + position: [100, 100], + disabled: stubData.disabled ?? false, + }; +} + +type TaskData = { + data: IDataObject; + outputIndex?: number; + nodeConnectionType?: NodeConnectionType; +}; + +export function toITaskData(taskData: TaskData[]): ITaskData { + const result: ITaskData = { + executionStatus: 'success', + executionTime: 0, + startTime: 0, + source: [], + data: {}, + }; + + // NOTE: Here to make TS happy. + result.data = result.data ?? {}; + for (const taskDatum of taskData) { + const type = taskDatum.nodeConnectionType ?? NodeConnectionType.Main; + const outputIndex = taskDatum.outputIndex ?? 0; + + result.data[type] = result.data[type] ?? []; + const dataConnection = result.data[type]; + dataConnection[outputIndex] = [{ json: taskDatum.data }]; + } + + for (const [type, dataConnection] of Object.entries(result.data)) { + for (const [index, maybe] of dataConnection.entries()) { + //result.data[type][index] = + // maybe ?? randomInt(2) === 0 + // ? null + // : // NOTE: The FE sends an empty array instead of null. I have yet to + // // figure out if there is a different when executing a workflow. + // []; + result.data[type][index] = maybe ?? null; + } + //result.data[type] = dataConnection.map((maybe) => + // maybe ? maybe.map((maybe) => maybe ?? null) : null, + //); + } + + return result; +} + +export const nodeTypes = { + getByName: jest.fn(), + getByNameAndVersion: jest.fn(), + getKnownTypes: jest.fn(), +}; + +export const defaultWorkflowParameter = { + active: false, + nodeTypes, +}; diff --git a/packages/core/src/__tests__/utils-2.test.ts b/packages/core/src/__tests__/utils-2.test.ts new file mode 100644 index 0000000000000..cbd57e2b1d313 --- /dev/null +++ b/packages/core/src/__tests__/utils-2.test.ts @@ -0,0 +1,300 @@ +import { recreateNodeExecutionStack } from '@/utils-2'; +import { createNodeData, defaultWorkflowParameter, toITaskData } from './helpers'; +import { DirectedGraph, findSubgraph2, StartNodeData } from '@/utils'; +import type { IPinData, IRunData } from 'n8n-workflow'; + +// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ +// If you update the tests please update the diagrams as well. +// +// Map +// 0 means the output has no data. +// 1 means the output has data. +// ►► denotes the node that the user wants to execute to. +// XX denotes that the node is disabled + +describe('recreateNodeExecutionStack', () => { + // ►► + // ┌───────┐ ┌────┐ + // │Trigger│1─────►│Node│ + // └───────┘ └────┘ + test('simple', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(trigger, node) + .addConnections({ from: trigger, to: node }); + + const workflow = findSubgraph2(graph, node, trigger).toWorkflow({ + ...defaultWorkflowParameter, + }); + const startNodes: StartNodeData[] = [ + { + node: node, + sourceData: { + previousNode: trigger, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + }, + ]; + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + }; + const pinData = {}; + + // + // ACT + // + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(workflow, startNodes, node.name, runData, pinData); + + // + // ASSERT + // + expect(nodeExecutionStack).toEqual([ + { + data: { main: [[{ json: { value: 1 } }]] }, + node: { + disabled: false, + id: 'uuid-1234', + name: 'node', + parameters: {}, + position: [100, 100], + type: 'test.set', + typeVersion: 1, + }, + source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, + }, + ]); + + expect(waitingExecution).toEqual({ node: { '0': { main: [[{ json: { value: 1 } }]] } } }); + expect(waitingExecutionSource).toEqual({ + node: { + '0': { + main: [ + { previousNode: 'trigger', previousNodeOutput: undefined, previousNodeRun: undefined }, + ], + }, + }, + }); + }); + + // ►► + // ┌───────┐ ┌────┐ + // │Trigger│0─────►│Node│ + // └───────┘ └────┘ + test('simple', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node = createNodeData({ name: 'node' }); + + const workflow = new DirectedGraph() + .addNodes(trigger, node) + .addConnections({ from: trigger, to: node }) + .toWorkflow({ ...defaultWorkflowParameter }); + const startNodes: StartNodeData[] = [{ node: trigger }]; + const runData: IRunData = {}; + const pinData: IPinData = {}; + + // + // ACT + // + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(workflow, startNodes, node.name, runData, pinData); + + // + // ASSERT + // + expect(nodeExecutionStack).toHaveLength(1); + expect(nodeExecutionStack).toEqual([ + { + data: { main: [[{ json: {} }]] }, + node: { + disabled: false, + id: 'uuid-1234', + name: 'trigger', + parameters: {}, + position: [100, 100], + type: 'test.set', + typeVersion: 1, + }, + source: null, + }, + ]); + + expect(waitingExecution).toEqual({ node: { '0': { main: [null] } } }); + expect(waitingExecutionSource).toEqual({ node: { '0': { main: [null] } } }); + }); + + // PD ►► + // ┌───────┐ ┌────┐ + // │Trigger│1─────►│Node│ + // └───────┘ └────┘ + test('pinned data', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node = createNodeData({ name: 'node' }); + + const workflow = new DirectedGraph() + .addNodes(trigger, node) + .addConnections({ from: trigger, to: node }) + .toWorkflow({ ...defaultWorkflowParameter }); + const startNodes: StartNodeData[] = [ + { + node, + sourceData: { previousNode: trigger, previousNodeRun: 0, previousNodeOutput: 0 }, + }, + ]; + const runData: IRunData = {}; + const pinData: IPinData = { + [trigger.name]: [{ json: { value: 1 } }], + }; + + // + // ACT + // + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(workflow, startNodes, node.name, runData, pinData); + + // + // ASSERT + // + expect(nodeExecutionStack).toHaveLength(1); + expect(nodeExecutionStack).toEqual([ + { + data: { main: [[{ json: { value: 1 } }]] }, + node: { + disabled: false, + id: 'uuid-1234', + name: 'node', + parameters: {}, + position: [100, 100], + type: 'test.set', + typeVersion: 1, + }, + source: { + main: [{ previousNode: trigger.name, previousNodeRun: 0, previousNodeOutput: 0 }], + }, + }, + ]); + + expect(waitingExecution).toEqual({ node: { '0': { main: [null] } } }); + expect(waitingExecutionSource).toEqual({ node: { '0': { main: [null] } } }); + }); + + // XX ►► + // ┌───────┐ ┌─────┐ ┌─────┐ + // │Trigger│1─┬──►│Node1│──┬───►│Node3│ + // └───────┘ │ └─────┘ │ └─────┘ + // │ │ + // │ ┌─────┐ │ + // └──►│Node2│1─┘ + // └─────┘ + test('disabled nodes', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1', disabled: true }); + const node2 = createNodeData({ name: 'node2' }); + const node3 = createNodeData({ name: 'node3' }); + + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2, node3) + .addConnections( + { from: trigger, to: node1 }, + { from: trigger, to: node2 }, + { from: node1, to: node3 }, + { from: node2, to: node3 }, + ); + + const workflow = findSubgraph2(graph, node3, trigger).toWorkflow({ + ...defaultWorkflowParameter, + }); + const startNodes: StartNodeData[] = [ + { + node: node3, + sourceData: { + previousNode: trigger, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + }, + { + node: node3, + sourceData: { + previousNode: node2, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + }, + ]; + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + [node2.name]: [toITaskData([{ data: { value: 1 } }])], + [node3.name]: [toITaskData([{ data: { value: 1 } }])], + }; + const pinData = {}; + + // + // ACT + // + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(workflow, startNodes, node2.name, runData, pinData); + + // + // ASSERT + // + + expect(nodeExecutionStack).toEqual([ + { + data: { main: [[{ json: { value: 1 } }]] }, + node: { + disabled: false, + id: 'uuid-1234', + name: 'node3', + parameters: {}, + position: [100, 100], + type: 'test.set', + typeVersion: 1, + }, + source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, + }, + { + data: { main: [[{ json: { value: 1 } }]] }, + node: { + disabled: false, + id: 'uuid-1234', + name: 'node3', + parameters: {}, + position: [100, 100], + type: 'test.set', + typeVersion: 1, + }, + source: { main: [{ previousNode: 'node2', previousNodeOutput: 0, previousNodeRun: 0 }] }, + }, + ]); + + expect(waitingExecution).toEqual({ + node2: { '0': { main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]] } }, + }); + expect(waitingExecutionSource).toEqual({ + node2: { + '0': { + main: [ + { previousNode: 'trigger', previousNodeOutput: undefined, previousNodeRun: undefined }, + { previousNode: 'trigger', previousNodeOutput: undefined, previousNodeRun: undefined }, + ], + }, + }, + }); + }); +}); diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts new file mode 100644 index 0000000000000..4a079cdc4434a --- /dev/null +++ b/packages/core/src/__tests__/utils.test.ts @@ -0,0 +1,638 @@ +// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ +// If you update the tests please update the diagrams as well. +// +// Map +// 0 means the output has no data. +// 1 means the output has data. +// ►► denotes the node that the user wants to execute to. +// XX denotes that the node is disabled +// +// TODO: rename all nodes to generic names, don't use if, merge, etc. + +import type { IConnections, INode, IPinData, IRunData } from 'n8n-workflow'; +import { NodeConnectionType, Workflow } from 'n8n-workflow'; +import { DirectedGraph, findStartNodes, findSubgraph, findSubgraph2, isDirty } from '../utils'; +import { createNodeData, toITaskData, nodeTypes, defaultWorkflowParameter } from './helpers'; + +test('toITaskData', function () { + expect(toITaskData([{ data: { value: 1 } }])).toEqual({ + executionStatus: 'success', + executionTime: 0, + source: [], + startTime: 0, + data: { + main: [[{ json: { value: 1 } }]], + }, + }); + + expect(toITaskData([{ data: { value: 1 }, outputIndex: 1 }])).toEqual({ + executionStatus: 'success', + executionTime: 0, + source: [], + startTime: 0, + data: { + main: [null, [{ json: { value: 1 } }]], + }, + }); + + expect( + toITaskData([ + { data: { value: 1 }, outputIndex: 1, nodeConnectionType: NodeConnectionType.AiAgent }, + ]), + ).toEqual({ + executionStatus: 'success', + executionTime: 0, + source: [], + startTime: 0, + data: { + [NodeConnectionType.AiAgent]: [null, [{ json: { value: 1 } }]], + }, + }); + + expect( + toITaskData([ + { data: { value: 1 }, outputIndex: 0 }, + { data: { value: 2 }, outputIndex: 1 }, + ]), + ).toEqual({ + executionStatus: 'success', + executionTime: 0, + startTime: 0, + source: [], + data: { + main: [ + [ + { + json: { value: 1 }, + }, + ], + [ + { + json: { value: 2 }, + }, + ], + ], + }, + }); +}); + +type Connection = { + from: INode; + to: INode; + type?: NodeConnectionType; + outputIndex?: number; + inputIndex?: number; +}; + +function toIConnections(connections: Connection[]): IConnections { + const result: IConnections = {}; + + for (const connection of connections) { + const type = connection.type ?? NodeConnectionType.Main; + const outputIndex = connection.outputIndex ?? 0; + const inputIndex = connection.inputIndex ?? 0; + + result[connection.from.name] = result[connection.from.name] ?? { + [type]: [], + }; + const resultConnection = result[connection.from.name]; + resultConnection[type][outputIndex] = resultConnection[type][outputIndex] ?? []; + const group = resultConnection[type][outputIndex]; + + group.push({ + node: connection.to.name, + type, + index: inputIndex, + }); + } + + return result; +} + +test('toIConnections', () => { + const node1 = createNodeData({ name: 'Basic Node 1' }); + const node2 = createNodeData({ name: 'Basic Node 2' }); + + expect( + toIConnections([{ from: node1, to: node2, type: NodeConnectionType.Main, outputIndex: 0 }]), + ).toEqual({ + [node1.name]: { + // output group + main: [ + // first output + [ + // first connection + { + node: node2.name, + type: NodeConnectionType.Main, + index: 0, + }, + ], + ], + }, + }); +}); + +//export interface INodeTypes { +// getByName(nodeType: string): INodeType | IVersionedNodeType; +// getByNameAndVersion(nodeType: string, version?: number): INodeType; +// getKnownTypes(): IDataObject; +//} + +describe('isDirty', () => { + test("if the node has pinned data it's not dirty", () => { + const node = createNodeData({ name: 'Basic Node' }); + + const pinData: IPinData = { + [node.name]: [{ json: { value: 1 } }], + }; + + expect(isDirty(node, undefined, pinData)).toBe(false); + }); + + test("if the node has run data it's not dirty", () => { + const node = createNodeData({ name: 'Basic Node' }); + + const runData: IRunData = { + [node.name]: [toITaskData([{ data: { value: 1 } }])], + }; + + expect(isDirty(node, runData)).toBe(false); + }); +}); + +describe('findStartNodes', () => { + test('simple', () => { + const node = createNodeData({ name: 'Basic Node' }); + + const graph = new DirectedGraph().addNode(node); + + expect(findStartNodes(graph, node, node)).toStrictEqual([{ node, sourceData: undefined }]); + }); + + test('less simple', () => { + const node1 = createNodeData({ name: 'Basic Node 1' }); + const node2 = createNodeData({ name: 'Basic Node 2' }); + + const graph = new DirectedGraph() + .addNodes(node1, node2) + .addConnection({ from: node1, to: node2 }); + + expect(findStartNodes(graph, node1, node2)).toStrictEqual([ + { node: node1, sourceData: undefined }, + ]); + + const runData: IRunData = { + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + }; + + expect(findStartNodes(graph, node1, node2, runData)).toStrictEqual([ + { + node: node2, + sourceData: { previousNode: node1, previousNodeOutput: 0, previousNodeRun: 0 }, + }, + ]); + }); + + // + // ┌────┐ + // │ │1───┐ ┌────┐ + // │ if │ ├───►│noOp│ + // │ │1───┘ └────┘ + // └────┘ + // + // All nodes have run data. `findStartNodes` should return noOp twice + // because it has 2 input connections. + test('multiple outputs', () => { + const ifNode = createNodeData({ name: 'If' }); + const noOp = createNodeData({ name: 'NoOp' }); + + const graph = new DirectedGraph() + .addNodes(ifNode, noOp) + .addConnections( + { from: ifNode, to: noOp, outputIndex: 0, inputIndex: 0 }, + { from: ifNode, to: noOp, outputIndex: 1, inputIndex: 0 }, + ); + + const runData: IRunData = { + [ifNode.name]: [ + toITaskData([ + { data: { value: 1 }, outputIndex: 0 }, + { data: { value: 1 }, outputIndex: 1 }, + ]), + ], + [noOp.name]: [toITaskData([{ data: { value: 1 } }])], + }; + + const startNodes = findStartNodes(graph, ifNode, noOp, runData); + + expect(startNodes).toHaveLength(2); + expect(startNodes).toContainEqual({ + node: noOp, + sourceData: { + previousNode: ifNode, + previousNodeOutput: 0, + previousNodeRun: 0, + }, + }); + expect(startNodes).toContainEqual({ + node: noOp, + sourceData: { + previousNode: ifNode, + previousNodeOutput: 1, + previousNodeRun: 0, + }, + }); + }); + + test('medium', () => { + const trigger = createNodeData({ name: 'Trigger' }); + const ifNode = createNodeData({ name: 'If' }); + const merge1 = createNodeData({ name: 'Merge1' }); + const merge2 = createNodeData({ name: 'Merge2' }); + const noOp = createNodeData({ name: 'NoOp' }); + + const graph = new DirectedGraph() + .addNodes(trigger, ifNode, merge1, merge2, noOp) + .addConnections( + { from: trigger, to: ifNode }, + { from: ifNode, to: merge1, outputIndex: 0, inputIndex: 0 }, + { from: ifNode, to: merge1, outputIndex: 1, inputIndex: 1 }, + { from: ifNode, to: merge2, outputIndex: 0, inputIndex: 1 }, + { from: ifNode, to: merge2, outputIndex: 1, inputIndex: 0 }, + { from: merge1, to: noOp }, + { from: merge2, to: noOp }, + ); + + // no run data means the trigger is the start node + expect(findStartNodes(graph, trigger, noOp)).toEqual([ + { node: trigger, sourceData: undefined }, + ]); + + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + [ifNode.name]: [toITaskData([{ data: { value: 1 } }])], + [merge1.name]: [toITaskData([{ data: { value: 1 } }])], + [merge2.name]: [toITaskData([{ data: { value: 1 } }])], + [noOp.name]: [toITaskData([{ data: { value: 1 } }])], + }; + + const startNodes = findStartNodes(graph, trigger, noOp, runData); + expect(startNodes).toHaveLength(2); + + // run data for everything + expect(startNodes).toContainEqual({ + node: noOp, + sourceData: { + previousNode: merge1, + previousNodeOutput: 0, + previousNodeRun: 0, + }, + }); + + expect(startNodes).toContainEqual({ + node: noOp, + sourceData: { + previousNode: merge2, + previousNodeOutput: 0, + previousNodeRun: 0, + }, + }); + }); + + // + // ┌────┐ ┌─────┐ + // │ │1───────►│ │ + // │ if │ │merge│O + // │ │O───────►│ │ + // └────┘ └─────┘ + // + // The merge node only gets data on one input, so the it should only be once + // in the start nodes + test('multiple connections', () => { + const ifNode = createNodeData({ name: 'if' }); + const merge = createNodeData({ name: 'merge' }); + + const graph = new DirectedGraph() + .addNodes(ifNode, merge) + .addConnections( + { from: ifNode, to: merge, outputIndex: 0, inputIndex: 0 }, + { from: ifNode, to: merge, outputIndex: 1, inputIndex: 1 }, + ); + + const startNodes = findStartNodes(graph, ifNode, merge, { + [ifNode.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], + }); + + expect(startNodes).toHaveLength(1); + expect(startNodes).toContainEqual({ + node: merge, + sourceData: { + previousNode: ifNode, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + }); + }); + + // + // ┌────┐ ┌─────┐ + // │ │0───────►│ │ + // │ if │ │merge│O + // │ │1───────►│ │ + // └────┘ └─────┘ + // + // The merge node only gets data on one input, so the it should only be once + // in the start nodes + test('multiple connections', () => { + const ifNode = createNodeData({ name: 'if' }); + const merge = createNodeData({ name: 'merge' }); + + const graph = new DirectedGraph() + .addNodes(ifNode, merge) + .addConnections( + { from: ifNode, to: merge, outputIndex: 0, inputIndex: 0 }, + { from: ifNode, to: merge, outputIndex: 1, inputIndex: 1 }, + ); + + const startNodes = findStartNodes(graph, ifNode, merge, { + [ifNode.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 1 }])], + }); + + expect(startNodes).toHaveLength(1); + expect(startNodes).toContainEqual({ + node: merge, + sourceData: { + previousNode: ifNode, + previousNodeRun: 0, + previousNodeOutput: 1, + }, + }); + }); + + // ┌────┐ ┌─────┐ + // │ │1───────►│ │ + // │ if │ │merge│O + // │ │1───────►│ │ + // └────┘ └─────┘ + // + // The merge node gets data on both inputs, so the it should be in the start + // nodes twice. + test('multiple connections', () => { + const ifNode = createNodeData({ name: 'if' }); + const merge = createNodeData({ name: 'merge' }); + + const graph = new DirectedGraph() + .addNodes(ifNode, merge) + .addConnections( + { from: ifNode, to: merge, outputIndex: 0 }, + { from: ifNode, to: merge, outputIndex: 1 }, + ); + + const startNodes = findStartNodes(graph, ifNode, merge, { + [ifNode.name]: [ + toITaskData([ + { data: { value: 1 }, outputIndex: 0 }, + { data: { value: 1 }, outputIndex: 1 }, + ]), + ], + }); + + expect(startNodes).toHaveLength(2); + expect(startNodes).toContainEqual({ + node: merge, + sourceData: { + previousNode: ifNode, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + }); + expect(startNodes).toContainEqual({ + node: merge, + sourceData: { + previousNode: ifNode, + previousNodeRun: 0, + previousNodeOutput: 1, + }, + }); + }); + + // ► + // ┌───────┐ ┌────┐ ┌─────┐ + // │ │ │ │0───────►│ │ + // │Trigger│1──────►│ if │ │merge│ + // │ │ │ │1───────►│ │ + // └───────┘ └────┘ └─────┘ + test('multiple connections with trigger', () => { + const trigger = createNodeData({ name: 'trigger' }); + const ifNode = createNodeData({ name: 'if' }); + const merge = createNodeData({ name: 'merge' }); + + const graph = new DirectedGraph() + .addNodes(trigger, ifNode, merge) + .addConnections( + { from: trigger, to: ifNode }, + { from: ifNode, to: merge, outputIndex: 0 }, + { from: ifNode, to: merge, outputIndex: 1 }, + ); + + const startNodes = findStartNodes(graph, ifNode, merge, { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + [ifNode.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 1 }])], + }); + + expect(startNodes).toHaveLength(1); + expect(startNodes).toContainEqual({ + node: merge, + sourceData: { + previousNode: ifNode, + previousNodeRun: 0, + previousNodeOutput: 1, + }, + }); + }); +}); + +describe('findSubgraph', () => { + test('simple', () => { + const from = createNodeData({ name: 'From' }); + const to = createNodeData({ name: 'To' }); + + const graph = new Workflow({ + nodes: [from, to], + connections: toIConnections([{ from, to }]), + active: false, + nodeTypes, + }); + + const subgraph = findSubgraph(graph, to); + + expect(subgraph).toHaveLength(2); + expect(subgraph).toContainEqual(to); + expect(subgraph).toContainEqual(from); + }); + + // + // ┌────┐ ┌────┐ + // │ │O───────►│ │ + // │ if │ │noOp│ + // │ │O───────►│ │ + // └────┘ └────┘ + // + test('multiple connections', () => { + const ifNode = createNodeData({ name: 'If' }); + const noOp = createNodeData({ name: 'noOp' }); + + const workflow = new Workflow({ + nodes: [ifNode, noOp], + connections: toIConnections([ + { from: ifNode, to: noOp, outputIndex: 0 }, + { from: ifNode, to: noOp, outputIndex: 1 }, + ]), + active: false, + nodeTypes, + }); + + const subgraph = findSubgraph(workflow, noOp); + + expect(subgraph).toHaveLength(2); + expect(subgraph).toContainEqual(noOp); + expect(subgraph).toContainEqual(ifNode); + }); +}); + +describe('findSubgraph2', () => { + test('simple', () => { + const from = createNodeData({ name: 'From' }); + const to = createNodeData({ name: 'To' }); + + const graph = new DirectedGraph().addNodes(from, to).addConnections({ from, to }); + + const subgraph = findSubgraph2(graph, to, from); + + expect(subgraph).toEqual(graph); + }); + + // ┌────┐ ┌────┐ + // │ │O───────►│ │ + // │ if │ │noOp│ + // │ │O───────►│ │ + // └────┘ └────┘ + test('multiple connections', () => { + const ifNode = createNodeData({ name: 'If' }); + const noOp = createNodeData({ name: 'noOp' }); + + const graph = new DirectedGraph() + .addNodes(ifNode, noOp) + .addConnections( + { from: ifNode, to: noOp, outputIndex: 0 }, + { from: ifNode, to: noOp, outputIndex: 1 }, + ); + + const subgraph = findSubgraph2(graph, noOp, ifNode); + + expect(subgraph).toEqual(graph); + }); + + test('disregard nodes after destination', () => { + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const node3 = createNodeData({ name: 'node3' }); + + const graph = new DirectedGraph() + .addNodes(node1, node2, node3) + .addConnections({ from: node1, to: node2 }, { from: node2, to: node3 }); + + const subgraph = findSubgraph2(graph, node2, node1); + + expect(subgraph).toEqual( + new DirectedGraph().addNodes(node1, node2).addConnections({ from: node1, to: node2 }), + ); + }); + + test('skip disabled nodes', () => { + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2', disabled: true }); + const node3 = createNodeData({ name: 'node3' }); + + const graph = new DirectedGraph() + .addNodes(node1, node2, node3) + .addConnections({ from: node1, to: node2 }, { from: node2, to: node3 }); + + const subgraph = findSubgraph2(graph, node3, node1); + + expect(subgraph).toEqual( + new DirectedGraph().addNodes(node1, node3).addConnections({ from: node1, to: node3 }), + ); + }); +}); + +describe('DirectedGraph', () => { + test('roundtrip', () => { + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const node3 = createNodeData({ name: 'Node3' }); + + const graph = new DirectedGraph() + .addNodes(node1, node2, node3) + .addConnections( + { from: node1, to: node2 }, + { from: node2, to: node3 }, + { from: node3, to: node1 }, + ); + + expect(DirectedGraph.fromWorkflow(graph.toWorkflow({ ...defaultWorkflowParameter }))).toEqual( + graph, + ); + }); + + describe('getChildren', () => { + // + // ┌─────┐ ┌─────┐ + // │Node1│O─────►│Node1│ + // └─────┘ └─────┘ + // + test('simple', () => { + const from = createNodeData({ name: 'Node1' }); + const to = createNodeData({ name: 'Node2' }); + + const graph = new DirectedGraph().addNodes(from, to).addConnections({ from, to }); + + const children = graph.getChildren(from); + expect(children).toHaveLength(1); + expect(children).toContainEqual({ + from, + to, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }); + }); + + test('medium', () => { + const from = createNodeData({ name: 'Node1' }); + const to = createNodeData({ name: 'Node2' }); + + const graph = new DirectedGraph() + .addNodes(from, to) + .addConnections({ from, to, outputIndex: 0 }, { from, to, outputIndex: 1 }); + + const children = graph.getChildren(from); + expect(children).toHaveLength(2); + expect(children).toContainEqual({ + from, + to, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }); + expect(children).toContainEqual({ + from, + to, + inputIndex: 0, + outputIndex: 1, + type: NodeConnectionType.Main, + }); + }); + }); +}); diff --git a/packages/core/src/utils-2.ts b/packages/core/src/utils-2.ts new file mode 100644 index 0000000000000..db26331d90971 --- /dev/null +++ b/packages/core/src/utils-2.ts @@ -0,0 +1,168 @@ +import type { + IConnection, + IExecuteData, + INode, + INodeConnections, + INodeExecutionData, + IPinData, + IRunData, + ISourceData, + ITaskDataConnectionsSource, + IWaitingForExecution, + IWaitingForExecutionSource, + Workflow, +} from 'n8n-workflow'; + +import * as a from 'assert'; +import { getIncomingData, StartNodeData } from './utils'; + +// eslint-disable-next-line @typescript-eslint/promise-function-async, complexity +export function recreateNodeExecutionStack( + workflow: Workflow, + // TODO: turn this into StartNodeData from utils + startNodes: StartNodeData[], + destinationNodeName: string, + runData: IRunData, + pinData: IPinData, +): { + nodeExecutionStack: IExecuteData[]; + waitingExecution: IWaitingForExecution; + waitingExecutionSource: IWaitingForExecutionSource; +} { + // Initialize the nodeExecutionStack and waitingExecution with + // the data from runData + const nodeExecutionStack: IExecuteData[] = []; + const waitingExecution: IWaitingForExecution = {}; + const waitingExecutionSource: IWaitingForExecutionSource = {}; + + // TODO: Don't hard code this! + const runIndex = 0; + + let incomingNodeConnections: INodeConnections | undefined; + let connection: IConnection; + + for (const startNode of startNodes) { + incomingNodeConnections = workflow.connectionsByDestinationNode[startNode.node.name]; + + const incomingData: INodeExecutionData[][] = []; + let incomingSourceData: ITaskDataConnectionsSource | null = null; + + if (incomingNodeConnections === undefined) { + incomingData.push([{ json: {} }]); + } else { + a.ok(incomingNodeConnections.main, 'the main input group is defined'); + + // Get the data of the incoming connections + incomingSourceData = { main: [] }; + for (const connections of incomingNodeConnections.main) { + for (let inputIndex = 0; inputIndex < connections.length; inputIndex++) { + connection = connections[inputIndex]; + + if (connection.node !== startNode.sourceData?.previousNode.name) { + continue; + } + + const node = workflow.getNode(connection.node); + + a.ok( + node, + `Could not find node(${connection.node}). The node is referenced by the connection "${startNode.node.name}->${connection.node}".`, + ); + + a.notEqual( + node.disabled, + true, + `Start node(${startNode.node.name}) has an incoming connection to a node that is disabled. This is not supported. The connection in question is "${startNode.node.name}->${connection.node}". Are you sure you called "findSubgraph2"?`, + ); + + if (pinData[node.name]) { + incomingData.push(pinData[node.name]); + } else { + a.ok( + runData[connection.node], + `Start node(${startNode.node.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${startNode.node.name}->${connection.node}". Are you sure the start nodes come from the "findStartNodes" function?`, + ); + + const nodeIncomingData = getIncomingData( + runData, + connection.node, + runIndex, + connection.type, + connection.index, + ); + + if (nodeIncomingData) { + incomingData.push(nodeIncomingData); + } + } + + incomingSourceData.main.push( + // NOTE: `sourceData` cannot be null or undefined. This can only + // happen if the startNode has no incoming connections, but we're + // iterating over the incoming connections here. + { + ...startNode.sourceData, + previousNode: startNode.sourceData.previousNode.name, + }, + ); + } + } + } + + const executeData: IExecuteData = { + node: workflow.getNode(startNode.node.name) as INode, + data: { main: incomingData }, + source: incomingSourceData, + }; + + nodeExecutionStack.push(executeData); + + if (destinationNodeName) { + // Check if the destinationNode has to be added as waiting + // because some input data is already fully available + incomingNodeConnections = workflow.connectionsByDestinationNode[destinationNodeName]; + if (incomingNodeConnections !== undefined) { + for (const connections of incomingNodeConnections.main) { + for (let inputIndex = 0; inputIndex < connections.length; inputIndex++) { + connection = connections[inputIndex]; + + if (waitingExecution[destinationNodeName] === undefined) { + waitingExecution[destinationNodeName] = {}; + waitingExecutionSource[destinationNodeName] = {}; + } + if (waitingExecution[destinationNodeName][runIndex] === undefined) { + waitingExecution[destinationNodeName][runIndex] = {}; + waitingExecutionSource[destinationNodeName][runIndex] = {}; + } + if (waitingExecution[destinationNodeName][runIndex][connection.type] === undefined) { + waitingExecution[destinationNodeName][runIndex][connection.type] = []; + waitingExecutionSource[destinationNodeName][runIndex][connection.type] = []; + } + + if (runData[connection.node] !== undefined) { + // Input data exists so add as waiting + // incomingDataDestination.push(runData[connection.node!][runIndex].data![connection.type][connection.index]); + waitingExecution[destinationNodeName][runIndex][connection.type].push( + runData[connection.node][runIndex].data![connection.type][connection.index], + ); + waitingExecutionSource[destinationNodeName][runIndex][connection.type].push({ + previousNode: connection.node, + previousNodeOutput: connection.index || undefined, + previousNodeRun: runIndex || undefined, + } as ISourceData); + } else { + waitingExecution[destinationNodeName][runIndex][connection.type].push(null); + waitingExecutionSource[destinationNodeName][runIndex][connection.type].push(null); + } + } + } + } + } + } + + return { + nodeExecutionStack, + waitingExecution, + waitingExecutionSource, + }; +} diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts new file mode 100644 index 0000000000000..9aa1154b3c41a --- /dev/null +++ b/packages/core/src/utils.ts @@ -0,0 +1,626 @@ +import * as a from 'assert'; +import type { + IConnections, + INode, + INodeExecutionData, + IPinData, + IRunData, + WorkflowParameters, +} from 'n8n-workflow'; +import { NodeConnectionType, Workflow } from 'n8n-workflow'; + +export function findSubgraph(workflow: Workflow, destinationNode: INode): INode[] { + const result = workflow + .getParentNodes(destinationNode.name) + .reduce( + ([set], name) => { + set.add(name); + return [set]; + }, + [new Set()], + ) + .flatMap((set) => [...set]) + .map((name) => workflow.getNode(name)) + .filter((node) => node !== null) + .filter((node) => !node.disabled); + + result.push(destinationNode); + + return result; +} + +function findSubgraph2Recursive( + graph: DirectedGraph, + destinationNode: INode, + current: INode, + trigger: INode, + newGraph: DirectedGraph, + currentBranch: Connection[], +) { + //console.log('currentBranch', currentBranch); + + if (current === trigger) { + console.log(`${current.name}: is trigger`); + for (const connection of currentBranch) { + newGraph.addNodes(connection.from, connection.to); + newGraph.addConnection(connection); + } + + return; + } + + let parentConnections = graph.getDirectParents(current); + + if (parentConnections.length === 0) { + console.log(`${current.name}: no parents`); + return; + } + + const isCycleWithDestinationNode = + current === destinationNode && currentBranch.some((c) => c.to === destinationNode); + + if (isCycleWithDestinationNode) { + console.log(`${current.name}: isCycleWithDestinationNode`); + return; + } + + const isCycleWithCurrentNode = currentBranch.some((c) => c.to === current); + + if (isCycleWithCurrentNode) { + console.log(`${current.name}: isCycleWithCurrentNode`); + // TODO: write function that adds nodes when adding connections + for (const connection of currentBranch) { + newGraph.addNodes(connection.from, connection.to); + newGraph.addConnection(connection); + } + return; + } + + if (current.disabled) { + console.log(`${current.name}: is disabled`); + const incomingConnections = graph.getDirectParents(current); + const outgoingConnections = graph + .getDirectChildren(current) + // NOTE: When a node is disabled only the first output gets data + .filter((connection) => connection.outputIndex === 0); + + parentConnections = []; + + for (const incomingConnection of incomingConnections) { + for (const outgoingConnection of outgoingConnections) { + const newConnection = { + ...incomingConnection, + to: outgoingConnection.to, + inputIndex: outgoingConnection.inputIndex, + }; + + parentConnections.push(newConnection); + currentBranch.pop(); + currentBranch.push(newConnection); + } + } + } + + for (const parentConnection of parentConnections) { + findSubgraph2Recursive(graph, destinationNode, parentConnection.from, trigger, newGraph, [ + ...currentBranch, + parentConnection, + ]); + } +} + +function findAllParentTriggers(workflow: Workflow, destinationNode: string) { + // Traverse from the destination node back until we found all trigger nodes. + // Do this recursively, because why not. + const parentNodes = workflow + .getParentNodes(destinationNode) + .map((name) => { + const node = workflow.getNode(name); + + if (!node) { + return null; + } + + return { + node, + nodeType: workflow.nodeTypes.getByNameAndVersion(node.type, node.typeVersion), + }; + }) + .filter((value) => value !== null) + .filter(({ nodeType }) => nodeType.description.group.includes('trigger')) + .map(({ node }) => node); + + return parentNodes; +} + +// TODO: write unit tests for this +export function findTriggerForPartialExecution( + workflow: Workflow, + destinationNode: string, +): INode | undefined { + const parentTriggers = findAllParentTriggers(workflow, destinationNode).filter( + (trigger) => !trigger.disabled, + ); + const pinnedTriggers = parentTriggers + // TODO: add the other filters here from `findAllPinnedActivators` + .filter((trigger) => workflow.pinData?.[trigger.name]) + .sort((n) => (n.type.endsWith('webhook') ? -1 : 1)); + + if (pinnedTriggers.length) { + return pinnedTriggers[0]; + } else { + return parentTriggers[0]; + } +} + +//function findAllPinnedActivators(workflow: Workflow, pinData?: IPinData) { +// return Object.values(workflow.nodes) +// .filter( +// (node) => +// !node.disabled && +// pinData?.[node.name] && +// ['trigger', 'webhook'].some((suffix) => node.type.toLowerCase().endsWith(suffix)) && +// node.type !== 'n8n-nodes-base.respondToWebhook', +// ) +// .sort((a) => (a.type.endsWith('webhook') ? -1 : 1)); +//} + +// TODO: deduplicate this with +// packages/cli/src/workflows/workflowExecution.service.ts +//function selectPinnedActivatorStarter( +// workflow: Workflow, +// startNodes?: string[], +// pinData?: IPinData, +//) { +// if (!pinData || !startNodes) return null; +// +// const allPinnedActivators = findAllPinnedActivators(workflow, pinData); +// +// if (allPinnedActivators.length === 0) return null; +// +// const [firstPinnedActivator] = allPinnedActivators; +// +// // full manual execution +// +// if (startNodes?.length === 0) return firstPinnedActivator ?? null; +// +// // partial manual execution +// +// /** +// * If the partial manual execution has 2+ start nodes, we search only the zeroth +// * start node's parents for a pinned activator. If we had 2+ start nodes without +// * a common ancestor and so if we end up finding multiple pinned activators, we +// * would still need to return one to comply with existing usage. +// */ +// const [firstStartNodeName] = startNodes; +// +// const parentNodeNames = +// //new Workflow({ +// // nodes: workflow.nodes, +// // connections: workflow.connections, +// // active: workflow.active, +// // nodeTypes: this.nodeTypes, +// // }). +// workflow.getParentNodes(firstStartNodeName); +// +// if (parentNodeNames.length > 0) { +// const parentNodeName = parentNodeNames.find((p) => p === firstPinnedActivator.name); +// +// return allPinnedActivators.find((pa) => pa.name === parentNodeName) ?? null; +// } +// +// return allPinnedActivators.find((pa) => pa.name === firstStartNodeName) ?? null; +//} + +// TODO: implement dirty checking for options and properties and parent nodes +// being disabled +export function isDirty(node: INode, runData: IRunData = {}, pinData: IPinData = {}): boolean { + //- it’s properties or options changed since last execution, or + + const propertiesOrOptionsChanged = false; + + if (propertiesOrOptionsChanged) { + return true; + } + + const parentNodeGotDisabled = false; + + if (parentNodeGotDisabled) { + return true; + } + + //- it has an error, or + + const hasAnError = false; + + if (hasAnError) { + return true; + } + + //- it does neither have run data nor pinned data + + const hasPinnedData = pinData[node.name] !== undefined; + + if (hasPinnedData) { + return false; + } + + const hasRunData = runData?.[node.name]; + + if (hasRunData) { + return false; + } + + return true; +} + +interface ISourceData { + previousNode: INode; + previousNodeOutput: number; // If undefined "0" gets used + previousNodeRun: number; // If undefined "0" gets used +} + +// TODO: rename to something more general, like path segment +export interface StartNodeData { + node: INode; + sourceData?: ISourceData; +} + +export function getDirectChildren(workflow: Workflow, parent: INode): StartNodeData[] { + const directChildren: StartNodeData[] = []; + + for (const [_connectionGroupName, inputs] of Object.entries( + workflow.connectionsBySourceNode[parent.name] ?? {}, + )) { + for (const [outputIndex, connections] of inputs.entries()) { + for (const connection of connections) { + const node = workflow.getNode(connection.node); + + a.ok(node, `Node(${connection.node}) does not exist in workflow.`); + + directChildren.push({ + node, + sourceData: { + previousNode: parent, + previousNodeOutput: outputIndex, + // TODO: I don't have this here. This part is working without run + // data, so there are not runs. + previousNodeRun: 0, + }, + }); + } + } + } + + return directChildren; +} + +export function getIncomingData( + runData: IRunData, + nodeName: string, + runIndex: number, + connectionType: NodeConnectionType, + outputIndex: number, +): INodeExecutionData[] | null | undefined { + a.ok(runData[nodeName], `Can't find node with name '${nodeName}' in runData.`); + a.ok( + runData[nodeName][runIndex], + `Can't find a run for index '${runIndex}' for node name '${nodeName}'`, + ); + a.ok( + runData[nodeName][runIndex].data, + `Can't find data for index '${runIndex}' for node name '${nodeName}'`, + ); + + return runData[nodeName][runIndex].data[connectionType][outputIndex]; +} + +type Key = `${string}-${number}-${string}`; + +function makeKey(from: ISourceData | undefined, to: INode): Key { + return `${from?.previousNode.name ?? 'start'}-${from?.previousNodeOutput ?? 0}-${to.name}`; +} + +function findStartNodesRecursive( + graph: DirectedGraph, + current: INode, + destination: INode, + runData: IRunData, + pinData: IPinData, + startNodes: Map, + source?: ISourceData, +) { + const nodeIsDirty = isDirty(current, runData, pinData); + + if (nodeIsDirty) { + startNodes.set(makeKey(source, current), { + node: current, + sourceData: source, + }); + return startNodes; + } + + if (current === destination) { + startNodes.set(makeKey(source, current), { node: current, sourceData: source }); + return startNodes; + } + + const outGoingConnections = graph.getDirectChildren(current); + + for (const outGoingConnection of outGoingConnections) { + // NOTE: can't use `Workflow.getNodeConnectionIndexes` here, because it + // only returns the first connection it finds. But I need all connections. + //const nodeConnection = workflow.getNodeConnectionIndexes(child.name, current.name); + //a.ok(nodeConnection, `Child(${child.name}) and parent(${current.name}) are not connected.`); + + const nodeRunData = getIncomingData( + runData, + outGoingConnection.from.name, + // // NOTE: I don't support multiple runs for now. + 0, + outGoingConnection.type, + outGoingConnection.outputIndex, + ); + const hasRunData = + nodeRunData === null || nodeRunData === undefined || nodeRunData.length === 0; + + if (hasRunData) { + continue; + } + + //if (child.sourceData) { + // const nodeRunData = getIncomingData( + // runData, + // child.sourceData.previousNode.name, + // child.sourceData.previousNodeRun, + // NodeConnectionType.Main, + // child.sourceData.previousNodeOutput, + // ); + // const hasRunData = + // nodeRunData === null || nodeRunData === undefined || nodeRunData.length === 0; + // + // if (hasRunData) { + // continue; + // } + //} + + //a.ok(child.sourceData, `Child(${child.node.name}) has no sourceDate.`); + + findStartNodesRecursive( + graph, + outGoingConnection.to, + destination, + runData, + pinData, + startNodes, + { + previousNode: current, + // NOTE: It's always 0 until I fix the bug that removes the run data for + // old runs. The FE only sends data for one run for each node. + previousNodeRun: 0, + // FIXME: Reimplement getChildNodes so I can keep track of which output + // was used to get to this child. + previousNodeOutput: outGoingConnection.outputIndex, + }, + ); + } + + return startNodes; +} + +export function findStartNodes( + graph: DirectedGraph, + trigger: INode, + destination: INode, + runData: IRunData = {}, + pinData: IPinData = {}, +): StartNodeData[] { + const startNodes = findStartNodesRecursive( + graph, + trigger, + destination, + runData, + pinData, + new Map(), + ); + return [...startNodes.values()]; +} + +export function findCycles(_workflow: Workflow) { + // TODO: implement depth first search or Tarjan's Algorithm + return []; +} + +type Connection = { + from: INode; + to: INode; + type: NodeConnectionType; + outputIndex: number; + inputIndex: number; +}; +// fromName-outputType-outputIndex-inputIndex-toName +type DirectedGraphKey = `${string}-${NodeConnectionType}-${number}-${number}-${string}`; +export class DirectedGraph { + private nodes: Map = new Map(); + + private connections: Map = new Map(); + + getNodes() { + return new Map(this.nodes.entries()); + } + + addNode(node: INode) { + this.nodes.set(node.name, node); + return this; + } + + addNodes(...nodes: INode[]) { + for (const node of nodes) { + this.addNode(node); + } + return this; + } + + addConnection(connectionInput: { + from: INode; + to: INode; + type?: NodeConnectionType; + outputIndex?: number; + inputIndex?: number; + }) { + const { from, to } = connectionInput; + + const fromExists = this.nodes.get(from.name) === from; + const toExists = this.nodes.get(to.name) === to; + + a.ok(fromExists); + a.ok(toExists); + + const connection: Connection = { + ...connectionInput, + type: connectionInput.type ?? NodeConnectionType.Main, + outputIndex: connectionInput.outputIndex ?? 0, + inputIndex: connectionInput.inputIndex ?? 0, + }; + + this.connections.set(this.makeKey(connection), connection); + return this; + } + + addConnections( + ...connectionInputs: Array<{ + from: INode; + to: INode; + type?: NodeConnectionType; + outputIndex?: number; + inputIndex?: number; + }> + ) { + for (const connectionInput of connectionInputs) { + this.addConnection(connectionInput); + } + return this; + } + + getDirectChildren(node: INode) { + const nodeExists = this.nodes.get(node.name) === node; + a.ok(nodeExists); + + const directChildren: Connection[] = []; + + for (const connection of this.connections.values()) { + if (connection.from !== node) { + continue; + } + + directChildren.push(connection); + } + + return directChildren; + } + + getChildren(node: INode): Connection[] { + const directChildren = this.getDirectChildren(node); + + return [...directChildren, ...directChildren.flatMap((child) => this.getChildren(child.to))]; + } + + getDirectParents(node: INode) { + const nodeExists = this.nodes.get(node.name) === node; + a.ok(nodeExists); + + const directParents: Connection[] = []; + + for (const connection of this.connections.values()) { + if (connection.to !== node) { + continue; + } + + directParents.push(connection); + } + + return directParents; + } + + toWorkflow(parameters: Omit): Workflow { + return new Workflow({ + ...parameters, + nodes: [...this.nodes.values()], + connections: this.toIConnections(), + }); + } + + static fromWorkflow(workflow: Workflow): DirectedGraph { + const graph = new DirectedGraph(); + + graph.addNodes(...Object.values(workflow.nodes)); + + for (const [fromNodeName, iConnection] of Object.entries(workflow.connectionsBySourceNode)) { + const from = workflow.getNode(fromNodeName); + a.ok(from); + + for (const [outputType, outputs] of Object.entries(iConnection)) { + // TODO: parse + //const type = outputType as NodeConnectionType + + for (const [outputIndex, conns] of outputs.entries()) { + for (const conn of conns) { + // TODO: What's with the input type? + const { node: toNodeName, type: _inputType, index: inputIndex } = conn; + const to = workflow.getNode(toNodeName); + a.ok(to); + + graph.addConnection({ + from, + to, + type: outputType as NodeConnectionType, + outputIndex, + inputIndex, + }); + } + } + } + } + + return graph; + } + + private toIConnections() { + const result: IConnections = {}; + + for (const connection of this.connections.values()) { + const { from, to, type, outputIndex, inputIndex } = connection; + + result[from.name] = result[from.name] ?? { + [type]: [], + }; + const resultConnection = result[from.name]; + resultConnection[type][outputIndex] = resultConnection[type][outputIndex] ?? []; + const group = resultConnection[type][outputIndex]; + + group.push({ + node: to.name, + type, + index: inputIndex, + }); + } + + return result; + } + + private makeKey(connection: Connection): DirectedGraphKey { + return `${connection.from.name}-${connection.type}-${connection.outputIndex}-${connection.inputIndex}-${connection.to.name}`; + } +} + +export function findSubgraph2( + graph: DirectedGraph, + destinationNode: INode, + trigger: INode, +): DirectedGraph { + const newGraph = new DirectedGraph(); + + findSubgraph2Recursive(graph, destinationNode, destinationNode, trigger, newGraph, []); + + return newGraph; +} diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 17f24c3096230..6dbf6ec7eccba 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2080,6 +2080,7 @@ export interface ISourceData { export interface StartNodeData { name: string; + // TODO: What is this for? sourceData: ISourceData | null; } @@ -2226,14 +2227,14 @@ export interface IWorkflowExecuteAdditionalData { } export type WorkflowExecuteMode = - | 'cli' - | 'error' + | 'cli' // unused + | 'error' // unused, but maybe used for error workflows | 'integrated' | 'internal' | 'manual' - | 'retry' - | 'trigger' - | 'webhook'; + | 'retry' // unused + | 'trigger' // unused + | 'webhook'; // unused export type WorkflowActivateMode = | 'init' From 1f99236b3d12d1870ad686b34c0a3a0bc38d2844 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 30 Jul 2024 14:49:57 +0200 Subject: [PATCH 02/54] refactor and simplify recreateNodeExecutionStack --- packages/core/src/__tests__/utils-2.test.ts | 142 +++++++------ packages/core/src/utils-2.ts | 209 ++++++++++---------- packages/core/src/utils.ts | 2 +- 3 files changed, 177 insertions(+), 176 deletions(-) diff --git a/packages/core/src/__tests__/utils-2.test.ts b/packages/core/src/__tests__/utils-2.test.ts index cbd57e2b1d313..5296b7dbd7882 100644 --- a/packages/core/src/__tests__/utils-2.test.ts +++ b/packages/core/src/__tests__/utils-2.test.ts @@ -1,14 +1,16 @@ import { recreateNodeExecutionStack } from '@/utils-2'; -import { createNodeData, defaultWorkflowParameter, toITaskData } from './helpers'; -import { DirectedGraph, findSubgraph2, StartNodeData } from '@/utils'; -import type { IPinData, IRunData } from 'n8n-workflow'; +import { createNodeData, toITaskData } from './helpers'; +import type { StartNodeData } from '@/utils'; +import { DirectedGraph, findSubgraph2 } from '@/utils'; +import { type IPinData, type IRunData } from 'n8n-workflow'; +import { AssertionError } from 'assert'; // NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ // If you update the tests please update the diagrams as well. // // Map -// 0 means the output has no data. -// 1 means the output has data. +// 0 means the output has no run data. +// 1 means the output has run data. // ►► denotes the node that the user wants to execute to. // XX denotes that the node is disabled @@ -17,7 +19,7 @@ describe('recreateNodeExecutionStack', () => { // ┌───────┐ ┌────┐ // │Trigger│1─────►│Node│ // └───────┘ └────┘ - test('simple', () => { + test('all nodes except destination node have data', () => { // // ARRANGE // @@ -28,12 +30,10 @@ describe('recreateNodeExecutionStack', () => { .addNodes(trigger, node) .addConnections({ from: trigger, to: node }); - const workflow = findSubgraph2(graph, node, trigger).toWorkflow({ - ...defaultWorkflowParameter, - }); + const workflow = findSubgraph2(graph, node, trigger); const startNodes: StartNodeData[] = [ { - node: node, + node, sourceData: { previousNode: trigger, previousNodeRun: 0, @@ -50,7 +50,7 @@ describe('recreateNodeExecutionStack', () => { // ACT // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(workflow, startNodes, node.name, runData, pinData); + recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); // // ASSERT @@ -58,15 +58,7 @@ describe('recreateNodeExecutionStack', () => { expect(nodeExecutionStack).toEqual([ { data: { main: [[{ json: { value: 1 } }]] }, - node: { - disabled: false, - id: 'uuid-1234', - name: 'node', - parameters: {}, - position: [100, 100], - type: 'test.set', - typeVersion: 1, - }, + node, source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, }, ]); @@ -87,7 +79,7 @@ describe('recreateNodeExecutionStack', () => { // ┌───────┐ ┌────┐ // │Trigger│0─────►│Node│ // └───────┘ └────┘ - test('simple', () => { + test('no nodes have data', () => { // // ARRANGE // @@ -96,8 +88,7 @@ describe('recreateNodeExecutionStack', () => { const workflow = new DirectedGraph() .addNodes(trigger, node) - .addConnections({ from: trigger, to: node }) - .toWorkflow({ ...defaultWorkflowParameter }); + .addConnections({ from: trigger, to: node }); const startNodes: StartNodeData[] = [{ node: trigger }]; const runData: IRunData = {}; const pinData: IPinData = {}; @@ -106,7 +97,7 @@ describe('recreateNodeExecutionStack', () => { // ACT // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(workflow, startNodes, node.name, runData, pinData); + recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); // // ASSERT @@ -115,15 +106,7 @@ describe('recreateNodeExecutionStack', () => { expect(nodeExecutionStack).toEqual([ { data: { main: [[{ json: {} }]] }, - node: { - disabled: false, - id: 'uuid-1234', - name: 'trigger', - parameters: {}, - position: [100, 100], - type: 'test.set', - typeVersion: 1, - }, + node: trigger, source: null, }, ]); @@ -136,7 +119,7 @@ describe('recreateNodeExecutionStack', () => { // ┌───────┐ ┌────┐ // │Trigger│1─────►│Node│ // └───────┘ └────┘ - test('pinned data', () => { + test('node before destination node has pinned data', () => { // // ARRANGE // @@ -145,8 +128,7 @@ describe('recreateNodeExecutionStack', () => { const workflow = new DirectedGraph() .addNodes(trigger, node) - .addConnections({ from: trigger, to: node }) - .toWorkflow({ ...defaultWorkflowParameter }); + .addConnections({ from: trigger, to: node }); const startNodes: StartNodeData[] = [ { node, @@ -162,7 +144,7 @@ describe('recreateNodeExecutionStack', () => { // ACT // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(workflow, startNodes, node.name, runData, pinData); + recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); // // ASSERT @@ -171,15 +153,7 @@ describe('recreateNodeExecutionStack', () => { expect(nodeExecutionStack).toEqual([ { data: { main: [[{ json: { value: 1 } }]] }, - node: { - disabled: false, - id: 'uuid-1234', - name: 'node', - parameters: {}, - position: [100, 100], - type: 'test.set', - typeVersion: 1, - }, + node, source: { main: [{ previousNode: trigger.name, previousNodeRun: 0, previousNodeOutput: 0 }], }, @@ -192,18 +166,57 @@ describe('recreateNodeExecutionStack', () => { // XX ►► // ┌───────┐ ┌─────┐ ┌─────┐ - // │Trigger│1─┬──►│Node1│──┬───►│Node3│ + // │Trigger│1────►│Node1│──────►│Node2│ + // └───────┘ └─────┘ └─────┘ + test('throws if a disabled node is found', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1', disabled: true }); + const node2 = createNodeData({ name: 'node2' }); + + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2) + .addConnections({ from: trigger, to: node1 }, { from: node1, to: node2 }); + + const startNodes: StartNodeData[] = [ + { + node: node2, + sourceData: { + previousNode: node1, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + }, + ]; + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + }; + const pinData = {}; + + // + // ACT & ASSERT + // + expect(() => + recreateNodeExecutionStack(graph, startNodes, node2, runData, pinData), + ).toThrowError(AssertionError); + }); + + // ►► + // ┌───────┐ ┌─────┐ ┌─────┐ + // │Trigger│1─┬──►│Node1│1─┬───►│Node3│ // └───────┘ │ └─────┘ │ └─────┘ // │ │ // │ ┌─────┐ │ // └──►│Node2│1─┘ // └─────┘ - test('disabled nodes', () => { + test('multiple incoming connections', () => { // // ARRANGE // const trigger = createNodeData({ name: 'trigger' }); - const node1 = createNodeData({ name: 'node1', disabled: true }); + const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); const node3 = createNodeData({ name: 'node3' }); @@ -216,14 +229,11 @@ describe('recreateNodeExecutionStack', () => { { from: node2, to: node3 }, ); - const workflow = findSubgraph2(graph, node3, trigger).toWorkflow({ - ...defaultWorkflowParameter, - }); const startNodes: StartNodeData[] = [ { node: node3, sourceData: { - previousNode: trigger, + previousNode: node1, previousNodeRun: 0, previousNodeOutput: 0, }, @@ -239,8 +249,8 @@ describe('recreateNodeExecutionStack', () => { ]; const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + [node1.name]: [toITaskData([{ data: { value: 1 } }])], [node2.name]: [toITaskData([{ data: { value: 1 } }])], - [node3.name]: [toITaskData([{ data: { value: 1 } }])], }; const pinData = {}; @@ -248,7 +258,7 @@ describe('recreateNodeExecutionStack', () => { // ACT // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(workflow, startNodes, node2.name, runData, pinData); + recreateNodeExecutionStack(graph, startNodes, node2, runData, pinData); // // ASSERT @@ -257,28 +267,12 @@ describe('recreateNodeExecutionStack', () => { expect(nodeExecutionStack).toEqual([ { data: { main: [[{ json: { value: 1 } }]] }, - node: { - disabled: false, - id: 'uuid-1234', - name: 'node3', - parameters: {}, - position: [100, 100], - type: 'test.set', - typeVersion: 1, - }, - source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, + node: node3, + source: { main: [{ previousNode: 'node1', previousNodeOutput: 0, previousNodeRun: 0 }] }, }, { data: { main: [[{ json: { value: 1 } }]] }, - node: { - disabled: false, - id: 'uuid-1234', - name: 'node3', - parameters: {}, - position: [100, 100], - type: 'test.set', - typeVersion: 1, - }, + node: node3, source: { main: [{ previousNode: 'node2', previousNodeOutput: 0, previousNodeRun: 0 }] }, }, ]); diff --git a/packages/core/src/utils-2.ts b/packages/core/src/utils-2.ts index db26331d90971..463d8da862984 100644 --- a/packages/core/src/utils-2.ts +++ b/packages/core/src/utils-2.ts @@ -1,27 +1,26 @@ -import type { - IConnection, - IExecuteData, - INode, - INodeConnections, - INodeExecutionData, - IPinData, - IRunData, - ISourceData, - ITaskDataConnectionsSource, - IWaitingForExecution, - IWaitingForExecutionSource, - Workflow, +import { + NodeConnectionType, + type IExecuteData, + type INode, + type INodeExecutionData, + type IPinData, + type IRunData, + type ISourceData, + type ITaskDataConnectionsSource, + type IWaitingForExecution, + type IWaitingForExecutionSource, } from 'n8n-workflow'; import * as a from 'assert'; -import { getIncomingData, StartNodeData } from './utils'; +import type { DirectedGraph, StartNodeData } from './utils'; +import { getIncomingData } from './utils'; // eslint-disable-next-line @typescript-eslint/promise-function-async, complexity export function recreateNodeExecutionStack( - workflow: Workflow, + graph: DirectedGraph, // TODO: turn this into StartNodeData from utils startNodes: StartNodeData[], - destinationNodeName: string, + destinationNode: INode, runData: IRunData, pinData: IPinData, ): { @@ -29,6 +28,32 @@ export function recreateNodeExecutionStack( waitingExecution: IWaitingForExecution; waitingExecutionSource: IWaitingForExecutionSource; } { + // Validate invariants. + + // The graph needs to be free of disabled nodes. If it's not it hasn't been + // passed through findSubgraph2. + for (const node of graph.getNodes().values()) { + a.notEqual( + node.disabled, + true, + `Graph contains disabled nodes. This is not supported. Make sure to pass the graph through "findSubgraph2" before calling "recreateNodeExecutionStack". The node in question is "${node.name}"`, + ); + } + + // The start nodes's sources need to have run data or pinned data. If they + // don't then they should not be the start nodes, but some node before them + // should be. Probably they are not coming from findStartNodes, make sure to + // use that function to get the start nodes. + for (const startNode of startNodes) { + if (startNode.sourceData) { + a.ok( + runData[startNode.sourceData.previousNode.name] || + pinData[startNode.sourceData.previousNode.name], + `Start nodes have sources that don't have run data. That is not supported. Make sure to get the start nodes by calling "findStartNodes". The node in question is "${startNode.node.name}" and their source is "${startNode.sourceData.previousNode.name}".`, + ); + } + } + // Initialize the nodeExecutionStack and waitingExecution with // the data from runData const nodeExecutionStack: IExecuteData[] = []; @@ -38,122 +63,104 @@ export function recreateNodeExecutionStack( // TODO: Don't hard code this! const runIndex = 0; - let incomingNodeConnections: INodeConnections | undefined; - let connection: IConnection; - for (const startNode of startNodes) { - incomingNodeConnections = workflow.connectionsByDestinationNode[startNode.node.name]; + const incomingStartNodeConnections = graph + .getDirectParents(startNode.node) + .filter((c) => c.type === NodeConnectionType.Main); const incomingData: INodeExecutionData[][] = []; let incomingSourceData: ITaskDataConnectionsSource | null = null; - if (incomingNodeConnections === undefined) { + if (incomingStartNodeConnections.length === 0) { incomingData.push([{ json: {} }]); } else { - a.ok(incomingNodeConnections.main, 'the main input group is defined'); - // Get the data of the incoming connections incomingSourceData = { main: [] }; - for (const connections of incomingNodeConnections.main) { - for (let inputIndex = 0; inputIndex < connections.length; inputIndex++) { - connection = connections[inputIndex]; + // TODO: Get rid of this whole loop. All data necessary to recreate the + // stack should exist in sourceData. The only thing that is currently + // missing is the inputIndex and that's the sole reason why we iterate + // over all incoming connections. + for (const connection of incomingStartNodeConnections) { + if (connection.from.name !== startNode.sourceData?.previousNode.name) { + continue; + } - if (connection.node !== startNode.sourceData?.previousNode.name) { - continue; - } + const node = connection.from; - const node = workflow.getNode(connection.node); + a.equal(startNode.sourceData.previousNode, node); + if (pinData[node.name]) { + incomingData.push(pinData[node.name]); + } else { a.ok( - node, - `Could not find node(${connection.node}). The node is referenced by the connection "${startNode.node.name}->${connection.node}".`, + runData[connection.from.name], + `Start node(${startNode.node.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${connection.from.name}->${connection.to.name}". Are you sure the start nodes come from the "findStartNodes" function?`, ); - a.notEqual( - node.disabled, - true, - `Start node(${startNode.node.name}) has an incoming connection to a node that is disabled. This is not supported. The connection in question is "${startNode.node.name}->${connection.node}". Are you sure you called "findSubgraph2"?`, + const nodeIncomingData = getIncomingData( + runData, + connection.from.name, + runIndex, + connection.type, + connection.inputIndex, ); - if (pinData[node.name]) { - incomingData.push(pinData[node.name]); - } else { - a.ok( - runData[connection.node], - `Start node(${startNode.node.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${startNode.node.name}->${connection.node}". Are you sure the start nodes come from the "findStartNodes" function?`, - ); - - const nodeIncomingData = getIncomingData( - runData, - connection.node, - runIndex, - connection.type, - connection.index, - ); - - if (nodeIncomingData) { - incomingData.push(nodeIncomingData); - } + if (nodeIncomingData) { + incomingData.push(nodeIncomingData); } - - incomingSourceData.main.push( - // NOTE: `sourceData` cannot be null or undefined. This can only - // happen if the startNode has no incoming connections, but we're - // iterating over the incoming connections here. - { - ...startNode.sourceData, - previousNode: startNode.sourceData.previousNode.name, - }, - ); } + + incomingSourceData.main.push({ + ...startNode.sourceData, + previousNode: startNode.sourceData.previousNode.name, + }); } } const executeData: IExecuteData = { - node: workflow.getNode(startNode.node.name) as INode, + node: startNode.node, data: { main: incomingData }, source: incomingSourceData, }; nodeExecutionStack.push(executeData); - if (destinationNodeName) { + if (destinationNode) { + const destinationNodeName = destinationNode.name; // Check if the destinationNode has to be added as waiting // because some input data is already fully available - incomingNodeConnections = workflow.connectionsByDestinationNode[destinationNodeName]; - if (incomingNodeConnections !== undefined) { - for (const connections of incomingNodeConnections.main) { - for (let inputIndex = 0; inputIndex < connections.length; inputIndex++) { - connection = connections[inputIndex]; - - if (waitingExecution[destinationNodeName] === undefined) { - waitingExecution[destinationNodeName] = {}; - waitingExecutionSource[destinationNodeName] = {}; - } - if (waitingExecution[destinationNodeName][runIndex] === undefined) { - waitingExecution[destinationNodeName][runIndex] = {}; - waitingExecutionSource[destinationNodeName][runIndex] = {}; - } - if (waitingExecution[destinationNodeName][runIndex][connection.type] === undefined) { - waitingExecution[destinationNodeName][runIndex][connection.type] = []; - waitingExecutionSource[destinationNodeName][runIndex][connection.type] = []; - } - - if (runData[connection.node] !== undefined) { - // Input data exists so add as waiting - // incomingDataDestination.push(runData[connection.node!][runIndex].data![connection.type][connection.index]); - waitingExecution[destinationNodeName][runIndex][connection.type].push( - runData[connection.node][runIndex].data![connection.type][connection.index], - ); - waitingExecutionSource[destinationNodeName][runIndex][connection.type].push({ - previousNode: connection.node, - previousNodeOutput: connection.index || undefined, - previousNodeRun: runIndex || undefined, - } as ISourceData); - } else { - waitingExecution[destinationNodeName][runIndex][connection.type].push(null); - waitingExecutionSource[destinationNodeName][runIndex][connection.type].push(null); - } + const incomingDestinationNodeConnections = graph + .getDirectParents(destinationNode) + .filter((c) => c.type === NodeConnectionType.Main); + if (incomingDestinationNodeConnections !== undefined) { + for (const connection of incomingDestinationNodeConnections) { + if (waitingExecution[destinationNodeName] === undefined) { + waitingExecution[destinationNodeName] = {}; + waitingExecutionSource[destinationNodeName] = {}; + } + if (waitingExecution[destinationNodeName][runIndex] === undefined) { + waitingExecution[destinationNodeName][runIndex] = {}; + waitingExecutionSource[destinationNodeName][runIndex] = {}; + } + if (waitingExecution[destinationNodeName][runIndex][connection.type] === undefined) { + waitingExecution[destinationNodeName][runIndex][connection.type] = []; + waitingExecutionSource[destinationNodeName][runIndex][connection.type] = []; + } + + if (runData[connection.from.name] !== undefined) { + // Input data exists so add as waiting + // incomingDataDestination.push(runData[connection.node!][runIndex].data![connection.type][connection.index]); + waitingExecution[destinationNodeName][runIndex][connection.type].push( + runData[connection.from.name][runIndex].data![connection.type][connection.inputIndex], + ); + waitingExecutionSource[destinationNodeName][runIndex][connection.type].push({ + previousNode: connection.from.name, + previousNodeOutput: connection.inputIndex || undefined, + previousNodeRun: runIndex || undefined, + } as ISourceData); + } else { + waitingExecution[destinationNodeName][runIndex][connection.type].push(null); + waitingExecutionSource[destinationNodeName][runIndex][connection.type].push(null); } } } diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 9aa1154b3c41a..aa0243a37ac57 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -431,7 +431,7 @@ export function findCycles(_workflow: Workflow) { return []; } -type Connection = { +export type Connection = { from: INode; to: INode; type: NodeConnectionType; From 6e6069dc27beb70b638c5a3f71ac4fb8aa47016e Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 30 Jul 2024 16:55:44 +0200 Subject: [PATCH 03/54] make sure all functions terminate for graphs with cycles --- packages/core/src/__tests__/utils.test.ts | 180 ++++++++++++++++++++++ packages/core/src/utils.ts | 78 ++++++---- 2 files changed, 224 insertions(+), 34 deletions(-) diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts index 4a079cdc4434a..a68888e59a2e1 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/__tests__/utils.test.ts @@ -451,6 +451,51 @@ describe('findStartNodes', () => { }, }); }); + + // ►► + //┌───────┐ ┌─────┐ ┌─────┐ + //│Trigger│1──┬───┤Node1│1──┬──┤Node2│ + //└───────┘ │ └─────┘ │ └─────┘ + // │ │ + // └─────────────┘ + test('terminates when called with graph that contains cycles', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2) + .addConnections( + { from: trigger, to: node1 }, + { from: node1, to: node1 }, + { from: node1, to: node2 }, + ); + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + }; + const pinData: IPinData = {}; + + // + // ACT + // + const startNodes = findStartNodes(graph, trigger, node2, runData, pinData); + + // + // ASSERT + // + expect(startNodes).toHaveLength(1); + expect(startNodes).toContainEqual({ + node: node2, + sourceData: { + previousNode: node1, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + }); + }); }); describe('findSubgraph', () => { @@ -565,6 +610,106 @@ describe('findSubgraph2', () => { new DirectedGraph().addNodes(node1, node3).addConnections({ from: node1, to: node3 }), ); }); + + // ►► + //┌───────┐ ┌─────┐ ┌─────┐ + //│Trigger├───┬───┤Node1├───┬──┤Node2│ + //└───────┘ │ └─────┘ │ └─────┘ + // │ │ + // └─────────────┘ + test('terminates when called with graph that contains cycles', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2) + .addConnections( + { from: trigger, to: node1 }, + { from: node1, to: node1 }, + { from: node1, to: node2 }, + ); + + // + // ACT + // + const subgraph = findSubgraph2(graph, node2, trigger); + + // + // ASSERT + // + expect(subgraph).toEqual(graph); + }); + + // ►► + // ┌───────┐ ┌─────┐ + // │Trigger├───┬─┤Node1│ + // └───────┘ │ └─────┘ + // │ + // ┌─────┐ │ + // │Node2├─────┘ + // └─────┘ + test('terminates when called with graph that contains cycles', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2) + .addConnections({ from: trigger, to: node1 }, { from: node2, to: node1 }); + + // + // ACT + // + const subgraph = findSubgraph2(graph, node1, trigger); + + // + // ASSERT + // + expect(subgraph).toEqual( + new DirectedGraph().addNodes(trigger, node1).addConnections({ from: trigger, to: node1 }), + ); + }); + + // ►► + // ┌───────┐ ┌───────────┐ ┌───────────┐ + // │Trigger├─┬─►│Destination├───┤AnotherNode├───┐ + // └───────┘ │ └───────────┘ └───────────┘ │ + // │ │ + // └──────────────────────────────────┘ + test('terminates if the destination node is part of a cycle', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const destination = createNodeData({ name: 'destination' }); + const anotherNode = createNodeData({ name: 'anotherNode' }); + const graph = new DirectedGraph() + .addNodes(trigger, destination, anotherNode) + .addConnections( + { from: trigger, to: destination }, + { from: destination, to: anotherNode }, + { from: anotherNode, to: destination }, + ); + + // + // ACT + // + const subgraph = findSubgraph2(graph, destination, trigger); + + // + // ASSERT + // + expect(subgraph).toEqual( + new DirectedGraph() + .addNodes(trigger, destination) + .addConnections({ from: trigger, to: destination }), + ); + }); }); describe('DirectedGraph', () => { @@ -634,5 +779,40 @@ describe('DirectedGraph', () => { type: NodeConnectionType.Main, }); }); + + test('terminates if the graph has cycles', () => { + // + // ARRANGE + // + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const graph = new DirectedGraph() + .addNodes(node1, node2) + .addConnections({ from: node1, to: node2 }, { from: node2, to: node2 }); + + // + // ACT + // + const children = graph.getChildren(node1); + + // + // ASSERT + // + expect(children).toHaveLength(2); + expect(children).toContainEqual({ + from: node1, + to: node2, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }); + expect(children).toContainEqual({ + from: node2, + to: node2, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }); + }); }); }); diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index aa0243a37ac57..d9d5c4b05a961 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -39,6 +39,7 @@ function findSubgraph2Recursive( ) { //console.log('currentBranch', currentBranch); + // If the current node is the chosen ‘trigger keep this branch. if (current === trigger) { console.log(`${current.name}: is trigger`); for (const connection of currentBranch) { @@ -51,21 +52,22 @@ function findSubgraph2Recursive( let parentConnections = graph.getDirectParents(current); + // If the current node has no parents, don’t keep this branch. if (parentConnections.length === 0) { console.log(`${current.name}: no parents`); return; } + // If the current node is the destination node again, don’t keep this branch. const isCycleWithDestinationNode = current === destinationNode && currentBranch.some((c) => c.to === destinationNode); - if (isCycleWithDestinationNode) { console.log(`${current.name}: isCycleWithDestinationNode`); return; } + // If the current node was already visited, keep this branch. const isCycleWithCurrentNode = currentBranch.some((c) => c.to === current); - if (isCycleWithCurrentNode) { console.log(`${current.name}: isCycleWithCurrentNode`); // TODO: write function that adds nodes when adding connections @@ -76,6 +78,10 @@ function findSubgraph2Recursive( return; } + // If the current node is disabled, don’t keep this node, but keep the + // branch. + // Take every incoming connection and connect it to every node that is + // connected to the current node’s first output if (current.disabled) { console.log(`${current.name}: is disabled`); const incomingConnections = graph.getDirectParents(current); @@ -101,6 +107,7 @@ function findSubgraph2Recursive( } } + // Recurse on each parent. for (const parentConnection of parentConnections) { findSubgraph2Recursive(graph, destinationNode, parentConnection.from, trigger, newGraph, [ ...currentBranch, @@ -328,10 +335,13 @@ function findStartNodesRecursive( runData: IRunData, pinData: IPinData, startNodes: Map, + seen: Set, source?: ISourceData, ) { const nodeIsDirty = isDirty(current, runData, pinData); + // If the current node is dirty stop following this branch, we found a start + // node. if (nodeIsDirty) { startNodes.set(makeKey(source, current), { node: current, @@ -340,52 +350,39 @@ function findStartNodesRecursive( return startNodes; } + // If the current node is the destination node stop following this branch, we + // found a start node. if (current === destination) { startNodes.set(makeKey(source, current), { node: current, sourceData: source }); return startNodes; } - const outGoingConnections = graph.getDirectChildren(current); + // If we detect a cycle stop following the branch, there is no start node on + // this branch. + if (seen.has(current)) { + return startNodes; + } + // Recurse with every direct child that is part of the sub graph. + const outGoingConnections = graph.getDirectChildren(current); for (const outGoingConnection of outGoingConnections) { - // NOTE: can't use `Workflow.getNodeConnectionIndexes` here, because it - // only returns the first connection it finds. But I need all connections. - //const nodeConnection = workflow.getNodeConnectionIndexes(child.name, current.name); - //a.ok(nodeConnection, `Child(${child.name}) and parent(${current.name}) are not connected.`); - const nodeRunData = getIncomingData( runData, outGoingConnection.from.name, - // // NOTE: I don't support multiple runs for now. + // NOTE: It's always 0 until I fix the bug that removes the run data for + // old runs. The FE only sends data for one run for each node. 0, outGoingConnection.type, outGoingConnection.outputIndex, ); - const hasRunData = - nodeRunData === null || nodeRunData === undefined || nodeRunData.length === 0; - if (hasRunData) { + // If the node has multiple outputs, only follow the outputs that have run data. + const hasNoRunData = + nodeRunData === null || nodeRunData === undefined || nodeRunData.length === 0; + if (hasNoRunData) { continue; } - //if (child.sourceData) { - // const nodeRunData = getIncomingData( - // runData, - // child.sourceData.previousNode.name, - // child.sourceData.previousNodeRun, - // NodeConnectionType.Main, - // child.sourceData.previousNodeOutput, - // ); - // const hasRunData = - // nodeRunData === null || nodeRunData === undefined || nodeRunData.length === 0; - // - // if (hasRunData) { - // continue; - // } - //} - - //a.ok(child.sourceData, `Child(${child.node.name}) has no sourceDate.`); - findStartNodesRecursive( graph, outGoingConnection.to, @@ -393,13 +390,12 @@ function findStartNodesRecursive( runData, pinData, startNodes, + new Set(seen).add(current), { previousNode: current, // NOTE: It's always 0 until I fix the bug that removes the run data for // old runs. The FE only sends data for one run for each node. previousNodeRun: 0, - // FIXME: Reimplement getChildNodes so I can keep track of which output - // was used to get to this child. previousNodeOutput: outGoingConnection.outputIndex, }, ); @@ -422,6 +418,7 @@ export function findStartNodes( runData, pinData, new Map(), + new Set(), ); return [...startNodes.values()]; } @@ -519,10 +516,23 @@ export class DirectedGraph { return directChildren; } - getChildren(node: INode): Connection[] { + private getChildrenRecursive(node: INode, seen: Set): Connection[] { + if (seen.has(node)) { + return []; + } + const directChildren = this.getDirectChildren(node); - return [...directChildren, ...directChildren.flatMap((child) => this.getChildren(child.to))]; + return [ + ...directChildren, + ...directChildren.flatMap((child) => + this.getChildrenRecursive(child.to, new Set(seen).add(node)), + ), + ]; + } + + getChildren(node: INode): Connection[] { + return this.getChildrenRecursive(node, new Set()); } getDirectParents(node: INode) { From 7e4d658c550cb45287fb53abd70427fe0ed9bc2b Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 31 Jul 2024 08:44:36 +0200 Subject: [PATCH 04/54] remove deprecated and unused functions --- packages/core/src/__tests__/utils.test.ts | 48 -------------- packages/core/src/utils.ts | 76 +++++++++-------------- 2 files changed, 28 insertions(+), 96 deletions(-) diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts index a68888e59a2e1..a47fd2630dbe2 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/__tests__/utils.test.ts @@ -498,54 +498,6 @@ describe('findStartNodes', () => { }); }); -describe('findSubgraph', () => { - test('simple', () => { - const from = createNodeData({ name: 'From' }); - const to = createNodeData({ name: 'To' }); - - const graph = new Workflow({ - nodes: [from, to], - connections: toIConnections([{ from, to }]), - active: false, - nodeTypes, - }); - - const subgraph = findSubgraph(graph, to); - - expect(subgraph).toHaveLength(2); - expect(subgraph).toContainEqual(to); - expect(subgraph).toContainEqual(from); - }); - - // - // ┌────┐ ┌────┐ - // │ │O───────►│ │ - // │ if │ │noOp│ - // │ │O───────►│ │ - // └────┘ └────┘ - // - test('multiple connections', () => { - const ifNode = createNodeData({ name: 'If' }); - const noOp = createNodeData({ name: 'noOp' }); - - const workflow = new Workflow({ - nodes: [ifNode, noOp], - connections: toIConnections([ - { from: ifNode, to: noOp, outputIndex: 0 }, - { from: ifNode, to: noOp, outputIndex: 1 }, - ]), - active: false, - nodeTypes, - }); - - const subgraph = findSubgraph(workflow, noOp); - - expect(subgraph).toHaveLength(2); - expect(subgraph).toContainEqual(noOp); - expect(subgraph).toContainEqual(ifNode); - }); -}); - describe('findSubgraph2', () => { test('simple', () => { const from = createNodeData({ name: 'From' }); diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index d9d5c4b05a961..21b983d6305ae 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -9,26 +9,6 @@ import type { } from 'n8n-workflow'; import { NodeConnectionType, Workflow } from 'n8n-workflow'; -export function findSubgraph(workflow: Workflow, destinationNode: INode): INode[] { - const result = workflow - .getParentNodes(destinationNode.name) - .reduce( - ([set], name) => { - set.add(name); - return [set]; - }, - [new Set()], - ) - .flatMap((set) => [...set]) - .map((name) => workflow.getNode(name)) - .filter((node) => node !== null) - .filter((node) => !node.disabled); - - result.push(destinationNode); - - return result; -} - function findSubgraph2Recursive( graph: DirectedGraph, destinationNode: INode, @@ -273,34 +253,34 @@ export interface StartNodeData { sourceData?: ISourceData; } -export function getDirectChildren(workflow: Workflow, parent: INode): StartNodeData[] { - const directChildren: StartNodeData[] = []; - - for (const [_connectionGroupName, inputs] of Object.entries( - workflow.connectionsBySourceNode[parent.name] ?? {}, - )) { - for (const [outputIndex, connections] of inputs.entries()) { - for (const connection of connections) { - const node = workflow.getNode(connection.node); - - a.ok(node, `Node(${connection.node}) does not exist in workflow.`); - - directChildren.push({ - node, - sourceData: { - previousNode: parent, - previousNodeOutput: outputIndex, - // TODO: I don't have this here. This part is working without run - // data, so there are not runs. - previousNodeRun: 0, - }, - }); - } - } - } - - return directChildren; -} +//export function getDirectChildren(workflow: Workflow, parent: INode): StartNodeData[] { +// const directChildren: StartNodeData[] = []; +// +// for (const [_connectionGroupName, inputs] of Object.entries( +// workflow.connectionsBySourceNode[parent.name] ?? {}, +// )) { +// for (const [outputIndex, connections] of inputs.entries()) { +// for (const connection of connections) { +// const node = workflow.getNode(connection.node); +// +// a.ok(node, `Node(${connection.node}) does not exist in workflow.`); +// +// directChildren.push({ +// node, +// sourceData: { +// previousNode: parent, +// previousNodeOutput: outputIndex, +// // TODO: I don't have this here. This part is working without run +// // data, so there are not runs. +// previousNodeRun: 0, +// }, +// }); +// } +// } +// } +// +// return directChildren; +//} export function getIncomingData( runData: IRunData, From 861402e8a96d5d258ddfcbc3cc44b749633ee528 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 31 Jul 2024 08:44:50 +0200 Subject: [PATCH 05/54] document all tests with diagrams --- packages/core/src/__tests__/utils-2.test.ts | 53 +-- packages/core/src/__tests__/utils.test.ts | 448 +++++++++++--------- 2 files changed, 285 insertions(+), 216 deletions(-) diff --git a/packages/core/src/__tests__/utils-2.test.ts b/packages/core/src/__tests__/utils-2.test.ts index 5296b7dbd7882..6cf48ad9a0199 100644 --- a/packages/core/src/__tests__/utils-2.test.ts +++ b/packages/core/src/__tests__/utils-2.test.ts @@ -1,3 +1,13 @@ +// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ +// If you update the tests please update the diagrams as well. +// +// Map +// 0 means the output has no run data +// 1 means the output has run data +// ►► denotes the node that the user wants to execute to +// XX denotes that the node is disabled +// PD denotes that the node has pinned data + import { recreateNodeExecutionStack } from '@/utils-2'; import { createNodeData, toITaskData } from './helpers'; import type { StartNodeData } from '@/utils'; @@ -5,19 +15,10 @@ import { DirectedGraph, findSubgraph2 } from '@/utils'; import { type IPinData, type IRunData } from 'n8n-workflow'; import { AssertionError } from 'assert'; -// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ -// If you update the tests please update the diagrams as well. -// -// Map -// 0 means the output has no run data. -// 1 means the output has run data. -// ►► denotes the node that the user wants to execute to. -// XX denotes that the node is disabled - describe('recreateNodeExecutionStack', () => { - // ►► - // ┌───────┐ ┌────┐ - // │Trigger│1─────►│Node│ + // ►► + // ┌───────┐1 ┌────┐ + // │Trigger├──────►│Node│ // └───────┘ └────┘ test('all nodes except destination node have data', () => { // @@ -75,9 +76,9 @@ describe('recreateNodeExecutionStack', () => { }); }); - // ►► - // ┌───────┐ ┌────┐ - // │Trigger│0─────►│Node│ + // ►► + // ┌───────┐0 ┌────┐ + // │Trigger├──────►│Node│ // └───────┘ └────┘ test('no nodes have data', () => { // @@ -115,9 +116,9 @@ describe('recreateNodeExecutionStack', () => { expect(waitingExecutionSource).toEqual({ node: { '0': { main: [null] } } }); }); - // PD ►► - // ┌───────┐ ┌────┐ - // │Trigger│1─────►│Node│ + // PinData ►► + // ┌───────┐1 ┌────┐ + // │Trigger├──────►│Node│ // └───────┘ └────┘ test('node before destination node has pinned data', () => { // @@ -164,9 +165,9 @@ describe('recreateNodeExecutionStack', () => { expect(waitingExecutionSource).toEqual({ node: { '0': { main: [null] } } }); }); - // XX ►► - // ┌───────┐ ┌─────┐ ┌─────┐ - // │Trigger│1────►│Node1│──────►│Node2│ + // XX ►► + // ┌───────┐1 ┌─────┐ ┌─────┐ + // │Trigger├─────►│Node1├──────►│Node2│ // └───────┘ └─────┘ └─────┘ test('throws if a disabled node is found', () => { // @@ -203,13 +204,13 @@ describe('recreateNodeExecutionStack', () => { ).toThrowError(AssertionError); }); - // ►► - // ┌───────┐ ┌─────┐ ┌─────┐ - // │Trigger│1─┬──►│Node1│1─┬───►│Node3│ + // ►► + // ┌───────┐1 ┌─────┐1 ┌─────┐ + // │Trigger├──┬──►│Node1├──┬───►│Node3│ // └───────┘ │ └─────┘ │ └─────┘ // │ │ - // │ ┌─────┐ │ - // └──►│Node2│1─┘ + // │ ┌─────┐1 │ + // └──►│Node2├──┘ // └─────┘ test('multiple incoming connections', () => { // diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts index a47fd2630dbe2..15e5b11980d09 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/__tests__/utils.test.ts @@ -2,17 +2,16 @@ // If you update the tests please update the diagrams as well. // // Map -// 0 means the output has no data. -// 1 means the output has data. -// ►► denotes the node that the user wants to execute to. +// 0 means the output has no run data +// 1 means the output has run data +// ►► denotes the node that the user wants to execute to // XX denotes that the node is disabled -// -// TODO: rename all nodes to generic names, don't use if, merge, etc. +// PD denotes that the node has pinned data import type { IConnections, INode, IPinData, IRunData } from 'n8n-workflow'; -import { NodeConnectionType, Workflow } from 'n8n-workflow'; -import { DirectedGraph, findStartNodes, findSubgraph, findSubgraph2, isDirty } from '../utils'; -import { createNodeData, toITaskData, nodeTypes, defaultWorkflowParameter } from './helpers'; +import { NodeConnectionType } from 'n8n-workflow'; +import { DirectedGraph, findStartNodes, findSubgraph2, isDirty } from '../utils'; +import { createNodeData, toITaskData, defaultWorkflowParameter } from './helpers'; test('toITaskData', function () { expect(toITaskData([{ data: { value: 1 } }])).toEqual({ @@ -162,189 +161,225 @@ describe('isDirty', () => { }); describe('findStartNodes', () => { - test('simple', () => { + // ►► + // ┌───────┐ + // │trigger│ + // └───────┘ + test('finds the start node if there is only a trigger', () => { const node = createNodeData({ name: 'Basic Node' }); - const graph = new DirectedGraph().addNode(node); - expect(findStartNodes(graph, node, node)).toStrictEqual([{ node, sourceData: undefined }]); - }); + const startNodes = findStartNodes(graph, node, node); - test('less simple', () => { - const node1 = createNodeData({ name: 'Basic Node 1' }); - const node2 = createNodeData({ name: 'Basic Node 2' }); + expect(startNodes).toHaveLength(1); + expect(startNodes).toContainEqual({ node, sourceData: undefined }); + }); + // ►► + // ┌───────┐ ┌───────────┐ + // │trigger├────►│destination│ + // └───────┘ └───────────┘ + test('finds the start node in a simple graph', () => { + const trigger = createNodeData({ name: 'trigger' }); + const destination = createNodeData({ name: 'destination' }); const graph = new DirectedGraph() - .addNodes(node1, node2) - .addConnection({ from: node1, to: node2 }); + .addNodes(trigger, destination) + .addConnection({ from: trigger, to: destination }); - expect(findStartNodes(graph, node1, node2)).toStrictEqual([ - { node: node1, sourceData: undefined }, - ]); + // if the trigger has no run data + { + const startNodes = findStartNodes(graph, trigger, destination); - const runData: IRunData = { - [node1.name]: [toITaskData([{ data: { value: 1 } }])], - }; + expect(startNodes).toHaveLength(1); + expect(startNodes).toContainEqual({ node: trigger, sourceData: undefined }); + } - expect(findStartNodes(graph, node1, node2, runData)).toStrictEqual([ - { - node: node2, - sourceData: { previousNode: node1, previousNodeOutput: 0, previousNodeRun: 0 }, - }, - ]); + // if the trigger has run data + { + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + }; + + const startNodes = findStartNodes(graph, trigger, destination, runData); + + expect(startNodes).toHaveLength(1); + expect(startNodes).toContainEqual({ + node: destination, + sourceData: { previousNode: trigger, previousNodeOutput: 0, previousNodeRun: 0 }, + }); + } }); - // - // ┌────┐ - // │ │1───┐ ┌────┐ - // │ if │ ├───►│noOp│ - // │ │1───┘ └────┘ - // └────┘ - // - // All nodes have run data. `findStartNodes` should return noOp twice + // ┌───────┐ ►► + // │ │1──┐ ┌────┐ + // │trigger│ ├─►│node│ + // │ │1──┘ └────┘ + // └───────┘ + // All nodes have run data. `findStartNodes` should return node twice // because it has 2 input connections. test('multiple outputs', () => { - const ifNode = createNodeData({ name: 'If' }); - const noOp = createNodeData({ name: 'NoOp' }); - + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node = createNodeData({ name: 'node' }); const graph = new DirectedGraph() - .addNodes(ifNode, noOp) + .addNodes(trigger, node) .addConnections( - { from: ifNode, to: noOp, outputIndex: 0, inputIndex: 0 }, - { from: ifNode, to: noOp, outputIndex: 1, inputIndex: 0 }, + { from: trigger, to: node, outputIndex: 0, inputIndex: 0 }, + { from: trigger, to: node, outputIndex: 1, inputIndex: 0 }, ); - const runData: IRunData = { - [ifNode.name]: [ + [trigger.name]: [ toITaskData([ { data: { value: 1 }, outputIndex: 0 }, { data: { value: 1 }, outputIndex: 1 }, ]), ], - [noOp.name]: [toITaskData([{ data: { value: 1 } }])], + [node.name]: [toITaskData([{ data: { value: 1 } }])], }; - const startNodes = findStartNodes(graph, ifNode, noOp, runData); + // + // ACT + // + const startNodes = findStartNodes(graph, trigger, node, runData); + // + // ASSERT + // expect(startNodes).toHaveLength(2); expect(startNodes).toContainEqual({ - node: noOp, + node, sourceData: { - previousNode: ifNode, + previousNode: trigger, previousNodeOutput: 0, previousNodeRun: 0, }, }); expect(startNodes).toContainEqual({ - node: noOp, + node, sourceData: { - previousNode: ifNode, + previousNode: trigger, previousNodeOutput: 1, previousNodeRun: 0, }, }); }); - test('medium', () => { - const trigger = createNodeData({ name: 'Trigger' }); - const ifNode = createNodeData({ name: 'If' }); - const merge1 = createNodeData({ name: 'Merge1' }); - const merge2 = createNodeData({ name: 'Merge2' }); - const noOp = createNodeData({ name: 'NoOp' }); - + // ┌─────┐ ┌─────┐ ►► + //┌───────┐ │ ├────┬────────►│ │ ┌─────┐ + //│trigger├───►│node1│ │ │node2├────┬───►│node4│ + //└───────┘ │ ├────┼────┬───►│ │ │ └─────┘ + // └─────┘ │ │ └─────┘ │ + // │ │ │ + // │ │ │ + // │ │ │ + // │ │ ┌─────┐ │ + // │ └───►│ │ │ + // │ │node3├────┘ + // └────────►│ │ + // └─────┘ + test('complex example with multiple outputs and inputs', () => { + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const node3 = createNodeData({ name: 'node3' }); + const node4 = createNodeData({ name: 'node4' }); const graph = new DirectedGraph() - .addNodes(trigger, ifNode, merge1, merge2, noOp) + .addNodes(trigger, node1, node2, node3, node4) .addConnections( - { from: trigger, to: ifNode }, - { from: ifNode, to: merge1, outputIndex: 0, inputIndex: 0 }, - { from: ifNode, to: merge1, outputIndex: 1, inputIndex: 1 }, - { from: ifNode, to: merge2, outputIndex: 0, inputIndex: 1 }, - { from: ifNode, to: merge2, outputIndex: 1, inputIndex: 0 }, - { from: merge1, to: noOp }, - { from: merge2, to: noOp }, + { from: trigger, to: node1 }, + { from: node1, to: node2, outputIndex: 0, inputIndex: 0 }, + { from: node1, to: node2, outputIndex: 1, inputIndex: 1 }, + { from: node1, to: node3, outputIndex: 0, inputIndex: 1 }, + { from: node1, to: node3, outputIndex: 1, inputIndex: 0 }, + { from: node2, to: node4 }, + { from: node3, to: node4 }, ); - // no run data means the trigger is the start node - expect(findStartNodes(graph, trigger, noOp)).toEqual([ - { node: trigger, sourceData: undefined }, - ]); - - const runData: IRunData = { - [trigger.name]: [toITaskData([{ data: { value: 1 } }])], - [ifNode.name]: [toITaskData([{ data: { value: 1 } }])], - [merge1.name]: [toITaskData([{ data: { value: 1 } }])], - [merge2.name]: [toITaskData([{ data: { value: 1 } }])], - [noOp.name]: [toITaskData([{ data: { value: 1 } }])], - }; - - const startNodes = findStartNodes(graph, trigger, noOp, runData); - expect(startNodes).toHaveLength(2); - - // run data for everything - expect(startNodes).toContainEqual({ - node: noOp, - sourceData: { - previousNode: merge1, - previousNodeOutput: 0, - previousNodeRun: 0, - }, - }); + { + const startNodes = findStartNodes(graph, trigger, node4); + expect(startNodes).toHaveLength(1); + // no run data means the trigger is the start node + expect(startNodes).toContainEqual({ node: trigger, sourceData: undefined }); + } + + { + // run data for everything + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [node2.name]: [toITaskData([{ data: { value: 1 } }])], + [node3.name]: [toITaskData([{ data: { value: 1 } }])], + [node4.name]: [toITaskData([{ data: { value: 1 } }])], + }; + + const startNodes = findStartNodes(graph, trigger, node4, runData); + expect(startNodes).toHaveLength(2); + + expect(startNodes).toContainEqual({ + node: node4, + sourceData: { + previousNode: node2, + previousNodeOutput: 0, + previousNodeRun: 0, + }, + }); - expect(startNodes).toContainEqual({ - node: noOp, - sourceData: { - previousNode: merge2, - previousNodeOutput: 0, - previousNodeRun: 0, - }, - }); + expect(startNodes).toContainEqual({ + node: node4, + sourceData: { + previousNode: node3, + previousNodeOutput: 0, + previousNodeRun: 0, + }, + }); + } }); - // - // ┌────┐ ┌─────┐ - // │ │1───────►│ │ - // │ if │ │merge│O - // │ │O───────►│ │ - // └────┘ └─────┘ - // + // ►► + // ┌───────┐1 ┌────┐ + // │ ├────────►│ │ + // │trigger│ │node│ + // │ ├────────►│ │ + // └───────┘0 └────┘ // The merge node only gets data on one input, so the it should only be once // in the start nodes - test('multiple connections', () => { - const ifNode = createNodeData({ name: 'if' }); - const merge = createNodeData({ name: 'merge' }); + test('multiple connections with the first one having data', () => { + const trigger = createNodeData({ name: 'trigger' }); + const node = createNodeData({ name: 'node' }); const graph = new DirectedGraph() - .addNodes(ifNode, merge) + .addNodes(trigger, node) .addConnections( - { from: ifNode, to: merge, outputIndex: 0, inputIndex: 0 }, - { from: ifNode, to: merge, outputIndex: 1, inputIndex: 1 }, + { from: trigger, to: node, outputIndex: 0, inputIndex: 0 }, + { from: trigger, to: node, outputIndex: 1, inputIndex: 1 }, ); - const startNodes = findStartNodes(graph, ifNode, merge, { - [ifNode.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], + const startNodes = findStartNodes(graph, trigger, node, { + [trigger.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], }); expect(startNodes).toHaveLength(1); expect(startNodes).toContainEqual({ - node: merge, + node, sourceData: { - previousNode: ifNode, + previousNode: trigger, previousNodeRun: 0, previousNodeOutput: 0, }, }); }); - // - // ┌────┐ ┌─────┐ - // │ │0───────►│ │ - // │ if │ │merge│O - // │ │1───────►│ │ - // └────┘ └─────┘ - // + // ►► + // ┌───────┐0 ┌────┐ + // │ ├────────►│ │ + // │trigger│ │node│ + // │ ├────────►│ │ + // └───────┘1 └────┘ // The merge node only gets data on one input, so the it should only be once // in the start nodes - test('multiple connections', () => { + test('multiple connections with the second one having data', () => { const ifNode = createNodeData({ name: 'if' }); const merge = createNodeData({ name: 'merge' }); @@ -370,15 +405,15 @@ describe('findStartNodes', () => { }); }); - // ┌────┐ ┌─────┐ - // │ │1───────►│ │ - // │ if │ │merge│O - // │ │1───────►│ │ - // └────┘ └─────┘ - // + // ►► + // ┌───────┐1 ┌────┐ + // │ ├────────►│ │ + // │trigger│ │node│ + // │ ├────────►│ │ + // └───────┘1 └────┘ // The merge node gets data on both inputs, so the it should be in the start // nodes twice. - test('multiple connections', () => { + test('multiple connections with both having data', () => { const ifNode = createNodeData({ name: 'if' }); const merge = createNodeData({ name: 'merge' }); @@ -417,35 +452,35 @@ describe('findStartNodes', () => { }); }); - // ► - // ┌───────┐ ┌────┐ ┌─────┐ - // │ │ │ │0───────►│ │ - // │Trigger│1──────►│ if │ │merge│ - // │ │ │ │1───────►│ │ - // └───────┘ └────┘ └─────┘ + // ►► + // ┌───────┐ ┌─────┐0 ┌─────┐ + // │ │1 │ ├────────►│ │ + // │trigger├───────►│node1│ │node2│ + // │ │ │ ├────────►│ │ + // └───────┘ └─────┘1 └─────┘ test('multiple connections with trigger', () => { const trigger = createNodeData({ name: 'trigger' }); - const ifNode = createNodeData({ name: 'if' }); - const merge = createNodeData({ name: 'merge' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); const graph = new DirectedGraph() - .addNodes(trigger, ifNode, merge) + .addNodes(trigger, node1, node2) .addConnections( - { from: trigger, to: ifNode }, - { from: ifNode, to: merge, outputIndex: 0 }, - { from: ifNode, to: merge, outputIndex: 1 }, + { from: trigger, to: node1 }, + { from: node1, to: node2, outputIndex: 0 }, + { from: node1, to: node2, outputIndex: 1 }, ); - const startNodes = findStartNodes(graph, ifNode, merge, { + const startNodes = findStartNodes(graph, node1, node2, { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], - [ifNode.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 1 }])], + [node1.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 1 }])], }); expect(startNodes).toHaveLength(1); expect(startNodes).toContainEqual({ - node: merge, + node: node2, sourceData: { - previousNode: ifNode, + previousNode: node1, previousNodeRun: 0, previousNodeOutput: 1, }, @@ -453,8 +488,8 @@ describe('findStartNodes', () => { }); // ►► - //┌───────┐ ┌─────┐ ┌─────┐ - //│Trigger│1──┬───┤Node1│1──┬──┤Node2│ + //┌───────┐1 ┌─────┐1 ┌─────┐ + //│Trigger├───┬──►│Node1├───┬─►│Node2│ //└───────┘ │ └─────┘ │ └─────┘ // │ │ // └─────────────┘ @@ -499,22 +534,29 @@ describe('findStartNodes', () => { }); describe('findSubgraph2', () => { + // ►► + // ┌───────┐ ┌───────────┐ + // │trigger├────►│destination│ + // └───────┘ └───────────┘ test('simple', () => { - const from = createNodeData({ name: 'From' }); - const to = createNodeData({ name: 'To' }); + const trigger = createNodeData({ name: 'trigger' }); + const destination = createNodeData({ name: 'destination' }); - const graph = new DirectedGraph().addNodes(from, to).addConnections({ from, to }); + const graph = new DirectedGraph() + .addNodes(trigger, destination) + .addConnections({ from: trigger, to: destination }); - const subgraph = findSubgraph2(graph, to, from); + const subgraph = findSubgraph2(graph, destination, trigger); expect(subgraph).toEqual(graph); }); - // ┌────┐ ┌────┐ - // │ │O───────►│ │ - // │ if │ │noOp│ - // │ │O───────►│ │ - // └────┘ └────┘ + // ►► + // ┌───────┐ ┌───────────┐ + // │ ├────────►│ │ + // │trigger│ │destination│ + // │ ├────────►│ │ + // └───────┘ └───────────┘ test('multiple connections', () => { const ifNode = createNodeData({ name: 'If' }); const noOp = createNodeData({ name: 'noOp' }); @@ -531,44 +573,60 @@ describe('findSubgraph2', () => { expect(subgraph).toEqual(graph); }); + // ►► + // ┌───────┐ ┌───────────┐ + // │ ├────────►│ │ ┌────┐ + // │trigger│ │destination├─────►│node│ + // │ ├────────►│ │ └────┘ + // └───────┘ └───────────┘ test('disregard nodes after destination', () => { - const node1 = createNodeData({ name: 'node1' }); - const node2 = createNodeData({ name: 'node2' }); - const node3 = createNodeData({ name: 'node3' }); + const trigger = createNodeData({ name: 'trigger' }); + const destination = createNodeData({ name: 'destination' }); + const node = createNodeData({ name: 'node' }); const graph = new DirectedGraph() - .addNodes(node1, node2, node3) - .addConnections({ from: node1, to: node2 }, { from: node2, to: node3 }); + .addNodes(trigger, destination, node) + .addConnections({ from: trigger, to: destination }, { from: destination, to: node }); - const subgraph = findSubgraph2(graph, node2, node1); + const subgraph = findSubgraph2(graph, destination, trigger); expect(subgraph).toEqual( - new DirectedGraph().addNodes(node1, node2).addConnections({ from: node1, to: node2 }), + new DirectedGraph() + .addNodes(trigger, destination) + .addConnections({ from: trigger, to: destination }), ); }); + // XX + // ┌───────┐ ┌────────┐ ►► + // │ ├────────►│ │ ┌───────────┐ + // │trigger│ │disabled├─────►│destination│ + // │ ├────────►│ │ └───────────┘ + // └───────┘ └────────┘ test('skip disabled nodes', () => { - const node1 = createNodeData({ name: 'node1' }); - const node2 = createNodeData({ name: 'node2', disabled: true }); - const node3 = createNodeData({ name: 'node3' }); + const trigger = createNodeData({ name: 'trigger' }); + const disabled = createNodeData({ name: 'disabled', disabled: true }); + const destination = createNodeData({ name: 'destination' }); const graph = new DirectedGraph() - .addNodes(node1, node2, node3) - .addConnections({ from: node1, to: node2 }, { from: node2, to: node3 }); + .addNodes(trigger, disabled, destination) + .addConnections({ from: trigger, to: disabled }, { from: disabled, to: destination }); - const subgraph = findSubgraph2(graph, node3, node1); + const subgraph = findSubgraph2(graph, destination, trigger); expect(subgraph).toEqual( - new DirectedGraph().addNodes(node1, node3).addConnections({ from: node1, to: node3 }), + new DirectedGraph() + .addNodes(trigger, destination) + .addConnections({ from: trigger, to: destination }), ); }); - // ►► - //┌───────┐ ┌─────┐ ┌─────┐ - //│Trigger├───┬───┤Node1├───┬──┤Node2│ - //└───────┘ │ └─────┘ │ └─────┘ - // │ │ - // └─────────────┘ + // ►► + // ┌───────┐ ┌─────┐ ┌─────┐ + // │Trigger├───┬──►│Node1├───┬─►│Node2│ + // └───────┘ │ └─────┘ │ └─────┘ + // │ │ + // └─────────────┘ test('terminates when called with graph that contains cycles', () => { // // ARRANGE @@ -597,11 +655,11 @@ describe('findSubgraph2', () => { // ►► // ┌───────┐ ┌─────┐ - // │Trigger├───┬─┤Node1│ - // └───────┘ │ └─────┘ - // │ - // ┌─────┐ │ - // │Node2├─────┘ + // │Trigger├──┬─►│Node1│ + // └───────┘ │ └─────┘ + // │ + // ┌─────┐ │ + // │Node2├────┘ // └─────┘ test('terminates when called with graph that contains cycles', () => { // @@ -629,7 +687,7 @@ describe('findSubgraph2', () => { // ►► // ┌───────┐ ┌───────────┐ ┌───────────┐ - // │Trigger├─┬─►│Destination├───┤AnotherNode├───┐ + // │Trigger├─┬─►│Destination├──►│AnotherNode├───┐ // └───────┘ │ └───────────┘ └───────────┘ │ // │ │ // └──────────────────────────────────┘ @@ -665,6 +723,11 @@ describe('findSubgraph2', () => { }); describe('DirectedGraph', () => { + // ┌─────┐ ┌─────┐ ┌─────┐ + // ┌─►│node1├───►│node2├──►│node3├─┐ + // │ └─────┘ └─────┘ └─────┘ │ + // │ │ + // └───────────────────────────────┘ test('roundtrip', () => { const node1 = createNodeData({ name: 'Node1' }); const node2 = createNodeData({ name: 'Node2' }); @@ -684,15 +747,12 @@ describe('DirectedGraph', () => { }); describe('getChildren', () => { - // // ┌─────┐ ┌─────┐ - // │Node1│O─────►│Node1│ + // │node1├──────►│node2│ // └─────┘ └─────┘ - // test('simple', () => { const from = createNodeData({ name: 'Node1' }); const to = createNodeData({ name: 'Node2' }); - const graph = new DirectedGraph().addNodes(from, to).addConnections({ from, to }); const children = graph.getChildren(from); @@ -705,11 +765,14 @@ describe('DirectedGraph', () => { type: NodeConnectionType.Main, }); }); - + // ┌─────┐ + // │ ├────┐ ┌─────┐ + // │node1│ ├─►│node2│ + // │ ├────┘ └─────┘ + // └─────┘ test('medium', () => { const from = createNodeData({ name: 'Node1' }); const to = createNodeData({ name: 'Node2' }); - const graph = new DirectedGraph() .addNodes(from, to) .addConnections({ from, to, outputIndex: 0 }, { from, to, outputIndex: 1 }); @@ -732,6 +795,11 @@ describe('DirectedGraph', () => { }); }); + // ┌─────┐ ┌─────┐ + // ┌─►│node1├──────►│node2├──┐ + // │ └─────┘ └─────┘ │ + // │ │ + // └─────────────────────────┘ test('terminates if the graph has cycles', () => { // // ARRANGE From 4c5cb161c57baaa22a07e90a3e977ca8a146907c Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 31 Jul 2024 11:04:53 +0200 Subject: [PATCH 06/54] update call site --- packages/core/src/WorkflowExecute.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index 9e3eb3e18437a..38278acf0ee76 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -397,13 +397,7 @@ export class WorkflowExecute { // Initialize the nodeExecutionStack and waitingExecution with // the data from runData const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack( - subgraph.toWorkflow({ ...workflow }), - startNodes, - destinationNode.name, - runData, - pinData ?? {}, - ); + recreateNodeExecutionStack(subgraph, startNodes, destinationNode, runData, pinData ?? {}); //console.log(JSON.stringify(nodeExecutionStack, null, 2)); From 7da90c713b3a2ea164ae84ed8b39eaf105b6c3fb Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 31 Jul 2024 11:08:42 +0200 Subject: [PATCH 07/54] allow switching between old and new partial execution using client side local storage --- packages/cli/src/FeatureFlags.ts | 10 ---------- packages/cli/src/workflow-runner.ts | 6 ++---- .../cli/src/workflows/workflow-execution.service.ts | 2 ++ packages/cli/src/workflows/workflow.request.ts | 7 ++++++- packages/cli/src/workflows/workflows.controller.ts | 1 + packages/editor-ui/src/stores/workflows.store.ts | 4 +++- packages/workflow/src/Interfaces.ts | 1 + 7 files changed, 15 insertions(+), 16 deletions(-) delete mode 100644 packages/cli/src/FeatureFlags.ts diff --git a/packages/cli/src/FeatureFlags.ts b/packages/cli/src/FeatureFlags.ts deleted file mode 100644 index 49b95c1088400..0000000000000 --- a/packages/cli/src/FeatureFlags.ts +++ /dev/null @@ -1,10 +0,0 @@ -import fs from 'fs/promises'; - -export async function isPartialExecutionEnabled() { - try { - await fs.access('new-partial-execution'); - return true; - } catch (error) { - return false; - } -} diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index 71b577d21d459..b20ef3777f923 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -39,8 +39,6 @@ import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.serv import { EventService } from './events/event.service'; -import { isPartialExecutionEnabled } from './FeatureFlags'; - @Service() export class WorkflowRunner { private scalingService: ScalingService; @@ -320,8 +318,8 @@ export class WorkflowRunner { // Execute only the nodes between start and destination nodes const workflowExecute = new WorkflowExecute(additionalData, data.executionMode); - if (await isPartialExecutionEnabled()) { - console.debug('Partial execution is enabled'); + if (data.partialExecutionVersion === '1') { + console.debug('new partial execution is enabled'); workflowExecution = workflowExecute.runPartialWorkflow2( workflow, data.runData, diff --git a/packages/cli/src/workflows/workflow-execution.service.ts b/packages/cli/src/workflows/workflow-execution.service.ts index 87a78f4b5e183..4dc6d00f3486e 100644 --- a/packages/cli/src/workflows/workflow-execution.service.ts +++ b/packages/cli/src/workflows/workflow-execution.service.ts @@ -92,6 +92,7 @@ export class WorkflowExecutionService { { workflowData, runData, startNodes, destinationNode }: WorkflowRequest.ManualRunPayload, user: User, pushRef?: string, + partialExecutionVersion?: string, ) { const pinData = workflowData.pinData; const pinnedTrigger = this.selectPinnedActivatorStarter( @@ -135,6 +136,7 @@ export class WorkflowExecutionService { startNodes, workflowData, userId: user.id, + partialExecutionVersion: partialExecutionVersion ?? '0', }; const hasRunData = (node: INode) => runData !== undefined && !!runData[node.name]; diff --git a/packages/cli/src/workflows/workflow.request.ts b/packages/cli/src/workflows/workflow.request.ts index d05b5c1dabb20..5fc8cf1741b80 100644 --- a/packages/cli/src/workflows/workflow.request.ts +++ b/packages/cli/src/workflows/workflow.request.ts @@ -43,7 +43,12 @@ export declare namespace WorkflowRequest { type NewName = AuthenticatedRequest<{}, {}, {}, { name?: string }>; - type ManualRun = AuthenticatedRequest<{ workflowId: string }, {}, ManualRunPayload>; + type ManualRun = AuthenticatedRequest< + { workflowId: string }, + {}, + ManualRunPayload, + { partialExecutionVersion: string | undefined } + >; type Share = AuthenticatedRequest<{ workflowId: string }, {}, { shareWithIds: string[] }>; diff --git a/packages/cli/src/workflows/workflows.controller.ts b/packages/cli/src/workflows/workflows.controller.ts index 797797e386ca0..4a8d7bda55147 100644 --- a/packages/cli/src/workflows/workflows.controller.ts +++ b/packages/cli/src/workflows/workflows.controller.ts @@ -405,6 +405,7 @@ export class WorkflowsController { req.body, req.user, req.headers['push-ref'] as string, + req.query.partialExecutionVersion, ); } diff --git a/packages/editor-ui/src/stores/workflows.store.ts b/packages/editor-ui/src/stores/workflows.store.ts index 19ea7d94e8a10..45c9eba2b2bae 100644 --- a/packages/editor-ui/src/stores/workflows.store.ts +++ b/packages/editor-ui/src/stores/workflows.store.ts @@ -72,6 +72,7 @@ import { computed, ref } from 'vue'; import { useProjectsStore } from '@/stores/projects.store'; import type { ProjectSharingData } from '@/types/projects.types'; import type { PushPayload } from '@n8n/api-types'; +import { useLocalStorage } from '@vueuse/core'; const defaults: Omit & { settings: NonNullable } = { name: '', @@ -99,6 +100,7 @@ let cachedWorkflow: Workflow | null = null; export const useWorkflowsStore = defineStore(STORES.WORKFLOWS, () => { const uiStore = useUIStore(); + const partialExecutionVersion = useLocalStorage('PartialExecution.version', 0); const workflow = ref(createEmptyWorkflow()); const usedCredentials = ref>({}); @@ -1390,7 +1392,7 @@ export const useWorkflowsStore = defineStore(STORES.WORKFLOWS, () => { return await makeRestApiRequest( rootStore.restApiContext, 'POST', - `/workflows/${startRunData.workflowData.id}/run`, + `/workflows/${startRunData.workflowData.id}/run?partialExecutionVersion=${partialExecutionVersion.value}`, startRunData as unknown as IDataObject, ); } catch (error) { diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 6dbf6ec7eccba..bba0fd597a905 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2160,6 +2160,7 @@ export interface IWorkflowExecutionDataProcess { workflowData: IWorkflowBase; userId?: string; projectId?: string; + partialExecutionVersion?: string; } export interface ExecuteWorkflowOptions { From 26be6f3ba1682db63867f81484522169aadb5a96 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 31 Jul 2024 18:34:51 +0200 Subject: [PATCH 08/54] clean up tests --- packages/core/src/__tests__/utils.test.ts | 27 ++++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts index 15e5b11980d09..d18761887bd19 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/__tests__/utils.test.ts @@ -352,8 +352,8 @@ describe('findStartNodes', () => { const graph = new DirectedGraph() .addNodes(trigger, node) .addConnections( - { from: trigger, to: node, outputIndex: 0, inputIndex: 0 }, - { from: trigger, to: node, outputIndex: 1, inputIndex: 1 }, + { from: trigger, to: node, inputIndex: 0, outputIndex: 0 }, + { from: trigger, to: node, inputIndex: 1, outputIndex: 1 }, ); const startNodes = findStartNodes(graph, trigger, node, { @@ -386,8 +386,8 @@ describe('findStartNodes', () => { const graph = new DirectedGraph() .addNodes(ifNode, merge) .addConnections( - { from: ifNode, to: merge, outputIndex: 0, inputIndex: 0 }, - { from: ifNode, to: merge, outputIndex: 1, inputIndex: 1 }, + { from: ifNode, to: merge, inputIndex: 0, outputIndex: 0 }, + { from: ifNode, to: merge, inputIndex: 1, outputIndex: 1 }, ); const startNodes = findStartNodes(graph, ifNode, merge, { @@ -414,18 +414,18 @@ describe('findStartNodes', () => { // The merge node gets data on both inputs, so the it should be in the start // nodes twice. test('multiple connections with both having data', () => { - const ifNode = createNodeData({ name: 'if' }); - const merge = createNodeData({ name: 'merge' }); + const trigger = createNodeData({ name: 'trigger' }); + const node = createNodeData({ name: 'node' }); const graph = new DirectedGraph() - .addNodes(ifNode, merge) + .addNodes(trigger, node) .addConnections( - { from: ifNode, to: merge, outputIndex: 0 }, - { from: ifNode, to: merge, outputIndex: 1 }, + { from: trigger, to: node, inputIndex: 0, outputIndex: 0 }, + { from: trigger, to: node, inputIndex: 1, outputIndex: 1 }, ); - const startNodes = findStartNodes(graph, ifNode, merge, { - [ifNode.name]: [ + const startNodes = findStartNodes(graph, trigger, node, { + [trigger.name]: [ toITaskData([ { data: { value: 1 }, outputIndex: 0 }, { data: { value: 1 }, outputIndex: 1 }, @@ -435,13 +435,14 @@ describe('findStartNodes', () => { expect(startNodes).toHaveLength(2); expect(startNodes).toContainEqual({ - node: merge, + node, sourceData: { - previousNode: ifNode, + previousNode: trigger, previousNodeRun: 0, previousNodeOutput: 0, }, }); + }); expect(startNodes).toContainEqual({ node: merge, sourceData: { From 6268f2c0dd5f6f3e8a70b4a7b9d697307922fb02 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 15:03:28 +0200 Subject: [PATCH 09/54] clean up comments --- jest.config.js | 4 +- packages/core/src/WorkflowExecute.ts | 157 ++++++++++----------------- packages/workflow/src/Interfaces.ts | 1 - 3 files changed, 60 insertions(+), 102 deletions(-) diff --git a/jest.config.js b/jest.config.js index 1a6d7c31a22bc..3caac38ef9dbf 100644 --- a/jest.config.js +++ b/jest.config.js @@ -32,8 +32,8 @@ const config = { return acc; }, {}), setupFilesAfterEnv: ['jest-expect-message'], - collectCoverage: true, - coverageReporters: ['html'], + collectCoverage: process.env.COVERAGE_ENABLED === 'true', + coverageReporters: ['text-summary'], collectCoverageFrom: ['src/**/*.ts'], }; diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index 38278acf0ee76..cff187039bd01 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-use-before-define */ /* eslint-disable @typescript-eslint/prefer-optional-chain */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ @@ -55,9 +54,11 @@ import { DirectedGraph, findCycles, findStartNodes, + // TODO: rename to findSubgraph findSubgraph2, findTriggerForPartialExecution, } from './utils'; +// TODO: move into it's own folder import { recreateNodeExecutionStack } from './utils-2'; export class WorkflowExecute { @@ -295,8 +296,6 @@ export class WorkflowExecute { } } - console.log(JSON.stringify(nodeExecutionStack, null, 2)); - this.runExecutionData = { startData: { destinationNode, @@ -318,7 +317,10 @@ export class WorkflowExecute { return this.processRunExecutionData(workflow); } - // eslint-disable-next-line @typescript-eslint/promise-function-async, complexity + // IMPORTANT: Do not add "async" to this function, it will then convert the + // PCancelable to a regular Promise and does so not allow canceling + // active executions anymore + // eslint-disable-next-line @typescript-eslint/promise-function-async runPartialWorkflow2( workflow: Workflow, runData: IRunData, @@ -326,111 +328,68 @@ export class WorkflowExecute { destinationNodeName?: string, pinData?: IPinData, ): PCancelable { - debugger; - try { - const graph = DirectedGraph.fromWorkflow(workflow); - - if (destinationNodeName === undefined) { - throw new ApplicationError('destinationNodeName is undefined'); - } - - const destinationNode = workflow.getNode(destinationNodeName); - - if (destinationNode === null) { - throw new ApplicationError( - `Could not find a node with the name ${destinationNodeName} in the workflow.`, - ); - } - - // 1. Find the Trigger - - const trigger = findTriggerForPartialExecution(workflow, destinationNodeName); - - if (trigger === undefined) { - throw new ApplicationError( - 'The destination node is not connected to any trigger. Partial executions need a trigger.', - ); - } - - // 2. Find the Subgraph - - // TODO: filter out the branches that connect to other triggers than the one - // selected above. - const subgraph = findSubgraph2(graph, destinationNode, trigger); - //const filteredNodes = findSubgraph(workflow, destinationNode); - const filteredNodes = subgraph.getNodes(); - console.log('filteredNodes', filteredNodes); - - // 3. Find the Start Nodes - - const startNodes = findStartNodes(subgraph, trigger, destinationNode, runData); - console.log('startNodes', JSON.stringify(startNodes, null, 2)); - - // 4. Detect Cycles - - const cycles = findCycles(workflow); - - // 5. Handle Cycles - - if (cycles.length) { - // TODO: handle - } + if (destinationNodeName === undefined) { + throw new ApplicationError('destinationNodeName is undefined'); + } - // 6. Clean Run Data + const destinationNode = workflow.getNode(destinationNodeName); + if (destinationNode === null) { + throw new ApplicationError( + `Could not find a node with the name ${destinationNodeName} in the workflow.`, + ); + } - // 7. Recreate Execution Stack + // 1. Find the Trigger + const trigger = findTriggerForPartialExecution(workflow, destinationNodeName); + if (trigger === undefined) { + throw new ApplicationError( + 'The destination node is not connected to any trigger. Partial executions need a trigger.', + ); + } - this.status = 'running'; + // 2. Find the Subgraph + const subgraph = findSubgraph2(DirectedGraph.fromWorkflow(workflow), destinationNode, trigger); + const filteredNodes = subgraph.getNodes(); - //_startNodes = startNodes.map((sn) => ({ - // name: sn.node.name, - // sourceData: sn.sourceData - // ? { - // ...sn.sourceData, - // previousNode: sn.sourceData.previousNode.name, - // } - // : null, - //})); + // 3. Find the Start Nodes + const startNodes = findStartNodes(subgraph, trigger, destinationNode, runData); - //console.log('_startNodes', _startNodes); + // 4. Detect Cycles + const cycles = findCycles(workflow); - // Initialize the nodeExecutionStack and waitingExecution with - // the data from runData - const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(subgraph, startNodes, destinationNode, runData, pinData ?? {}); + // 5. Handle Cycles + if (cycles.length) { + // TODO: handle + } - //console.log(JSON.stringify(nodeExecutionStack, null, 2)); + // 6. Clean Run Data + // TODO: - // 8. Execute + // 7. Recreate Execution Stack + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(subgraph, startNodes, destinationNode, runData, pinData ?? {}); - this.runExecutionData = { - startData: { - destinationNode: destinationNodeName, - runNodeFilter: Array.from(filteredNodes.values()).map((node) => node.name), - }, - resultData: { - runData, - pinData, - }, - executionData: { - contextData: {}, - nodeExecutionStack, - metadata: {}, - waitingExecution, - waitingExecutionSource, - }, - }; + // 8. Execute + this.status = 'running'; + this.runExecutionData = { + startData: { + destinationNode: destinationNodeName, + runNodeFilter: Array.from(filteredNodes.values()).map((node) => node.name), + }, + resultData: { + runData, + pinData, + }, + executionData: { + contextData: {}, + nodeExecutionStack, + metadata: {}, + waitingExecution, + waitingExecutionSource, + }, + }; - return this.processRunExecutionData( - //workflow, - subgraph.toWorkflow({ - ...workflow, - }), - ); - } catch (error) { - console.error(error); - throw error; - } + return this.processRunExecutionData(subgraph.toWorkflow({ ...workflow })); } /** diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index bba0fd597a905..5e761b61e3fd9 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2080,7 +2080,6 @@ export interface ISourceData { export interface StartNodeData { name: string; - // TODO: What is this for? sourceData: ISourceData | null; } From 92656d2ab6cc864749a9f01e03621ad2ab522eb0 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 15:15:50 +0200 Subject: [PATCH 10/54] fix e2e tests --- cypress/e2e/19-execution.cy.ts | 10 +++++----- cypress/e2e/28-debug.cy.ts | 2 +- cypress/e2e/30-editor-after-route-changes.cy.ts | 2 +- cypress/pages/workflow-executions-tab.ts | 2 +- cypress/utils/executions.ts | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cypress/e2e/19-execution.cy.ts b/cypress/e2e/19-execution.cy.ts index d6b8d08fd5f4b..81e11b1b63c91 100644 --- a/cypress/e2e/19-execution.cy.ts +++ b/cypress/e2e/19-execution.cy.ts @@ -503,7 +503,7 @@ describe('Execution', () => { workflowPage.getters.clearExecutionDataButton().should('be.visible'); - cy.intercept('POST', '/rest/workflows/**/run').as('workflowRun'); + cy.intercept('POST', '/rest/workflows/**/run?**').as('workflowRun'); workflowPage.getters .canvasNodeByName('do something with them') @@ -525,7 +525,7 @@ describe('Execution', () => { workflowPage.getters.zoomToFitButton().click(); - cy.intercept('POST', '/rest/workflows/**/run').as('workflowRun'); + cy.intercept('POST', '/rest/workflows/**/run?**').as('workflowRun'); workflowPage.getters .canvasNodeByName('If') @@ -547,7 +547,7 @@ describe('Execution', () => { workflowPage.getters.clearExecutionDataButton().should('be.visible'); - cy.intercept('POST', '/rest/workflows/**/run').as('workflowRun'); + cy.intercept('POST', '/rest/workflows/**/run?**').as('workflowRun'); workflowPage.getters .canvasNodeByName('NoOp2') @@ -576,7 +576,7 @@ describe('Execution', () => { it('should successfully execute partial executions with nodes attached to the second output', () => { cy.createFixtureWorkflow('Test_Workflow_pairedItem_incomplete_manual_bug.json'); - cy.intercept('POST', '/rest/workflows/**/run').as('workflowRun'); + cy.intercept('POST', '/rest/workflows/**/run?**').as('workflowRun'); workflowPage.getters.zoomToFitButton().click(); workflowPage.getters.executeWorkflowButton().click(); @@ -596,7 +596,7 @@ describe('Execution', () => { it('should execute workflow partially up to the node that has issues', () => { cy.createFixtureWorkflow('Test_workflow_partial_execution_with_missing_credentials.json'); - cy.intercept('POST', '/rest/workflows/**/run').as('workflowRun'); + cy.intercept('POST', '/rest/workflows/**/run?**').as('workflowRun'); workflowPage.getters.zoomToFitButton().click(); workflowPage.getters.executeWorkflowButton().click(); diff --git a/cypress/e2e/28-debug.cy.ts b/cypress/e2e/28-debug.cy.ts index 5d2bd76cac3a4..bc1f03c1628b3 100644 --- a/cypress/e2e/28-debug.cy.ts +++ b/cypress/e2e/28-debug.cy.ts @@ -18,7 +18,7 @@ describe('Debug', () => { it('should be able to debug executions', () => { cy.intercept('GET', '/rest/executions?filter=*').as('getExecutions'); cy.intercept('GET', '/rest/executions/*').as('getExecution'); - cy.intercept('POST', '/rest/workflows/**/run').as('postWorkflowRun'); + cy.intercept('POST', '/rest/workflows/**/run?**').as('postWorkflowRun'); cy.signinAsOwner(); diff --git a/cypress/e2e/30-editor-after-route-changes.cy.ts b/cypress/e2e/30-editor-after-route-changes.cy.ts index 65987806764de..f0381a32a29d4 100644 --- a/cypress/e2e/30-editor-after-route-changes.cy.ts +++ b/cypress/e2e/30-editor-after-route-changes.cy.ts @@ -142,7 +142,7 @@ describe('Editor actions should work', () => { it('after switching between Editor and Debug', () => { cy.intercept('GET', '/rest/executions?filter=*').as('getExecutions'); cy.intercept('GET', '/rest/executions/*').as('getExecution'); - cy.intercept('POST', '/rest/workflows/**/run').as('postWorkflowRun'); + cy.intercept('POST', '/rest/workflows/**/run?**').as('postWorkflowRun'); editWorkflowAndDeactivate(); workflowPage.actions.executeWorkflow(); diff --git a/cypress/pages/workflow-executions-tab.ts b/cypress/pages/workflow-executions-tab.ts index 93c9af86ff80a..5e8c36c055552 100644 --- a/cypress/pages/workflow-executions-tab.ts +++ b/cypress/pages/workflow-executions-tab.ts @@ -35,7 +35,7 @@ export class WorkflowExecutionsTab extends BasePage { }, createManualExecutions: (count: number) => { for (let i = 0; i < count; i++) { - cy.intercept('POST', '/rest/workflows/**/run').as('workflowExecution'); + cy.intercept('POST', '/rest/workflows/**/run?**').as('workflowExecution'); workflowPage.actions.executeWorkflow(); cy.wait('@workflowExecution'); } diff --git a/cypress/utils/executions.ts b/cypress/utils/executions.ts index eb0dbfc251f7e..0b4814fdc9f38 100644 --- a/cypress/utils/executions.ts +++ b/cypress/utils/executions.ts @@ -89,7 +89,7 @@ export function runMockWorkflowExecution({ }) { const executionId = nanoid(8); - cy.intercept('POST', '/rest/workflows/**/run', { + cy.intercept('POST', '/rest/workflows/**/run?**', { statusCode: 201, body: { data: { From c7970dcbf75097f5c0603c35dbc7d2c260928050 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 15:16:26 +0200 Subject: [PATCH 11/54] document executionData --- packages/workflow/src/Interfaces.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 5e761b61e3fd9..92ccfa8d978e5 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2150,6 +2150,10 @@ export interface IWorkflowExecutionDataProcess { destinationNode?: string; restartExecutionId?: string; executionMode: WorkflowExecuteMode; + /** + * The data that is sent in the body of the webhook that started this + * execution. + */ executionData?: IRunExecutionData; runData?: IRunData; pinData?: IPinData; From 11bf1f0d0ecbb652c53cd2752ef4d17863ef69fa Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 15:18:45 +0200 Subject: [PATCH 12/54] rename findSubgraph2 to findSubgraph --- packages/core/src/WorkflowExecute.ts | 5 ++--- packages/core/src/__tests__/utils-2.test.ts | 4 ++-- packages/core/src/__tests__/utils.test.ts | 16 ++++++++-------- packages/core/src/utils.ts | 8 ++++---- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index cff187039bd01..d7ff711e22dd6 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -54,8 +54,7 @@ import { DirectedGraph, findCycles, findStartNodes, - // TODO: rename to findSubgraph - findSubgraph2, + findSubgraph, findTriggerForPartialExecution, } from './utils'; // TODO: move into it's own folder @@ -348,7 +347,7 @@ export class WorkflowExecute { } // 2. Find the Subgraph - const subgraph = findSubgraph2(DirectedGraph.fromWorkflow(workflow), destinationNode, trigger); + const subgraph = findSubgraph(DirectedGraph.fromWorkflow(workflow), destinationNode, trigger); const filteredNodes = subgraph.getNodes(); // 3. Find the Start Nodes diff --git a/packages/core/src/__tests__/utils-2.test.ts b/packages/core/src/__tests__/utils-2.test.ts index 6cf48ad9a0199..826de8f22b0a6 100644 --- a/packages/core/src/__tests__/utils-2.test.ts +++ b/packages/core/src/__tests__/utils-2.test.ts @@ -11,7 +11,7 @@ import { recreateNodeExecutionStack } from '@/utils-2'; import { createNodeData, toITaskData } from './helpers'; import type { StartNodeData } from '@/utils'; -import { DirectedGraph, findSubgraph2 } from '@/utils'; +import { DirectedGraph, findSubgraph } from '@/utils'; import { type IPinData, type IRunData } from 'n8n-workflow'; import { AssertionError } from 'assert'; @@ -31,7 +31,7 @@ describe('recreateNodeExecutionStack', () => { .addNodes(trigger, node) .addConnections({ from: trigger, to: node }); - const workflow = findSubgraph2(graph, node, trigger); + const workflow = findSubgraph(graph, node, trigger); const startNodes: StartNodeData[] = [ { node, diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts index d18761887bd19..128ea9c5da7f5 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/__tests__/utils.test.ts @@ -10,7 +10,7 @@ import type { IConnections, INode, IPinData, IRunData } from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; -import { DirectedGraph, findStartNodes, findSubgraph2, isDirty } from '../utils'; +import { DirectedGraph, findStartNodes, findSubgraph, isDirty } from '../utils'; import { createNodeData, toITaskData, defaultWorkflowParameter } from './helpers'; test('toITaskData', function () { @@ -547,7 +547,7 @@ describe('findSubgraph2', () => { .addNodes(trigger, destination) .addConnections({ from: trigger, to: destination }); - const subgraph = findSubgraph2(graph, destination, trigger); + const subgraph = findSubgraph(graph, destination, trigger); expect(subgraph).toEqual(graph); }); @@ -569,7 +569,7 @@ describe('findSubgraph2', () => { { from: ifNode, to: noOp, outputIndex: 1 }, ); - const subgraph = findSubgraph2(graph, noOp, ifNode); + const subgraph = findSubgraph(graph, noOp, ifNode); expect(subgraph).toEqual(graph); }); @@ -589,7 +589,7 @@ describe('findSubgraph2', () => { .addNodes(trigger, destination, node) .addConnections({ from: trigger, to: destination }, { from: destination, to: node }); - const subgraph = findSubgraph2(graph, destination, trigger); + const subgraph = findSubgraph(graph, destination, trigger); expect(subgraph).toEqual( new DirectedGraph() @@ -613,7 +613,7 @@ describe('findSubgraph2', () => { .addNodes(trigger, disabled, destination) .addConnections({ from: trigger, to: disabled }, { from: disabled, to: destination }); - const subgraph = findSubgraph2(graph, destination, trigger); + const subgraph = findSubgraph(graph, destination, trigger); expect(subgraph).toEqual( new DirectedGraph() @@ -646,7 +646,7 @@ describe('findSubgraph2', () => { // // ACT // - const subgraph = findSubgraph2(graph, node2, trigger); + const subgraph = findSubgraph(graph, node2, trigger); // // ASSERT @@ -676,7 +676,7 @@ describe('findSubgraph2', () => { // // ACT // - const subgraph = findSubgraph2(graph, node1, trigger); + const subgraph = findSubgraph(graph, node1, trigger); // // ASSERT @@ -710,7 +710,7 @@ describe('findSubgraph2', () => { // // ACT // - const subgraph = findSubgraph2(graph, destination, trigger); + const subgraph = findSubgraph(graph, destination, trigger); // // ASSERT diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 21b983d6305ae..01385a34f1d25 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -9,7 +9,7 @@ import type { } from 'n8n-workflow'; import { NodeConnectionType, Workflow } from 'n8n-workflow'; -function findSubgraph2Recursive( +function findSubgraphRecursive( graph: DirectedGraph, destinationNode: INode, current: INode, @@ -89,7 +89,7 @@ function findSubgraph2Recursive( // Recurse on each parent. for (const parentConnection of parentConnections) { - findSubgraph2Recursive(graph, destinationNode, parentConnection.from, trigger, newGraph, [ + findSubgraphRecursive(graph, destinationNode, parentConnection.from, trigger, newGraph, [ ...currentBranch, parentConnection, ]); @@ -603,14 +603,14 @@ export class DirectedGraph { } } -export function findSubgraph2( +export function findSubgraph( graph: DirectedGraph, destinationNode: INode, trigger: INode, ): DirectedGraph { const newGraph = new DirectedGraph(); - findSubgraph2Recursive(graph, destinationNode, destinationNode, trigger, newGraph, []); + findSubgraphRecursive(graph, destinationNode, destinationNode, trigger, newGraph, []); return newGraph; } From 59f82c8e1a4dd9d58ae698f141b78167eeb184fc Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 15:35:52 +0200 Subject: [PATCH 13/54] add skipped failing tests to continue working on nodes with multiple inputs --- packages/core/src/__tests__/utils-2.test.ts | 95 ++++++++++++++++++++- packages/core/src/__tests__/utils.test.ts | 38 ++++++++- packages/core/src/utils-2.ts | 6 +- packages/core/src/utils.ts | 5 ++ 4 files changed, 139 insertions(+), 5 deletions(-) diff --git a/packages/core/src/__tests__/utils-2.test.ts b/packages/core/src/__tests__/utils-2.test.ts index 826de8f22b0a6..569730002f1aa 100644 --- a/packages/core/src/__tests__/utils-2.test.ts +++ b/packages/core/src/__tests__/utils-2.test.ts @@ -220,7 +220,6 @@ describe('recreateNodeExecutionStack', () => { const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); const node3 = createNodeData({ name: 'node3' }); - const graph = new DirectedGraph() .addNodes(trigger, node1, node2, node3) .addConnections( @@ -292,4 +291,98 @@ describe('recreateNodeExecutionStack', () => { }, }); }); + + // TODO: This does not work as expected right now. The node execution stack + // will contain node3, but only with data from input 1, instead of data from + // input 1 and 2. + // I need to spent time to understand the node execution stack, waiting + // executions and waiting execution sources and write a spec for this and + // then re-implement it from the spec. + // Changing `StartNodeData.sourceData` to contain sources from multiple nodes + // could be helpful: + // { name: string, sourceData: ISourceData[] } + // ┌─────┐1 ►► + // ┌─►│node1├───┐ ┌─────┐ + // ┌───────┐1 │ └─────┘ └──►│ │ + // │Trigger├──┤ │node3│ + // └───────┘ │ ┌─────┐1 ┌──►│ │ + // └─►│node2├───┘ └─────┘ + // └─────┘ + // eslint-disable-next-line n8n-local-rules/no-skipped-tests + test.skip('multiple inputs', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const node3 = createNodeData({ name: 'node3' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2, node3) + .addConnections( + { from: trigger, to: node1 }, + { from: trigger, to: node2 }, + { from: node1, to: node3, outputIndex: 0 }, + { from: node2, to: node3, outputIndex: 1 }, + ); + const startNodes: StartNodeData[] = [ + { + node: node3, + sourceData: { + previousNode: node1, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + }, + ]; + const runData: IRunData = { + [trigger.name]: [toITaskData([{ data: { value: 1 } }])], + [node1.name]: [toITaskData([{ data: { value: 1 } }])], + [node2.name]: [toITaskData([{ data: { value: 1 } }])], + }; + const pinData: IPinData = { + [trigger.name]: [{ json: { value: 1 } }], + }; + + // + // ACT + // + const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = + recreateNodeExecutionStack(graph, startNodes, node3, runData, pinData); + + // + // ASSERT + // + expect(nodeExecutionStack).toHaveLength(1); + expect(nodeExecutionStack).toContainEqual({ + data: { main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]] }, + node: node3, + source: { + main: [ + { previousNode: 'node1', previousNodeOutput: 0, previousNodeRun: 0 }, + // TODO: this should be node2, but this would only work if I refactor + // sourceData to contain multiple sources per node. + { previousNode: 'node1', previousNodeOutput: 0, previousNodeRun: 0 }, + ], + }, + }); + + expect(waitingExecution).toEqual({ + node3: { + '0': { + main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]], + }, + }, + }); + expect(waitingExecutionSource).toEqual({ + node3: { + '0': { + main: [ + { previousNode: 'node1', previousNodeOutput: undefined, previousNodeRun: undefined }, + { previousNode: 'node2', previousNodeOutput: undefined, previousNodeRun: undefined }, + ], + }, + }, + }); + }); }); diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts index 128ea9c5da7f5..52b8f31552d4b 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/__tests__/utils.test.ts @@ -443,12 +443,44 @@ describe('findStartNodes', () => { }, }); }); + + // TODO: This is not working yet as expected. + // It should only have `node` once as a start node. + // The spec needs to be updated before this is fixed. + // ►► + // ┌───────┐ ┌────┐ + // │ │1 ┌────►│ │ + // │trigger├───┤ │node│ + // │ │ └────►│ │ + // └───────┘ └────┘ + // eslint-disable-next-line n8n-local-rules/no-skipped-tests + test.skip('multiple connections with both having data', () => { + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const node3 = createNodeData({ name: 'node3' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2, node3) + .addConnections( + { from: trigger, to: node1 }, + { from: trigger, to: node2 }, + { from: node1, to: node3 }, + { from: node2, to: node3 }, + ); + + const startNodes = findStartNodes(graph, trigger, node3, { + [trigger.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], + [node1.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], + [node2.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], + }); + + expect(startNodes).toHaveLength(1); expect(startNodes).toContainEqual({ - node: merge, + node: node3, sourceData: { - previousNode: ifNode, + previousNode: node1, previousNodeRun: 0, - previousNodeOutput: 1, + previousNodeOutput: 0, }, }); }); diff --git a/packages/core/src/utils-2.ts b/packages/core/src/utils-2.ts index 463d8da862984..713d8902cfac3 100644 --- a/packages/core/src/utils-2.ts +++ b/packages/core/src/utils-2.ts @@ -15,7 +15,6 @@ import * as a from 'assert'; import type { DirectedGraph, StartNodeData } from './utils'; import { getIncomingData } from './utils'; -// eslint-disable-next-line @typescript-eslint/promise-function-async, complexity export function recreateNodeExecutionStack( graph: DirectedGraph, // TODO: turn this into StartNodeData from utils @@ -81,6 +80,10 @@ export function recreateNodeExecutionStack( // missing is the inputIndex and that's the sole reason why we iterate // over all incoming connections. for (const connection of incomingStartNodeConnections) { + // TODO: do not skip connections that don't match the source data, this + // causes problems with nodes that have multiple inputs. + // The proper fix would be to remodel source data to contain all sources + // not just the first one it finds. if (connection.from.name !== startNode.sourceData?.previousNode.name) { continue; } @@ -125,6 +128,7 @@ export function recreateNodeExecutionStack( nodeExecutionStack.push(executeData); + // NOTE: Do we need this? if (destinationNode) { const destinationNodeName = destinationNode.name; // Check if the destinationNode has to be added as waiting diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 01385a34f1d25..f1feb5e912a95 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -246,6 +246,11 @@ interface ISourceData { previousNodeOutput: number; // If undefined "0" gets used previousNodeRun: number; // If undefined "0" gets used } +// TODO: This is how ISourceData should look like. +//interface NewSourceData { +// connection: Connection; +// previousNodeRun: number; // If undefined "0" gets used +//} // TODO: rename to something more general, like path segment export interface StartNodeData { From 1dcdd6c025a18d01c095fb284ea5c5dbfc78a930 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 16:05:15 +0200 Subject: [PATCH 14/54] reorganize code --- .../PartialExecutionUtils/DirectedGraph.ts | 199 ++++++ .../__tests__/DirectedGraph.test.ts | 129 ++++ .../__tests__/findStartNodes.test.ts} | 440 +------------ .../__tests__/findSubgraph.test.ts | 203 ++++++ .../__tests__/helpers.ts | 35 +- .../recreateNodeExecutionStack.test.ts} | 12 +- .../__tests__/toIConnections.test.ts | 26 + .../__tests__/toITaskData.test.ts | 64 ++ .../src/PartialExecutionUtils/findCycles.ts | 6 + .../PartialExecutionUtils/findStartNodes.ts | 163 +++++ .../src/PartialExecutionUtils/findSubgraph.ts | 102 +++ .../findTriggerForPartialExecution.ts | 104 +++ .../PartialExecutionUtils/getIncomingData.ts | 22 + .../core/src/PartialExecutionUtils/index.ts | 8 + .../recreateNodeExecutionStack.ts} | 5 +- packages/core/src/WorkflowExecute.ts | 5 +- packages/core/src/utils.ts | 621 ------------------ 17 files changed, 1078 insertions(+), 1066 deletions(-) create mode 100644 packages/core/src/PartialExecutionUtils/DirectedGraph.ts create mode 100644 packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts rename packages/core/src/{__tests__/utils.test.ts => PartialExecutionUtils/__tests__/findStartNodes.test.ts} (52%) create mode 100644 packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts rename packages/core/src/{ => PartialExecutionUtils}/__tests__/helpers.ts (66%) rename packages/core/src/{__tests__/utils-2.test.ts => PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts} (96%) create mode 100644 packages/core/src/PartialExecutionUtils/__tests__/toIConnections.test.ts create mode 100644 packages/core/src/PartialExecutionUtils/__tests__/toITaskData.test.ts create mode 100644 packages/core/src/PartialExecutionUtils/findCycles.ts create mode 100644 packages/core/src/PartialExecutionUtils/findStartNodes.ts create mode 100644 packages/core/src/PartialExecutionUtils/findSubgraph.ts create mode 100644 packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts create mode 100644 packages/core/src/PartialExecutionUtils/getIncomingData.ts create mode 100644 packages/core/src/PartialExecutionUtils/index.ts rename packages/core/src/{utils-2.ts => PartialExecutionUtils/recreateNodeExecutionStack.ts} (97%) delete mode 100644 packages/core/src/utils.ts diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts new file mode 100644 index 0000000000000..01e39d97b2b11 --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -0,0 +1,199 @@ +import * as a from 'assert'; +import type { IConnections, INode, WorkflowParameters } from 'n8n-workflow'; +import { NodeConnectionType, Workflow } from 'n8n-workflow'; + +export type Connection = { + from: INode; + to: INode; + type: NodeConnectionType; + outputIndex: number; + inputIndex: number; +}; +// fromName-outputType-outputIndex-inputIndex-toName +type DirectedGraphKey = `${string}-${NodeConnectionType}-${number}-${number}-${string}`; + +export class DirectedGraph { + private nodes: Map = new Map(); + + private connections: Map = new Map(); + + getNodes() { + return new Map(this.nodes.entries()); + } + + addNode(node: INode) { + this.nodes.set(node.name, node); + return this; + } + + addNodes(...nodes: INode[]) { + for (const node of nodes) { + this.addNode(node); + } + return this; + } + + addConnection(connectionInput: { + from: INode; + to: INode; + type?: NodeConnectionType; + outputIndex?: number; + inputIndex?: number; + }) { + const { from, to } = connectionInput; + + const fromExists = this.nodes.get(from.name) === from; + const toExists = this.nodes.get(to.name) === to; + + a.ok(fromExists); + a.ok(toExists); + + const connection: Connection = { + ...connectionInput, + type: connectionInput.type ?? NodeConnectionType.Main, + outputIndex: connectionInput.outputIndex ?? 0, + inputIndex: connectionInput.inputIndex ?? 0, + }; + + this.connections.set(this.makeKey(connection), connection); + return this; + } + + addConnections( + ...connectionInputs: Array<{ + from: INode; + to: INode; + type?: NodeConnectionType; + outputIndex?: number; + inputIndex?: number; + }> + ) { + for (const connectionInput of connectionInputs) { + this.addConnection(connectionInput); + } + return this; + } + + getDirectChildren(node: INode) { + const nodeExists = this.nodes.get(node.name) === node; + a.ok(nodeExists); + + const directChildren: Connection[] = []; + + for (const connection of this.connections.values()) { + if (connection.from !== node) { + continue; + } + + directChildren.push(connection); + } + + return directChildren; + } + + private getChildrenRecursive(node: INode, seen: Set): Connection[] { + if (seen.has(node)) { + return []; + } + + const directChildren = this.getDirectChildren(node); + + return [ + ...directChildren, + ...directChildren.flatMap((child) => + this.getChildrenRecursive(child.to, new Set(seen).add(node)), + ), + ]; + } + + getChildren(node: INode): Connection[] { + return this.getChildrenRecursive(node, new Set()); + } + + getDirectParents(node: INode) { + const nodeExists = this.nodes.get(node.name) === node; + a.ok(nodeExists); + + const directParents: Connection[] = []; + + for (const connection of this.connections.values()) { + if (connection.to !== node) { + continue; + } + + directParents.push(connection); + } + + return directParents; + } + + toWorkflow(parameters: Omit): Workflow { + return new Workflow({ + ...parameters, + nodes: [...this.nodes.values()], + connections: this.toIConnections(), + }); + } + + static fromWorkflow(workflow: Workflow): DirectedGraph { + const graph = new DirectedGraph(); + + graph.addNodes(...Object.values(workflow.nodes)); + + for (const [fromNodeName, iConnection] of Object.entries(workflow.connectionsBySourceNode)) { + const from = workflow.getNode(fromNodeName); + a.ok(from); + + for (const [outputType, outputs] of Object.entries(iConnection)) { + // TODO: parse + //const type = outputType as NodeConnectionType + + for (const [outputIndex, conns] of outputs.entries()) { + for (const conn of conns) { + // TODO: What's with the input type? + const { node: toNodeName, type: _inputType, index: inputIndex } = conn; + const to = workflow.getNode(toNodeName); + a.ok(to); + + graph.addConnection({ + from, + to, + type: outputType as NodeConnectionType, + outputIndex, + inputIndex, + }); + } + } + } + } + + return graph; + } + + private toIConnections() { + const result: IConnections = {}; + + for (const connection of this.connections.values()) { + const { from, to, type, outputIndex, inputIndex } = connection; + + result[from.name] = result[from.name] ?? { + [type]: [], + }; + const resultConnection = result[from.name]; + resultConnection[type][outputIndex] = resultConnection[type][outputIndex] ?? []; + const group = resultConnection[type][outputIndex]; + + group.push({ + node: to.name, + type, + index: inputIndex, + }); + } + + return result; + } + + private makeKey(connection: Connection): DirectedGraphKey { + return `${connection.from.name}-${connection.type}-${connection.outputIndex}-${connection.inputIndex}-${connection.to.name}`; + } +} diff --git a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts new file mode 100644 index 0000000000000..52009bef913f5 --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts @@ -0,0 +1,129 @@ +// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ +// If you update the tests, please update the diagrams as well. +// If you add a test, please create a new diagram. +// +// Map +// 0 means the output has no run data +// 1 means the output has run data +// ►► denotes the node that the user wants to execute to +// XX denotes that the node is disabled +// PD denotes that the node has pinned data + +import { NodeConnectionType } from 'n8n-workflow'; +import { DirectedGraph } from '../DirectedGraph'; +import { createNodeData, defaultWorkflowParameter } from './helpers'; + +describe('DirectedGraph', () => { + // ┌─────┐ ┌─────┐ ┌─────┐ + // ┌─►│node1├───►│node2├──►│node3├─┐ + // │ └─────┘ └─────┘ └─────┘ │ + // │ │ + // └───────────────────────────────┘ + test('roundtrip', () => { + const node1 = createNodeData({ name: 'Node1' }); + const node2 = createNodeData({ name: 'Node2' }); + const node3 = createNodeData({ name: 'Node3' }); + + const graph = new DirectedGraph() + .addNodes(node1, node2, node3) + .addConnections( + { from: node1, to: node2 }, + { from: node2, to: node3 }, + { from: node3, to: node1 }, + ); + + expect(DirectedGraph.fromWorkflow(graph.toWorkflow({ ...defaultWorkflowParameter }))).toEqual( + graph, + ); + }); + + describe('getChildren', () => { + // ┌─────┐ ┌─────┐ + // │node1├──────►│node2│ + // └─────┘ └─────┘ + test('simple', () => { + const from = createNodeData({ name: 'Node1' }); + const to = createNodeData({ name: 'Node2' }); + const graph = new DirectedGraph().addNodes(from, to).addConnections({ from, to }); + + const children = graph.getChildren(from); + expect(children).toHaveLength(1); + expect(children).toContainEqual({ + from, + to, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }); + }); + // ┌─────┐ + // │ ├────┐ ┌─────┐ + // │node1│ ├─►│node2│ + // │ ├────┘ └─────┘ + // └─────┘ + test('medium', () => { + const from = createNodeData({ name: 'Node1' }); + const to = createNodeData({ name: 'Node2' }); + const graph = new DirectedGraph() + .addNodes(from, to) + .addConnections({ from, to, outputIndex: 0 }, { from, to, outputIndex: 1 }); + + const children = graph.getChildren(from); + expect(children).toHaveLength(2); + expect(children).toContainEqual({ + from, + to, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }); + expect(children).toContainEqual({ + from, + to, + inputIndex: 0, + outputIndex: 1, + type: NodeConnectionType.Main, + }); + }); + + // ┌─────┐ ┌─────┐ + // ┌─►│node1├──────►│node2├──┐ + // │ └─────┘ └─────┘ │ + // │ │ + // └─────────────────────────┘ + test('terminates if the graph has cycles', () => { + // + // ARRANGE + // + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const graph = new DirectedGraph() + .addNodes(node1, node2) + .addConnections({ from: node1, to: node2 }, { from: node2, to: node2 }); + + // + // ACT + // + const children = graph.getChildren(node1); + + // + // ASSERT + // + expect(children).toHaveLength(2); + expect(children).toContainEqual({ + from: node1, + to: node2, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }); + expect(children).toContainEqual({ + from: node2, + to: node2, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }); + }); + }); +}); diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts similarity index 52% rename from packages/core/src/__tests__/utils.test.ts rename to packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts index 52b8f31552d4b..93befc1cff884 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts @@ -1,5 +1,6 @@ // NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ -// If you update the tests please update the diagrams as well. +// If you update the tests, please update the diagrams as well. +// If you add a test, please create a new diagram. // // Map // 0 means the output has no run data @@ -8,135 +9,10 @@ // XX denotes that the node is disabled // PD denotes that the node has pinned data -import type { IConnections, INode, IPinData, IRunData } from 'n8n-workflow'; -import { NodeConnectionType } from 'n8n-workflow'; -import { DirectedGraph, findStartNodes, findSubgraph, isDirty } from '../utils'; -import { createNodeData, toITaskData, defaultWorkflowParameter } from './helpers'; - -test('toITaskData', function () { - expect(toITaskData([{ data: { value: 1 } }])).toEqual({ - executionStatus: 'success', - executionTime: 0, - source: [], - startTime: 0, - data: { - main: [[{ json: { value: 1 } }]], - }, - }); - - expect(toITaskData([{ data: { value: 1 }, outputIndex: 1 }])).toEqual({ - executionStatus: 'success', - executionTime: 0, - source: [], - startTime: 0, - data: { - main: [null, [{ json: { value: 1 } }]], - }, - }); - - expect( - toITaskData([ - { data: { value: 1 }, outputIndex: 1, nodeConnectionType: NodeConnectionType.AiAgent }, - ]), - ).toEqual({ - executionStatus: 'success', - executionTime: 0, - source: [], - startTime: 0, - data: { - [NodeConnectionType.AiAgent]: [null, [{ json: { value: 1 } }]], - }, - }); - - expect( - toITaskData([ - { data: { value: 1 }, outputIndex: 0 }, - { data: { value: 2 }, outputIndex: 1 }, - ]), - ).toEqual({ - executionStatus: 'success', - executionTime: 0, - startTime: 0, - source: [], - data: { - main: [ - [ - { - json: { value: 1 }, - }, - ], - [ - { - json: { value: 2 }, - }, - ], - ], - }, - }); -}); - -type Connection = { - from: INode; - to: INode; - type?: NodeConnectionType; - outputIndex?: number; - inputIndex?: number; -}; - -function toIConnections(connections: Connection[]): IConnections { - const result: IConnections = {}; - - for (const connection of connections) { - const type = connection.type ?? NodeConnectionType.Main; - const outputIndex = connection.outputIndex ?? 0; - const inputIndex = connection.inputIndex ?? 0; - - result[connection.from.name] = result[connection.from.name] ?? { - [type]: [], - }; - const resultConnection = result[connection.from.name]; - resultConnection[type][outputIndex] = resultConnection[type][outputIndex] ?? []; - const group = resultConnection[type][outputIndex]; - - group.push({ - node: connection.to.name, - type, - index: inputIndex, - }); - } - - return result; -} - -test('toIConnections', () => { - const node1 = createNodeData({ name: 'Basic Node 1' }); - const node2 = createNodeData({ name: 'Basic Node 2' }); - - expect( - toIConnections([{ from: node1, to: node2, type: NodeConnectionType.Main, outputIndex: 0 }]), - ).toEqual({ - [node1.name]: { - // output group - main: [ - // first output - [ - // first connection - { - node: node2.name, - type: NodeConnectionType.Main, - index: 0, - }, - ], - ], - }, - }); -}); - -//export interface INodeTypes { -// getByName(nodeType: string): INodeType | IVersionedNodeType; -// getByNameAndVersion(nodeType: string, version?: number): INodeType; -// getKnownTypes(): IDataObject; -//} +import type { IPinData, IRunData } from 'n8n-workflow'; +import { createNodeData, toITaskData } from './helpers'; +import { findStartNodes, isDirty } from '../findStartNodes'; +import { DirectedGraph } from '../DirectedGraph'; describe('isDirty', () => { test("if the node has pinned data it's not dirty", () => { @@ -565,307 +441,3 @@ describe('findStartNodes', () => { }); }); }); - -describe('findSubgraph2', () => { - // ►► - // ┌───────┐ ┌───────────┐ - // │trigger├────►│destination│ - // └───────┘ └───────────┘ - test('simple', () => { - const trigger = createNodeData({ name: 'trigger' }); - const destination = createNodeData({ name: 'destination' }); - - const graph = new DirectedGraph() - .addNodes(trigger, destination) - .addConnections({ from: trigger, to: destination }); - - const subgraph = findSubgraph(graph, destination, trigger); - - expect(subgraph).toEqual(graph); - }); - - // ►► - // ┌───────┐ ┌───────────┐ - // │ ├────────►│ │ - // │trigger│ │destination│ - // │ ├────────►│ │ - // └───────┘ └───────────┘ - test('multiple connections', () => { - const ifNode = createNodeData({ name: 'If' }); - const noOp = createNodeData({ name: 'noOp' }); - - const graph = new DirectedGraph() - .addNodes(ifNode, noOp) - .addConnections( - { from: ifNode, to: noOp, outputIndex: 0 }, - { from: ifNode, to: noOp, outputIndex: 1 }, - ); - - const subgraph = findSubgraph(graph, noOp, ifNode); - - expect(subgraph).toEqual(graph); - }); - - // ►► - // ┌───────┐ ┌───────────┐ - // │ ├────────►│ │ ┌────┐ - // │trigger│ │destination├─────►│node│ - // │ ├────────►│ │ └────┘ - // └───────┘ └───────────┘ - test('disregard nodes after destination', () => { - const trigger = createNodeData({ name: 'trigger' }); - const destination = createNodeData({ name: 'destination' }); - const node = createNodeData({ name: 'node' }); - - const graph = new DirectedGraph() - .addNodes(trigger, destination, node) - .addConnections({ from: trigger, to: destination }, { from: destination, to: node }); - - const subgraph = findSubgraph(graph, destination, trigger); - - expect(subgraph).toEqual( - new DirectedGraph() - .addNodes(trigger, destination) - .addConnections({ from: trigger, to: destination }), - ); - }); - - // XX - // ┌───────┐ ┌────────┐ ►► - // │ ├────────►│ │ ┌───────────┐ - // │trigger│ │disabled├─────►│destination│ - // │ ├────────►│ │ └───────────┘ - // └───────┘ └────────┘ - test('skip disabled nodes', () => { - const trigger = createNodeData({ name: 'trigger' }); - const disabled = createNodeData({ name: 'disabled', disabled: true }); - const destination = createNodeData({ name: 'destination' }); - - const graph = new DirectedGraph() - .addNodes(trigger, disabled, destination) - .addConnections({ from: trigger, to: disabled }, { from: disabled, to: destination }); - - const subgraph = findSubgraph(graph, destination, trigger); - - expect(subgraph).toEqual( - new DirectedGraph() - .addNodes(trigger, destination) - .addConnections({ from: trigger, to: destination }), - ); - }); - - // ►► - // ┌───────┐ ┌─────┐ ┌─────┐ - // │Trigger├───┬──►│Node1├───┬─►│Node2│ - // └───────┘ │ └─────┘ │ └─────┘ - // │ │ - // └─────────────┘ - test('terminates when called with graph that contains cycles', () => { - // - // ARRANGE - // - const trigger = createNodeData({ name: 'trigger' }); - const node1 = createNodeData({ name: 'node1' }); - const node2 = createNodeData({ name: 'node2' }); - const graph = new DirectedGraph() - .addNodes(trigger, node1, node2) - .addConnections( - { from: trigger, to: node1 }, - { from: node1, to: node1 }, - { from: node1, to: node2 }, - ); - - // - // ACT - // - const subgraph = findSubgraph(graph, node2, trigger); - - // - // ASSERT - // - expect(subgraph).toEqual(graph); - }); - - // ►► - // ┌───────┐ ┌─────┐ - // │Trigger├──┬─►│Node1│ - // └───────┘ │ └─────┘ - // │ - // ┌─────┐ │ - // │Node2├────┘ - // └─────┘ - test('terminates when called with graph that contains cycles', () => { - // - // ARRANGE - // - const trigger = createNodeData({ name: 'trigger' }); - const node1 = createNodeData({ name: 'node1' }); - const node2 = createNodeData({ name: 'node2' }); - const graph = new DirectedGraph() - .addNodes(trigger, node1, node2) - .addConnections({ from: trigger, to: node1 }, { from: node2, to: node1 }); - - // - // ACT - // - const subgraph = findSubgraph(graph, node1, trigger); - - // - // ASSERT - // - expect(subgraph).toEqual( - new DirectedGraph().addNodes(trigger, node1).addConnections({ from: trigger, to: node1 }), - ); - }); - - // ►► - // ┌───────┐ ┌───────────┐ ┌───────────┐ - // │Trigger├─┬─►│Destination├──►│AnotherNode├───┐ - // └───────┘ │ └───────────┘ └───────────┘ │ - // │ │ - // └──────────────────────────────────┘ - test('terminates if the destination node is part of a cycle', () => { - // - // ARRANGE - // - const trigger = createNodeData({ name: 'trigger' }); - const destination = createNodeData({ name: 'destination' }); - const anotherNode = createNodeData({ name: 'anotherNode' }); - const graph = new DirectedGraph() - .addNodes(trigger, destination, anotherNode) - .addConnections( - { from: trigger, to: destination }, - { from: destination, to: anotherNode }, - { from: anotherNode, to: destination }, - ); - - // - // ACT - // - const subgraph = findSubgraph(graph, destination, trigger); - - // - // ASSERT - // - expect(subgraph).toEqual( - new DirectedGraph() - .addNodes(trigger, destination) - .addConnections({ from: trigger, to: destination }), - ); - }); -}); - -describe('DirectedGraph', () => { - // ┌─────┐ ┌─────┐ ┌─────┐ - // ┌─►│node1├───►│node2├──►│node3├─┐ - // │ └─────┘ └─────┘ └─────┘ │ - // │ │ - // └───────────────────────────────┘ - test('roundtrip', () => { - const node1 = createNodeData({ name: 'Node1' }); - const node2 = createNodeData({ name: 'Node2' }); - const node3 = createNodeData({ name: 'Node3' }); - - const graph = new DirectedGraph() - .addNodes(node1, node2, node3) - .addConnections( - { from: node1, to: node2 }, - { from: node2, to: node3 }, - { from: node3, to: node1 }, - ); - - expect(DirectedGraph.fromWorkflow(graph.toWorkflow({ ...defaultWorkflowParameter }))).toEqual( - graph, - ); - }); - - describe('getChildren', () => { - // ┌─────┐ ┌─────┐ - // │node1├──────►│node2│ - // └─────┘ └─────┘ - test('simple', () => { - const from = createNodeData({ name: 'Node1' }); - const to = createNodeData({ name: 'Node2' }); - const graph = new DirectedGraph().addNodes(from, to).addConnections({ from, to }); - - const children = graph.getChildren(from); - expect(children).toHaveLength(1); - expect(children).toContainEqual({ - from, - to, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }); - }); - // ┌─────┐ - // │ ├────┐ ┌─────┐ - // │node1│ ├─►│node2│ - // │ ├────┘ └─────┘ - // └─────┘ - test('medium', () => { - const from = createNodeData({ name: 'Node1' }); - const to = createNodeData({ name: 'Node2' }); - const graph = new DirectedGraph() - .addNodes(from, to) - .addConnections({ from, to, outputIndex: 0 }, { from, to, outputIndex: 1 }); - - const children = graph.getChildren(from); - expect(children).toHaveLength(2); - expect(children).toContainEqual({ - from, - to, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }); - expect(children).toContainEqual({ - from, - to, - inputIndex: 0, - outputIndex: 1, - type: NodeConnectionType.Main, - }); - }); - - // ┌─────┐ ┌─────┐ - // ┌─►│node1├──────►│node2├──┐ - // │ └─────┘ └─────┘ │ - // │ │ - // └─────────────────────────┘ - test('terminates if the graph has cycles', () => { - // - // ARRANGE - // - const node1 = createNodeData({ name: 'node1' }); - const node2 = createNodeData({ name: 'node2' }); - const graph = new DirectedGraph() - .addNodes(node1, node2) - .addConnections({ from: node1, to: node2 }, { from: node2, to: node2 }); - - // - // ACT - // - const children = graph.getChildren(node1); - - // - // ASSERT - // - expect(children).toHaveLength(2); - expect(children).toContainEqual({ - from: node1, - to: node2, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }); - expect(children).toContainEqual({ - from: node2, - to: node2, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }); - }); - }); -}); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts new file mode 100644 index 0000000000000..32b50c25a9546 --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts @@ -0,0 +1,203 @@ +// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ +// If you update the tests, please update the diagrams as well. +// If you add a test, please create a new diagram. +// +// Map +// 0 means the output has no run data +// 1 means the output has run data +// ►► denotes the node that the user wants to execute to +// XX denotes that the node is disabled +// PD denotes that the node has pinned data + +import { DirectedGraph } from '../DirectedGraph'; +import { findSubgraph } from '../findSubgraph'; +import { createNodeData } from './helpers'; + +describe('findSubgraph2', () => { + // ►► + // ┌───────┐ ┌───────────┐ + // │trigger├────►│destination│ + // └───────┘ └───────────┘ + test('simple', () => { + const trigger = createNodeData({ name: 'trigger' }); + const destination = createNodeData({ name: 'destination' }); + + const graph = new DirectedGraph() + .addNodes(trigger, destination) + .addConnections({ from: trigger, to: destination }); + + const subgraph = findSubgraph(graph, destination, trigger); + + expect(subgraph).toEqual(graph); + }); + + // ►► + // ┌───────┐ ┌───────────┐ + // │ ├────────►│ │ + // │trigger│ │destination│ + // │ ├────────►│ │ + // └───────┘ └───────────┘ + test('multiple connections', () => { + const ifNode = createNodeData({ name: 'If' }); + const noOp = createNodeData({ name: 'noOp' }); + + const graph = new DirectedGraph() + .addNodes(ifNode, noOp) + .addConnections( + { from: ifNode, to: noOp, outputIndex: 0 }, + { from: ifNode, to: noOp, outputIndex: 1 }, + ); + + const subgraph = findSubgraph(graph, noOp, ifNode); + + expect(subgraph).toEqual(graph); + }); + + // ►► + // ┌───────┐ ┌───────────┐ + // │ ├────────►│ │ ┌────┐ + // │trigger│ │destination├─────►│node│ + // │ ├────────►│ │ └────┘ + // └───────┘ └───────────┘ + test('disregard nodes after destination', () => { + const trigger = createNodeData({ name: 'trigger' }); + const destination = createNodeData({ name: 'destination' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(trigger, destination, node) + .addConnections({ from: trigger, to: destination }, { from: destination, to: node }); + + const subgraph = findSubgraph(graph, destination, trigger); + + expect(subgraph).toEqual( + new DirectedGraph() + .addNodes(trigger, destination) + .addConnections({ from: trigger, to: destination }), + ); + }); + + // XX + // ┌───────┐ ┌────────┐ ►► + // │ ├────────►│ │ ┌───────────┐ + // │trigger│ │disabled├─────►│destination│ + // │ ├────────►│ │ └───────────┘ + // └───────┘ └────────┘ + test('skip disabled nodes', () => { + const trigger = createNodeData({ name: 'trigger' }); + const disabled = createNodeData({ name: 'disabled', disabled: true }); + const destination = createNodeData({ name: 'destination' }); + + const graph = new DirectedGraph() + .addNodes(trigger, disabled, destination) + .addConnections({ from: trigger, to: disabled }, { from: disabled, to: destination }); + + const subgraph = findSubgraph(graph, destination, trigger); + + expect(subgraph).toEqual( + new DirectedGraph() + .addNodes(trigger, destination) + .addConnections({ from: trigger, to: destination }), + ); + }); + + // ►► + // ┌───────┐ ┌─────┐ ┌─────┐ + // │Trigger├───┬──►│Node1├───┬─►│Node2│ + // └───────┘ │ └─────┘ │ └─────┘ + // │ │ + // └─────────────┘ + test('terminates when called with graph that contains cycles', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2) + .addConnections( + { from: trigger, to: node1 }, + { from: node1, to: node1 }, + { from: node1, to: node2 }, + ); + + // + // ACT + // + const subgraph = findSubgraph(graph, node2, trigger); + + // + // ASSERT + // + expect(subgraph).toEqual(graph); + }); + + // ►► + // ┌───────┐ ┌─────┐ + // │Trigger├──┬─►│Node1│ + // └───────┘ │ └─────┘ + // │ + // ┌─────┐ │ + // │Node2├────┘ + // └─────┘ + test('terminates when called with graph that contains cycles', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const node1 = createNodeData({ name: 'node1' }); + const node2 = createNodeData({ name: 'node2' }); + const graph = new DirectedGraph() + .addNodes(trigger, node1, node2) + .addConnections({ from: trigger, to: node1 }, { from: node2, to: node1 }); + + // + // ACT + // + const subgraph = findSubgraph(graph, node1, trigger); + + // + // ASSERT + // + expect(subgraph).toEqual( + new DirectedGraph().addNodes(trigger, node1).addConnections({ from: trigger, to: node1 }), + ); + }); + + // ►► + // ┌───────┐ ┌───────────┐ ┌───────────┐ + // │Trigger├─┬─►│Destination├──►│AnotherNode├───┐ + // └───────┘ │ └───────────┘ └───────────┘ │ + // │ │ + // └──────────────────────────────────┘ + test('terminates if the destination node is part of a cycle', () => { + // + // ARRANGE + // + const trigger = createNodeData({ name: 'trigger' }); + const destination = createNodeData({ name: 'destination' }); + const anotherNode = createNodeData({ name: 'anotherNode' }); + const graph = new DirectedGraph() + .addNodes(trigger, destination, anotherNode) + .addConnections( + { from: trigger, to: destination }, + { from: destination, to: anotherNode }, + { from: anotherNode, to: destination }, + ); + + // + // ACT + // + const subgraph = findSubgraph(graph, destination, trigger); + + // + // ASSERT + // + expect(subgraph).toEqual( + new DirectedGraph() + .addNodes(trigger, destination) + .addConnections({ from: trigger, to: destination }), + ); + }); +}); diff --git a/packages/core/src/__tests__/helpers.ts b/packages/core/src/PartialExecutionUtils/__tests__/helpers.ts similarity index 66% rename from packages/core/src/__tests__/helpers.ts rename to packages/core/src/PartialExecutionUtils/__tests__/helpers.ts index fa9039b5a8e61..641254242c56d 100644 --- a/packages/core/src/__tests__/helpers.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/helpers.ts @@ -1,5 +1,5 @@ import { NodeConnectionType } from 'n8n-workflow'; -import type { INodeParameters, INode, ITaskData, IDataObject } from 'n8n-workflow'; +import type { INodeParameters, INode, ITaskData, IDataObject, IConnections } from 'n8n-workflow'; interface StubNode { name: string; @@ -73,3 +73,36 @@ export const defaultWorkflowParameter = { active: false, nodeTypes, }; + +type Connection = { + from: INode; + to: INode; + type?: NodeConnectionType; + outputIndex?: number; + inputIndex?: number; +}; + +export function toIConnections(connections: Connection[]): IConnections { + const result: IConnections = {}; + + for (const connection of connections) { + const type = connection.type ?? NodeConnectionType.Main; + const outputIndex = connection.outputIndex ?? 0; + const inputIndex = connection.inputIndex ?? 0; + + result[connection.from.name] = result[connection.from.name] ?? { + [type]: [], + }; + const resultConnection = result[connection.from.name]; + resultConnection[type][outputIndex] = resultConnection[type][outputIndex] ?? []; + const group = resultConnection[type][outputIndex]; + + group.push({ + node: connection.to.name, + type, + index: inputIndex, + }); + } + + return result; +} diff --git a/packages/core/src/__tests__/utils-2.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts similarity index 96% rename from packages/core/src/__tests__/utils-2.test.ts rename to packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts index 569730002f1aa..3bbfd520017dc 100644 --- a/packages/core/src/__tests__/utils-2.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts @@ -1,5 +1,6 @@ // NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ -// If you update the tests please update the diagrams as well. +// If you update the tests, please update the diagrams as well. +// If you add a test, please create a new diagram. // // Map // 0 means the output has no run data @@ -8,12 +9,13 @@ // XX denotes that the node is disabled // PD denotes that the node has pinned data -import { recreateNodeExecutionStack } from '@/utils-2'; -import { createNodeData, toITaskData } from './helpers'; -import type { StartNodeData } from '@/utils'; -import { DirectedGraph, findSubgraph } from '@/utils'; +import { recreateNodeExecutionStack } from '@/PartialExecutionUtils/recreateNodeExecutionStack'; import { type IPinData, type IRunData } from 'n8n-workflow'; import { AssertionError } from 'assert'; +import { DirectedGraph } from '../DirectedGraph'; +import { findSubgraph } from '../findSubgraph'; +import type { StartNodeData } from '../findStartNodes'; +import { createNodeData, toITaskData } from './helpers'; describe('recreateNodeExecutionStack', () => { // ►► diff --git a/packages/core/src/PartialExecutionUtils/__tests__/toIConnections.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/toIConnections.test.ts new file mode 100644 index 0000000000000..a2524bf3ce4ef --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/__tests__/toIConnections.test.ts @@ -0,0 +1,26 @@ +import { NodeConnectionType } from 'n8n-workflow'; +import { createNodeData, toIConnections } from './helpers'; + +test('toIConnections', () => { + const node1 = createNodeData({ name: 'Basic Node 1' }); + const node2 = createNodeData({ name: 'Basic Node 2' }); + + expect( + toIConnections([{ from: node1, to: node2, type: NodeConnectionType.Main, outputIndex: 0 }]), + ).toEqual({ + [node1.name]: { + // output group + main: [ + // first output + [ + // first connection + { + node: node2.name, + type: NodeConnectionType.Main, + index: 0, + }, + ], + ], + }, + }); +}); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/toITaskData.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/toITaskData.test.ts new file mode 100644 index 0000000000000..e255836339849 --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/__tests__/toITaskData.test.ts @@ -0,0 +1,64 @@ +import { NodeConnectionType } from 'n8n-workflow'; +import { toITaskData } from './helpers'; + +test('toITaskData', function () { + expect(toITaskData([{ data: { value: 1 } }])).toEqual({ + executionStatus: 'success', + executionTime: 0, + source: [], + startTime: 0, + data: { + main: [[{ json: { value: 1 } }]], + }, + }); + + expect(toITaskData([{ data: { value: 1 }, outputIndex: 1 }])).toEqual({ + executionStatus: 'success', + executionTime: 0, + source: [], + startTime: 0, + data: { + main: [null, [{ json: { value: 1 } }]], + }, + }); + + expect( + toITaskData([ + { data: { value: 1 }, outputIndex: 1, nodeConnectionType: NodeConnectionType.AiAgent }, + ]), + ).toEqual({ + executionStatus: 'success', + executionTime: 0, + source: [], + startTime: 0, + data: { + [NodeConnectionType.AiAgent]: [null, [{ json: { value: 1 } }]], + }, + }); + + expect( + toITaskData([ + { data: { value: 1 }, outputIndex: 0 }, + { data: { value: 2 }, outputIndex: 1 }, + ]), + ).toEqual({ + executionStatus: 'success', + executionTime: 0, + startTime: 0, + source: [], + data: { + main: [ + [ + { + json: { value: 1 }, + }, + ], + [ + { + json: { value: 2 }, + }, + ], + ], + }, + }); +}); diff --git a/packages/core/src/PartialExecutionUtils/findCycles.ts b/packages/core/src/PartialExecutionUtils/findCycles.ts new file mode 100644 index 0000000000000..388518ae52d55 --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/findCycles.ts @@ -0,0 +1,6 @@ +import type { Workflow } from 'n8n-workflow'; + +export function findCycles(_workflow: Workflow) { + // TODO: implement depth first search or Tarjan's Algorithm + return []; +} diff --git a/packages/core/src/PartialExecutionUtils/findStartNodes.ts b/packages/core/src/PartialExecutionUtils/findStartNodes.ts new file mode 100644 index 0000000000000..8715b662929e5 --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/findStartNodes.ts @@ -0,0 +1,163 @@ +import type { INode, IPinData, IRunData } from 'n8n-workflow'; +import type { DirectedGraph } from './DirectedGraph'; +import { getIncomingData } from './getIncomingData'; + +interface ISourceData { + previousNode: INode; + previousNodeOutput: number; // If undefined "0" gets used + previousNodeRun: number; // If undefined "0" gets used +} +// TODO: This is how ISourceData should look like. +//interface NewSourceData { +// connection: Connection; +// previousNodeRun: number; // If undefined "0" gets used +//} + +// TODO: rename to something more general, like path segment +export interface StartNodeData { + node: INode; + sourceData?: ISourceData; +} + +type Key = `${string}-${number}-${string}`; + +// TODO: implement dirty checking for options and properties and parent nodes +// being disabled +export function isDirty(node: INode, runData: IRunData = {}, pinData: IPinData = {}): boolean { + //- it’s properties or options changed since last execution, or + + const propertiesOrOptionsChanged = false; + + if (propertiesOrOptionsChanged) { + return true; + } + + const parentNodeGotDisabled = false; + + if (parentNodeGotDisabled) { + return true; + } + + //- it has an error, or + + const hasAnError = false; + + if (hasAnError) { + return true; + } + + //- it does neither have run data nor pinned data + + const hasPinnedData = pinData[node.name] !== undefined; + + if (hasPinnedData) { + return false; + } + + const hasRunData = runData?.[node.name]; + + if (hasRunData) { + return false; + } + + return true; +} + +function makeKey(from: ISourceData | undefined, to: INode): Key { + return `${from?.previousNode.name ?? 'start'}-${from?.previousNodeOutput ?? 0}-${to.name}`; +} + +function findStartNodesRecursive( + graph: DirectedGraph, + current: INode, + destination: INode, + runData: IRunData, + pinData: IPinData, + startNodes: Map, + seen: Set, + source?: ISourceData, +) { + const nodeIsDirty = isDirty(current, runData, pinData); + + // If the current node is dirty stop following this branch, we found a start + // node. + if (nodeIsDirty) { + startNodes.set(makeKey(source, current), { + node: current, + sourceData: source, + }); + return startNodes; + } + + // If the current node is the destination node stop following this branch, we + // found a start node. + if (current === destination) { + startNodes.set(makeKey(source, current), { node: current, sourceData: source }); + return startNodes; + } + + // If we detect a cycle stop following the branch, there is no start node on + // this branch. + if (seen.has(current)) { + return startNodes; + } + + // Recurse with every direct child that is part of the sub graph. + const outGoingConnections = graph.getDirectChildren(current); + for (const outGoingConnection of outGoingConnections) { + const nodeRunData = getIncomingData( + runData, + outGoingConnection.from.name, + // NOTE: It's always 0 until I fix the bug that removes the run data for + // old runs. The FE only sends data for one run for each node. + 0, + outGoingConnection.type, + outGoingConnection.outputIndex, + ); + + // If the node has multiple outputs, only follow the outputs that have run data. + const hasNoRunData = + nodeRunData === null || nodeRunData === undefined || nodeRunData.length === 0; + if (hasNoRunData) { + continue; + } + + findStartNodesRecursive( + graph, + outGoingConnection.to, + destination, + runData, + pinData, + startNodes, + new Set(seen).add(current), + { + previousNode: current, + // NOTE: It's always 0 until I fix the bug that removes the run data for + // old runs. The FE only sends data for one run for each node. + previousNodeRun: 0, + previousNodeOutput: outGoingConnection.outputIndex, + }, + ); + } + + return startNodes; +} + +export function findStartNodes( + graph: DirectedGraph, + trigger: INode, + destination: INode, + runData: IRunData = {}, + pinData: IPinData = {}, +): StartNodeData[] { + const startNodes = findStartNodesRecursive( + graph, + trigger, + destination, + runData, + pinData, + new Map(), + new Set(), + ); + return [...startNodes.values()]; +} diff --git a/packages/core/src/PartialExecutionUtils/findSubgraph.ts b/packages/core/src/PartialExecutionUtils/findSubgraph.ts new file mode 100644 index 0000000000000..3695de863e8fe --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/findSubgraph.ts @@ -0,0 +1,102 @@ +import type { INode } from 'n8n-workflow'; +import type { Connection } from './DirectedGraph'; +import { DirectedGraph } from './DirectedGraph'; + +function findSubgraphRecursive( + graph: DirectedGraph, + destinationNode: INode, + current: INode, + trigger: INode, + newGraph: DirectedGraph, + currentBranch: Connection[], +) { + //console.log('currentBranch', currentBranch); + + // If the current node is the chosen ‘trigger keep this branch. + if (current === trigger) { + console.log(`${current.name}: is trigger`); + for (const connection of currentBranch) { + newGraph.addNodes(connection.from, connection.to); + newGraph.addConnection(connection); + } + + return; + } + + let parentConnections = graph.getDirectParents(current); + + // If the current node has no parents, don’t keep this branch. + if (parentConnections.length === 0) { + console.log(`${current.name}: no parents`); + return; + } + + // If the current node is the destination node again, don’t keep this branch. + const isCycleWithDestinationNode = + current === destinationNode && currentBranch.some((c) => c.to === destinationNode); + if (isCycleWithDestinationNode) { + console.log(`${current.name}: isCycleWithDestinationNode`); + return; + } + + // If the current node was already visited, keep this branch. + const isCycleWithCurrentNode = currentBranch.some((c) => c.to === current); + if (isCycleWithCurrentNode) { + console.log(`${current.name}: isCycleWithCurrentNode`); + // TODO: write function that adds nodes when adding connections + for (const connection of currentBranch) { + newGraph.addNodes(connection.from, connection.to); + newGraph.addConnection(connection); + } + return; + } + + // If the current node is disabled, don’t keep this node, but keep the + // branch. + // Take every incoming connection and connect it to every node that is + // connected to the current node’s first output + if (current.disabled) { + console.log(`${current.name}: is disabled`); + const incomingConnections = graph.getDirectParents(current); + const outgoingConnections = graph + .getDirectChildren(current) + // NOTE: When a node is disabled only the first output gets data + .filter((connection) => connection.outputIndex === 0); + + parentConnections = []; + + for (const incomingConnection of incomingConnections) { + for (const outgoingConnection of outgoingConnections) { + const newConnection = { + ...incomingConnection, + to: outgoingConnection.to, + inputIndex: outgoingConnection.inputIndex, + }; + + parentConnections.push(newConnection); + currentBranch.pop(); + currentBranch.push(newConnection); + } + } + } + + // Recurse on each parent. + for (const parentConnection of parentConnections) { + findSubgraphRecursive(graph, destinationNode, parentConnection.from, trigger, newGraph, [ + ...currentBranch, + parentConnection, + ]); + } +} + +export function findSubgraph( + graph: DirectedGraph, + destinationNode: INode, + trigger: INode, +): DirectedGraph { + const newGraph = new DirectedGraph(); + + findSubgraphRecursive(graph, destinationNode, destinationNode, trigger, newGraph, []); + + return newGraph; +} diff --git a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts new file mode 100644 index 0000000000000..baf28624498dc --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts @@ -0,0 +1,104 @@ +import type { INode, Workflow } from 'n8n-workflow'; + +function findAllParentTriggers(workflow: Workflow, destinationNode: string) { + // Traverse from the destination node back until we found all trigger nodes. + // Do this recursively, because why not. + const parentNodes = workflow + .getParentNodes(destinationNode) + .map((name) => { + const node = workflow.getNode(name); + + if (!node) { + return null; + } + + return { + node, + nodeType: workflow.nodeTypes.getByNameAndVersion(node.type, node.typeVersion), + }; + }) + .filter((value) => value !== null) + .filter(({ nodeType }) => nodeType.description.group.includes('trigger')) + .map(({ node }) => node); + + return parentNodes; +} + +// TODO: write unit tests for this +export function findTriggerForPartialExecution( + workflow: Workflow, + destinationNode: string, +): INode | undefined { + const parentTriggers = findAllParentTriggers(workflow, destinationNode).filter( + (trigger) => !trigger.disabled, + ); + const pinnedTriggers = parentTriggers + // TODO: add the other filters here from `findAllPinnedActivators` + .filter((trigger) => workflow.pinData?.[trigger.name]) + .sort((n) => (n.type.endsWith('webhook') ? -1 : 1)); + + if (pinnedTriggers.length) { + return pinnedTriggers[0]; + } else { + return parentTriggers[0]; + } +} + +//function findAllPinnedActivators(workflow: Workflow, pinData?: IPinData) { +// return Object.values(workflow.nodes) +// .filter( +// (node) => +// !node.disabled && +// pinData?.[node.name] && +// ['trigger', 'webhook'].some((suffix) => node.type.toLowerCase().endsWith(suffix)) && +// node.type !== 'n8n-nodes-base.respondToWebhook', +// ) +// .sort((a) => (a.type.endsWith('webhook') ? -1 : 1)); +//} + +// TODO: deduplicate this with +// packages/cli/src/workflows/workflowExecution.service.ts +//function selectPinnedActivatorStarter( +// workflow: Workflow, +// startNodes?: string[], +// pinData?: IPinData, +//) { +// if (!pinData || !startNodes) return null; +// +// const allPinnedActivators = findAllPinnedActivators(workflow, pinData); +// +// if (allPinnedActivators.length === 0) return null; +// +// const [firstPinnedActivator] = allPinnedActivators; +// +// // full manual execution +// +// if (startNodes?.length === 0) return firstPinnedActivator ?? null; +// +// // partial manual execution +// +// /** +// * If the partial manual execution has 2+ start nodes, we search only the zeroth +// * start node's parents for a pinned activator. If we had 2+ start nodes without +// * a common ancestor and so if we end up finding multiple pinned activators, we +// * would still need to return one to comply with existing usage. +// */ +// const [firstStartNodeName] = startNodes; +// +// const parentNodeNames = +// //new Workflow({ +// // nodes: workflow.nodes, +// // connections: workflow.connections, +// // active: workflow.active, +// // nodeTypes: this.nodeTypes, +// // }). +// workflow.getParentNodes(firstStartNodeName); +// +// if (parentNodeNames.length > 0) { +// const parentNodeName = parentNodeNames.find((p) => p === firstPinnedActivator.name); +// +// return allPinnedActivators.find((pa) => pa.name === parentNodeName) ?? null; +// } +// +// return allPinnedActivators.find((pa) => pa.name === firstStartNodeName) ?? null; +//} diff --git a/packages/core/src/PartialExecutionUtils/getIncomingData.ts b/packages/core/src/PartialExecutionUtils/getIncomingData.ts new file mode 100644 index 0000000000000..2f5f22cd35836 --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/getIncomingData.ts @@ -0,0 +1,22 @@ +import * as a from 'assert'; +import type { INodeExecutionData, IRunData, NodeConnectionType } from 'n8n-workflow'; + +export function getIncomingData( + runData: IRunData, + nodeName: string, + runIndex: number, + connectionType: NodeConnectionType, + outputIndex: number, +): INodeExecutionData[] | null | undefined { + a.ok(runData[nodeName], `Can't find node with name '${nodeName}' in runData.`); + a.ok( + runData[nodeName][runIndex], + `Can't find a run for index '${runIndex}' for node name '${nodeName}'`, + ); + a.ok( + runData[nodeName][runIndex].data, + `Can't find data for index '${runIndex}' for node name '${nodeName}'`, + ); + + return runData[nodeName][runIndex].data[connectionType][outputIndex]; +} diff --git a/packages/core/src/PartialExecutionUtils/index.ts b/packages/core/src/PartialExecutionUtils/index.ts new file mode 100644 index 0000000000000..b43b8f3896979 --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/index.ts @@ -0,0 +1,8 @@ +export { DirectedGraph, Connection } from './DirectedGraph'; +export { getIncomingData } from './getIncomingData'; + +export { findTriggerForPartialExecution } from './findTriggerForPartialExecution'; +export { findStartNodes } from './findStartNodes'; +export { findSubgraph } from './findSubgraph'; +export { findCycles } from './findCycles'; +export { recreateNodeExecutionStack } from './recreateNodeExecutionStack'; diff --git a/packages/core/src/utils-2.ts b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts similarity index 97% rename from packages/core/src/utils-2.ts rename to packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts index 713d8902cfac3..8c56e8436eda4 100644 --- a/packages/core/src/utils-2.ts +++ b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts @@ -12,8 +12,9 @@ import { } from 'n8n-workflow'; import * as a from 'assert'; -import type { DirectedGraph, StartNodeData } from './utils'; -import { getIncomingData } from './utils'; +import type { DirectedGraph } from './DirectedGraph'; +import type { StartNodeData } from './findStartNodes'; +import { getIncomingData } from './getIncomingData'; export function recreateNodeExecutionStack( graph: DirectedGraph, diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index d7ff711e22dd6..6cd8c505490ca 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -50,15 +50,14 @@ import get from 'lodash/get'; import * as NodeExecuteFunctions from './NodeExecuteFunctions'; import * as a from 'assert'; +import { recreateNodeExecutionStack } from './PartialExecutionUtils/recreateNodeExecutionStack'; import { DirectedGraph, findCycles, findStartNodes, findSubgraph, findTriggerForPartialExecution, -} from './utils'; -// TODO: move into it's own folder -import { recreateNodeExecutionStack } from './utils-2'; +} from './PartialExecutionUtils'; export class WorkflowExecute { private status: ExecutionStatus = 'new'; diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts deleted file mode 100644 index f1feb5e912a95..0000000000000 --- a/packages/core/src/utils.ts +++ /dev/null @@ -1,621 +0,0 @@ -import * as a from 'assert'; -import type { - IConnections, - INode, - INodeExecutionData, - IPinData, - IRunData, - WorkflowParameters, -} from 'n8n-workflow'; -import { NodeConnectionType, Workflow } from 'n8n-workflow'; - -function findSubgraphRecursive( - graph: DirectedGraph, - destinationNode: INode, - current: INode, - trigger: INode, - newGraph: DirectedGraph, - currentBranch: Connection[], -) { - //console.log('currentBranch', currentBranch); - - // If the current node is the chosen ‘trigger keep this branch. - if (current === trigger) { - console.log(`${current.name}: is trigger`); - for (const connection of currentBranch) { - newGraph.addNodes(connection.from, connection.to); - newGraph.addConnection(connection); - } - - return; - } - - let parentConnections = graph.getDirectParents(current); - - // If the current node has no parents, don’t keep this branch. - if (parentConnections.length === 0) { - console.log(`${current.name}: no parents`); - return; - } - - // If the current node is the destination node again, don’t keep this branch. - const isCycleWithDestinationNode = - current === destinationNode && currentBranch.some((c) => c.to === destinationNode); - if (isCycleWithDestinationNode) { - console.log(`${current.name}: isCycleWithDestinationNode`); - return; - } - - // If the current node was already visited, keep this branch. - const isCycleWithCurrentNode = currentBranch.some((c) => c.to === current); - if (isCycleWithCurrentNode) { - console.log(`${current.name}: isCycleWithCurrentNode`); - // TODO: write function that adds nodes when adding connections - for (const connection of currentBranch) { - newGraph.addNodes(connection.from, connection.to); - newGraph.addConnection(connection); - } - return; - } - - // If the current node is disabled, don’t keep this node, but keep the - // branch. - // Take every incoming connection and connect it to every node that is - // connected to the current node’s first output - if (current.disabled) { - console.log(`${current.name}: is disabled`); - const incomingConnections = graph.getDirectParents(current); - const outgoingConnections = graph - .getDirectChildren(current) - // NOTE: When a node is disabled only the first output gets data - .filter((connection) => connection.outputIndex === 0); - - parentConnections = []; - - for (const incomingConnection of incomingConnections) { - for (const outgoingConnection of outgoingConnections) { - const newConnection = { - ...incomingConnection, - to: outgoingConnection.to, - inputIndex: outgoingConnection.inputIndex, - }; - - parentConnections.push(newConnection); - currentBranch.pop(); - currentBranch.push(newConnection); - } - } - } - - // Recurse on each parent. - for (const parentConnection of parentConnections) { - findSubgraphRecursive(graph, destinationNode, parentConnection.from, trigger, newGraph, [ - ...currentBranch, - parentConnection, - ]); - } -} - -function findAllParentTriggers(workflow: Workflow, destinationNode: string) { - // Traverse from the destination node back until we found all trigger nodes. - // Do this recursively, because why not. - const parentNodes = workflow - .getParentNodes(destinationNode) - .map((name) => { - const node = workflow.getNode(name); - - if (!node) { - return null; - } - - return { - node, - nodeType: workflow.nodeTypes.getByNameAndVersion(node.type, node.typeVersion), - }; - }) - .filter((value) => value !== null) - .filter(({ nodeType }) => nodeType.description.group.includes('trigger')) - .map(({ node }) => node); - - return parentNodes; -} - -// TODO: write unit tests for this -export function findTriggerForPartialExecution( - workflow: Workflow, - destinationNode: string, -): INode | undefined { - const parentTriggers = findAllParentTriggers(workflow, destinationNode).filter( - (trigger) => !trigger.disabled, - ); - const pinnedTriggers = parentTriggers - // TODO: add the other filters here from `findAllPinnedActivators` - .filter((trigger) => workflow.pinData?.[trigger.name]) - .sort((n) => (n.type.endsWith('webhook') ? -1 : 1)); - - if (pinnedTriggers.length) { - return pinnedTriggers[0]; - } else { - return parentTriggers[0]; - } -} - -//function findAllPinnedActivators(workflow: Workflow, pinData?: IPinData) { -// return Object.values(workflow.nodes) -// .filter( -// (node) => -// !node.disabled && -// pinData?.[node.name] && -// ['trigger', 'webhook'].some((suffix) => node.type.toLowerCase().endsWith(suffix)) && -// node.type !== 'n8n-nodes-base.respondToWebhook', -// ) -// .sort((a) => (a.type.endsWith('webhook') ? -1 : 1)); -//} - -// TODO: deduplicate this with -// packages/cli/src/workflows/workflowExecution.service.ts -//function selectPinnedActivatorStarter( -// workflow: Workflow, -// startNodes?: string[], -// pinData?: IPinData, -//) { -// if (!pinData || !startNodes) return null; -// -// const allPinnedActivators = findAllPinnedActivators(workflow, pinData); -// -// if (allPinnedActivators.length === 0) return null; -// -// const [firstPinnedActivator] = allPinnedActivators; -// -// // full manual execution -// -// if (startNodes?.length === 0) return firstPinnedActivator ?? null; -// -// // partial manual execution -// -// /** -// * If the partial manual execution has 2+ start nodes, we search only the zeroth -// * start node's parents for a pinned activator. If we had 2+ start nodes without -// * a common ancestor and so if we end up finding multiple pinned activators, we -// * would still need to return one to comply with existing usage. -// */ -// const [firstStartNodeName] = startNodes; -// -// const parentNodeNames = -// //new Workflow({ -// // nodes: workflow.nodes, -// // connections: workflow.connections, -// // active: workflow.active, -// // nodeTypes: this.nodeTypes, -// // }). -// workflow.getParentNodes(firstStartNodeName); -// -// if (parentNodeNames.length > 0) { -// const parentNodeName = parentNodeNames.find((p) => p === firstPinnedActivator.name); -// -// return allPinnedActivators.find((pa) => pa.name === parentNodeName) ?? null; -// } -// -// return allPinnedActivators.find((pa) => pa.name === firstStartNodeName) ?? null; -//} - -// TODO: implement dirty checking for options and properties and parent nodes -// being disabled -export function isDirty(node: INode, runData: IRunData = {}, pinData: IPinData = {}): boolean { - //- it’s properties or options changed since last execution, or - - const propertiesOrOptionsChanged = false; - - if (propertiesOrOptionsChanged) { - return true; - } - - const parentNodeGotDisabled = false; - - if (parentNodeGotDisabled) { - return true; - } - - //- it has an error, or - - const hasAnError = false; - - if (hasAnError) { - return true; - } - - //- it does neither have run data nor pinned data - - const hasPinnedData = pinData[node.name] !== undefined; - - if (hasPinnedData) { - return false; - } - - const hasRunData = runData?.[node.name]; - - if (hasRunData) { - return false; - } - - return true; -} - -interface ISourceData { - previousNode: INode; - previousNodeOutput: number; // If undefined "0" gets used - previousNodeRun: number; // If undefined "0" gets used -} -// TODO: This is how ISourceData should look like. -//interface NewSourceData { -// connection: Connection; -// previousNodeRun: number; // If undefined "0" gets used -//} - -// TODO: rename to something more general, like path segment -export interface StartNodeData { - node: INode; - sourceData?: ISourceData; -} - -//export function getDirectChildren(workflow: Workflow, parent: INode): StartNodeData[] { -// const directChildren: StartNodeData[] = []; -// -// for (const [_connectionGroupName, inputs] of Object.entries( -// workflow.connectionsBySourceNode[parent.name] ?? {}, -// )) { -// for (const [outputIndex, connections] of inputs.entries()) { -// for (const connection of connections) { -// const node = workflow.getNode(connection.node); -// -// a.ok(node, `Node(${connection.node}) does not exist in workflow.`); -// -// directChildren.push({ -// node, -// sourceData: { -// previousNode: parent, -// previousNodeOutput: outputIndex, -// // TODO: I don't have this here. This part is working without run -// // data, so there are not runs. -// previousNodeRun: 0, -// }, -// }); -// } -// } -// } -// -// return directChildren; -//} - -export function getIncomingData( - runData: IRunData, - nodeName: string, - runIndex: number, - connectionType: NodeConnectionType, - outputIndex: number, -): INodeExecutionData[] | null | undefined { - a.ok(runData[nodeName], `Can't find node with name '${nodeName}' in runData.`); - a.ok( - runData[nodeName][runIndex], - `Can't find a run for index '${runIndex}' for node name '${nodeName}'`, - ); - a.ok( - runData[nodeName][runIndex].data, - `Can't find data for index '${runIndex}' for node name '${nodeName}'`, - ); - - return runData[nodeName][runIndex].data[connectionType][outputIndex]; -} - -type Key = `${string}-${number}-${string}`; - -function makeKey(from: ISourceData | undefined, to: INode): Key { - return `${from?.previousNode.name ?? 'start'}-${from?.previousNodeOutput ?? 0}-${to.name}`; -} - -function findStartNodesRecursive( - graph: DirectedGraph, - current: INode, - destination: INode, - runData: IRunData, - pinData: IPinData, - startNodes: Map, - seen: Set, - source?: ISourceData, -) { - const nodeIsDirty = isDirty(current, runData, pinData); - - // If the current node is dirty stop following this branch, we found a start - // node. - if (nodeIsDirty) { - startNodes.set(makeKey(source, current), { - node: current, - sourceData: source, - }); - return startNodes; - } - - // If the current node is the destination node stop following this branch, we - // found a start node. - if (current === destination) { - startNodes.set(makeKey(source, current), { node: current, sourceData: source }); - return startNodes; - } - - // If we detect a cycle stop following the branch, there is no start node on - // this branch. - if (seen.has(current)) { - return startNodes; - } - - // Recurse with every direct child that is part of the sub graph. - const outGoingConnections = graph.getDirectChildren(current); - for (const outGoingConnection of outGoingConnections) { - const nodeRunData = getIncomingData( - runData, - outGoingConnection.from.name, - // NOTE: It's always 0 until I fix the bug that removes the run data for - // old runs. The FE only sends data for one run for each node. - 0, - outGoingConnection.type, - outGoingConnection.outputIndex, - ); - - // If the node has multiple outputs, only follow the outputs that have run data. - const hasNoRunData = - nodeRunData === null || nodeRunData === undefined || nodeRunData.length === 0; - if (hasNoRunData) { - continue; - } - - findStartNodesRecursive( - graph, - outGoingConnection.to, - destination, - runData, - pinData, - startNodes, - new Set(seen).add(current), - { - previousNode: current, - // NOTE: It's always 0 until I fix the bug that removes the run data for - // old runs. The FE only sends data for one run for each node. - previousNodeRun: 0, - previousNodeOutput: outGoingConnection.outputIndex, - }, - ); - } - - return startNodes; -} - -export function findStartNodes( - graph: DirectedGraph, - trigger: INode, - destination: INode, - runData: IRunData = {}, - pinData: IPinData = {}, -): StartNodeData[] { - const startNodes = findStartNodesRecursive( - graph, - trigger, - destination, - runData, - pinData, - new Map(), - new Set(), - ); - return [...startNodes.values()]; -} - -export function findCycles(_workflow: Workflow) { - // TODO: implement depth first search or Tarjan's Algorithm - return []; -} - -export type Connection = { - from: INode; - to: INode; - type: NodeConnectionType; - outputIndex: number; - inputIndex: number; -}; -// fromName-outputType-outputIndex-inputIndex-toName -type DirectedGraphKey = `${string}-${NodeConnectionType}-${number}-${number}-${string}`; -export class DirectedGraph { - private nodes: Map = new Map(); - - private connections: Map = new Map(); - - getNodes() { - return new Map(this.nodes.entries()); - } - - addNode(node: INode) { - this.nodes.set(node.name, node); - return this; - } - - addNodes(...nodes: INode[]) { - for (const node of nodes) { - this.addNode(node); - } - return this; - } - - addConnection(connectionInput: { - from: INode; - to: INode; - type?: NodeConnectionType; - outputIndex?: number; - inputIndex?: number; - }) { - const { from, to } = connectionInput; - - const fromExists = this.nodes.get(from.name) === from; - const toExists = this.nodes.get(to.name) === to; - - a.ok(fromExists); - a.ok(toExists); - - const connection: Connection = { - ...connectionInput, - type: connectionInput.type ?? NodeConnectionType.Main, - outputIndex: connectionInput.outputIndex ?? 0, - inputIndex: connectionInput.inputIndex ?? 0, - }; - - this.connections.set(this.makeKey(connection), connection); - return this; - } - - addConnections( - ...connectionInputs: Array<{ - from: INode; - to: INode; - type?: NodeConnectionType; - outputIndex?: number; - inputIndex?: number; - }> - ) { - for (const connectionInput of connectionInputs) { - this.addConnection(connectionInput); - } - return this; - } - - getDirectChildren(node: INode) { - const nodeExists = this.nodes.get(node.name) === node; - a.ok(nodeExists); - - const directChildren: Connection[] = []; - - for (const connection of this.connections.values()) { - if (connection.from !== node) { - continue; - } - - directChildren.push(connection); - } - - return directChildren; - } - - private getChildrenRecursive(node: INode, seen: Set): Connection[] { - if (seen.has(node)) { - return []; - } - - const directChildren = this.getDirectChildren(node); - - return [ - ...directChildren, - ...directChildren.flatMap((child) => - this.getChildrenRecursive(child.to, new Set(seen).add(node)), - ), - ]; - } - - getChildren(node: INode): Connection[] { - return this.getChildrenRecursive(node, new Set()); - } - - getDirectParents(node: INode) { - const nodeExists = this.nodes.get(node.name) === node; - a.ok(nodeExists); - - const directParents: Connection[] = []; - - for (const connection of this.connections.values()) { - if (connection.to !== node) { - continue; - } - - directParents.push(connection); - } - - return directParents; - } - - toWorkflow(parameters: Omit): Workflow { - return new Workflow({ - ...parameters, - nodes: [...this.nodes.values()], - connections: this.toIConnections(), - }); - } - - static fromWorkflow(workflow: Workflow): DirectedGraph { - const graph = new DirectedGraph(); - - graph.addNodes(...Object.values(workflow.nodes)); - - for (const [fromNodeName, iConnection] of Object.entries(workflow.connectionsBySourceNode)) { - const from = workflow.getNode(fromNodeName); - a.ok(from); - - for (const [outputType, outputs] of Object.entries(iConnection)) { - // TODO: parse - //const type = outputType as NodeConnectionType - - for (const [outputIndex, conns] of outputs.entries()) { - for (const conn of conns) { - // TODO: What's with the input type? - const { node: toNodeName, type: _inputType, index: inputIndex } = conn; - const to = workflow.getNode(toNodeName); - a.ok(to); - - graph.addConnection({ - from, - to, - type: outputType as NodeConnectionType, - outputIndex, - inputIndex, - }); - } - } - } - } - - return graph; - } - - private toIConnections() { - const result: IConnections = {}; - - for (const connection of this.connections.values()) { - const { from, to, type, outputIndex, inputIndex } = connection; - - result[from.name] = result[from.name] ?? { - [type]: [], - }; - const resultConnection = result[from.name]; - resultConnection[type][outputIndex] = resultConnection[type][outputIndex] ?? []; - const group = resultConnection[type][outputIndex]; - - group.push({ - node: to.name, - type, - index: inputIndex, - }); - } - - return result; - } - - private makeKey(connection: Connection): DirectedGraphKey { - return `${connection.from.name}-${connection.type}-${connection.outputIndex}-${connection.inputIndex}-${connection.to.name}`; - } -} - -export function findSubgraph( - graph: DirectedGraph, - destinationNode: INode, - trigger: INode, -): DirectedGraph { - const newGraph = new DirectedGraph(); - - findSubgraphRecursive(graph, destinationNode, destinationNode, trigger, newGraph, []); - - return newGraph; -} From c4a5e56540ddf4d551d22f6716f529bbfec18262 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 16:06:00 +0200 Subject: [PATCH 15/54] remove console.logs --- packages/core/src/PartialExecutionUtils/findSubgraph.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/findSubgraph.ts b/packages/core/src/PartialExecutionUtils/findSubgraph.ts index 3695de863e8fe..9f975e59dabee 100644 --- a/packages/core/src/PartialExecutionUtils/findSubgraph.ts +++ b/packages/core/src/PartialExecutionUtils/findSubgraph.ts @@ -10,11 +10,8 @@ function findSubgraphRecursive( newGraph: DirectedGraph, currentBranch: Connection[], ) { - //console.log('currentBranch', currentBranch); - // If the current node is the chosen ‘trigger keep this branch. if (current === trigger) { - console.log(`${current.name}: is trigger`); for (const connection of currentBranch) { newGraph.addNodes(connection.from, connection.to); newGraph.addConnection(connection); @@ -27,7 +24,6 @@ function findSubgraphRecursive( // If the current node has no parents, don’t keep this branch. if (parentConnections.length === 0) { - console.log(`${current.name}: no parents`); return; } @@ -35,14 +31,12 @@ function findSubgraphRecursive( const isCycleWithDestinationNode = current === destinationNode && currentBranch.some((c) => c.to === destinationNode); if (isCycleWithDestinationNode) { - console.log(`${current.name}: isCycleWithDestinationNode`); return; } // If the current node was already visited, keep this branch. const isCycleWithCurrentNode = currentBranch.some((c) => c.to === current); if (isCycleWithCurrentNode) { - console.log(`${current.name}: isCycleWithCurrentNode`); // TODO: write function that adds nodes when adding connections for (const connection of currentBranch) { newGraph.addNodes(connection.from, connection.to); @@ -56,7 +50,6 @@ function findSubgraphRecursive( // Take every incoming connection and connect it to every node that is // connected to the current node’s first output if (current.disabled) { - console.log(`${current.name}: is disabled`); const incomingConnections = graph.getDirectParents(current); const outgoingConnections = graph .getDirectChildren(current) From d5b96db7ebdeb2717522ea5a6a9105966b25970f Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 19:40:27 +0200 Subject: [PATCH 16/54] clean up comments again --- packages/cli/new-partial-execution | 0 packages/cli/package.json | 2 +- packages/cli/src/license.ts | 2 -- packages/cli/src/workflow-runner.ts | 31 ++++++++--------------------- 4 files changed, 9 insertions(+), 26 deletions(-) delete mode 100644 packages/cli/new-partial-execution diff --git a/packages/cli/new-partial-execution b/packages/cli/new-partial-execution deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/packages/cli/package.json b/packages/cli/package.json index 9b9b2afa00790..1b33555a962f3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -22,7 +22,7 @@ "lint": "eslint . --quiet", "lintfix": "eslint . --fix", "start": "run-script-os", - "start:default": "node --inspect bin/n8n", + "start:default": "cd bin && ./n8n", "start:windows": "cd bin && n8n", "test": "pnpm test:sqlite", "test:sqlite": "N8N_LOG_LEVEL=silent DB_TYPE=sqlite jest", diff --git a/packages/cli/src/license.ts b/packages/cli/src/license.ts index 400a2c7623f24..75a57efd2ca33 100644 --- a/packages/cli/src/license.ts +++ b/packages/cli/src/license.ts @@ -230,7 +230,6 @@ export class License { } isFeatureEnabled(feature: BooleanLicenseFeature) { - return true; return this.manager?.hasFeatureEnabled(feature) ?? false; } @@ -371,7 +370,6 @@ export class License { } getTeamProjectLimit() { - return UNLIMITED_LICENSE_QUOTA; return this.getFeatureValue(LICENSE_QUOTAS.TEAM_PROJECT_LIMIT) ?? 0; } diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index b20ef3777f923..cdfa309e89734 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -109,11 +109,12 @@ export class WorkflowRunner { } } - /** Run the workflow */ + /** Run the workflow + * @param realtime This is used in queue mode to change the priority of an execution, making sure they are picked up quicker. + */ async run( data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, - // TODO: Figure out what this is for realtime?: boolean, restartExecutionId?: string, responsePromise?: IDeferredPromise, @@ -139,7 +140,6 @@ export class WorkflowRunner { this.activeExecutions.attachResponsePromise(executionId, responsePromise); } - // NOTE: queue mode if (this.executionsMode === 'queue' && data.executionMode !== 'manual') { // Do not run "manual" executions in bull because sending events to the // frontend would not be possible @@ -200,9 +200,8 @@ export class WorkflowRunner { ): Promise { const workflowId = data.workflowData.id; if (loadStaticData === true && workflowId) { - // TODO: Can we assign static data to a variable instead of mutating `data`? - // NOTE: This is the workflow and node specific data that can be saved - // and retrieved with the code node. + // This is the workflow and node specific data that can be saved and + // retrieved with the code node. data.workflowData.staticData = await this.workflowStaticDataService.getStaticDataById(workflowId); } @@ -220,8 +219,6 @@ export class WorkflowRunner { let pinData: IPinData | undefined; if (data.executionMode === 'manual') { - // TODO: Find out why pin data exists on both objects and if we need both - // or if one can be cleaned up. pinData = data.pinData ?? data.workflowData.pinData; } @@ -236,8 +233,6 @@ export class WorkflowRunner { settings: workflowSettings, pinData, }); - // NOTE: This seems like a catchall so we can pass anything deep into the - // workflow execution engine. const additionalData = await WorkflowExecuteAdditionalData.getBase( data.userId, undefined, @@ -253,8 +248,6 @@ export class WorkflowRunner { { executionId }, ); let workflowExecution: PCancelable; - // NOTE: This is were we update the status of the execution in the - // database. And this is where the race condition happens. await this.executionRepository.updateStatus(executionId, 'running'); try { @@ -266,8 +259,6 @@ export class WorkflowRunner { }, ]; - // TODO: Why the detour through the WorkflowExecuteAdditionalData to call - // ActiveExecutions? additionalData.setExecutionStatus = WorkflowExecuteAdditionalData.setExecutionStatus.bind({ executionId, }); @@ -277,12 +268,7 @@ export class WorkflowRunner { }); if (data.executionData !== undefined) { - // TODO: What's the difference between `data.executionData` and `data.runData`? - // I think this is the data coming from a webhook or a trigger, e.g. the - // body of a POST request or the message of a queue message. - console.trace('data.executionData', JSON.stringify(data.executionData, null, 2)); - - console.debug(`Execution ID ${executionId} had Execution data. Running with payload.`, { + this.logger.debug(`Execution ID ${executionId} had Execution data. Running with payload.`, { executionId, }); const workflowExecute = new WorkflowExecute( @@ -297,7 +283,7 @@ export class WorkflowRunner { data.startNodes.length === 0 ) { // Full Execution - console.debug(`Execution ID ${executionId} will run executing all nodes.`, { + this.logger.debug(`Execution ID ${executionId} will run executing all nodes.`, { executionId, }); // Execute all nodes @@ -314,12 +300,11 @@ export class WorkflowRunner { ); } else { // Partial Execution - console.debug(`Execution ID ${executionId} is a partial execution.`, { executionId }); + this.logger.debug(`Execution ID ${executionId} is a partial execution.`, { executionId }); // Execute only the nodes between start and destination nodes const workflowExecute = new WorkflowExecute(additionalData, data.executionMode); if (data.partialExecutionVersion === '1') { - console.debug('new partial execution is enabled'); workflowExecution = workflowExecute.runPartialWorkflow2( workflow, data.runData, From 72edb2fe9824ee328afaf100dc2b8474894b39e1 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Thu, 1 Aug 2024 21:07:11 +0200 Subject: [PATCH 17/54] allow setting the default partial execution flow with an env variable PARTIAL_EXECUTION_VERSION_DEFAULT can be 0 or 1 --- packages/cli/src/config/schema.ts | 9 +++++++++ packages/cli/src/workflows/workflows.controller.ts | 4 +++- packages/editor-ui/src/stores/workflows.store.ts | 5 ++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 1f1132b630ce5..0db300eaf0901 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -640,4 +640,13 @@ export const schema = { env: 'N8N_PROXY_HOPS', doc: 'Number of reverse-proxies n8n is running behind', }, + + featureFlags: { + partialExecutionVersionDefault: { + format: String, + default: '0', + env: 'PARTIAL_EXECUTION_VERSION_DEFAULT', + doc: 'Set this to 1 to enable the new partial execution logic by default.', + }, + }, }; diff --git a/packages/cli/src/workflows/workflows.controller.ts b/packages/cli/src/workflows/workflows.controller.ts index 4a8d7bda55147..57f46002c2edb 100644 --- a/packages/cli/src/workflows/workflows.controller.ts +++ b/packages/cli/src/workflows/workflows.controller.ts @@ -405,7 +405,9 @@ export class WorkflowsController { req.body, req.user, req.headers['push-ref'] as string, - req.query.partialExecutionVersion, + req.query.partialExecutionVersion === '-1' + ? config.getEnv('featureFlags.partialExecutionVersionDefault') + : req.query.partialExecutionVersion, ); } diff --git a/packages/editor-ui/src/stores/workflows.store.ts b/packages/editor-ui/src/stores/workflows.store.ts index 45c9eba2b2bae..02faa7ec46817 100644 --- a/packages/editor-ui/src/stores/workflows.store.ts +++ b/packages/editor-ui/src/stores/workflows.store.ts @@ -100,7 +100,10 @@ let cachedWorkflow: Workflow | null = null; export const useWorkflowsStore = defineStore(STORES.WORKFLOWS, () => { const uiStore = useUIStore(); - const partialExecutionVersion = useLocalStorage('PartialExecution.version', 0); + // -1 means the backend chooses the default + // 0 is the old flow + // 1 is the new flow + const partialExecutionVersion = useLocalStorage('PartialExecution.version', -1); const workflow = ref(createEmptyWorkflow()); const usedCredentials = ref>({}); From 5ef2d9f1e3e8d8572a796159aa76028186b51ba1 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Fri, 30 Aug 2024 14:56:52 +0200 Subject: [PATCH 18/54] remove unused import --- packages/core/src/WorkflowExecute.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index 6cd8c505490ca..2ee062ede0c18 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -49,7 +49,6 @@ import { import get from 'lodash/get'; import * as NodeExecuteFunctions from './NodeExecuteFunctions'; -import * as a from 'assert'; import { recreateNodeExecutionStack } from './PartialExecutionUtils/recreateNodeExecutionStack'; import { DirectedGraph, From 2527d0e823e7dd4e4d16cec546eba8ddc8393db1 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Fri, 30 Aug 2024 15:48:12 +0200 Subject: [PATCH 19/54] clean up comments --- packages/cli/src/workflow-runner.ts | 2 -- packages/workflow/src/Interfaces.ts | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index cdfa309e89734..0d75f7e83eeef 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -200,8 +200,6 @@ export class WorkflowRunner { ): Promise { const workflowId = data.workflowData.id; if (loadStaticData === true && workflowId) { - // This is the workflow and node specific data that can be saved and - // retrieved with the code node. data.workflowData.staticData = await this.workflowStaticDataService.getStaticDataById(workflowId); } diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 92ccfa8d978e5..f794692bf9646 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2231,14 +2231,14 @@ export interface IWorkflowExecuteAdditionalData { } export type WorkflowExecuteMode = - | 'cli' // unused - | 'error' // unused, but maybe used for error workflows + | 'cli' + | 'error' | 'integrated' | 'internal' | 'manual' - | 'retry' // unused - | 'trigger' // unused - | 'webhook'; // unused + | 'retry' + | 'trigger' + | 'webhook'; export type WorkflowActivateMode = | 'init' From 2632f6a04524272b4035b5347791890e2dae04e8 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Mon, 2 Sep 2024 11:47:27 +0200 Subject: [PATCH 20/54] intermediate commit to keep track of changes --- execution stack notation.md | 19 ++ jest.config.js | 2 +- .../PartialExecutionUtils/DirectedGraph.ts | 20 +- .../__tests__/findStartNodes.test.ts | 239 +++++++++++++----- .../recreateNodeExecutionStack.test.ts | 138 ++++++++-- .../PartialExecutionUtils/findStartNodes.ts | 66 +++-- .../recreateNodeExecutionStack.ts | 87 ++++--- packages/core/src/WorkflowExecute.ts | 10 + packages/workflow/src/Interfaces.ts | 10 +- 9 files changed, 444 insertions(+), 147 deletions(-) create mode 100644 execution stack notation.md diff --git a/execution stack notation.md b/execution stack notation.md new file mode 100644 index 0000000000000..306733f1de64d --- /dev/null +++ b/execution stack notation.md @@ -0,0 +1,19 @@ +# execution stack notation.md + +ES = Execution Stack +WN = Waiting Nodes +WNS = Waiting Node Sources + +(ES[], WN[], WNS[]) + +([t], [ ], [ ]) +([n1, n2], [ ], [ ]) +([n2], [n3], [n1]) +([n3], [ ], [ ]) <-- +([n4], [ ], [ ]) + +# start node notation +SN = Start Node +SNS = Start Node Source + + diff --git a/jest.config.js b/jest.config.js index 3caac38ef9dbf..78c205957f9c3 100644 --- a/jest.config.js +++ b/jest.config.js @@ -33,7 +33,7 @@ const config = { }, {}), setupFilesAfterEnv: ['jest-expect-message'], collectCoverage: process.env.COVERAGE_ENABLED === 'true', - coverageReporters: ['text-summary'], + coverageReporters: ['html'], collectCoverageFrom: ['src/**/*.ts'], }; diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index 01e39d97b2b11..68986ac27b7e4 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -15,7 +15,7 @@ type DirectedGraphKey = `${string}-${NodeConnectionType}-${number}-${number}-${s export class DirectedGraph { private nodes: Map = new Map(); - private connections: Map = new Map(); + private connections: Map = new Map(); getNodes() { return new Map(this.nodes.entries()); @@ -127,6 +127,24 @@ export class DirectedGraph { return directParents; } + getConnection( + from: INode, + outputIndex: number, + type: NodeConnectionType, + inputIndex: number, + to: INode, + ): Connection | undefined { + return this.connections.get( + this.makeKey({ + from, + outputIndex, + type, + inputIndex, + to, + }), + ); + } + toWorkflow(parameters: Omit): Workflow { return new Workflow({ ...parameters, diff --git a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts index 93befc1cff884..3cf28caab0cde 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts @@ -9,7 +9,7 @@ // XX denotes that the node is disabled // PD denotes that the node has pinned data -import type { IPinData, IRunData } from 'n8n-workflow'; +import { NodeConnectionType, type IPinData, type IRunData } from 'n8n-workflow'; import { createNodeData, toITaskData } from './helpers'; import { findStartNodes, isDirty } from '../findStartNodes'; import { DirectedGraph } from '../DirectedGraph'; @@ -48,7 +48,7 @@ describe('findStartNodes', () => { const startNodes = findStartNodes(graph, node, node); expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ node, sourceData: undefined }); + expect(startNodes).toContainEqual({ node, sourceData: [] }); }); // ►► @@ -67,7 +67,7 @@ describe('findStartNodes', () => { const startNodes = findStartNodes(graph, trigger, destination); expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ node: trigger, sourceData: undefined }); + expect(startNodes).toContainEqual({ node: trigger, sourceData: [] }); } // if the trigger has run data @@ -81,7 +81,18 @@ describe('findStartNodes', () => { expect(startNodes).toHaveLength(1); expect(startNodes).toContainEqual({ node: destination, - sourceData: { previousNode: trigger, previousNodeOutput: 0, previousNodeRun: 0 }, + sourceData: [ + { + connection: { + from: trigger, + to: destination, + type: NodeConnectionType.Main, + outputIndex: 0, + inputIndex: 0, + }, + previousNodeRun: 0, + }, + ], }); } }); @@ -126,19 +137,39 @@ describe('findStartNodes', () => { expect(startNodes).toHaveLength(2); expect(startNodes).toContainEqual({ node, - sourceData: { - previousNode: trigger, - previousNodeOutput: 0, - previousNodeRun: 0, - }, + sourceData: [ + { + connection: { + from: trigger, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: trigger, + //previousNodeOutput: 0, + }, + ], }); expect(startNodes).toContainEqual({ node, - sourceData: { - previousNode: trigger, - previousNodeOutput: 1, - previousNodeRun: 0, - }, + sourceData: [ + { + connection: { + from: trigger, + outputIndex: 1, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: trigger, + //previousNodeOutput: 0, + }, + ], }); }); @@ -177,7 +208,7 @@ describe('findStartNodes', () => { const startNodes = findStartNodes(graph, trigger, node4); expect(startNodes).toHaveLength(1); // no run data means the trigger is the start node - expect(startNodes).toContainEqual({ node: trigger, sourceData: undefined }); + expect(startNodes).toContainEqual({ node: trigger, sourceData: [] }); } { @@ -195,20 +226,40 @@ describe('findStartNodes', () => { expect(startNodes).toContainEqual({ node: node4, - sourceData: { - previousNode: node2, - previousNodeOutput: 0, - previousNodeRun: 0, - }, + sourceData: [ + { + connection: { + from: node2, + to: node4, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: node2, + //previousNodeOutput: 0, + }, + ], }); expect(startNodes).toContainEqual({ node: node4, - sourceData: { - previousNode: node3, - previousNodeOutput: 0, - previousNodeRun: 0, - }, + sourceData: [ + { + connection: { + from: node3, + to: node4, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: node3, + //previousNodeOutput: 0, + }, + ], }); } }); @@ -239,11 +290,21 @@ describe('findStartNodes', () => { expect(startNodes).toHaveLength(1); expect(startNodes).toContainEqual({ node, - sourceData: { - previousNode: trigger, - previousNodeRun: 0, - previousNodeOutput: 0, - }, + sourceData: [ + { + connection: { + from: trigger, + to: node, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: trigger, + //previousNodeOutput: 0, + }, + ], }); }); @@ -256,28 +317,38 @@ describe('findStartNodes', () => { // The merge node only gets data on one input, so the it should only be once // in the start nodes test('multiple connections with the second one having data', () => { - const ifNode = createNodeData({ name: 'if' }); - const merge = createNodeData({ name: 'merge' }); + const trigger = createNodeData({ name: 'trigger' }); + const node = createNodeData({ name: 'node' }); const graph = new DirectedGraph() - .addNodes(ifNode, merge) + .addNodes(trigger, node) .addConnections( - { from: ifNode, to: merge, inputIndex: 0, outputIndex: 0 }, - { from: ifNode, to: merge, inputIndex: 1, outputIndex: 1 }, + { from: trigger, to: node, inputIndex: 0, outputIndex: 0 }, + { from: trigger, to: node, inputIndex: 1, outputIndex: 1 }, ); - const startNodes = findStartNodes(graph, ifNode, merge, { - [ifNode.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 1 }])], + const startNodes = findStartNodes(graph, trigger, node, { + [trigger.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 1 }])], }); expect(startNodes).toHaveLength(1); expect(startNodes).toContainEqual({ - node: merge, - sourceData: { - previousNode: ifNode, - previousNodeRun: 0, - previousNodeOutput: 1, - }, + node, + sourceData: [ + { + connection: { + from: trigger, + to: node, + inputIndex: 1, + outputIndex: 1, + type: NodeConnectionType.Main, + }, + previousNodeRun: 0, + //currentNodeInput: 1, + //previousNode: trigger, + //previousNodeOutput: 1, + }, + ], }); }); @@ -289,7 +360,7 @@ describe('findStartNodes', () => { // └───────┘1 └────┘ // The merge node gets data on both inputs, so the it should be in the start // nodes twice. - test('multiple connections with both having data', () => { + test.only('multiple connections with both having data', () => { const trigger = createNodeData({ name: 'trigger' }); const node = createNodeData({ name: 'node' }); @@ -309,14 +380,39 @@ describe('findStartNodes', () => { ], }); - expect(startNodes).toHaveLength(2); + expect(startNodes).toHaveLength(1); + // TODO: this is wrong, technically this should contain one start node + // and the source data should be an array. expect(startNodes).toContainEqual({ node, - sourceData: { - previousNode: trigger, - previousNodeRun: 0, - previousNodeOutput: 0, - }, + sourceData: [ + { + connection: { + from: trigger, + to: node, + inputIndex: 0, + outputIndex: 0, + type: NodeConnectionType.Main, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: trigger, + //previousNodeOutput: 0, + }, + { + connection: { + from: trigger, + to: node, + inputIndex: 1, + outputIndex: 1, + type: NodeConnectionType.Main, + }, + previousNodeRun: 0, + //currentNodeInput: 1, + //previousNode: trigger, + //previousNodeOutput: 1, + }, + ], }); }); @@ -330,7 +426,7 @@ describe('findStartNodes', () => { // │ │ └────►│ │ // └───────┘ └────┘ // eslint-disable-next-line n8n-local-rules/no-skipped-tests - test.skip('multiple connections with both having data', () => { + test.only('multiple connections with both having data', () => { const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -388,11 +484,22 @@ describe('findStartNodes', () => { expect(startNodes).toHaveLength(1); expect(startNodes).toContainEqual({ node: node2, - sourceData: { - previousNode: node1, - previousNodeRun: 0, - previousNodeOutput: 1, - }, + sourceData: [ + { + connection: { + from: node1, + to: node2, + // TODO: Shouldn't this be 1 instead of 0? + inputIndex: 0, + outputIndex: 1, + type: NodeConnectionType.Main, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: node1, + //previousNodeOutput: 1, + }, + ], }); }); @@ -433,11 +540,21 @@ describe('findStartNodes', () => { expect(startNodes).toHaveLength(1); expect(startNodes).toContainEqual({ node: node2, - sourceData: { - previousNode: node1, - previousNodeRun: 0, - previousNodeOutput: 0, - }, + sourceData: [ + { + connection: { + from: node1, + to: node2, + outputIndex: 0, + inputIndex: 0, + type: NodeConnectionType.Main, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: node1, + //previousNodeOutput: 0, + }, + ], }); }); }); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts index 3bbfd520017dc..29f756c5db18c 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts @@ -10,7 +10,7 @@ // PD denotes that the node has pinned data import { recreateNodeExecutionStack } from '@/PartialExecutionUtils/recreateNodeExecutionStack'; -import { type IPinData, type IRunData } from 'n8n-workflow'; +import { NodeConnectionType, type IPinData, type IRunData } from 'n8n-workflow'; import { AssertionError } from 'assert'; import { DirectedGraph } from '../DirectedGraph'; import { findSubgraph } from '../findSubgraph'; @@ -38,9 +38,17 @@ describe('recreateNodeExecutionStack', () => { { node, sourceData: { - previousNode: trigger, + connection: { + from: trigger, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }, previousNodeRun: 0, - previousNodeOutput: 0, + //currentNodeInput: 0, + //previousNode: trigger, + //previousNodeOutput: 0, }, }, ]; @@ -62,7 +70,17 @@ describe('recreateNodeExecutionStack', () => { { data: { main: [[{ json: { value: 1 } }]] }, node, - source: { main: [{ previousNode: 'trigger', previousNodeOutput: 0, previousNodeRun: 0 }] }, + source: { + main: [ + { + // TODO: not part of IScourceDate, but maybe it should be? + //currentNodeInput: 0, + previousNode: 'trigger', + previousNodeOutput: 0, + previousNodeRun: 0, + }, + ], + }, }, ]); @@ -135,7 +153,19 @@ describe('recreateNodeExecutionStack', () => { const startNodes: StartNodeData[] = [ { node, - sourceData: { previousNode: trigger, previousNodeRun: 0, previousNodeOutput: 0 }, + sourceData: { + connection: { + from: trigger, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: trigger, + //previousNodeOutput: 0, + }, }, ]; const runData: IRunData = {}; @@ -158,7 +188,15 @@ describe('recreateNodeExecutionStack', () => { data: { main: [[{ json: { value: 1 } }]] }, node, source: { - main: [{ previousNode: trigger.name, previousNodeRun: 0, previousNodeOutput: 0 }], + main: [ + { + // TODO: not part of IScourceDate, but maybe it should be? + //currentNodeInput: 0, + previousNode: trigger.name, + previousNodeRun: 0, + previousNodeOutput: 0, + }, + ], }, }, ]); @@ -187,9 +225,17 @@ describe('recreateNodeExecutionStack', () => { { node: node2, sourceData: { - previousNode: node1, + connection: { + from: node1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node2, + }, previousNodeRun: 0, - previousNodeOutput: 0, + //currentNodeInput: 0, + //previousNode: node1, + //previousNodeOutput: 0, }, }, ]; @@ -235,17 +281,33 @@ describe('recreateNodeExecutionStack', () => { { node: node3, sourceData: { - previousNode: node1, + connection: { + from: node1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node3, + }, previousNodeRun: 0, - previousNodeOutput: 0, + //currentNodeInput: 0, + //previousNode: node1, + //previousNodeOutput: 0, }, }, { node: node3, sourceData: { - previousNode: node2, + connection: { + from: node2, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node3, + }, previousNodeRun: 0, - previousNodeOutput: 0, + //currentNodeInput: 0, + //previousNode: node2, + //previousNodeOutput: 0, }, }, ]; @@ -270,12 +332,32 @@ describe('recreateNodeExecutionStack', () => { { data: { main: [[{ json: { value: 1 } }]] }, node: node3, - source: { main: [{ previousNode: 'node1', previousNodeOutput: 0, previousNodeRun: 0 }] }, + source: { + main: [ + { + // TODO: not part of IScourceDate, but maybe it should be? + //currentNodeInput: 0, + previousNode: 'node1', + previousNodeOutput: 0, + previousNodeRun: 0, + }, + ], + }, }, { data: { main: [[{ json: { value: 1 } }]] }, node: node3, - source: { main: [{ previousNode: 'node2', previousNodeOutput: 0, previousNodeRun: 0 }] }, + source: { + main: [ + { + // TODO: not part of IScourceDate, but maybe it should be? + //currentNodeInput: 0, + previousNode: 'node2', + previousNodeOutput: 0, + previousNodeRun: 0, + }, + ], + }, }, ]); @@ -331,9 +413,33 @@ describe('recreateNodeExecutionStack', () => { { node: node3, sourceData: { - previousNode: node1, + connection: { + from: node1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node3, + }, + previousNodeRun: 0, + //currentNodeInput: 0, + //previousNode: node1, + //previousNodeOutput: 0, + }, + }, + { + node: node3, + sourceData: { + connection: { + from: node2, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 1, + to: node3, + }, previousNodeRun: 0, - previousNodeOutput: 0, + //currentNodeInput: 0, + //previousNode: node1, + //previousNodeOutput: 0, }, }, ]; diff --git a/packages/core/src/PartialExecutionUtils/findStartNodes.ts b/packages/core/src/PartialExecutionUtils/findStartNodes.ts index 8715b662929e5..b56fe73760b16 100644 --- a/packages/core/src/PartialExecutionUtils/findStartNodes.ts +++ b/packages/core/src/PartialExecutionUtils/findStartNodes.ts @@ -1,25 +1,20 @@ import type { INode, IPinData, IRunData } from 'n8n-workflow'; -import type { DirectedGraph } from './DirectedGraph'; +import type { Connection, DirectedGraph } from './DirectedGraph'; import { getIncomingData } from './getIncomingData'; -interface ISourceData { - previousNode: INode; - previousNodeOutput: number; // If undefined "0" gets used - previousNodeRun: number; // If undefined "0" gets used -} // TODO: This is how ISourceData should look like. -//interface NewSourceData { -// connection: Connection; -// previousNodeRun: number; // If undefined "0" gets used -//} +type NewSourceData = { + connection: Connection; + previousNodeRun: number; // If undefined "0" gets used +}; // TODO: rename to something more general, like path segment export interface StartNodeData { node: INode; - sourceData?: ISourceData; + sourceData: NewSourceData[]; } -type Key = `${string}-${number}-${string}`; +type Key = `${number}-${string}`; // TODO: implement dirty checking for options and properties and parent nodes // being disabled @@ -63,8 +58,8 @@ export function isDirty(node: INode, runData: IRunData = {}, pinData: IPinData = return true; } -function makeKey(from: ISourceData | undefined, to: INode): Key { - return `${from?.previousNode.name ?? 'start'}-${from?.previousNodeOutput ?? 0}-${to.name}`; +function makeKey(sourceConnection: Connection | undefined, to: INode): Key { + return `${sourceConnection?.outputIndex ?? 0}-${to.name}`; } function findStartNodesRecursive( @@ -73,26 +68,43 @@ function findStartNodesRecursive( destination: INode, runData: IRunData, pinData: IPinData, - startNodes: Map, + startNodes: Map, seen: Set, - source?: ISourceData, + source?: NewSourceData, ) { const nodeIsDirty = isDirty(current, runData, pinData); // If the current node is dirty stop following this branch, we found a start // node. if (nodeIsDirty) { - startNodes.set(makeKey(source, current), { + const key = source ? source.connection : 'Trigger'; // makeKey(source?.connection, current); + const value = startNodes.get(key) ?? { node: current, - sourceData: source, - }); + sourceData: [], + }; + + if (source) { + value.sourceData.push(source); + } + + startNodes.set(key, value); return startNodes; } // If the current node is the destination node stop following this branch, we // found a start node. if (current === destination) { - startNodes.set(makeKey(source, current), { node: current, sourceData: source }); + const key = source ? source.connection : 'Trigger'; // makeKey(source?.connection, current); + const value = startNodes.get(key) ?? { + node: current, + sourceData: [], + }; + + if (source) { + value.sourceData.push(source); + } + + startNodes.set(key, value); return startNodes; } @@ -131,11 +143,10 @@ function findStartNodesRecursive( startNodes, new Set(seen).add(current), { - previousNode: current, + connection: outGoingConnection, // NOTE: It's always 0 until I fix the bug that removes the run data for // old runs. The FE only sends data for one run for each node. previousNodeRun: 0, - previousNodeOutput: outGoingConnection.outputIndex, }, ); } @@ -156,8 +167,17 @@ export function findStartNodes( destination, runData, pinData, + // start nodes and their sources new Map(), + // seen new Set(), ); - return [...startNodes.values()]; + + const startNodesData: StartNodeData[] = []; + + for (const [_node, value] of startNodes) { + startNodesData.push(value); + } + + return startNodesData; } diff --git a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts index 8c56e8436eda4..6ff87d63612b2 100644 --- a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts +++ b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts @@ -18,7 +18,6 @@ import { getIncomingData } from './getIncomingData'; export function recreateNodeExecutionStack( graph: DirectedGraph, - // TODO: turn this into StartNodeData from utils startNodes: StartNodeData[], destinationNode: INode, runData: IRunData, @@ -47,9 +46,9 @@ export function recreateNodeExecutionStack( for (const startNode of startNodes) { if (startNode.sourceData) { a.ok( - runData[startNode.sourceData.previousNode.name] || - pinData[startNode.sourceData.previousNode.name], - `Start nodes have sources that don't have run data. That is not supported. Make sure to get the start nodes by calling "findStartNodes". The node in question is "${startNode.node.name}" and their source is "${startNode.sourceData.previousNode.name}".`, + runData[startNode.sourceData.connection.from.name] || + pinData[startNode.sourceData.connection.from.name], + `Start nodes have sources that don't have run data. That is not supported. Make sure to get the start nodes by calling "findStartNodes". The node in question is "${startNode.node.name}" and their source is "${startNode.sourceData.connection.from.name}".`, ); } } @@ -80,45 +79,53 @@ export function recreateNodeExecutionStack( // stack should exist in sourceData. The only thing that is currently // missing is the inputIndex and that's the sole reason why we iterate // over all incoming connections. - for (const connection of incomingStartNodeConnections) { - // TODO: do not skip connections that don't match the source data, this - // causes problems with nodes that have multiple inputs. - // The proper fix would be to remodel source data to contain all sources - // not just the first one it finds. - if (connection.from.name !== startNode.sourceData?.previousNode.name) { - continue; - } - - const node = connection.from; - - a.equal(startNode.sourceData.previousNode, node); - - if (pinData[node.name]) { - incomingData.push(pinData[node.name]); - } else { - a.ok( - runData[connection.from.name], - `Start node(${startNode.node.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${connection.from.name}->${connection.to.name}". Are you sure the start nodes come from the "findStartNodes" function?`, - ); - - const nodeIncomingData = getIncomingData( - runData, - connection.from.name, - runIndex, - connection.type, - connection.inputIndex, - ); + //for (const connection of incomingStartNodeConnections) { + // TODO: do not skip connections that don't match the source data, this + // causes problems with nodes that have multiple inputs. + // The proper fix would be to remodel source data to contain all sources + // not just the first one it finds. + //if (connection.from.name !== startNode.sourceData?.previousNode.name) { + // continue; + //} + + if (startNode.sourceData === undefined) { + continue; + } - if (nodeIncomingData) { - incomingData.push(nodeIncomingData); - } + //const node = connection.from; + const node = startNode.sourceData.connection.from; + + //a.equal(startNode.sourceData.previousNode, node); + + if (pinData[node.name]) { + incomingData.push(pinData[node.name]); + } else { + a.ok( + runData[node.name], + `Start node(${startNode.node.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${node.name}->${startNode.node.name}". Are you sure the start nodes come from the "findStartNodes" function?`, + ); + + const nodeIncomingData = getIncomingData( + runData, + node.name, + runIndex, + startNode.sourceData.connection.type, + startNode.sourceData.connection.outputIndex, + ); + + if (nodeIncomingData) { + incomingData.push(nodeIncomingData); } - - incomingSourceData.main.push({ - ...startNode.sourceData, - previousNode: startNode.sourceData.previousNode.name, - }); } + + incomingSourceData.main.push({ + //...startNode.sourceData, + previousNode: startNode.sourceData.connection.from.name, + //currentNodeInput: startNode.sourceData.connection.inputIndex, + previousNodeOutput: startNode.sourceData.connection.outputIndex, + previousNodeRun: 0, + }); + //} } const executeData: IExecuteData = { diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index 2ee062ede0c18..daf01060f624a 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -989,6 +989,16 @@ export class WorkflowExecute { executionLoop: while ( this.runExecutionData.executionData!.nodeExecutionStack.length !== 0 ) { + console.log('---------------------------------------------------'); + console.log( + 'nodeExecutionStack', + this.runExecutionData.executionData?.nodeExecutionStack.map((v) => ({ + nodeName: v.node.name, + sourceName: v.source?.main.map((v) => v?.previousNode), + })), + ); + console.log('waitingExecution', this.runExecutionData.executionData?.waitingExecution); + if ( this.additionalData.executionTimeoutTimestamp !== undefined && Date.now() >= this.additionalData.executionTimeoutTimestamp diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index f794692bf9646..92ccfa8d978e5 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2231,14 +2231,14 @@ export interface IWorkflowExecuteAdditionalData { } export type WorkflowExecuteMode = - | 'cli' - | 'error' + | 'cli' // unused + | 'error' // unused, but maybe used for error workflows | 'integrated' | 'internal' | 'manual' - | 'retry' - | 'trigger' - | 'webhook'; + | 'retry' // unused + | 'trigger' // unused + | 'webhook'; // unused export type WorkflowActivateMode = | 'init' From 1dc0ac983403c458ba541bb9073adaf87e451aeb Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Mon, 2 Sep 2024 21:49:41 +0200 Subject: [PATCH 21/54] another intermediate commit --- .../PartialExecutionUtils/DirectedGraph.ts | 16 ++ .../__tests__/getSourceDataGroups.test.ts | 197 +++++++++++++ .../recreateNodeExecutionStack.test.ts | 91 +++--- .../recreateNodeExecutionStack.ts | 261 +++++++++++++----- 4 files changed, 449 insertions(+), 116 deletions(-) create mode 100644 packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index 68986ac27b7e4..5759e1cf2c7e5 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -21,6 +21,22 @@ export class DirectedGraph { return new Map(this.nodes.entries()); } + getConnections(filter: { to?: INode } = {}) { + const filteredCopy: Connection[] = []; + + for (const connection of this.connections.values()) { + const toMatches = filter.to ? connection.to === filter.to : true; + + if (toMatches) { + filteredCopy.push(connection); + } + } + + return filteredCopy; + + //return new Map(this.connections.entries()); + } + addNode(node: INode) { this.nodes.set(node.name, node); return this; diff --git a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts new file mode 100644 index 0000000000000..d569cf63dac1a --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts @@ -0,0 +1,197 @@ +// NOTE: Diagrams in this file have been created with https://asciiflow.com/#/ +// If you update the tests, please update the diagrams as well. +// If you add a test, please create a new diagram. +// +// Map +// 0 means the output has no run data +// 1 means the output has run data +// PD denotes that the node has pinned data + +import { NodeConnectionType, type IRunData } from 'n8n-workflow'; +import { DirectedGraph } from '../DirectedGraph'; +import { createNodeData, toITaskData } from './helpers'; +import { getSourceDataGroups } from '../recreateNodeExecutionStack'; + +describe('getSourceDataGroups', () => { + //┌───────┐1 + //│source1├────┐ + //└───────┘ │ ┌────┐ + //┌───────┐1 ├──►│ │ + //│source2├────┘ │node│ + //└───────┘ ┌──►│ │ + //┌───────┐1 │ └────┘ + //│source3├────┘ + //└───────┘ + it('groups sources into possibly complete sets if all of them have data', () => { + const source1 = createNodeData({ name: 'source1' }); + const source2 = createNodeData({ name: 'source2' }); + const source3 = createNodeData({ name: 'source3' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(source1, source2, source3, node) + .addConnections( + { from: source1, to: node, inputIndex: 0 }, + { from: source2, to: node, inputIndex: 0 }, + { from: source3, to: node, inputIndex: 1 }, + ); + const runData: IRunData = { + [source1.name]: [toITaskData([{ data: { value: 1 } }])], + [source2.name]: [toITaskData([{ data: { value: 1 } }])], + [source3.name]: [toITaskData([{ data: { value: 1 } }])], + }; + const pinnedData: IRunData = {}; + + const groups = getSourceDataGroups(graph, node, runData, pinnedData); + + expect(groups).toHaveLength(2); + + const group1 = groups[0]; + expect(group1).toHaveLength(2); + expect(group1[0]).toEqual({ + from: source1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + expect(group1[1]).toEqual({ + from: source3, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 1, + to: node, + }); + + const group2 = groups[1]; + expect(group2).toHaveLength(1); + expect(group2[0]).toEqual({ + from: source2, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + }); + + //┌───────┐PD + //│source1├────┐ + //└───────┘ │ ┌────┐ + //┌───────┐PD ├──►│ │ + //│source2├────┘ │node│ + //└───────┘ ┌──►│ │ + //┌───────┐PD │ └────┘ + //│source3├────┘ + //└───────┘ + it('groups sources into possibly complete sets if all of them have data', () => { + const source1 = createNodeData({ name: 'source1' }); + const source2 = createNodeData({ name: 'source2' }); + const source3 = createNodeData({ name: 'source3' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(source1, source2, source3, node) + .addConnections( + { from: source1, to: node, inputIndex: 0 }, + { from: source2, to: node, inputIndex: 0 }, + { from: source3, to: node, inputIndex: 1 }, + ); + const runData: IRunData = {}; + const pinnedData: IRunData = { + [source1.name]: [toITaskData([{ data: { value: 1 } }])], + [source2.name]: [toITaskData([{ data: { value: 1 } }])], + [source3.name]: [toITaskData([{ data: { value: 1 } }])], + }; + + const groups = getSourceDataGroups(graph, node, runData, pinnedData); + + expect(groups).toHaveLength(2); + + const group1 = groups[0]; + expect(group1).toHaveLength(2); + expect(group1[0]).toEqual({ + from: source1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + expect(group1[1]).toEqual({ + from: source3, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 1, + to: node, + }); + + const group2 = groups[1]; + expect(group2).toHaveLength(1); + expect(group2[0]).toEqual({ + from: source2, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + }); + + //┌───────┐0 + //│source1├────┐ + //└───────┘ │ ┌────┐ + //┌───────┐1 ├──►│ │ + //│source2├────┘ │node│ + //└───────┘ ┌──►│ │ + //┌───────┐1 │ └────┘ + //│source3├────┘ + //└───────┘ + it('groups sources into possibly complete sets if all of them have data', () => { + const source1 = createNodeData({ name: 'source1' }); + const source2 = createNodeData({ name: 'source2' }); + const source3 = createNodeData({ name: 'source3' }); + const node = createNodeData({ name: 'node' }); + + const graph = new DirectedGraph() + .addNodes(source1, source2, source3, node) + .addConnections( + { from: source1, to: node, inputIndex: 0 }, + { from: source2, to: node, inputIndex: 0 }, + { from: source3, to: node, inputIndex: 1 }, + ); + const runData: IRunData = { + [source2.name]: [toITaskData([{ data: { value: 1 } }])], + [source3.name]: [toITaskData([{ data: { value: 1 } }])], + }; + const pinnedData: IRunData = {}; + + const groups = getSourceDataGroups(graph, node, runData, pinnedData); + + expect(groups).toHaveLength(2); + + const group1 = groups[0]; + expect(group1).toHaveLength(2); + expect(group1[0]).toEqual({ + from: source2, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + expect(group1[1]).toEqual({ + from: source3, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 1, + to: node, + }); + + const group2 = groups[1]; + expect(group2).toHaveLength(1); + expect(group2[0]).toEqual({ + from: source1, + outputIndex: 0, + type: NodeConnectionType.Main, + inputIndex: 0, + to: node, + }); + }); +}); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts index 29f756c5db18c..7dc79ffbaaa13 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts @@ -66,6 +66,7 @@ describe('recreateNodeExecutionStack', () => { // // ASSERT // + expect(nodeExecutionStack).toHaveLength(1); expect(nodeExecutionStack).toEqual([ { data: { main: [[{ json: { value: 1 } }]] }, @@ -294,22 +295,22 @@ describe('recreateNodeExecutionStack', () => { //previousNodeOutput: 0, }, }, - { - node: node3, - sourceData: { - connection: { - from: node2, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node3, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node2, - //previousNodeOutput: 0, - }, - }, + //{ + // node: node3, + // sourceData: { + // connection: { + // from: node2, + // outputIndex: 0, + // type: NodeConnectionType.Main, + // inputIndex: 0, + // to: node3, + // }, + // previousNodeRun: 0, + // //currentNodeInput: 0, + // //previousNode: node2, + // //previousNodeOutput: 0, + // }, + //}, ]; const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], @@ -322,7 +323,7 @@ describe('recreateNodeExecutionStack', () => { // ACT // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = - recreateNodeExecutionStack(graph, startNodes, node2, runData, pinData); + recreateNodeExecutionStack(graph, startNodes, node3, runData, pinData); // // ASSERT @@ -362,14 +363,14 @@ describe('recreateNodeExecutionStack', () => { ]); expect(waitingExecution).toEqual({ - node2: { '0': { main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]] } }, + node3: { '0': { main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]] } }, }); expect(waitingExecutionSource).toEqual({ - node2: { + node3: { '0': { main: [ - { previousNode: 'trigger', previousNodeOutput: undefined, previousNodeRun: undefined }, - { previousNode: 'trigger', previousNodeOutput: undefined, previousNodeRun: undefined }, + { previousNode: 'node1', previousNodeOutput: undefined, previousNodeRun: undefined }, + { previousNode: 'node2', previousNodeOutput: undefined, previousNodeRun: undefined }, ], }, }, @@ -393,7 +394,7 @@ describe('recreateNodeExecutionStack', () => { // └─►│node2├───┘ └─────┘ // └─────┘ // eslint-disable-next-line n8n-local-rules/no-skipped-tests - test.skip('multiple inputs', () => { + test('multiple inputs', () => { // // ARRANGE // @@ -406,8 +407,8 @@ describe('recreateNodeExecutionStack', () => { .addConnections( { from: trigger, to: node1 }, { from: trigger, to: node2 }, - { from: node1, to: node3, outputIndex: 0 }, - { from: node2, to: node3, outputIndex: 1 }, + { from: node1, to: node3, inputIndex: 0 }, + { from: node2, to: node3, inputIndex: 1 }, ); const startNodes: StartNodeData[] = [ { @@ -426,22 +427,22 @@ describe('recreateNodeExecutionStack', () => { //previousNodeOutput: 0, }, }, - { - node: node3, - sourceData: { - connection: { - from: node2, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 1, - to: node3, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node1, - //previousNodeOutput: 0, - }, - }, + //{ + // node: node3, + // sourceData: { + // connection: { + // from: node2, + // outputIndex: 0, + // type: NodeConnectionType.Main, + // inputIndex: 1, + // to: node3, + // }, + // previousNodeRun: 0, + // //currentNodeInput: 0, + // //previousNode: node1, + // //previousNodeOutput: 0, + // }, + //}, ]; const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], @@ -462,15 +463,13 @@ describe('recreateNodeExecutionStack', () => { // ASSERT // expect(nodeExecutionStack).toHaveLength(1); - expect(nodeExecutionStack).toContainEqual({ + expect(nodeExecutionStack[0]).toEqual({ data: { main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]] }, node: node3, source: { main: [ { previousNode: 'node1', previousNodeOutput: 0, previousNodeRun: 0 }, - // TODO: this should be node2, but this would only work if I refactor - // sourceData to contain multiple sources per node. - { previousNode: 'node1', previousNodeOutput: 0, previousNodeRun: 0 }, + { previousNode: 'node2', previousNodeOutput: 0, previousNodeRun: 0 }, ], }, }); @@ -478,7 +477,7 @@ describe('recreateNodeExecutionStack', () => { expect(waitingExecution).toEqual({ node3: { '0': { - main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]], + main: [[{ json: { value: 1 } }]], }, }, }); @@ -487,7 +486,7 @@ describe('recreateNodeExecutionStack', () => { '0': { main: [ { previousNode: 'node1', previousNodeOutput: undefined, previousNodeRun: undefined }, - { previousNode: 'node2', previousNodeOutput: undefined, previousNodeRun: undefined }, + { previousNode: 'node2', previousNodeOutput: 1, previousNodeRun: undefined }, ], }, }, diff --git a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts index 6ff87d63612b2..a5e7e530d00f9 100644 --- a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts +++ b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts @@ -11,11 +11,139 @@ import { type IWaitingForExecutionSource, } from 'n8n-workflow'; -import * as a from 'assert'; -import type { DirectedGraph } from './DirectedGraph'; +import * as a from 'assert/strict'; +import type { Connection, DirectedGraph } from './DirectedGraph'; import type { StartNodeData } from './findStartNodes'; import { getIncomingData } from './getIncomingData'; +function sortByInputIndexThenByName(connection1: Connection, connection2: Connection): number { + if (connection1.inputIndex === connection2.inputIndex) { + return connection1.from.name.localeCompare(connection2.from.name); + } else { + return connection1.inputIndex - connection2.inputIndex; + } +} + +// INFO: I don't need to care about connections without data. I just have to +// make sure all connections have data, otherwise the node that was passed in +// is not a valid startNode, startNodes must have data on all incoming +// connections. +// TODO: assert that all incoming connections have run data of pinned data. +// TODO: remove all code handling connections without data + +/** + * Groups incoming connections to the node. The groups contain one connection + * per input, if possible, with run data or pinned data, if possible. + * + * The purpose of this is to get as many complete sets of data for executing + * nodes with multiple inputs. + * + * # Example 1: + * ┌───────┐1 + * │source1├────┐ + * └───────┘ │ ┌────┐ + * ┌───────┐1 ├──►│ │ + * │source2├────┘ │node│ + * └───────┘ ┌──►│ │ + * ┌───────┐1 │ └────┘ + * │source3├────┘ + * └───────┘ + * + * Given this workflow, and assuming all sources have run data or pinned data, + * it's possible to run `node` with the data of `source1` and `source3` and + * then one more time with the data from `source2`. + * + * It would also be possible to run `node` with the data of `source2` and + * `source3` and then one more time with the data from `source1`. + * + * To improve the determinism of this the connections are sorted by input and + * then by from-node name. + * + * So this will return 2 groups: + * 1. source1 and source3 + * 2. source2 + * + * # Example 1: + * ┌───────┐0 + * │source1├────┐ + * └───────┘ │ ┌────┐ + * ┌───────┐1 ├──►│ │ + * │source2├────┘ │node│ + * └───────┘ ┌──►│ │ + * ┌───────┐1 │ └────┘ + * │source3├────┘ + * └───────┘ + * + * Since `source1` has no run data and no pinned data it's skipped in favor of + * `source2` for the for input. + * + * So this will return 2 groups: + * 1. source2 and source3 + * 2. source1 + */ +export function getSourceDataGroups( + graph: DirectedGraph, + node: INode, + runData: IRunData, + pinnedData: IPinData, +): Connection[][] { + const connections = graph.getConnections({ to: node }); + + const sortedConnectionsWithData = []; + const sortedConnectionsWithoutData = []; + + for (const connection of connections) { + const hasData = runData[connection.from.name] || pinnedData[connection.from.name]; + + if (hasData) { + sortedConnectionsWithData.push(connection); + } else { + sortedConnectionsWithoutData.push(connection); + } + } + + sortedConnectionsWithData.sort(sortByInputIndexThenByName); + sortedConnectionsWithoutData.sort(sortByInputIndexThenByName); + + const groups: Connection[][] = []; + let currentGroup: Connection[] = []; + let currentInputIndex = -1; + + while (sortedConnectionsWithData.length > 0 || sortedConnectionsWithoutData.length > 0) { + const connectionWithDataIndex = sortedConnectionsWithData.findIndex( + // eslint-disable-next-line @typescript-eslint/no-loop-func + (c) => c.inputIndex > currentInputIndex, + ); + const connectionWithoutDataIndex = sortedConnectionsWithoutData.findIndex( + // eslint-disable-next-line @typescript-eslint/no-loop-func + (c) => c.inputIndex > currentInputIndex, + ); + const connection: Connection | undefined = + sortedConnectionsWithData[connectionWithDataIndex] ?? + sortedConnectionsWithoutData[connectionWithoutDataIndex]; + + if (connection === undefined) { + groups.push(currentGroup); + currentGroup = []; + currentInputIndex = -1; + continue; + } + + currentInputIndex = connection.inputIndex; + currentGroup.push(connection); + + if (connectionWithDataIndex >= 0) { + sortedConnectionsWithData.splice(connectionWithDataIndex, 1); + } else if (connectionWithoutDataIndex >= 0) { + sortedConnectionsWithoutData.splice(connectionWithoutDataIndex, 1); + } + } + + groups.push(currentGroup); + + return groups; +} + export function recreateNodeExecutionStack( graph: DirectedGraph, startNodes: StartNodeData[], @@ -43,15 +171,15 @@ export function recreateNodeExecutionStack( // don't then they should not be the start nodes, but some node before them // should be. Probably they are not coming from findStartNodes, make sure to // use that function to get the start nodes. - for (const startNode of startNodes) { - if (startNode.sourceData) { - a.ok( - runData[startNode.sourceData.connection.from.name] || - pinData[startNode.sourceData.connection.from.name], - `Start nodes have sources that don't have run data. That is not supported. Make sure to get the start nodes by calling "findStartNodes". The node in question is "${startNode.node.name}" and their source is "${startNode.sourceData.connection.from.name}".`, - ); - } - } + //for (const startNode of startNodes) { + // if (startNode.sourceData) { + // a.ok( + // runData[startNode.sourceData.connection.from.name] || + // pinData[startNode.sourceData.connection.from.name], + // `Start nodes have sources that don't have run data. That is not supported. Make sure to get the start nodes by calling "findStartNodes". The node in question is "${startNode.node.name}" and their source is "${startNode.sourceData.connection.from.name}".`, + // ); + // } + //} // Initialize the nodeExecutionStack and waitingExecution with // the data from runData @@ -67,74 +195,67 @@ export function recreateNodeExecutionStack( .getDirectParents(startNode.node) .filter((c) => c.type === NodeConnectionType.Main); - const incomingData: INodeExecutionData[][] = []; + let incomingData: INodeExecutionData[][] = []; let incomingSourceData: ITaskDataConnectionsSource | null = null; if (incomingStartNodeConnections.length === 0) { incomingData.push([{ json: {} }]); + + const executeData: IExecuteData = { + node: startNode.node, + data: { main: incomingData }, + source: incomingSourceData, + }; + + nodeExecutionStack.push(executeData); } else { - // Get the data of the incoming connections - incomingSourceData = { main: [] }; - // TODO: Get rid of this whole loop. All data necessary to recreate the - // stack should exist in sourceData. The only thing that is currently - // missing is the inputIndex and that's the sole reason why we iterate - // over all incoming connections. - //for (const connection of incomingStartNodeConnections) { - // TODO: do not skip connections that don't match the source data, this - // causes problems with nodes that have multiple inputs. - // The proper fix would be to remodel source data to contain all sources - // not just the first one it finds. - //if (connection.from.name !== startNode.sourceData?.previousNode.name) { - // continue; - //} - - if (startNode.sourceData === undefined) { - continue; - } + const sourceDataSets = getSourceDataGroups(graph, startNode.node, runData, pinData); - //const node = connection.from; - const node = startNode.sourceData.connection.from; - - //a.equal(startNode.sourceData.previousNode, node); - - if (pinData[node.name]) { - incomingData.push(pinData[node.name]); - } else { - a.ok( - runData[node.name], - `Start node(${startNode.node.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${node.name}->${startNode.node.name}". Are you sure the start nodes come from the "findStartNodes" function?`, - ); - - const nodeIncomingData = getIncomingData( - runData, - node.name, - runIndex, - startNode.sourceData.connection.type, - startNode.sourceData.connection.outputIndex, - ); - - if (nodeIncomingData) { - incomingData.push(nodeIncomingData); - } - } + for (const sourceData of sourceDataSets) { + incomingData = []; - incomingSourceData.main.push({ - //...startNode.sourceData, - previousNode: startNode.sourceData.connection.from.name, - //currentNodeInput: startNode.sourceData.connection.inputIndex, - previousNodeOutput: startNode.sourceData.connection.outputIndex, - previousNodeRun: 0, - }); - //} - } + incomingSourceData = { main: [] }; - const executeData: IExecuteData = { - node: startNode.node, - data: { main: incomingData }, - source: incomingSourceData, - }; + for (const incomingConnection of sourceData) { + const node = incomingConnection.from; - nodeExecutionStack.push(executeData); + if (pinData[node.name]) { + incomingData.push(pinData[node.name]); + } else { + a.ok( + runData[node.name], + `Start node(${incomingConnection.to.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${node.name}->${startNode.node.name}". Are you sure the start nodes come from the "findStartNodes" function?`, + ); + + const nodeIncomingData = getIncomingData( + runData, + node.name, + runIndex, + incomingConnection.type, + incomingConnection.outputIndex, + ); + + if (nodeIncomingData) { + incomingData.push(nodeIncomingData); + } + } + + incomingSourceData.main.push({ + previousNode: incomingConnection.from.name, + previousNodeOutput: incomingConnection.outputIndex, + previousNodeRun: 0, + }); + } + + const executeData: IExecuteData = { + node: startNode.node, + data: { main: incomingData }, + source: incomingSourceData, + }; + + nodeExecutionStack.push(executeData); + } + } // NOTE: Do we need this? if (destinationNode) { From 43adb5e99f72b1625ab45f8433cac76039a993da Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 3 Sep 2024 22:19:51 +0200 Subject: [PATCH 22/54] fix frontend stripping down the runData before sending it to the backend --- packages/editor-ui/src/composables/useRunWorkflow.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/editor-ui/src/composables/useRunWorkflow.ts b/packages/editor-ui/src/composables/useRunWorkflow.ts index 5e06e1d2039cb..a6caeafc772fd 100644 --- a/packages/editor-ui/src/composables/useRunWorkflow.ts +++ b/packages/editor-ui/src/composables/useRunWorkflow.ts @@ -34,6 +34,7 @@ import { useI18n } from '@/composables/useI18n'; import { get } from 'lodash-es'; import { useExecutionsStore } from '@/stores/executions.store'; import type { PushPayload } from '@n8n/api-types'; +import { useLocalStorage } from '@vueuse/core'; export function useRunWorkflow(useRunWorkflowOpts: { router: ReturnType }) { const nodeHelpers = useNodeHelpers(); @@ -213,9 +214,15 @@ export function useRunWorkflow(useRunWorkflowOpts: { router: ReturnType Date: Tue, 3 Sep 2024 22:24:14 +0200 Subject: [PATCH 23/54] simplify finding start nodes by extracting the logic for finding the source data that's necessary to reconstruct the execution stack --- .../__tests__/findStartNodes.test.ts | 262 +++--------------- .../__tests__/getSourceDataGroups.test.ts | 34 +-- .../recreateNodeExecutionStack.test.ts | 140 +--------- .../PartialExecutionUtils/findStartNodes.ts | 54 +--- .../recreateNodeExecutionStack.ts | 45 +-- 5 files changed, 84 insertions(+), 451 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts index 3cf28caab0cde..47572296f4ed9 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts @@ -9,7 +9,7 @@ // XX denotes that the node is disabled // PD denotes that the node has pinned data -import { NodeConnectionType, type IPinData, type IRunData } from 'n8n-workflow'; +import { type IPinData, type IRunData } from 'n8n-workflow'; import { createNodeData, toITaskData } from './helpers'; import { findStartNodes, isDirty } from '../findStartNodes'; import { DirectedGraph } from '../DirectedGraph'; @@ -48,7 +48,7 @@ describe('findStartNodes', () => { const startNodes = findStartNodes(graph, node, node); expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ node, sourceData: [] }); + expect(startNodes[0]).toEqual(node); }); // ►► @@ -67,7 +67,7 @@ describe('findStartNodes', () => { const startNodes = findStartNodes(graph, trigger, destination); expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ node: trigger, sourceData: [] }); + expect(startNodes[0]).toEqual(trigger); } // if the trigger has run data @@ -79,21 +79,7 @@ describe('findStartNodes', () => { const startNodes = findStartNodes(graph, trigger, destination, runData); expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ - node: destination, - sourceData: [ - { - connection: { - from: trigger, - to: destination, - type: NodeConnectionType.Main, - outputIndex: 0, - inputIndex: 0, - }, - previousNodeRun: 0, - }, - ], - }); + expect(startNodes[0]).toEqual(destination); } }); @@ -105,9 +91,7 @@ describe('findStartNodes', () => { // All nodes have run data. `findStartNodes` should return node twice // because it has 2 input connections. test('multiple outputs', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node = createNodeData({ name: 'node' }); const graph = new DirectedGraph() @@ -126,51 +110,12 @@ describe('findStartNodes', () => { [node.name]: [toITaskData([{ data: { value: 1 } }])], }; - // // ACT - // const startNodes = findStartNodes(graph, trigger, node, runData); - // // ASSERT - // - expect(startNodes).toHaveLength(2); - expect(startNodes).toContainEqual({ - node, - sourceData: [ - { - connection: { - from: trigger, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: trigger, - //previousNodeOutput: 0, - }, - ], - }); - expect(startNodes).toContainEqual({ - node, - sourceData: [ - { - connection: { - from: trigger, - outputIndex: 1, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: trigger, - //previousNodeOutput: 0, - }, - ], - }); + expect(startNodes).toHaveLength(1); + expect(startNodes[0]).toEqual(node); }); // ┌─────┐ ┌─────┐ ►► @@ -187,6 +132,7 @@ describe('findStartNodes', () => { // └────────►│ │ // └─────┘ test('complex example with multiple outputs and inputs', () => { + // ARRANGE const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -205,10 +151,13 @@ describe('findStartNodes', () => { ); { + // ACT const startNodes = findStartNodes(graph, trigger, node4); + + // ASSERT expect(startNodes).toHaveLength(1); // no run data means the trigger is the start node - expect(startNodes).toContainEqual({ node: trigger, sourceData: [] }); + expect(startNodes[0]).toEqual(trigger); } { @@ -221,46 +170,12 @@ describe('findStartNodes', () => { [node4.name]: [toITaskData([{ data: { value: 1 } }])], }; + // ACT const startNodes = findStartNodes(graph, trigger, node4, runData); - expect(startNodes).toHaveLength(2); - - expect(startNodes).toContainEqual({ - node: node4, - sourceData: [ - { - connection: { - from: node2, - to: node4, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node2, - //previousNodeOutput: 0, - }, - ], - }); - - expect(startNodes).toContainEqual({ - node: node4, - sourceData: [ - { - connection: { - from: node3, - to: node4, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node3, - //previousNodeOutput: 0, - }, - ], - }); + + // ASSERT + expect(startNodes).toHaveLength(1); + expect(startNodes[0]).toEqual(node4); } }); @@ -273,6 +188,7 @@ describe('findStartNodes', () => { // The merge node only gets data on one input, so the it should only be once // in the start nodes test('multiple connections with the first one having data', () => { + // ARRANGE const trigger = createNodeData({ name: 'trigger' }); const node = createNodeData({ name: 'node' }); @@ -283,29 +199,14 @@ describe('findStartNodes', () => { { from: trigger, to: node, inputIndex: 1, outputIndex: 1 }, ); + // ACT const startNodes = findStartNodes(graph, trigger, node, { [trigger.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], }); + // ASSERT expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ - node, - sourceData: [ - { - connection: { - from: trigger, - to: node, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: trigger, - //previousNodeOutput: 0, - }, - ], - }); + expect(startNodes[0]).toEqual(node); }); // ►► @@ -317,6 +218,7 @@ describe('findStartNodes', () => { // The merge node only gets data on one input, so the it should only be once // in the start nodes test('multiple connections with the second one having data', () => { + // ARRANGE const trigger = createNodeData({ name: 'trigger' }); const node = createNodeData({ name: 'node' }); @@ -327,29 +229,14 @@ describe('findStartNodes', () => { { from: trigger, to: node, inputIndex: 1, outputIndex: 1 }, ); + // ACT const startNodes = findStartNodes(graph, trigger, node, { [trigger.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 1 }])], }); + // ASSERT expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ - node, - sourceData: [ - { - connection: { - from: trigger, - to: node, - inputIndex: 1, - outputIndex: 1, - type: NodeConnectionType.Main, - }, - previousNodeRun: 0, - //currentNodeInput: 1, - //previousNode: trigger, - //previousNodeOutput: 1, - }, - ], - }); + expect(startNodes[0]).toEqual(node); }); // ►► @@ -360,7 +247,8 @@ describe('findStartNodes', () => { // └───────┘1 └────┘ // The merge node gets data on both inputs, so the it should be in the start // nodes twice. - test.only('multiple connections with both having data', () => { + test('multiple connections with both having data', () => { + // ARRANGE const trigger = createNodeData({ name: 'trigger' }); const node = createNodeData({ name: 'node' }); @@ -371,6 +259,7 @@ describe('findStartNodes', () => { { from: trigger, to: node, inputIndex: 1, outputIndex: 1 }, ); + // ACT const startNodes = findStartNodes(graph, trigger, node, { [trigger.name]: [ toITaskData([ @@ -380,40 +269,9 @@ describe('findStartNodes', () => { ], }); + // ASSERT expect(startNodes).toHaveLength(1); - // TODO: this is wrong, technically this should contain one start node - // and the source data should be an array. - expect(startNodes).toContainEqual({ - node, - sourceData: [ - { - connection: { - from: trigger, - to: node, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: trigger, - //previousNodeOutput: 0, - }, - { - connection: { - from: trigger, - to: node, - inputIndex: 1, - outputIndex: 1, - type: NodeConnectionType.Main, - }, - previousNodeRun: 0, - //currentNodeInput: 1, - //previousNode: trigger, - //previousNodeOutput: 1, - }, - ], - }); + expect(startNodes[0]).toEqual(node); }); // TODO: This is not working yet as expected. @@ -426,7 +284,8 @@ describe('findStartNodes', () => { // │ │ └────►│ │ // └───────┘ └────┘ // eslint-disable-next-line n8n-local-rules/no-skipped-tests - test.only('multiple connections with both having data', () => { + test('multiple connections with both having data', () => { + // ARRANGE const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -440,21 +299,16 @@ describe('findStartNodes', () => { { from: node2, to: node3 }, ); + // ACT const startNodes = findStartNodes(graph, trigger, node3, { [trigger.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], [node1.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], [node2.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 0 }])], }); + // ASSERT expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ - node: node3, - sourceData: { - previousNode: node1, - previousNodeRun: 0, - previousNodeOutput: 0, - }, - }); + expect(startNodes[0]).toEqual(node3); }); // ►► @@ -464,6 +318,7 @@ describe('findStartNodes', () => { // │ │ │ ├────────►│ │ // └───────┘ └─────┘1 └─────┘ test('multiple connections with trigger', () => { + // ARRANGE const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -476,31 +331,15 @@ describe('findStartNodes', () => { { from: node1, to: node2, outputIndex: 1 }, ); + // ACT const startNodes = findStartNodes(graph, node1, node2, { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], [node1.name]: [toITaskData([{ data: { value: 1 }, outputIndex: 1 }])], }); + // ASSERT expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ - node: node2, - sourceData: [ - { - connection: { - from: node1, - to: node2, - // TODO: Shouldn't this be 1 instead of 0? - inputIndex: 0, - outputIndex: 1, - type: NodeConnectionType.Main, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node1, - //previousNodeOutput: 1, - }, - ], - }); + expect(startNodes[0]).toEqual(node2); }); // ►► @@ -510,9 +349,7 @@ describe('findStartNodes', () => { // │ │ // └─────────────┘ test('terminates when called with graph that contains cycles', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -529,32 +366,11 @@ describe('findStartNodes', () => { }; const pinData: IPinData = {}; - // // ACT - // const startNodes = findStartNodes(graph, trigger, node2, runData, pinData); - // // ASSERT - // expect(startNodes).toHaveLength(1); - expect(startNodes).toContainEqual({ - node: node2, - sourceData: [ - { - connection: { - from: node1, - to: node2, - outputIndex: 0, - inputIndex: 0, - type: NodeConnectionType.Main, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node1, - //previousNodeOutput: 0, - }, - ], - }); + expect(startNodes[0]).toEqual(node2); }); }); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts index d569cf63dac1a..6636a3f01d7f8 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts @@ -7,6 +7,7 @@ // 1 means the output has run data // PD denotes that the node has pinned data +import type { IPinData } from 'n8n-workflow'; import { NodeConnectionType, type IRunData } from 'n8n-workflow'; import { DirectedGraph } from '../DirectedGraph'; import { createNodeData, toITaskData } from './helpers'; @@ -23,6 +24,7 @@ describe('getSourceDataGroups', () => { //│source3├────┘ //└───────┘ it('groups sources into possibly complete sets if all of them have data', () => { + // ARRANGE const source1 = createNodeData({ name: 'source1' }); const source2 = createNodeData({ name: 'source2' }); const source3 = createNodeData({ name: 'source3' }); @@ -40,10 +42,12 @@ describe('getSourceDataGroups', () => { [source2.name]: [toITaskData([{ data: { value: 1 } }])], [source3.name]: [toITaskData([{ data: { value: 1 } }])], }; - const pinnedData: IRunData = {}; + const pinnedData: IPinData = {}; + // ACT const groups = getSourceDataGroups(graph, node, runData, pinnedData); + // ASSERT expect(groups).toHaveLength(2); const group1 = groups[0]; @@ -84,6 +88,7 @@ describe('getSourceDataGroups', () => { //│source3├────┘ //└───────┘ it('groups sources into possibly complete sets if all of them have data', () => { + // ARRANGE const source1 = createNodeData({ name: 'source1' }); const source2 = createNodeData({ name: 'source2' }); const source3 = createNodeData({ name: 'source3' }); @@ -97,14 +102,16 @@ describe('getSourceDataGroups', () => { { from: source3, to: node, inputIndex: 1 }, ); const runData: IRunData = {}; - const pinnedData: IRunData = { - [source1.name]: [toITaskData([{ data: { value: 1 } }])], - [source2.name]: [toITaskData([{ data: { value: 1 } }])], - [source3.name]: [toITaskData([{ data: { value: 1 } }])], + const pinnedData: IPinData = { + [source1.name]: [{ json: { value: 1 } }], + [source2.name]: [{ json: { value: 2 } }], + [source3.name]: [{ json: { value: 3 } }], }; + // ACT const groups = getSourceDataGroups(graph, node, runData, pinnedData); + // ASSERT expect(groups).toHaveLength(2); const group1 = groups[0]; @@ -145,6 +152,7 @@ describe('getSourceDataGroups', () => { //│source3├────┘ //└───────┘ it('groups sources into possibly complete sets if all of them have data', () => { + // ARRANGE const source1 = createNodeData({ name: 'source1' }); const source2 = createNodeData({ name: 'source2' }); const source3 = createNodeData({ name: 'source3' }); @@ -161,11 +169,13 @@ describe('getSourceDataGroups', () => { [source2.name]: [toITaskData([{ data: { value: 1 } }])], [source3.name]: [toITaskData([{ data: { value: 1 } }])], }; - const pinnedData: IRunData = {}; + const pinnedData: IPinData = {}; + // ACT const groups = getSourceDataGroups(graph, node, runData, pinnedData); - expect(groups).toHaveLength(2); + // ASSERT + expect(groups).toHaveLength(1); const group1 = groups[0]; expect(group1).toHaveLength(2); @@ -183,15 +193,5 @@ describe('getSourceDataGroups', () => { inputIndex: 1, to: node, }); - - const group2 = groups[1]; - expect(group2).toHaveLength(1); - expect(group2[0]).toEqual({ - from: source1, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node, - }); }); }); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts index 7dc79ffbaaa13..1586c448d820c 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts @@ -10,11 +10,10 @@ // PD denotes that the node has pinned data import { recreateNodeExecutionStack } from '@/PartialExecutionUtils/recreateNodeExecutionStack'; -import { NodeConnectionType, type IPinData, type IRunData } from 'n8n-workflow'; +import { type IPinData, type IRunData } from 'n8n-workflow'; import { AssertionError } from 'assert'; import { DirectedGraph } from '../DirectedGraph'; import { findSubgraph } from '../findSubgraph'; -import type { StartNodeData } from '../findStartNodes'; import { createNodeData, toITaskData } from './helpers'; describe('recreateNodeExecutionStack', () => { @@ -34,24 +33,7 @@ describe('recreateNodeExecutionStack', () => { .addConnections({ from: trigger, to: node }); const workflow = findSubgraph(graph, node, trigger); - const startNodes: StartNodeData[] = [ - { - node, - sourceData: { - connection: { - from: trigger, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: trigger, - //previousNodeOutput: 0, - }, - }, - ]; + const startNodes = [node]; const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], }; @@ -74,7 +56,7 @@ describe('recreateNodeExecutionStack', () => { source: { main: [ { - // TODO: not part of IScourceDate, but maybe it should be? + // TODO: not part of ISourceDate, but maybe it should be? //currentNodeInput: 0, previousNode: 'trigger', previousNodeOutput: 0, @@ -111,7 +93,7 @@ describe('recreateNodeExecutionStack', () => { const workflow = new DirectedGraph() .addNodes(trigger, node) .addConnections({ from: trigger, to: node }); - const startNodes: StartNodeData[] = [{ node: trigger }]; + const startNodes = [trigger]; const runData: IRunData = {}; const pinData: IPinData = {}; @@ -151,24 +133,7 @@ describe('recreateNodeExecutionStack', () => { const workflow = new DirectedGraph() .addNodes(trigger, node) .addConnections({ from: trigger, to: node }); - const startNodes: StartNodeData[] = [ - { - node, - sourceData: { - connection: { - from: trigger, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: trigger, - //previousNodeOutput: 0, - }, - }, - ]; + const startNodes = [node]; const runData: IRunData = {}; const pinData: IPinData = { [trigger.name]: [{ json: { value: 1 } }], @@ -191,7 +156,7 @@ describe('recreateNodeExecutionStack', () => { source: { main: [ { - // TODO: not part of IScourceDate, but maybe it should be? + // TODO: not part of ISourceDate, but maybe it should be? //currentNodeInput: 0, previousNode: trigger.name, previousNodeRun: 0, @@ -222,24 +187,7 @@ describe('recreateNodeExecutionStack', () => { .addNodes(trigger, node1, node2) .addConnections({ from: trigger, to: node1 }, { from: node1, to: node2 }); - const startNodes: StartNodeData[] = [ - { - node: node2, - sourceData: { - connection: { - from: node1, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node2, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node1, - //previousNodeOutput: 0, - }, - }, - ]; + const startNodes = [node2]; const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], }; @@ -278,40 +226,7 @@ describe('recreateNodeExecutionStack', () => { { from: node2, to: node3 }, ); - const startNodes: StartNodeData[] = [ - { - node: node3, - sourceData: { - connection: { - from: node1, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node3, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node1, - //previousNodeOutput: 0, - }, - }, - //{ - // node: node3, - // sourceData: { - // connection: { - // from: node2, - // outputIndex: 0, - // type: NodeConnectionType.Main, - // inputIndex: 0, - // to: node3, - // }, - // previousNodeRun: 0, - // //currentNodeInput: 0, - // //previousNode: node2, - // //previousNodeOutput: 0, - // }, - //}, - ]; + const startNodes = [node3]; const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], [node1.name]: [toITaskData([{ data: { value: 1 } }])], @@ -336,7 +251,7 @@ describe('recreateNodeExecutionStack', () => { source: { main: [ { - // TODO: not part of IScourceDate, but maybe it should be? + // TODO: not part of ISourceDate, but maybe it should be? //currentNodeInput: 0, previousNode: 'node1', previousNodeOutput: 0, @@ -351,7 +266,7 @@ describe('recreateNodeExecutionStack', () => { source: { main: [ { - // TODO: not part of IScourceDate, but maybe it should be? + // TODO: not part of ISourceDate, but maybe it should be? //currentNodeInput: 0, previousNode: 'node2', previousNodeOutput: 0, @@ -410,40 +325,7 @@ describe('recreateNodeExecutionStack', () => { { from: node1, to: node3, inputIndex: 0 }, { from: node2, to: node3, inputIndex: 1 }, ); - const startNodes: StartNodeData[] = [ - { - node: node3, - sourceData: { - connection: { - from: node1, - outputIndex: 0, - type: NodeConnectionType.Main, - inputIndex: 0, - to: node3, - }, - previousNodeRun: 0, - //currentNodeInput: 0, - //previousNode: node1, - //previousNodeOutput: 0, - }, - }, - //{ - // node: node3, - // sourceData: { - // connection: { - // from: node2, - // outputIndex: 0, - // type: NodeConnectionType.Main, - // inputIndex: 1, - // to: node3, - // }, - // previousNodeRun: 0, - // //currentNodeInput: 0, - // //previousNode: node1, - // //previousNodeOutput: 0, - // }, - //}, - ]; + const startNodes = [node3]; const runData: IRunData = { [trigger.name]: [toITaskData([{ data: { value: 1 } }])], [node1.name]: [toITaskData([{ data: { value: 1 } }])], diff --git a/packages/core/src/PartialExecutionUtils/findStartNodes.ts b/packages/core/src/PartialExecutionUtils/findStartNodes.ts index b56fe73760b16..66b9a415646a7 100644 --- a/packages/core/src/PartialExecutionUtils/findStartNodes.ts +++ b/packages/core/src/PartialExecutionUtils/findStartNodes.ts @@ -14,8 +14,6 @@ export interface StartNodeData { sourceData: NewSourceData[]; } -type Key = `${number}-${string}`; - // TODO: implement dirty checking for options and properties and parent nodes // being disabled export function isDirty(node: INode, runData: IRunData = {}, pinData: IPinData = {}): boolean { @@ -58,53 +56,29 @@ export function isDirty(node: INode, runData: IRunData = {}, pinData: IPinData = return true; } -function makeKey(sourceConnection: Connection | undefined, to: INode): Key { - return `${sourceConnection?.outputIndex ?? 0}-${to.name}`; -} - function findStartNodesRecursive( graph: DirectedGraph, current: INode, destination: INode, runData: IRunData, pinData: IPinData, - startNodes: Map, + startNodes: Set, seen: Set, - source?: NewSourceData, -) { +): Set { const nodeIsDirty = isDirty(current, runData, pinData); // If the current node is dirty stop following this branch, we found a start // node. if (nodeIsDirty) { - const key = source ? source.connection : 'Trigger'; // makeKey(source?.connection, current); - const value = startNodes.get(key) ?? { - node: current, - sourceData: [], - }; - - if (source) { - value.sourceData.push(source); - } + startNodes.add(current); - startNodes.set(key, value); return startNodes; } // If the current node is the destination node stop following this branch, we // found a start node. if (current === destination) { - const key = source ? source.connection : 'Trigger'; // makeKey(source?.connection, current); - const value = startNodes.get(key) ?? { - node: current, - sourceData: [], - }; - - if (source) { - value.sourceData.push(source); - } - - startNodes.set(key, value); + startNodes.add(current); return startNodes; } @@ -142,12 +116,6 @@ function findStartNodesRecursive( pinData, startNodes, new Set(seen).add(current), - { - connection: outGoingConnection, - // NOTE: It's always 0 until I fix the bug that removes the run data for - // old runs. The FE only sends data for one run for each node. - previousNodeRun: 0, - }, ); } @@ -160,24 +128,18 @@ export function findStartNodes( destination: INode, runData: IRunData = {}, pinData: IPinData = {}, -): StartNodeData[] { +): INode[] { const startNodes = findStartNodesRecursive( graph, trigger, destination, runData, pinData, - // start nodes and their sources - new Map(), + // found start nodes + new Set(), // seen new Set(), ); - const startNodesData: StartNodeData[] = []; - - for (const [_node, value] of startNodes) { - startNodesData.push(value); - } - - return startNodesData; + return [...startNodes]; } diff --git a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts index a5e7e530d00f9..f51bf09b7e591 100644 --- a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts +++ b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts @@ -13,7 +13,6 @@ import { import * as a from 'assert/strict'; import type { Connection, DirectedGraph } from './DirectedGraph'; -import type { StartNodeData } from './findStartNodes'; import { getIncomingData } from './getIncomingData'; function sortByInputIndexThenByName(connection1: Connection, connection2: Connection): number { @@ -90,37 +89,27 @@ export function getSourceDataGroups( const connections = graph.getConnections({ to: node }); const sortedConnectionsWithData = []; - const sortedConnectionsWithoutData = []; for (const connection of connections) { const hasData = runData[connection.from.name] || pinnedData[connection.from.name]; if (hasData) { sortedConnectionsWithData.push(connection); - } else { - sortedConnectionsWithoutData.push(connection); } } sortedConnectionsWithData.sort(sortByInputIndexThenByName); - sortedConnectionsWithoutData.sort(sortByInputIndexThenByName); const groups: Connection[][] = []; let currentGroup: Connection[] = []; let currentInputIndex = -1; - while (sortedConnectionsWithData.length > 0 || sortedConnectionsWithoutData.length > 0) { + while (sortedConnectionsWithData.length > 0) { const connectionWithDataIndex = sortedConnectionsWithData.findIndex( // eslint-disable-next-line @typescript-eslint/no-loop-func (c) => c.inputIndex > currentInputIndex, ); - const connectionWithoutDataIndex = sortedConnectionsWithoutData.findIndex( - // eslint-disable-next-line @typescript-eslint/no-loop-func - (c) => c.inputIndex > currentInputIndex, - ); - const connection: Connection | undefined = - sortedConnectionsWithData[connectionWithDataIndex] ?? - sortedConnectionsWithoutData[connectionWithoutDataIndex]; + const connection: Connection | undefined = sortedConnectionsWithData[connectionWithDataIndex]; if (connection === undefined) { groups.push(currentGroup); @@ -134,8 +123,6 @@ export function getSourceDataGroups( if (connectionWithDataIndex >= 0) { sortedConnectionsWithData.splice(connectionWithDataIndex, 1); - } else if (connectionWithoutDataIndex >= 0) { - sortedConnectionsWithoutData.splice(connectionWithoutDataIndex, 1); } } @@ -146,7 +133,7 @@ export function getSourceDataGroups( export function recreateNodeExecutionStack( graph: DirectedGraph, - startNodes: StartNodeData[], + startNodes: INode[], destinationNode: INode, runData: IRunData, pinData: IPinData, @@ -167,20 +154,6 @@ export function recreateNodeExecutionStack( ); } - // The start nodes's sources need to have run data or pinned data. If they - // don't then they should not be the start nodes, but some node before them - // should be. Probably they are not coming from findStartNodes, make sure to - // use that function to get the start nodes. - //for (const startNode of startNodes) { - // if (startNode.sourceData) { - // a.ok( - // runData[startNode.sourceData.connection.from.name] || - // pinData[startNode.sourceData.connection.from.name], - // `Start nodes have sources that don't have run data. That is not supported. Make sure to get the start nodes by calling "findStartNodes". The node in question is "${startNode.node.name}" and their source is "${startNode.sourceData.connection.from.name}".`, - // ); - // } - //} - // Initialize the nodeExecutionStack and waitingExecution with // the data from runData const nodeExecutionStack: IExecuteData[] = []; @@ -192,7 +165,7 @@ export function recreateNodeExecutionStack( for (const startNode of startNodes) { const incomingStartNodeConnections = graph - .getDirectParents(startNode.node) + .getDirectParents(startNode) .filter((c) => c.type === NodeConnectionType.Main); let incomingData: INodeExecutionData[][] = []; @@ -202,14 +175,14 @@ export function recreateNodeExecutionStack( incomingData.push([{ json: {} }]); const executeData: IExecuteData = { - node: startNode.node, + node: startNode, data: { main: incomingData }, source: incomingSourceData, }; nodeExecutionStack.push(executeData); } else { - const sourceDataSets = getSourceDataGroups(graph, startNode.node, runData, pinData); + const sourceDataSets = getSourceDataGroups(graph, startNode, runData, pinData); for (const sourceData of sourceDataSets) { incomingData = []; @@ -224,7 +197,7 @@ export function recreateNodeExecutionStack( } else { a.ok( runData[node.name], - `Start node(${incomingConnection.to.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${node.name}->${startNode.node.name}". Are you sure the start nodes come from the "findStartNodes" function?`, + `Start node(${incomingConnection.to.name}) has an incoming connection with no run or pinned data. This is not supported. The connection in question is "${node.name}->${startNode.name}". Are you sure the start nodes come from the "findStartNodes" function?`, ); const nodeIncomingData = getIncomingData( @@ -248,7 +221,7 @@ export function recreateNodeExecutionStack( } const executeData: IExecuteData = { - node: startNode.node, + node: startNode, data: { main: incomingData }, source: incomingSourceData, }; @@ -257,7 +230,7 @@ export function recreateNodeExecutionStack( } } - // NOTE: Do we need this? + // TODO: Do we need this? if (destinationNode) { const destinationNodeName = destinationNode.name; // Check if the destinationNode has to be added as waiting From 1763826c2f371d8907039d6b4760304dc5a0124f Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:25:54 +0200 Subject: [PATCH 24/54] extract getSourceDataGroups out --- .../__tests__/getSourceDataGroups.test.ts | 2 +- .../getSourceDataGroups.ts | 112 +++++++++++++++++ .../recreateNodeExecutionStack.ts | 119 +----------------- 3 files changed, 115 insertions(+), 118 deletions(-) create mode 100644 packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts diff --git a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts index 6636a3f01d7f8..737d0a2754318 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/getSourceDataGroups.test.ts @@ -11,7 +11,7 @@ import type { IPinData } from 'n8n-workflow'; import { NodeConnectionType, type IRunData } from 'n8n-workflow'; import { DirectedGraph } from '../DirectedGraph'; import { createNodeData, toITaskData } from './helpers'; -import { getSourceDataGroups } from '../recreateNodeExecutionStack'; +import { getSourceDataGroups } from '../getSourceDataGroups'; describe('getSourceDataGroups', () => { //┌───────┐1 diff --git a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts b/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts new file mode 100644 index 0000000000000..7d8fcc5dcaebd --- /dev/null +++ b/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts @@ -0,0 +1,112 @@ +import { type INode, type IPinData, type IRunData } from 'n8n-workflow'; + +import type { Connection, DirectedGraph } from './DirectedGraph'; + +function sortByInputIndexThenByName(connection1: Connection, connection2: Connection): number { + if (connection1.inputIndex === connection2.inputIndex) { + return connection1.from.name.localeCompare(connection2.from.name); + } else { + return connection1.inputIndex - connection2.inputIndex; + } +} + +/** + * Groups incoming connections to the node. The groups contain one connection + * per input, if possible, with run data or pinned data, if possible. + * + * The purpose of this is to get as many complete sets of data for executing + * nodes with multiple inputs. + * + * # Example 1: + * ┌───────┐1 + * │source1├────┐ + * └───────┘ │ ┌────┐ + * ┌───────┐1 ├──►│ │ + * │source2├────┘ │node│ + * └───────┘ ┌──►│ │ + * ┌───────┐1 │ └────┘ + * │source3├────┘ + * └───────┘ + * + * Given this workflow, and assuming all sources have run data or pinned data, + * it's possible to run `node` with the data of `source1` and `source3` and + * then one more time with the data from `source2`. + * + * It would also be possible to run `node` with the data of `source2` and + * `source3` and then one more time with the data from `source1`. + * + * To improve the determinism of this the connections are sorted by input and + * then by from-node name. + * + * So this will return 2 groups: + * 1. source1 and source3 + * 2. source2 + * + * # Example 1: + * ┌───────┐0 + * │source1├────┐ + * └───────┘ │ ┌────┐ + * ┌───────┐1 ├──►│ │ + * │source2├────┘ │node│ + * └───────┘ ┌──►│ │ + * ┌───────┐1 │ └────┘ + * │source3├────┘ + * └───────┘ + * + * Since `source1` has no run data and no pinned data it's skipped in favor of + * `source2` for the for input. + * + * So this will return 2 groups: + * 1. source2 and source3 + * 2. source1 + */ +export function getSourceDataGroups( + graph: DirectedGraph, + node: INode, + runData: IRunData, + pinnedData: IPinData, +): Connection[][] { + const connections = graph.getConnections({ to: node }); + + const sortedConnectionsWithData = []; + + for (const connection of connections) { + const hasData = runData[connection.from.name] || pinnedData[connection.from.name]; + + if (hasData) { + sortedConnectionsWithData.push(connection); + } + } + + sortedConnectionsWithData.sort(sortByInputIndexThenByName); + + const groups: Connection[][] = []; + let currentGroup: Connection[] = []; + let currentInputIndex = -1; + + while (sortedConnectionsWithData.length > 0) { + const connectionWithDataIndex = sortedConnectionsWithData.findIndex( + // eslint-disable-next-line @typescript-eslint/no-loop-func + (c) => c.inputIndex > currentInputIndex, + ); + const connection: Connection | undefined = sortedConnectionsWithData[connectionWithDataIndex]; + + if (connection === undefined) { + groups.push(currentGroup); + currentGroup = []; + currentInputIndex = -1; + continue; + } + + currentInputIndex = connection.inputIndex; + currentGroup.push(connection); + + if (connectionWithDataIndex >= 0) { + sortedConnectionsWithData.splice(connectionWithDataIndex, 1); + } + } + + groups.push(currentGroup); + + return groups; +} diff --git a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts index f51bf09b7e591..4491b22b490a0 100644 --- a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts +++ b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts @@ -12,124 +12,9 @@ import { } from 'n8n-workflow'; import * as a from 'assert/strict'; -import type { Connection, DirectedGraph } from './DirectedGraph'; +import type { DirectedGraph } from './DirectedGraph'; import { getIncomingData } from './getIncomingData'; - -function sortByInputIndexThenByName(connection1: Connection, connection2: Connection): number { - if (connection1.inputIndex === connection2.inputIndex) { - return connection1.from.name.localeCompare(connection2.from.name); - } else { - return connection1.inputIndex - connection2.inputIndex; - } -} - -// INFO: I don't need to care about connections without data. I just have to -// make sure all connections have data, otherwise the node that was passed in -// is not a valid startNode, startNodes must have data on all incoming -// connections. -// TODO: assert that all incoming connections have run data of pinned data. -// TODO: remove all code handling connections without data - -/** - * Groups incoming connections to the node. The groups contain one connection - * per input, if possible, with run data or pinned data, if possible. - * - * The purpose of this is to get as many complete sets of data for executing - * nodes with multiple inputs. - * - * # Example 1: - * ┌───────┐1 - * │source1├────┐ - * └───────┘ │ ┌────┐ - * ┌───────┐1 ├──►│ │ - * │source2├────┘ │node│ - * └───────┘ ┌──►│ │ - * ┌───────┐1 │ └────┘ - * │source3├────┘ - * └───────┘ - * - * Given this workflow, and assuming all sources have run data or pinned data, - * it's possible to run `node` with the data of `source1` and `source3` and - * then one more time with the data from `source2`. - * - * It would also be possible to run `node` with the data of `source2` and - * `source3` and then one more time with the data from `source1`. - * - * To improve the determinism of this the connections are sorted by input and - * then by from-node name. - * - * So this will return 2 groups: - * 1. source1 and source3 - * 2. source2 - * - * # Example 1: - * ┌───────┐0 - * │source1├────┐ - * └───────┘ │ ┌────┐ - * ┌───────┐1 ├──►│ │ - * │source2├────┘ │node│ - * └───────┘ ┌──►│ │ - * ┌───────┐1 │ └────┘ - * │source3├────┘ - * └───────┘ - * - * Since `source1` has no run data and no pinned data it's skipped in favor of - * `source2` for the for input. - * - * So this will return 2 groups: - * 1. source2 and source3 - * 2. source1 - */ -export function getSourceDataGroups( - graph: DirectedGraph, - node: INode, - runData: IRunData, - pinnedData: IPinData, -): Connection[][] { - const connections = graph.getConnections({ to: node }); - - const sortedConnectionsWithData = []; - - for (const connection of connections) { - const hasData = runData[connection.from.name] || pinnedData[connection.from.name]; - - if (hasData) { - sortedConnectionsWithData.push(connection); - } - } - - sortedConnectionsWithData.sort(sortByInputIndexThenByName); - - const groups: Connection[][] = []; - let currentGroup: Connection[] = []; - let currentInputIndex = -1; - - while (sortedConnectionsWithData.length > 0) { - const connectionWithDataIndex = sortedConnectionsWithData.findIndex( - // eslint-disable-next-line @typescript-eslint/no-loop-func - (c) => c.inputIndex > currentInputIndex, - ); - const connection: Connection | undefined = sortedConnectionsWithData[connectionWithDataIndex]; - - if (connection === undefined) { - groups.push(currentGroup); - currentGroup = []; - currentInputIndex = -1; - continue; - } - - currentInputIndex = connection.inputIndex; - currentGroup.push(connection); - - if (connectionWithDataIndex >= 0) { - sortedConnectionsWithData.splice(connectionWithDataIndex, 1); - } - } - - groups.push(currentGroup); - - return groups; -} +import { getSourceDataGroups } from './getSourceDataGroups'; export function recreateNodeExecutionStack( graph: DirectedGraph, From 15112f70ab19f481855cefd10ebeefdda9cf14dd Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:27:31 +0200 Subject: [PATCH 25/54] remove notation file --- execution stack notation.md | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 execution stack notation.md diff --git a/execution stack notation.md b/execution stack notation.md deleted file mode 100644 index 306733f1de64d..0000000000000 --- a/execution stack notation.md +++ /dev/null @@ -1,19 +0,0 @@ -# execution stack notation.md - -ES = Execution Stack -WN = Waiting Nodes -WNS = Waiting Node Sources - -(ES[], WN[], WNS[]) - -([t], [ ], [ ]) -([n1, n2], [ ], [ ]) -([n2], [n3], [n1]) -([n3], [ ], [ ]) <-- -([n4], [ ], [ ]) - -# start node notation -SN = Start Node -SNS = Start Node Source - - From e5ef7f9f13555cb74c8f99a8dbd52f582099b51c Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:29:45 +0200 Subject: [PATCH 26/54] restore default coverage reports --- jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index 78c205957f9c3..3caac38ef9dbf 100644 --- a/jest.config.js +++ b/jest.config.js @@ -33,7 +33,7 @@ const config = { }, {}), setupFilesAfterEnv: ['jest-expect-message'], collectCoverage: process.env.COVERAGE_ENABLED === 'true', - coverageReporters: ['html'], + coverageReporters: ['text-summary'], collectCoverageFrom: ['src/**/*.ts'], }; From a45517a32f32fc46b81f6567e535d08f04c94dfb Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:39:55 +0200 Subject: [PATCH 27/54] improve comments --- packages/core/src/PartialExecutionUtils/DirectedGraph.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index 5759e1cf2c7e5..3b9674c25a13c 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -179,9 +179,6 @@ export class DirectedGraph { a.ok(from); for (const [outputType, outputs] of Object.entries(iConnection)) { - // TODO: parse - //const type = outputType as NodeConnectionType - for (const [outputIndex, conns] of outputs.entries()) { for (const conn of conns) { // TODO: What's with the input type? @@ -192,6 +189,7 @@ export class DirectedGraph { graph.addConnection({ from, to, + // TODO: parse outputType instead of casting it type: outputType as NodeConnectionType, outputIndex, inputIndex, From 147b70c9534065f614c1c06bff5170402b07c64a Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:43:00 +0200 Subject: [PATCH 28/54] remove old code --- .../core/src/PartialExecutionUtils/__tests__/helpers.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/__tests__/helpers.ts b/packages/core/src/PartialExecutionUtils/__tests__/helpers.ts index 641254242c56d..72b1efa30c7c7 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/helpers.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/helpers.ts @@ -47,17 +47,8 @@ export function toITaskData(taskData: TaskData[]): ITaskData { for (const [type, dataConnection] of Object.entries(result.data)) { for (const [index, maybe] of dataConnection.entries()) { - //result.data[type][index] = - // maybe ?? randomInt(2) === 0 - // ? null - // : // NOTE: The FE sends an empty array instead of null. I have yet to - // // figure out if there is a different when executing a workflow. - // []; result.data[type][index] = maybe ?? null; } - //result.data[type] = dataConnection.map((maybe) => - // maybe ? maybe.map((maybe) => maybe ?? null) : null, - //); } return result; From c819fc174d5c5827ef33fe7751736ab516addf80 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:45:08 +0200 Subject: [PATCH 29/54] update comments --- .../recreateNodeExecutionStack.test.ts | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts index 1586c448d820c..42cbe1f5ff9d6 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/recreateNodeExecutionStack.test.ts @@ -22,9 +22,7 @@ describe('recreateNodeExecutionStack', () => { // │Trigger├──────►│Node│ // └───────┘ └────┘ test('all nodes except destination node have data', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node = createNodeData({ name: 'node' }); @@ -39,15 +37,11 @@ describe('recreateNodeExecutionStack', () => { }; const pinData = {}; - // // ACT - // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); - // // ASSERT - // expect(nodeExecutionStack).toHaveLength(1); expect(nodeExecutionStack).toEqual([ { @@ -84,9 +78,7 @@ describe('recreateNodeExecutionStack', () => { // │Trigger├──────►│Node│ // └───────┘ └────┘ test('no nodes have data', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node = createNodeData({ name: 'node' }); @@ -97,15 +89,11 @@ describe('recreateNodeExecutionStack', () => { const runData: IRunData = {}; const pinData: IPinData = {}; - // // ACT - // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); - // // ASSERT - // expect(nodeExecutionStack).toHaveLength(1); expect(nodeExecutionStack).toEqual([ { @@ -124,9 +112,7 @@ describe('recreateNodeExecutionStack', () => { // │Trigger├──────►│Node│ // └───────┘ └────┘ test('node before destination node has pinned data', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node = createNodeData({ name: 'node' }); @@ -139,15 +125,11 @@ describe('recreateNodeExecutionStack', () => { [trigger.name]: [{ json: { value: 1 } }], }; - // // ACT - // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = recreateNodeExecutionStack(workflow, startNodes, node, runData, pinData); - // // ASSERT - // expect(nodeExecutionStack).toHaveLength(1); expect(nodeExecutionStack).toEqual([ { @@ -176,9 +158,7 @@ describe('recreateNodeExecutionStack', () => { // │Trigger├─────►│Node1├──────►│Node2│ // └───────┘ └─────┘ └─────┘ test('throws if a disabled node is found', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1', disabled: true }); const node2 = createNodeData({ name: 'node2' }); @@ -193,9 +173,7 @@ describe('recreateNodeExecutionStack', () => { }; const pinData = {}; - // // ACT & ASSERT - // expect(() => recreateNodeExecutionStack(graph, startNodes, node2, runData, pinData), ).toThrowError(AssertionError); @@ -210,9 +188,7 @@ describe('recreateNodeExecutionStack', () => { // └──►│Node2├──┘ // └─────┘ test('multiple incoming connections', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -234,15 +210,11 @@ describe('recreateNodeExecutionStack', () => { }; const pinData = {}; - // // ACT - // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = recreateNodeExecutionStack(graph, startNodes, node3, runData, pinData); - // // ASSERT - // expect(nodeExecutionStack).toEqual([ { @@ -292,15 +264,6 @@ describe('recreateNodeExecutionStack', () => { }); }); - // TODO: This does not work as expected right now. The node execution stack - // will contain node3, but only with data from input 1, instead of data from - // input 1 and 2. - // I need to spent time to understand the node execution stack, waiting - // executions and waiting execution sources and write a spec for this and - // then re-implement it from the spec. - // Changing `StartNodeData.sourceData` to contain sources from multiple nodes - // could be helpful: - // { name: string, sourceData: ISourceData[] } // ┌─────┐1 ►► // ┌─►│node1├───┐ ┌─────┐ // ┌───────┐1 │ └─────┘ └──►│ │ @@ -308,11 +271,8 @@ describe('recreateNodeExecutionStack', () => { // └───────┘ │ ┌─────┐1 ┌──►│ │ // └─►│node2├───┘ └─────┘ // └─────┘ - // eslint-disable-next-line n8n-local-rules/no-skipped-tests test('multiple inputs', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -335,15 +295,11 @@ describe('recreateNodeExecutionStack', () => { [trigger.name]: [{ json: { value: 1 } }], }; - // // ACT - // const { nodeExecutionStack, waitingExecution, waitingExecutionSource } = recreateNodeExecutionStack(graph, startNodes, node3, runData, pinData); - // // ASSERT - // expect(nodeExecutionStack).toHaveLength(1); expect(nodeExecutionStack[0]).toEqual({ data: { main: [[{ json: { value: 1 } }], [{ json: { value: 1 } }]] }, From 5f00779df8aed3536954ff10b57d470d5df6b953 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:47:01 +0200 Subject: [PATCH 30/54] update comments --- .../src/PartialExecutionUtils/findStartNodes.ts | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/findStartNodes.ts b/packages/core/src/PartialExecutionUtils/findStartNodes.ts index 66b9a415646a7..50c7d08a65877 100644 --- a/packages/core/src/PartialExecutionUtils/findStartNodes.ts +++ b/packages/core/src/PartialExecutionUtils/findStartNodes.ts @@ -1,19 +1,7 @@ import type { INode, IPinData, IRunData } from 'n8n-workflow'; -import type { Connection, DirectedGraph } from './DirectedGraph'; +import type { DirectedGraph } from './DirectedGraph'; import { getIncomingData } from './getIncomingData'; -// TODO: This is how ISourceData should look like. -type NewSourceData = { - connection: Connection; - previousNodeRun: number; // If undefined "0" gets used -}; - -// TODO: rename to something more general, like path segment -export interface StartNodeData { - node: INode; - sourceData: NewSourceData[]; -} - // TODO: implement dirty checking for options and properties and parent nodes // being disabled export function isDirty(node: INode, runData: IRunData = {}, pinData: IPinData = {}): boolean { @@ -135,7 +123,7 @@ export function findStartNodes( destination, runData, pinData, - // found start nodes + // start nodes found new Set(), // seen new Set(), From 415fface7c7db548873149136de65d6533304971 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:53:08 +0200 Subject: [PATCH 31/54] update comment --- packages/core/src/PartialExecutionUtils/findSubgraph.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/PartialExecutionUtils/findSubgraph.ts b/packages/core/src/PartialExecutionUtils/findSubgraph.ts index 9f975e59dabee..130a27e681809 100644 --- a/packages/core/src/PartialExecutionUtils/findSubgraph.ts +++ b/packages/core/src/PartialExecutionUtils/findSubgraph.ts @@ -10,7 +10,7 @@ function findSubgraphRecursive( newGraph: DirectedGraph, currentBranch: Connection[], ) { - // If the current node is the chosen ‘trigger keep this branch. + // If the current node is the chosen trigger keep this branch. if (current === trigger) { for (const connection of currentBranch) { newGraph.addNodes(connection.from, connection.to); From 78f95f7cb3dc83ba38c47d0fa14ee7f5f2689177 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:53:21 +0200 Subject: [PATCH 32/54] update comment and clean dead code --- .../findTriggerForPartialExecution.ts | 52 +------------------ 1 file changed, 2 insertions(+), 50 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts index baf28624498dc..ef5f0ae0b10be 100644 --- a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts +++ b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts @@ -1,8 +1,6 @@ import type { INode, Workflow } from 'n8n-workflow'; function findAllParentTriggers(workflow: Workflow, destinationNode: string) { - // Traverse from the destination node back until we found all trigger nodes. - // Do this recursively, because why not. const parentNodes = workflow .getParentNodes(destinationNode) .map((name) => { @@ -33,7 +31,8 @@ export function findTriggerForPartialExecution( (trigger) => !trigger.disabled, ); const pinnedTriggers = parentTriggers - // TODO: add the other filters here from `findAllPinnedActivators` + // TODO: add the other filters here from `findAllPinnedActivators`, see + // copy below. .filter((trigger) => workflow.pinData?.[trigger.name]) .sort((n) => (n.type.endsWith('webhook') ? -1 : 1)); @@ -55,50 +54,3 @@ export function findTriggerForPartialExecution( // ) // .sort((a) => (a.type.endsWith('webhook') ? -1 : 1)); //} - -// TODO: deduplicate this with -// packages/cli/src/workflows/workflowExecution.service.ts -//function selectPinnedActivatorStarter( -// workflow: Workflow, -// startNodes?: string[], -// pinData?: IPinData, -//) { -// if (!pinData || !startNodes) return null; -// -// const allPinnedActivators = findAllPinnedActivators(workflow, pinData); -// -// if (allPinnedActivators.length === 0) return null; -// -// const [firstPinnedActivator] = allPinnedActivators; -// -// // full manual execution -// -// if (startNodes?.length === 0) return firstPinnedActivator ?? null; -// -// // partial manual execution -// -// /** -// * If the partial manual execution has 2+ start nodes, we search only the zeroth -// * start node's parents for a pinned activator. If we had 2+ start nodes without -// * a common ancestor and so if we end up finding multiple pinned activators, we -// * would still need to return one to comply with existing usage. -// */ -// const [firstStartNodeName] = startNodes; -// -// const parentNodeNames = -// //new Workflow({ -// // nodes: workflow.nodes, -// // connections: workflow.connections, -// // active: workflow.active, -// // nodeTypes: this.nodeTypes, -// // }). -// workflow.getParentNodes(firstStartNodeName); -// -// if (parentNodeNames.length > 0) { -// const parentNodeName = parentNodeNames.find((p) => p === firstPinnedActivator.name); -// -// return allPinnedActivators.find((pa) => pa.name === parentNodeName) ?? null; -// } -// -// return allPinnedActivators.find((pa) => pa.name === firstStartNodeName) ?? null; -//} From 0c325e4b21103f709f956eecf5e18b462283dbef Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 11:57:26 +0200 Subject: [PATCH 33/54] update comment --- .../core/src/PartialExecutionUtils/getSourceDataGroups.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts b/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts index 7d8fcc5dcaebd..b8a1a5b11f302 100644 --- a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts +++ b/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts @@ -12,7 +12,7 @@ function sortByInputIndexThenByName(connection1: Connection, connection2: Connec /** * Groups incoming connections to the node. The groups contain one connection - * per input, if possible, with run data or pinned data, if possible. + * per input, if possible, with run data or pinned data. * * The purpose of this is to get as many complete sets of data for executing * nodes with multiple inputs. @@ -42,7 +42,7 @@ function sortByInputIndexThenByName(connection1: Connection, connection2: Connec * 1. source1 and source3 * 2. source2 * - * # Example 1: + * # Example 2: * ┌───────┐0 * │source1├────┐ * └───────┘ │ ┌────┐ @@ -56,9 +56,8 @@ function sortByInputIndexThenByName(connection1: Connection, connection2: Connec * Since `source1` has no run data and no pinned data it's skipped in favor of * `source2` for the for input. * - * So this will return 2 groups: + * So this will return 1 group: * 1. source2 and source3 - * 2. source1 */ export function getSourceDataGroups( graph: DirectedGraph, From cd5657355712d480bf2446e0f5eb0ccf2b89b70b Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 12:02:27 +0200 Subject: [PATCH 34/54] remove console logs --- packages/core/src/WorkflowExecute.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index daf01060f624a..2ee062ede0c18 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -989,16 +989,6 @@ export class WorkflowExecute { executionLoop: while ( this.runExecutionData.executionData!.nodeExecutionStack.length !== 0 ) { - console.log('---------------------------------------------------'); - console.log( - 'nodeExecutionStack', - this.runExecutionData.executionData?.nodeExecutionStack.map((v) => ({ - nodeName: v.node.name, - sourceName: v.source?.main.map((v) => v?.previousNode), - })), - ); - console.log('waitingExecution', this.runExecutionData.executionData?.waitingExecution); - if ( this.additionalData.executionTimeoutTimestamp !== undefined && Date.now() >= this.additionalData.executionTimeoutTimestamp From d102d74bd06f3fc8e468dc081f6b4878c095a141 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 4 Sep 2024 12:04:23 +0200 Subject: [PATCH 35/54] remove comments --- packages/workflow/src/Interfaces.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 92ccfa8d978e5..f794692bf9646 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2231,14 +2231,14 @@ export interface IWorkflowExecuteAdditionalData { } export type WorkflowExecuteMode = - | 'cli' // unused - | 'error' // unused, but maybe used for error workflows + | 'cli' + | 'error' | 'integrated' | 'internal' | 'manual' - | 'retry' // unused - | 'trigger' // unused - | 'webhook'; // unused + | 'retry' + | 'trigger' + | 'webhook'; export type WorkflowActivateMode = | 'init' From aeede0d04b9c0c541979ac1edc59e1b1154caa36 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 10 Sep 2024 15:54:16 +0200 Subject: [PATCH 36/54] simplify defintion --- packages/cli/src/workflows/workflow.request.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/workflows/workflow.request.ts b/packages/cli/src/workflows/workflow.request.ts index 5fc8cf1741b80..d45cfd14d36dd 100644 --- a/packages/cli/src/workflows/workflow.request.ts +++ b/packages/cli/src/workflows/workflow.request.ts @@ -47,7 +47,7 @@ export declare namespace WorkflowRequest { { workflowId: string }, {}, ManualRunPayload, - { partialExecutionVersion: string | undefined } + { partialExecutionVersion?: string } >; type Share = AuthenticatedRequest<{ workflowId: string }, {}, { shareWithIds: string[] }>; From 18ab57d565ca92e34c6b30123e935ddb19241e95 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 10 Sep 2024 15:55:56 +0200 Subject: [PATCH 37/54] add documentation for `partialExecutionVersion` --- packages/workflow/src/Interfaces.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index f794692bf9646..0d56f79009dfd 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2163,6 +2163,14 @@ export interface IWorkflowExecutionDataProcess { workflowData: IWorkflowBase; userId?: string; projectId?: string; + /** + * Defines which version of the partial execution flow is used. + * Possible values are: + * 0 - use the old flow + * 1 - use the new flow + * -1 - the backend chooses which flow based on the environment variable + * PARTIAL_EXECUTION_VERSION_DEFAULT + */ partialExecutionVersion?: string; } From 1a143e70e111a434c33d27154957fb5f1122ffd3 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 10 Sep 2024 15:56:07 +0200 Subject: [PATCH 38/54] remove unnecessary parameter --- packages/cli/src/workflow-runner.ts | 1 - packages/core/src/WorkflowExecute.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index 0d75f7e83eeef..c27baa5ba1f05 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -306,7 +306,6 @@ export class WorkflowRunner { workflowExecution = workflowExecute.runPartialWorkflow2( workflow, data.runData, - data.startNodes, data.destinationNode, data.pinData, ); diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index 2ee062ede0c18..81d237ce2d47e 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -321,7 +321,6 @@ export class WorkflowExecute { runPartialWorkflow2( workflow: Workflow, runData: IRunData, - _startNodes: StartNodeData[], destinationNodeName?: string, pinData?: IPinData, ): PCancelable { From ac6c54f426ec08644ada48830a3675886ae48283 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 10 Sep 2024 17:43:44 +0200 Subject: [PATCH 39/54] improve parameter name --- .../findTriggerForPartialExecution.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts index ef5f0ae0b10be..4df9508096859 100644 --- a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts +++ b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts @@ -1,8 +1,8 @@ import type { INode, Workflow } from 'n8n-workflow'; -function findAllParentTriggers(workflow: Workflow, destinationNode: string) { +function findAllParentTriggers(workflow: Workflow, destinationNodeName: string) { const parentNodes = workflow - .getParentNodes(destinationNode) + .getParentNodes(destinationNodeName) .map((name) => { const node = workflow.getNode(name); @@ -25,9 +25,9 @@ function findAllParentTriggers(workflow: Workflow, destinationNode: string) { // TODO: write unit tests for this export function findTriggerForPartialExecution( workflow: Workflow, - destinationNode: string, + destinationNodeName: string, ): INode | undefined { - const parentTriggers = findAllParentTriggers(workflow, destinationNode).filter( + const parentTriggers = findAllParentTriggers(workflow, destinationNodeName).filter( (trigger) => !trigger.disabled, ); const pinnedTriggers = parentTriggers From 6d4cf7a49b046799bdf663c5ea95f31725b30f27 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 10 Sep 2024 17:48:03 +0200 Subject: [PATCH 40/54] use an assertion instead of ignoring an error --- .../findTriggerForPartialExecution.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts index 4df9508096859..9a3c4c7d4ff02 100644 --- a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts +++ b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts @@ -1,4 +1,5 @@ import type { INode, Workflow } from 'n8n-workflow'; +import * as assert from 'assert/strict'; function findAllParentTriggers(workflow: Workflow, destinationNodeName: string) { const parentNodes = workflow @@ -6,9 +7,9 @@ function findAllParentTriggers(workflow: Workflow, destinationNodeName: string) .map((name) => { const node = workflow.getNode(name); - if (!node) { - return null; - } + // We got the node name from `workflow.getParentNodes`. The node must + // exist. + assert.ok(node); return { node, @@ -23,6 +24,7 @@ function findAllParentTriggers(workflow: Workflow, destinationNodeName: string) } // TODO: write unit tests for this +// TODO: rewrite this using DirectedGraph instead of workflow. export function findTriggerForPartialExecution( workflow: Workflow, destinationNodeName: string, From fa857dfd8dfeed693d528fdf38479d2d2325b8a1 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 10 Sep 2024 17:58:40 +0200 Subject: [PATCH 41/54] explain which way this array is sorted --- .../PartialExecutionUtils/findTriggerForPartialExecution.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts index 9a3c4c7d4ff02..baae6e7304f5c 100644 --- a/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts +++ b/packages/core/src/PartialExecutionUtils/findTriggerForPartialExecution.ts @@ -36,6 +36,9 @@ export function findTriggerForPartialExecution( // TODO: add the other filters here from `findAllPinnedActivators`, see // copy below. .filter((trigger) => workflow.pinData?.[trigger.name]) + // TODO: Make this sorting more predictable + // Put nodes which names end with 'webhook' first, while also reversing the + // order they had in the original array. .sort((n) => (n.type.endsWith('webhook') ? -1 : 1)); if (pinnedTriggers.length) { From aaeb0a798540f22cff883f9d3ad005177b3f0b80 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 10 Sep 2024 18:03:15 +0200 Subject: [PATCH 42/54] improve naming --- .../PartialExecutionUtils/DirectedGraph.ts | 20 +++++++++---------- .../src/PartialExecutionUtils/findSubgraph.ts | 4 ++-- .../getSourceDataGroups.ts | 16 +++++++++------ .../core/src/PartialExecutionUtils/index.ts | 4 +--- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index 3b9674c25a13c..41a149909f434 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -2,7 +2,7 @@ import * as a from 'assert'; import type { IConnections, INode, WorkflowParameters } from 'n8n-workflow'; import { NodeConnectionType, Workflow } from 'n8n-workflow'; -export type Connection = { +export type GraphConnection = { from: INode; to: INode; type: NodeConnectionType; @@ -15,14 +15,14 @@ type DirectedGraphKey = `${string}-${NodeConnectionType}-${number}-${number}-${s export class DirectedGraph { private nodes: Map = new Map(); - private connections: Map = new Map(); + private connections: Map = new Map(); getNodes() { return new Map(this.nodes.entries()); } getConnections(filter: { to?: INode } = {}) { - const filteredCopy: Connection[] = []; + const filteredCopy: GraphConnection[] = []; for (const connection of this.connections.values()) { const toMatches = filter.to ? connection.to === filter.to : true; @@ -64,7 +64,7 @@ export class DirectedGraph { a.ok(fromExists); a.ok(toExists); - const connection: Connection = { + const connection: GraphConnection = { ...connectionInput, type: connectionInput.type ?? NodeConnectionType.Main, outputIndex: connectionInput.outputIndex ?? 0, @@ -94,7 +94,7 @@ export class DirectedGraph { const nodeExists = this.nodes.get(node.name) === node; a.ok(nodeExists); - const directChildren: Connection[] = []; + const directChildren: GraphConnection[] = []; for (const connection of this.connections.values()) { if (connection.from !== node) { @@ -107,7 +107,7 @@ export class DirectedGraph { return directChildren; } - private getChildrenRecursive(node: INode, seen: Set): Connection[] { + private getChildrenRecursive(node: INode, seen: Set): GraphConnection[] { if (seen.has(node)) { return []; } @@ -122,7 +122,7 @@ export class DirectedGraph { ]; } - getChildren(node: INode): Connection[] { + getChildren(node: INode): GraphConnection[] { return this.getChildrenRecursive(node, new Set()); } @@ -130,7 +130,7 @@ export class DirectedGraph { const nodeExists = this.nodes.get(node.name) === node; a.ok(nodeExists); - const directParents: Connection[] = []; + const directParents: GraphConnection[] = []; for (const connection of this.connections.values()) { if (connection.to !== node) { @@ -149,7 +149,7 @@ export class DirectedGraph { type: NodeConnectionType, inputIndex: number, to: INode, - ): Connection | undefined { + ): GraphConnection | undefined { return this.connections.get( this.makeKey({ from, @@ -225,7 +225,7 @@ export class DirectedGraph { return result; } - private makeKey(connection: Connection): DirectedGraphKey { + private makeKey(connection: GraphConnection): DirectedGraphKey { return `${connection.from.name}-${connection.type}-${connection.outputIndex}-${connection.inputIndex}-${connection.to.name}`; } } diff --git a/packages/core/src/PartialExecutionUtils/findSubgraph.ts b/packages/core/src/PartialExecutionUtils/findSubgraph.ts index 130a27e681809..2241eedd2a88a 100644 --- a/packages/core/src/PartialExecutionUtils/findSubgraph.ts +++ b/packages/core/src/PartialExecutionUtils/findSubgraph.ts @@ -1,5 +1,5 @@ import type { INode } from 'n8n-workflow'; -import type { Connection } from './DirectedGraph'; +import type { GraphConnection } from './DirectedGraph'; import { DirectedGraph } from './DirectedGraph'; function findSubgraphRecursive( @@ -8,7 +8,7 @@ function findSubgraphRecursive( current: INode, trigger: INode, newGraph: DirectedGraph, - currentBranch: Connection[], + currentBranch: GraphConnection[], ) { // If the current node is the chosen trigger keep this branch. if (current === trigger) { diff --git a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts b/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts index b8a1a5b11f302..58f8f2f7459e1 100644 --- a/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts +++ b/packages/core/src/PartialExecutionUtils/getSourceDataGroups.ts @@ -1,8 +1,11 @@ import { type INode, type IPinData, type IRunData } from 'n8n-workflow'; -import type { Connection, DirectedGraph } from './DirectedGraph'; +import type { GraphConnection, DirectedGraph } from './DirectedGraph'; -function sortByInputIndexThenByName(connection1: Connection, connection2: Connection): number { +function sortByInputIndexThenByName( + connection1: GraphConnection, + connection2: GraphConnection, +): number { if (connection1.inputIndex === connection2.inputIndex) { return connection1.from.name.localeCompare(connection2.from.name); } else { @@ -64,7 +67,7 @@ export function getSourceDataGroups( node: INode, runData: IRunData, pinnedData: IPinData, -): Connection[][] { +): GraphConnection[][] { const connections = graph.getConnections({ to: node }); const sortedConnectionsWithData = []; @@ -79,8 +82,8 @@ export function getSourceDataGroups( sortedConnectionsWithData.sort(sortByInputIndexThenByName); - const groups: Connection[][] = []; - let currentGroup: Connection[] = []; + const groups: GraphConnection[][] = []; + let currentGroup: GraphConnection[] = []; let currentInputIndex = -1; while (sortedConnectionsWithData.length > 0) { @@ -88,7 +91,8 @@ export function getSourceDataGroups( // eslint-disable-next-line @typescript-eslint/no-loop-func (c) => c.inputIndex > currentInputIndex, ); - const connection: Connection | undefined = sortedConnectionsWithData[connectionWithDataIndex]; + const connection: GraphConnection | undefined = + sortedConnectionsWithData[connectionWithDataIndex]; if (connection === undefined) { groups.push(currentGroup); diff --git a/packages/core/src/PartialExecutionUtils/index.ts b/packages/core/src/PartialExecutionUtils/index.ts index b43b8f3896979..6a6f1a233aa11 100644 --- a/packages/core/src/PartialExecutionUtils/index.ts +++ b/packages/core/src/PartialExecutionUtils/index.ts @@ -1,6 +1,4 @@ -export { DirectedGraph, Connection } from './DirectedGraph'; -export { getIncomingData } from './getIncomingData'; - +export { DirectedGraph } from './DirectedGraph'; export { findTriggerForPartialExecution } from './findTriggerForPartialExecution'; export { findStartNodes } from './findStartNodes'; export { findSubgraph } from './findSubgraph'; From cc7cf3e667ec0d838e35576224f9fb192b98dbb8 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 10 Sep 2024 18:26:57 +0200 Subject: [PATCH 43/54] remove unused method --- .../PartialExecutionUtils/DirectedGraph.ts | 19 ---- .../__tests__/DirectedGraph.test.ts | 90 ------------------- 2 files changed, 109 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index 41a149909f434..7c87e9a0192b5 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -107,25 +107,6 @@ export class DirectedGraph { return directChildren; } - private getChildrenRecursive(node: INode, seen: Set): GraphConnection[] { - if (seen.has(node)) { - return []; - } - - const directChildren = this.getDirectChildren(node); - - return [ - ...directChildren, - ...directChildren.flatMap((child) => - this.getChildrenRecursive(child.to, new Set(seen).add(node)), - ), - ]; - } - - getChildren(node: INode): GraphConnection[] { - return this.getChildrenRecursive(node, new Set()); - } - getDirectParents(node: INode) { const nodeExists = this.nodes.get(node.name) === node; a.ok(nodeExists); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts index 52009bef913f5..594e88204a99a 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts @@ -36,94 +36,4 @@ describe('DirectedGraph', () => { graph, ); }); - - describe('getChildren', () => { - // ┌─────┐ ┌─────┐ - // │node1├──────►│node2│ - // └─────┘ └─────┘ - test('simple', () => { - const from = createNodeData({ name: 'Node1' }); - const to = createNodeData({ name: 'Node2' }); - const graph = new DirectedGraph().addNodes(from, to).addConnections({ from, to }); - - const children = graph.getChildren(from); - expect(children).toHaveLength(1); - expect(children).toContainEqual({ - from, - to, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }); - }); - // ┌─────┐ - // │ ├────┐ ┌─────┐ - // │node1│ ├─►│node2│ - // │ ├────┘ └─────┘ - // └─────┘ - test('medium', () => { - const from = createNodeData({ name: 'Node1' }); - const to = createNodeData({ name: 'Node2' }); - const graph = new DirectedGraph() - .addNodes(from, to) - .addConnections({ from, to, outputIndex: 0 }, { from, to, outputIndex: 1 }); - - const children = graph.getChildren(from); - expect(children).toHaveLength(2); - expect(children).toContainEqual({ - from, - to, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }); - expect(children).toContainEqual({ - from, - to, - inputIndex: 0, - outputIndex: 1, - type: NodeConnectionType.Main, - }); - }); - - // ┌─────┐ ┌─────┐ - // ┌─►│node1├──────►│node2├──┐ - // │ └─────┘ └─────┘ │ - // │ │ - // └─────────────────────────┘ - test('terminates if the graph has cycles', () => { - // - // ARRANGE - // - const node1 = createNodeData({ name: 'node1' }); - const node2 = createNodeData({ name: 'node2' }); - const graph = new DirectedGraph() - .addNodes(node1, node2) - .addConnections({ from: node1, to: node2 }, { from: node2, to: node2 }); - - // - // ACT - // - const children = graph.getChildren(node1); - - // - // ASSERT - // - expect(children).toHaveLength(2); - expect(children).toContainEqual({ - from: node1, - to: node2, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }); - expect(children).toContainEqual({ - from: node2, - to: node2, - inputIndex: 0, - outputIndex: 0, - type: NodeConnectionType.Main, - }); - }); - }); }); From d5546ac90ae400eee2e2a8de8242114f75a0be8c Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 09:51:05 +0200 Subject: [PATCH 44/54] remove comment and eslint ignore --- .../PartialExecutionUtils/__tests__/findStartNodes.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts index 47572296f4ed9..c830833d8df5b 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/findStartNodes.test.ts @@ -274,16 +274,12 @@ describe('findStartNodes', () => { expect(startNodes[0]).toEqual(node); }); - // TODO: This is not working yet as expected. - // It should only have `node` once as a start node. - // The spec needs to be updated before this is fixed. // ►► // ┌───────┐ ┌────┐ // │ │1 ┌────►│ │ // │trigger├───┤ │node│ // │ │ └────►│ │ // └───────┘ └────┘ - // eslint-disable-next-line n8n-local-rules/no-skipped-tests test('multiple connections with both having data', () => { // ARRANGE const trigger = createNodeData({ name: 'trigger' }); From 8adf6593070fc30b9a8fd3fcec022f5fb5c7f54d Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 09:51:34 +0200 Subject: [PATCH 45/54] unify arrange, act, assert trifecta --- .../__tests__/DirectedGraph.test.ts | 3 +++ .../__tests__/findSubgraph.test.ts | 18 ------------------ 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts index 594e88204a99a..0d8c6f9ed6bee 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts @@ -20,10 +20,12 @@ describe('DirectedGraph', () => { // │ │ // └───────────────────────────────┘ test('roundtrip', () => { + // ARRANGE const node1 = createNodeData({ name: 'Node1' }); const node2 = createNodeData({ name: 'Node2' }); const node3 = createNodeData({ name: 'Node3' }); + // ACT const graph = new DirectedGraph() .addNodes(node1, node2, node3) .addConnections( @@ -32,6 +34,7 @@ describe('DirectedGraph', () => { { from: node3, to: node1 }, ); + // ASSERT expect(DirectedGraph.fromWorkflow(graph.toWorkflow({ ...defaultWorkflowParameter }))).toEqual( graph, ); diff --git a/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts index 32b50c25a9546..d82f73e9e3355 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/findSubgraph.test.ts @@ -108,9 +108,7 @@ describe('findSubgraph2', () => { // │ │ // └─────────────┘ test('terminates when called with graph that contains cycles', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -122,14 +120,10 @@ describe('findSubgraph2', () => { { from: node1, to: node2 }, ); - // // ACT - // const subgraph = findSubgraph(graph, node2, trigger); - // // ASSERT - // expect(subgraph).toEqual(graph); }); @@ -142,9 +136,7 @@ describe('findSubgraph2', () => { // │Node2├────┘ // └─────┘ test('terminates when called with graph that contains cycles', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const node1 = createNodeData({ name: 'node1' }); const node2 = createNodeData({ name: 'node2' }); @@ -152,14 +144,10 @@ describe('findSubgraph2', () => { .addNodes(trigger, node1, node2) .addConnections({ from: trigger, to: node1 }, { from: node2, to: node1 }); - // // ACT - // const subgraph = findSubgraph(graph, node1, trigger); - // // ASSERT - // expect(subgraph).toEqual( new DirectedGraph().addNodes(trigger, node1).addConnections({ from: trigger, to: node1 }), ); @@ -172,9 +160,7 @@ describe('findSubgraph2', () => { // │ │ // └──────────────────────────────────┘ test('terminates if the destination node is part of a cycle', () => { - // // ARRANGE - // const trigger = createNodeData({ name: 'trigger' }); const destination = createNodeData({ name: 'destination' }); const anotherNode = createNodeData({ name: 'anotherNode' }); @@ -186,14 +172,10 @@ describe('findSubgraph2', () => { { from: anotherNode, to: destination }, ); - // // ACT - // const subgraph = findSubgraph(graph, destination, trigger); - // // ASSERT - // expect(subgraph).toEqual( new DirectedGraph() .addNodes(trigger, destination) From 7c3b1b6938c28be78751d748b1c377c239080073 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 09:52:14 +0200 Subject: [PATCH 46/54] fixup! remove unused method --- .../src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts index 0d8c6f9ed6bee..4049878eb2f25 100644 --- a/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts +++ b/packages/core/src/PartialExecutionUtils/__tests__/DirectedGraph.test.ts @@ -9,7 +9,6 @@ // XX denotes that the node is disabled // PD denotes that the node has pinned data -import { NodeConnectionType } from 'n8n-workflow'; import { DirectedGraph } from '../DirectedGraph'; import { createNodeData, defaultWorkflowParameter } from './helpers'; From 22dfdc1b4ea9e3c07e023949640093d36ec559ff Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 09:58:06 +0200 Subject: [PATCH 47/54] add docs to `isDirty` --- .../PartialExecutionUtils/findStartNodes.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/findStartNodes.ts b/packages/core/src/PartialExecutionUtils/findStartNodes.ts index 50c7d08a65877..28bf3f49092c0 100644 --- a/packages/core/src/PartialExecutionUtils/findStartNodes.ts +++ b/packages/core/src/PartialExecutionUtils/findStartNodes.ts @@ -2,33 +2,35 @@ import type { INode, IPinData, IRunData } from 'n8n-workflow'; import type { DirectedGraph } from './DirectedGraph'; import { getIncomingData } from './getIncomingData'; -// TODO: implement dirty checking for options and properties and parent nodes -// being disabled +/** + * A node is dirty if either of the following is true: + * - it's properties or options changed since last execution (not implemented yet) + * - one of it's parents is disabled + * - it has an error (not implemented yet) + * - it neither has run data nor pinned data + */ export function isDirty(node: INode, runData: IRunData = {}, pinData: IPinData = {}): boolean { - //- it’s properties or options changed since last execution, or - + // TODO: implement const propertiesOrOptionsChanged = false; if (propertiesOrOptionsChanged) { return true; } + // TODO: implement const parentNodeGotDisabled = false; if (parentNodeGotDisabled) { return true; } - //- it has an error, or - + // TODO: implement const hasAnError = false; if (hasAnError) { return true; } - //- it does neither have run data nor pinned data - const hasPinnedData = pinData[node.name] !== undefined; if (hasPinnedData) { From a706cd295d946e9e555567a992a4ae89d0494958 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 10:05:16 +0200 Subject: [PATCH 48/54] add jsdocs for `findStartNodes` --- .../PartialExecutionUtils/findStartNodes.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/core/src/PartialExecutionUtils/findStartNodes.ts b/packages/core/src/PartialExecutionUtils/findStartNodes.ts index 28bf3f49092c0..910045d7097ce 100644 --- a/packages/core/src/PartialExecutionUtils/findStartNodes.ts +++ b/packages/core/src/PartialExecutionUtils/findStartNodes.ts @@ -112,6 +112,24 @@ function findStartNodesRecursive( return startNodes; } +/** + * The start node is the node from which a partial execution starts. The start + * node will be executed or re-executed. + * The nodes are found by traversing the graph from the trigger to the + * destination and finding the earliest dirty nodes on every branch. + * + * The algorithm is: + * Starting from the trigger node. + * + * 1. if the current node is not a trigger and has no input data (on all + * connections) (not implemented yet, possibly not necessary) + * - stop following this branch, there is no start node on this branch + * 2. If the current node is dirty, or is the destination node + * - stop following this branch, we found a start node + * 3. If we detect a cycle + * - stop following the branch, there is no start node on this branch + * 4. Recurse with every direct child that is part of the sub graph + */ export function findStartNodes( graph: DirectedGraph, trigger: INode, From 3b2a0c3438a4bd7bc428546db1f65b3180130dd7 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 10:09:39 +0200 Subject: [PATCH 49/54] add jsdocs for `findSubgraph` --- .../src/PartialExecutionUtils/findSubgraph.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/core/src/PartialExecutionUtils/findSubgraph.ts b/packages/core/src/PartialExecutionUtils/findSubgraph.ts index 2241eedd2a88a..2b1ceb2998205 100644 --- a/packages/core/src/PartialExecutionUtils/findSubgraph.ts +++ b/packages/core/src/PartialExecutionUtils/findSubgraph.ts @@ -82,6 +82,24 @@ function findSubgraphRecursive( } } +/** + * Find all nodes that can lead from the trigger to the destination node, + * ignoring disabled nodes. + * + * The algorithm is: + * Start with Destination Node + * + * 1. if the current node is the chosen trigger keep this branch + * 2. if the current node has no parents, don’t keep this branch + * 3. if the current node is the destination node again, don’t keep this + * branch + * 4. if the current node was already visited, keep this branch + * 5. if the current node is disabled, don’t keep this node, but keep the + * branch + * - take every incoming connection and connect it to every node that is + * connected to the current node’s first output + * 6. Recurse on each parent + */ export function findSubgraph( graph: DirectedGraph, destinationNode: INode, From 2ad0600848fbf525c67a6841f8fbacf9b1f41eb2 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 10:24:27 +0200 Subject: [PATCH 50/54] add jsdocs for `recreateNodeExecutionStack` --- .../recreateNodeExecutionStack.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts index 4491b22b490a0..b1e3334440adb 100644 --- a/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts +++ b/packages/core/src/PartialExecutionUtils/recreateNodeExecutionStack.ts @@ -16,6 +16,20 @@ import type { DirectedGraph } from './DirectedGraph'; import { getIncomingData } from './getIncomingData'; import { getSourceDataGroups } from './getSourceDataGroups'; +/** + * Recreates the node execution stack, waiting executions and waiting + * execution sources from a directed graph, start nodes, the destination node, + * run and pinned data. + * + * This function aims to be able to recreate the internal state of the + * WorkflowExecute class at any point of time during an execution based on the + * data that is already available. Specifically it will recreate the + * `WorkflowExecute.runExecutionData.executionData` properties. + * + * This allows "restarting" an execution and having it only execute what's + * necessary to be able to execute the destination node accurately, e.g. as + * close as possible to what would happen in a production execution. + */ export function recreateNodeExecutionStack( graph: DirectedGraph, startNodes: INode[], From 3a488bdea48b5247df2d53fc337335e9919277ca Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 10:33:42 +0200 Subject: [PATCH 51/54] use assertions for programmer errors --- packages/core/src/WorkflowExecute.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index 81d237ce2d47e..e0898225e96b5 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -49,6 +49,7 @@ import { import get from 'lodash/get'; import * as NodeExecuteFunctions from './NodeExecuteFunctions'; +import * as assert from 'assert/strict'; import { recreateNodeExecutionStack } from './PartialExecutionUtils/recreateNodeExecutionStack'; import { DirectedGraph, @@ -324,16 +325,18 @@ export class WorkflowExecute { destinationNodeName?: string, pinData?: IPinData, ): PCancelable { - if (destinationNodeName === undefined) { - throw new ApplicationError('destinationNodeName is undefined'); - } + // TODO: Refactor the call-site to make `destinationNodeName` a required + // after removing the old partial execution flow. + assert.ok( + destinationNodeName, + 'a destinationNodeName is required for the new partial execution flow', + ); const destinationNode = workflow.getNode(destinationNodeName); - if (destinationNode === null) { - throw new ApplicationError( - `Could not find a node with the name ${destinationNodeName} in the workflow.`, - ); - } + assert.ok( + destinationNode, + `Could not find a node with the name ${destinationNodeName} in the workflow.`, + ); // 1. Find the Trigger const trigger = findTriggerForPartialExecution(workflow, destinationNodeName); From 11685f505eb7174cef099daa1472fcc6d403e5d1 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 10:44:14 +0200 Subject: [PATCH 52/54] add jsdoc to DirectedGraph --- .../PartialExecutionUtils/DirectedGraph.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index 7c87e9a0192b5..89103301beb21 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -12,6 +12,26 @@ export type GraphConnection = { // fromName-outputType-outputIndex-inputIndex-toName type DirectedGraphKey = `${string}-${NodeConnectionType}-${number}-${number}-${string}`; +/** + * Represents a directed graph as an adjacency list, e.g. one list for the + * vertices and one list for the edges. + * To integrate easier with the n8n codebase vertices are called nodes and + * edges are called connections. + * + * The reason why this exists next to the Workflow class is that the workflow + * class stored the graph in a deeply nested, normalized format. This format + * does not lend itself to editing the graph or build graphs incrementally. + * This class is closes this gap by having import and export functions: + * `fromWorkflow`, `toWorkflow`. + * + * Thus it allows to do something like this: + * ```ts + * const newWorkflow = DirectedGraph.fromWorkflow(workflow) + * .addNodes(node1, node2) + * .addConnection({ from: node1, to: node2 }) + * .toWorkflow(...workflow); + * ``` + */ export class DirectedGraph { private nodes: Map = new Map(); From a14e4bf65da7a1cc54fea6cbf869ffc912811188 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 14:10:13 +0200 Subject: [PATCH 53/54] fix grammar in docs Co-authored-by: Tomi Turtiainen <10324676+tomi@users.noreply.github.com> --- packages/core/src/PartialExecutionUtils/DirectedGraph.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index 89103301beb21..cf9035e026320 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -21,7 +21,7 @@ type DirectedGraphKey = `${string}-${NodeConnectionType}-${number}-${number}-${s * The reason why this exists next to the Workflow class is that the workflow * class stored the graph in a deeply nested, normalized format. This format * does not lend itself to editing the graph or build graphs incrementally. - * This class is closes this gap by having import and export functions: + * This closes this gap by having import and export functions: * `fromWorkflow`, `toWorkflow`. * * Thus it allows to do something like this: From 87cbab8f623d30c912112d8b2418fe8485458fbd Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Wed, 18 Sep 2024 14:12:12 +0200 Subject: [PATCH 54/54] remove commented out code --- packages/core/src/PartialExecutionUtils/DirectedGraph.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts index cf9035e026320..2485b895b0457 100644 --- a/packages/core/src/PartialExecutionUtils/DirectedGraph.ts +++ b/packages/core/src/PartialExecutionUtils/DirectedGraph.ts @@ -53,8 +53,6 @@ export class DirectedGraph { } return filteredCopy; - - //return new Map(this.connections.entries()); } addNode(node: INode) {