Add containerisation and step bundles info to the workflow run plan (#…

…992)

* Add containerisation and step bundles info to the workflow run plan

* omitempty new with group and step bundle fields

* Test containerisation JSON logs

* Debug docker json log failure

* Test step bundle in json logs

* Fix yaml lint issue

* Fix json log test flakyness

* Fix docker tests

* Pull stepman dep from master branch

_tests/integration/docker_test.go

@@ -4,8 +4,11 @@
 package integration
 
 import (
+	"encoding/json"
+	"strings"
 	"testing"
 
+	"github.com/bitrise-io/bitrise/models"
 	"github.com/bitrise-io/go-utils/command"
 	"github.com/ryanuber/go-glob"
 	"github.com/stretchr/testify/require"
@@ -158,3 +161,70 @@ func Test_Docker(t *testing.T) {
 		})
 	}
 }
+
+func Test_Docker_JSON_Logs(t *testing.T) {
+	testCases := map[string]struct {
+		workflowName           string
+		configPath             string
+		inventoryPath          string
+		requiredContainerImage string
+		requiredServiceImages  []string
+	}{
+		"With group with step execution and service containers": {
+			workflowName:           "docker-login-multiple-containers",
+			configPath:             "docker_multiple_containers_bitrise.yml",
+			inventoryPath:          "docker_multiple_containers_secrets.yml",
+			requiredContainerImage: "localhost:5001/healthy-image",
+			requiredServiceImages: []string{
+				"localhost:5002/healthy-image",
+				"localhost:5003/healthy-image",
+			},
+		},
+	}
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			cmd := command.New(binPath(), "run", testCase.workflowName, "--config", testCase.configPath, "--inventory", testCase.inventoryPath, "--output-format", "json")
+			out, _ := cmd.RunAndReturnTrimmedCombinedOutput()
+			//require.NoError(t, err, out)
+			checkRequiredContainers(t, out, testCase.requiredContainerImage, testCase.requiredServiceImages)
+		})
+	}
+}
+
+func checkRequiredContainers(t *testing.T, log string, requiredContainerImage string, requiredServiceImages []string) {
+	lines := strings.Split(log, "\n")
+	require.True(t, len(lines) > 0)
+
+	var bitriseStartedEvent models.WorkflowRunPlan
+	for _, line := range lines {
+		var eventLogStruct struct {
+			EventType string                 `json:"event_type"`
+			Content   models.WorkflowRunPlan `json:"content"`
+		}
+		require.NoError(t, json.Unmarshal([]byte(line), &eventLogStruct))
+		if eventLogStruct.EventType == "bitrise_started" {
+			bitriseStartedEvent = eventLogStruct.Content
+			break
+		}
+	}
+
+	var usedContainerImages []string
+	var usedServiceImages []string
+
+	for _, workflowPlans := range bitriseStartedEvent.ExecutionPlan {
+		for _, stepPlans := range workflowPlans.Steps {
+			if stepPlans.WithGroupUUID != "" {
+				withGroupPlan := bitriseStartedEvent.WithGroupPlans[stepPlans.WithGroupUUID]
+
+				usedContainerImages = append(usedContainerImages, withGroupPlan.Container.Image)
+				for _, servicePlan := range withGroupPlan.Services {
+					usedServiceImages = append(usedServiceImages, servicePlan.Image)
+				}
+			}
+		}
+	}
+
+	require.Equal(t, 1, len(usedContainerImages), log)
+	require.EqualValues(t, requiredContainerImage, usedContainerImages[0], log)
+	require.EqualValues(t, requiredServiceImages, usedServiceImages, log)
+}

_tests/integration/modular_config_module.yml

@@ -1,8 +1,13 @@
+step_bundles:
+  print:
+    steps:
+    - script:
+        inputs:
+        - content: echo "Hello $NAME!"
+
 workflows:
   print_hello_world:
     envs:
     - NAME: World
     steps:
-    - script:
-        inputs:
-        - content: echo "Hello $NAME!"
+    - bundle::print: { }

_tests/integration/modular_config_test.go

@@ -1,14 +1,17 @@
 package integration
 
 import (
+	"encoding/json"
 	"os"
+	"strings"
 	"testing"
 
+	"github.com/bitrise-io/bitrise/models"
 	"github.com/bitrise-io/go-utils/command"
 	"github.com/stretchr/testify/require"
 )
 
-func Test_ModularConfig_Run(t *testing.T) {
+func Test_ModularConfig(t *testing.T) {
 	configPth := "modular_config_main.yml"
 	deployDir := os.Getenv("BITRISE_DEPLOY_DIR")
 
@@ -41,3 +44,45 @@ func Test_ModularConfig_Run(t *testing.T) {
 	require.NoError(t, err, out)
 	require.Contains(t, out, "Hello World!")
 }
+
+func Test_ModularConfig_Run_JSON_Logs(t *testing.T) {
+	configPth := "modular_config_main.yml"
+
+	cmd := command.New(binPath(), "run", "print_hello_world", "--config", configPth, "--output-format", "json")
+	out, err := cmd.RunAndReturnTrimmedCombinedOutput()
+	require.NoError(t, err, out)
+	require.Contains(t, out, "Hello World!")
+	checkRequiredStepBundle(t, out, "print")
+}
+
+func checkRequiredStepBundle(t *testing.T, log string, requiredStepBundle string) {
+	lines := strings.Split(log, "\n")
+	require.True(t, len(lines) > 0)
+
+	var bitriseStartedEvent models.WorkflowRunPlan
+	for _, line := range lines {
+		var eventLogStruct struct {
+			EventType string                 `json:"event_type"`
+			Content   models.WorkflowRunPlan `json:"content"`
+		}
+		require.NoError(t, json.Unmarshal([]byte(line), &eventLogStruct))
+		if eventLogStruct.EventType == "bitrise_started" {
+			bitriseStartedEvent = eventLogStruct.Content
+			break
+		}
+	}
+
+	var usedStepBundles []string
+
+	for _, workflowPlans := range bitriseStartedEvent.ExecutionPlan {
+		for _, stepPlans := range workflowPlans.Steps {
+			if stepPlans.StepBundleUUID != "" {
+				stepBundlePlan := bitriseStartedEvent.StepBundlePlans[stepPlans.StepBundleUUID]
+				usedStepBundles = append(usedStepBundles, stepBundlePlan.ID)
+			}
+		}
+	}
+
+	require.Equal(t, 1, len(usedStepBundles), log)
+	require.EqualValues(t, requiredStepBundle, usedStepBundles[0], log)
+}

cli/run.go

@@ -307,7 +307,11 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun
 		ProjectType:    r.config.Config.ProjectType,
 	}
 
-	plan, err := createWorkflowRunPlan(r.config.Modes, r.config.Workflow, r.config.Config.Workflows, r.config.Config.StepBundles, func() string { return uuid.Must(uuid.NewV4()).String() })
+	plan, err := createWorkflowRunPlan(
+		r.config.Modes, r.config.Workflow, r.config.Config.Workflows,
+		r.config.Config.StepBundles, r.config.Config.Containers, r.config.Config.Services,
+		func() string { return uuid.Must(uuid.NewV4()).String() },
+	)
 	if err != nil {
 		return models.BuildRunResultsModel{}, fmt.Errorf("failed to create workflow execution plan: %w", err)
 	}
@@ -495,8 +499,14 @@ func registerRunModes(modes models.WorkflowRunModes) error {
 	return nil
 }
 
-func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel, stepBundles map[string]models.StepBundleModel, uuidProvider func() string) (models.WorkflowRunPlan, error) {
+func createWorkflowRunPlan(
+	modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel,
+	stepBundles map[string]models.StepBundleModel, containers map[string]models.Container, services map[string]models.Container,
+	uuidProvider func() string,
+) (models.WorkflowRunPlan, error) {
 	var executionPlan []models.WorkflowExecutionPlan
+	withGroupPlans := map[string]models.WithGroupPlan{}
+	stepBundlePlans := map[string]models.StepBundlePlan{}
 
 	workflowList := walkWorkflows(targetWorkflow, workflows, nil)
 	for _, workflowID := range workflowList {
@@ -530,6 +540,21 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string,
 
 				groupID := uuidProvider()
 
+				var containerPlan models.ContainerPlan
+				if with.ContainerID != "" {
+					containerPlan.Image = containers[with.ContainerID].Image
+				}
+
+				var servicePlans []models.ContainerPlan
+				for _, serviceID := range with.ServiceIDs {
+					servicePlans = append(servicePlans, models.ContainerPlan{Image: services[serviceID].Image})
+				}
+
+				withGroupPlans[groupID] = models.WithGroupPlan{
+					Services:  servicePlans,
+					Container: containerPlan,
+				}
+
 				for _, stepListStepItem := range with.Steps {
 					stepID, step, err := stepListStepItem.GetStepIDAndStep()
 					if err != nil {
@@ -560,6 +585,10 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string,
 				bundleEnvs := append(bundleDefinition.Environments, bundleOverride.Environments...)
 				bundleUUID := uuidProvider()
 
+				stepBundlePlans[bundleUUID] = models.StepBundlePlan{
+					ID: bundleID,
+				}
+
 				for idx, stepListStepItem := range bundleDefinition.Steps {
 					stepID, step, err := stepListStepItem.GetStepIDAndStep()
 					if err != nil {
@@ -611,6 +640,8 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string,
 		NoOutputTimeoutMode:     modes.NoOutputTimeout > 0,
 		SecretFilteringMode:     modes.SecretFilteringMode,
 		SecretEnvsFilteringMode: modes.SecretEnvsFilteringMode,
+		WithGroupPlans:          withGroupPlans,
+		StepBundlePlans:         stepBundlePlans,
 		ExecutionPlan:           executionPlan,
 	}, nil
 }

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/bitrise-io/go-utils v1.0.13
 	github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22
 	github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef
-	github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1
+	github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/gofrs/uuid v4.3.1+incompatible
 	github.com/hashicorp/go-version v1.4.0

go.sum

@@ -24,8 +24,8 @@ github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22 h1:/SD9xE4LlX/Ju9YZ+n/yW/uDs7h
 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ=
 github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef h1:R5FOa8RHjqZwMN9g1FQ8W7nXxQAG7iwq1Cw+mUk5S9A=
 github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef/go.mod h1:27ldH2bkCdYN5CEJ6x92EK+gkd5EcDBkA7dMrSKQFYU=
-github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1 h1:/hVIftPENx0k6JuXDgPbpWXCx1Lm4GbvsUVfgr6618A=
-github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1/go.mod h1:netRLDQD95IzWZbzmn7CBolzNqH1tErRKS31BrZKt9s=
+github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf h1:Sw+nLHrAqcPE7jbsIgFMvaRsvQOZAA95xFCPUSb0ZKE=
+github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf/go.mod h1:Lq9nEqKerBD35w3eSU8lf83F7uZPkXfmRSZEUDJN40w=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=

models/workflow_run_plan.go

@@ -25,11 +25,11 @@ type StepExecutionPlan struct {
 
 	Step stepmanModels.StepModel `json:"-"`
 	// With (container) group
-	WithGroupUUID string   `json:"-"`
+	WithGroupUUID string   `json:"with_group_uuid,omitempty"`
 	ContainerID   string   `json:"-"`
 	ServiceIDs    []string `json:"-"`
 	// Step Bundle group
-	StepBundleUUID string                              `json:"-"`
+	StepBundleUUID string                              `json:"step_bundle_uuid,omitempty"`
 	StepBundleEnvs []envmanModels.EnvironmentItemModel `json:"-"`
 }
 
@@ -41,6 +41,19 @@ type WorkflowExecutionPlan struct {
 	IsSteplibOfflineMode bool                `json:"-"`
 }
 
+type ContainerPlan struct {
+	Image string `json:"image"`
+}
+
+type WithGroupPlan struct {
+	Services  []ContainerPlan `json:"services,omitempty"`
+	Container ContainerPlan   `json:"container,omitempty"`
+}
+
+type StepBundlePlan struct {
+	ID string `json:"id"`
+}
+
 type WorkflowRunPlan struct {
 	Version          string `json:"version"`
 	LogFormatVersion string `json:"log_format_version"`
@@ -53,5 +66,7 @@ type WorkflowRunPlan struct {
 	SecretFilteringMode     bool `json:"secret_filtering_mode"`
 	SecretEnvsFilteringMode bool `json:"secret_envs_filtering_mode"`
 
-	ExecutionPlan []WorkflowExecutionPlan `json:"execution_plan"`
+	WithGroupPlans  map[string]WithGroupPlan  `json:"with_groups,omitempty"`
+	StepBundlePlans map[string]StepBundlePlan `json:"step_bundles,omitempty"`
+	ExecutionPlan   []WorkflowExecutionPlan   `json:"execution_plan"`
 }

tools/tools.go

@@ -134,18 +134,6 @@ func StepmanSetup(collection string) error {
 	return stepman.Setup(collection, "", log)
 }
 
-// StepmanUpdate ...
-func StepmanUpdate(collection string) error {
-	log := log.NewLogger(log.GetGlobalLoggerOpts())
-	return stepman.UpdateLibrary(collection, log)
-}
-
-// StepmanActivate ...
-func StepmanActivate(collection, stepID, stepVersion, dir, ymlPth string, isOfflineMode bool) error {
-	log := log.NewLogger(log.GetGlobalLoggerOpts())
-	return stepman.Activate(collection, stepID, stepVersion, dir, ymlPth, false, log, isOfflineMode)
-}
-
 // StepmanStepInfo ...
 func StepmanStepInfo(collection, stepID, stepVersion string) (stepmanModels.StepInfoModel, error) {
 	log := log.NewLogger(log.GetGlobalLoggerOpts())