From eff4a49817eee448e03575fa6da5d7ca2c88c2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20G=C3=B6drei?= Date: Fri, 16 Aug 2024 14:14:12 +0200 Subject: [PATCH] Add containerisation and step bundles info to the workflow run plan (#992) * Add containerisation and step bundles info to the workflow run plan * omitempty new with group and step bundle fields * Test containerisation JSON logs * Debug docker json log failure * Test step bundle in json logs * Fix yaml lint issue * Fix json log test flakyness * Fix docker tests * Pull stepman dep from master branch --- _tests/integration/docker_test.go | 70 +++++++++ _tests/integration/modular_config_module.yml | 11 +- _tests/integration/modular_config_test.go | 47 +++++- cli/run.go | 35 ++++- go.mod | 2 +- go.sum | 4 +- models/workflow_run_plan.go | 21 ++- tools/tools.go | 12 -- .../bitrise-io/stepman/activator/activator.go | 8 - .../bitrise-io/stepman/activator/git_ref.go | 3 +- .../bitrise-io/stepman/activator/path_ref.go | 11 +- .../{cli => activator/steplib}/activate.go | 142 +++++------------- .../stepman/activator/steplib_ref.go | 64 +++++--- .../bitrise-io/stepman/cli/commands.go | 1 - .../bitrise-io/stepman/cli/errors.go | 5 - .../bitrise-io/stepman/cli/step_list.go | 3 +- vendor/modules.txt | 3 +- 17 files changed, 262 insertions(+), 180 deletions(-) rename vendor/github.com/bitrise-io/stepman/{cli => activator/steplib}/activate.go (60%) delete mode 100644 vendor/github.com/bitrise-io/stepman/cli/errors.go diff --git a/_tests/integration/docker_test.go b/_tests/integration/docker_test.go index 3b5b6d31b..73df48524 100644 --- a/_tests/integration/docker_test.go +++ b/_tests/integration/docker_test.go @@ -4,8 +4,11 @@ package integration import ( + "encoding/json" + "strings" "testing" + "github.com/bitrise-io/bitrise/models" "github.com/bitrise-io/go-utils/command" "github.com/ryanuber/go-glob" "github.com/stretchr/testify/require" @@ -158,3 +161,70 @@ func Test_Docker(t *testing.T) { }) } } + +func Test_Docker_JSON_Logs(t *testing.T) { + testCases := map[string]struct { + workflowName string + configPath string + inventoryPath string + requiredContainerImage string + requiredServiceImages []string + }{ + "With group with step execution and service containers": { + workflowName: "docker-login-multiple-containers", + configPath: "docker_multiple_containers_bitrise.yml", + inventoryPath: "docker_multiple_containers_secrets.yml", + requiredContainerImage: "localhost:5001/healthy-image", + requiredServiceImages: []string{ + "localhost:5002/healthy-image", + "localhost:5003/healthy-image", + }, + }, + } + for testName, testCase := range testCases { + t.Run(testName, func(t *testing.T) { + cmd := command.New(binPath(), "run", testCase.workflowName, "--config", testCase.configPath, "--inventory", testCase.inventoryPath, "--output-format", "json") + out, _ := cmd.RunAndReturnTrimmedCombinedOutput() + //require.NoError(t, err, out) + checkRequiredContainers(t, out, testCase.requiredContainerImage, testCase.requiredServiceImages) + }) + } +} + +func checkRequiredContainers(t *testing.T, log string, requiredContainerImage string, requiredServiceImages []string) { + lines := strings.Split(log, "\n") + require.True(t, len(lines) > 0) + + var bitriseStartedEvent models.WorkflowRunPlan + for _, line := range lines { + var eventLogStruct struct { + EventType string `json:"event_type"` + Content models.WorkflowRunPlan `json:"content"` + } + require.NoError(t, json.Unmarshal([]byte(line), &eventLogStruct)) + if eventLogStruct.EventType == "bitrise_started" { + bitriseStartedEvent = eventLogStruct.Content + break + } + } + + var usedContainerImages []string + var usedServiceImages []string + + for _, workflowPlans := range bitriseStartedEvent.ExecutionPlan { + for _, stepPlans := range workflowPlans.Steps { + if stepPlans.WithGroupUUID != "" { + withGroupPlan := bitriseStartedEvent.WithGroupPlans[stepPlans.WithGroupUUID] + + usedContainerImages = append(usedContainerImages, withGroupPlan.Container.Image) + for _, servicePlan := range withGroupPlan.Services { + usedServiceImages = append(usedServiceImages, servicePlan.Image) + } + } + } + } + + require.Equal(t, 1, len(usedContainerImages), log) + require.EqualValues(t, requiredContainerImage, usedContainerImages[0], log) + require.EqualValues(t, requiredServiceImages, usedServiceImages, log) +} diff --git a/_tests/integration/modular_config_module.yml b/_tests/integration/modular_config_module.yml index d79f299de..2b1affea0 100644 --- a/_tests/integration/modular_config_module.yml +++ b/_tests/integration/modular_config_module.yml @@ -1,8 +1,13 @@ +step_bundles: + print: + steps: + - script: + inputs: + - content: echo "Hello $NAME!" + workflows: print_hello_world: envs: - NAME: World steps: - - script: - inputs: - - content: echo "Hello $NAME!" + - bundle::print: { } diff --git a/_tests/integration/modular_config_test.go b/_tests/integration/modular_config_test.go index 78655db09..fc11d0822 100644 --- a/_tests/integration/modular_config_test.go +++ b/_tests/integration/modular_config_test.go @@ -1,14 +1,17 @@ package integration import ( + "encoding/json" "os" + "strings" "testing" + "github.com/bitrise-io/bitrise/models" "github.com/bitrise-io/go-utils/command" "github.com/stretchr/testify/require" ) -func Test_ModularConfig_Run(t *testing.T) { +func Test_ModularConfig(t *testing.T) { configPth := "modular_config_main.yml" deployDir := os.Getenv("BITRISE_DEPLOY_DIR") @@ -41,3 +44,45 @@ func Test_ModularConfig_Run(t *testing.T) { require.NoError(t, err, out) require.Contains(t, out, "Hello World!") } + +func Test_ModularConfig_Run_JSON_Logs(t *testing.T) { + configPth := "modular_config_main.yml" + + cmd := command.New(binPath(), "run", "print_hello_world", "--config", configPth, "--output-format", "json") + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + require.NoError(t, err, out) + require.Contains(t, out, "Hello World!") + checkRequiredStepBundle(t, out, "print") +} + +func checkRequiredStepBundle(t *testing.T, log string, requiredStepBundle string) { + lines := strings.Split(log, "\n") + require.True(t, len(lines) > 0) + + var bitriseStartedEvent models.WorkflowRunPlan + for _, line := range lines { + var eventLogStruct struct { + EventType string `json:"event_type"` + Content models.WorkflowRunPlan `json:"content"` + } + require.NoError(t, json.Unmarshal([]byte(line), &eventLogStruct)) + if eventLogStruct.EventType == "bitrise_started" { + bitriseStartedEvent = eventLogStruct.Content + break + } + } + + var usedStepBundles []string + + for _, workflowPlans := range bitriseStartedEvent.ExecutionPlan { + for _, stepPlans := range workflowPlans.Steps { + if stepPlans.StepBundleUUID != "" { + stepBundlePlan := bitriseStartedEvent.StepBundlePlans[stepPlans.StepBundleUUID] + usedStepBundles = append(usedStepBundles, stepBundlePlan.ID) + } + } + } + + require.Equal(t, 1, len(usedStepBundles), log) + require.EqualValues(t, requiredStepBundle, usedStepBundles[0], log) +} diff --git a/cli/run.go b/cli/run.go index 7ea90ec19..2c42b1ef4 100644 --- a/cli/run.go +++ b/cli/run.go @@ -307,7 +307,11 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun ProjectType: r.config.Config.ProjectType, } - plan, err := createWorkflowRunPlan(r.config.Modes, r.config.Workflow, r.config.Config.Workflows, r.config.Config.StepBundles, func() string { return uuid.Must(uuid.NewV4()).String() }) + plan, err := createWorkflowRunPlan( + r.config.Modes, r.config.Workflow, r.config.Config.Workflows, + r.config.Config.StepBundles, r.config.Config.Containers, r.config.Config.Services, + func() string { return uuid.Must(uuid.NewV4()).String() }, + ) if err != nil { return models.BuildRunResultsModel{}, fmt.Errorf("failed to create workflow execution plan: %w", err) } @@ -495,8 +499,14 @@ func registerRunModes(modes models.WorkflowRunModes) error { return nil } -func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel, stepBundles map[string]models.StepBundleModel, uuidProvider func() string) (models.WorkflowRunPlan, error) { +func createWorkflowRunPlan( + modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel, + stepBundles map[string]models.StepBundleModel, containers map[string]models.Container, services map[string]models.Container, + uuidProvider func() string, +) (models.WorkflowRunPlan, error) { var executionPlan []models.WorkflowExecutionPlan + withGroupPlans := map[string]models.WithGroupPlan{} + stepBundlePlans := map[string]models.StepBundlePlan{} workflowList := walkWorkflows(targetWorkflow, workflows, nil) for _, workflowID := range workflowList { @@ -530,6 +540,21 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, groupID := uuidProvider() + var containerPlan models.ContainerPlan + if with.ContainerID != "" { + containerPlan.Image = containers[with.ContainerID].Image + } + + var servicePlans []models.ContainerPlan + for _, serviceID := range with.ServiceIDs { + servicePlans = append(servicePlans, models.ContainerPlan{Image: services[serviceID].Image}) + } + + withGroupPlans[groupID] = models.WithGroupPlan{ + Services: servicePlans, + Container: containerPlan, + } + for _, stepListStepItem := range with.Steps { stepID, step, err := stepListStepItem.GetStepIDAndStep() if err != nil { @@ -560,6 +585,10 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, bundleEnvs := append(bundleDefinition.Environments, bundleOverride.Environments...) bundleUUID := uuidProvider() + stepBundlePlans[bundleUUID] = models.StepBundlePlan{ + ID: bundleID, + } + for idx, stepListStepItem := range bundleDefinition.Steps { stepID, step, err := stepListStepItem.GetStepIDAndStep() if err != nil { @@ -611,6 +640,8 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, NoOutputTimeoutMode: modes.NoOutputTimeout > 0, SecretFilteringMode: modes.SecretFilteringMode, SecretEnvsFilteringMode: modes.SecretEnvsFilteringMode, + WithGroupPlans: withGroupPlans, + StepBundlePlans: stepBundlePlans, ExecutionPlan: executionPlan, }, nil } diff --git a/go.mod b/go.mod index 5300e6547..15a30f28b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/bitrise-io/go-utils v1.0.13 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22 github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef - github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1 + github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf github.com/go-git/go-git/v5 v5.12.0 github.com/gofrs/uuid v4.3.1+incompatible github.com/hashicorp/go-version v1.4.0 diff --git a/go.sum b/go.sum index 422151a79..2a292c922 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22 h1:/SD9xE4LlX/Ju9YZ+n/yW/uDs7h github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ= github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef h1:R5FOa8RHjqZwMN9g1FQ8W7nXxQAG7iwq1Cw+mUk5S9A= github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef/go.mod h1:27ldH2bkCdYN5CEJ6x92EK+gkd5EcDBkA7dMrSKQFYU= -github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1 h1:/hVIftPENx0k6JuXDgPbpWXCx1Lm4GbvsUVfgr6618A= -github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1/go.mod h1:netRLDQD95IzWZbzmn7CBolzNqH1tErRKS31BrZKt9s= +github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf h1:Sw+nLHrAqcPE7jbsIgFMvaRsvQOZAA95xFCPUSb0ZKE= +github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf/go.mod h1:Lq9nEqKerBD35w3eSU8lf83F7uZPkXfmRSZEUDJN40w= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= diff --git a/models/workflow_run_plan.go b/models/workflow_run_plan.go index 2dff2d1f8..0fbd4a528 100644 --- a/models/workflow_run_plan.go +++ b/models/workflow_run_plan.go @@ -25,11 +25,11 @@ type StepExecutionPlan struct { Step stepmanModels.StepModel `json:"-"` // With (container) group - WithGroupUUID string `json:"-"` + WithGroupUUID string `json:"with_group_uuid,omitempty"` ContainerID string `json:"-"` ServiceIDs []string `json:"-"` // Step Bundle group - StepBundleUUID string `json:"-"` + StepBundleUUID string `json:"step_bundle_uuid,omitempty"` StepBundleEnvs []envmanModels.EnvironmentItemModel `json:"-"` } @@ -41,6 +41,19 @@ type WorkflowExecutionPlan struct { IsSteplibOfflineMode bool `json:"-"` } +type ContainerPlan struct { + Image string `json:"image"` +} + +type WithGroupPlan struct { + Services []ContainerPlan `json:"services,omitempty"` + Container ContainerPlan `json:"container,omitempty"` +} + +type StepBundlePlan struct { + ID string `json:"id"` +} + type WorkflowRunPlan struct { Version string `json:"version"` LogFormatVersion string `json:"log_format_version"` @@ -53,5 +66,7 @@ type WorkflowRunPlan struct { SecretFilteringMode bool `json:"secret_filtering_mode"` SecretEnvsFilteringMode bool `json:"secret_envs_filtering_mode"` - ExecutionPlan []WorkflowExecutionPlan `json:"execution_plan"` + WithGroupPlans map[string]WithGroupPlan `json:"with_groups,omitempty"` + StepBundlePlans map[string]StepBundlePlan `json:"step_bundles,omitempty"` + ExecutionPlan []WorkflowExecutionPlan `json:"execution_plan"` } diff --git a/tools/tools.go b/tools/tools.go index 1180e90d5..04a28820d 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -134,18 +134,6 @@ func StepmanSetup(collection string) error { return stepman.Setup(collection, "", log) } -// StepmanUpdate ... -func StepmanUpdate(collection string) error { - log := log.NewLogger(log.GetGlobalLoggerOpts()) - return stepman.UpdateLibrary(collection, log) -} - -// StepmanActivate ... -func StepmanActivate(collection, stepID, stepVersion, dir, ymlPth string, isOfflineMode bool) error { - log := log.NewLogger(log.GetGlobalLoggerOpts()) - return stepman.Activate(collection, stepID, stepVersion, dir, ymlPth, false, log, isOfflineMode) -} - // StepmanStepInfo ... func StepmanStepInfo(collection, stepID, stepVersion string) (stepmanModels.StepInfoModel, error) { log := log.NewLogger(log.GetGlobalLoggerOpts()) diff --git a/vendor/github.com/bitrise-io/stepman/activator/activator.go b/vendor/github.com/bitrise-io/stepman/activator/activator.go index dbfe5720e..47dbf732d 100644 --- a/vendor/github.com/bitrise-io/stepman/activator/activator.go +++ b/vendor/github.com/bitrise-io/stepman/activator/activator.go @@ -3,14 +3,6 @@ package activator type ActivatedStep struct { StepYMLPath string - // TODO: this is a mess and only makes sense in the context of a path:: ref - // This should be cleaned up when all step actions are moved here from the CLI, - // but I don't want to blow up my PR with that change. - OrigStepYMLPath string - - // TODO: is this always the same as the `activatedStepDir` function param in all activations? Can we clean this up? - WorkDir string - // DidStepLibUpdate indicates that the local steplib cache was updated while resolving the exact step version. DidStepLibUpdate bool } diff --git a/vendor/github.com/bitrise-io/stepman/activator/git_ref.go b/vendor/github.com/bitrise-io/stepman/activator/git_ref.go index 706b6b0a4..35d582820 100644 --- a/vendor/github.com/bitrise-io/stepman/activator/git_ref.go +++ b/vendor/github.com/bitrise-io/stepman/activator/git_ref.go @@ -51,7 +51,6 @@ even if the repository is open source!`) return ActivatedStep{ StepYMLPath: stepYMLPath, - OrigStepYMLPath: "", // TODO: temporary during refactors, see definition - WorkDir: activatedStepDir, + DidStepLibUpdate: false, }, nil } diff --git a/vendor/github.com/bitrise-io/stepman/activator/path_ref.go b/vendor/github.com/bitrise-io/stepman/activator/path_ref.go index 3f456118f..a94b167e4 100644 --- a/vendor/github.com/bitrise-io/stepman/activator/path_ref.go +++ b/vendor/github.com/bitrise-io/stepman/activator/path_ref.go @@ -17,7 +17,7 @@ func ActivatePathRefStep( workDir string, ) (ActivatedStep, error) { log.Debugf("Local step found: (path:%s)", id.IDorURI) - // TODO: id.IDorURI is a path to the step dir in this case + // id.IDorURI is a path to the step dir in this case stepAbsLocalPth, err := pathutil.AbsPath(id.IDorURI) if err != nil { return ActivatedStep{}, err @@ -40,8 +40,8 @@ func ActivatePathRefStep( return ActivatedStep{}, fmt.Errorf("step.yml doesn't exist at %s", origStepYMLPth) } - stepYMLPath := filepath.Join(workDir, "current_step.yml") - if err := command.CopyFile(origStepYMLPth, stepYMLPath); err != nil { + activatedStepYMLPath := filepath.Join(workDir, "current_step.yml") + if err := command.CopyFile(origStepYMLPth, activatedStepYMLPath); err != nil { return ActivatedStep{}, err } @@ -50,8 +50,7 @@ func ActivatePathRefStep( } return ActivatedStep{ - StepYMLPath: stepYMLPath, - OrigStepYMLPath: origStepYMLPth, - WorkDir: activatedStepDir, + StepYMLPath: activatedStepYMLPath, + DidStepLibUpdate: false, }, nil } diff --git a/vendor/github.com/bitrise-io/stepman/cli/activate.go b/vendor/github.com/bitrise-io/stepman/activator/steplib/activate.go similarity index 60% rename from vendor/github.com/bitrise-io/stepman/cli/activate.go rename to vendor/github.com/bitrise-io/stepman/activator/steplib/activate.go index d71b102e7..0f192a5dd 100644 --- a/vendor/github.com/bitrise-io/stepman/cli/activate.go +++ b/vendor/github.com/bitrise-io/stepman/activator/steplib/activate.go @@ -1,4 +1,4 @@ -package cli +package steplib import ( "fmt" @@ -7,92 +7,28 @@ import ( "slices" "github.com/bitrise-io/go-utils/command" - "github.com/bitrise-io/go-utils/log" "github.com/bitrise-io/go-utils/pathutil" "github.com/bitrise-io/stepman/models" "github.com/bitrise-io/stepman/stepman" - "github.com/urfave/cli" ) -var activateCommand = cli.Command{ - Name: "activate", - Usage: "Copy the step with specified --id, and --version, into provided path. If --version flag is not set, the latest version of the step will be used. If --copyyml flag is set, step.yml will be copied to the given path.", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: CollectionKey + ", " + collectionKeyShort, - Usage: "Collection of step.", - EnvVar: CollectionPathEnvKey, - }, - cli.StringFlag{ - Name: IDKey + ", " + idKeyShort, - Usage: "Step id.", - }, - cli.StringFlag{ - Name: VersionKey + ", " + versionKeyShort, - Usage: "Step version.", - }, - cli.StringFlag{ - Name: PathKey + ", " + pathKeyShort, - Usage: "Path where the step will copied.", - }, - cli.StringFlag{ - Name: CopyYMLKey + ", " + copyYMLKeyShort, - Usage: "Path where the activated step's step.yml will be copied.", - }, - cli.BoolFlag{ - Name: UpdateKey + ", " + updateKeyShort, - Usage: "If flag is set, and collection doesn't contains the specified step, the collection will updated.", - }, - }, - Action: func(c *cli.Context) error { - if err := activate(c); err != nil { - failf("Command failed: %s", err) - } - return nil - }, -} - -func activate(c *cli.Context) error { - stepLibURI := c.String(CollectionKey) - if stepLibURI == "" { - return fmt.Errorf("no steplib specified") - } - - id := c.String(IDKey) - if id == "" { - return fmt.Errorf("no step ID specified") - } - - path := c.String(PathKey) - if path == "" { - return fmt.Errorf("no destination path specified") - } - - version := c.String(VersionKey) - copyYML := c.String(CopyYMLKey) - update := c.Bool(UpdateKey) - logger := log.NewDefaultLogger(false) - isOfflineMode := false +var errStepNotAvailableOfflineMode error = fmt.Errorf("step not available in offline mode") - return Activate(stepLibURI, id, version, path, copyYML, update, logger, isOfflineMode) -} - -// Activate ... -func Activate(stepLibURI, id, version, destination, destinationStepYML string, updateLibrary bool, log stepman.Logger, isOfflineMode bool) error { - stepLib, err := stepman.ReadStepSpec(stepLibURI) +func ActivateStep(stepLibURI, id, version, destination, destinationStepYML string, log stepman.Logger, isOfflineMode bool) error { + stepCollection, err := stepman.ReadStepSpec(stepLibURI) if err != nil { return fmt.Errorf("failed to read %s steplib: %s", stepLibURI, err) } - step, version, err := queryStep(stepLib, stepLibURI, id, version, updateLibrary, log) + step, version, err := queryStep(stepCollection, stepLibURI, id, version) if err != nil { return fmt.Errorf("failed to find step: %s", err) } - srcFolder, err := activateStep(stepLib, stepLibURI, id, version, step, log, isOfflineMode) + srcFolder, err := activateStep(stepCollection, stepLibURI, id, version, step, log, isOfflineMode) if err != nil { if err == errStepNotAvailableOfflineMode { - availableVersions := listCachedStepVersion(log, stepLib, stepLibURI, id) + availableVersions := ListCachedStepVersions(log, stepCollection, stepLibURI, id) versionList := "Other versions available in the local cache:" for _, version := range availableVersions { versionList = versionList + fmt.Sprintf("\n- %s", version) @@ -118,17 +54,9 @@ func Activate(stepLibURI, id, version, destination, destinationStepYML string, u return nil } -func queryStep(stepLib models.StepCollectionModel, stepLibURI string, id, version string, updateLibrary bool, log stepman.Logger) (models.StepModel, string, error) { +func queryStep(stepLib models.StepCollectionModel, stepLibURI string, id, version string) (models.StepModel, string, error) { step, stepFound, versionFound := stepLib.GetStep(id, version) - if (!stepFound || !versionFound) && updateLibrary { - var err error - stepLib, err = stepman.UpdateLibrary(stepLibURI, log) - if err != nil { - return models.StepModel{}, "", fmt.Errorf("failed to update %s steplib: %s", stepLibURI, err) - } - step, stepFound, versionFound = stepLib.GetStep(id, version) - } if !stepFound { return models.StepModel{}, "", fmt.Errorf("%s steplib does not contain %s step", stepLibURI, id) } @@ -172,33 +100,6 @@ func activateStep(stepLib models.StepCollectionModel, stepLibURI, id, version st return stepCacheDir, nil } -func listCachedStepVersion(log stepman.Logger, stepLib models.StepCollectionModel, stepLibURI, stepID string) []string { - versions := []models.Semver{} - - for version, step := range stepLib.Steps[stepID].Versions { - _, err := activateStep(stepLib, stepLibURI, stepID, version, step, log, true) - if err != nil { - continue - } - - v, err := models.ParseSemver(version) - if err != nil { - log.Warnf("failed to parse version (%s): %s", version, err) - } - - versions = append(versions, v) - } - - slices.SortFunc(versions, models.CmpSemver) - - versionsStr := make([]string, len(versions)) - for i, v := range versions { - versionsStr[i] = v.String() - } - - return versionsStr -} - func copyStep(src, dst string) error { if exist, err := pathutil.IsPathExists(dst); err != nil { return fmt.Errorf("failed to check if %s path exist: %s", dst, err) @@ -233,3 +134,30 @@ func copyStepYML(libraryURL, id, version, dest string) error { } return nil } + +func ListCachedStepVersions(log stepman.Logger, stepLib models.StepCollectionModel, stepLibURI, stepID string) []string { + versions := []models.Semver{} + + for version, step := range stepLib.Steps[stepID].Versions { + _, err := activateStep(stepLib, stepLibURI, stepID, version, step, log, true) + if err != nil { + continue + } + + v, err := models.ParseSemver(version) + if err != nil { + log.Warnf("failed to parse version (%s): %s", version, err) + } + + versions = append(versions, v) + } + + slices.SortFunc(versions, models.CmpSemver) + + versionsStr := make([]string, len(versions)) + for i, v := range versions { + versionsStr[i] = v.String() + } + + return versionsStr +} diff --git a/vendor/github.com/bitrise-io/stepman/activator/steplib_ref.go b/vendor/github.com/bitrise-io/stepman/activator/steplib_ref.go index 69cb41ce1..30e1efdab 100644 --- a/vendor/github.com/bitrise-io/stepman/activator/steplib_ref.go +++ b/vendor/github.com/bitrise-io/stepman/activator/steplib_ref.go @@ -5,6 +5,7 @@ import ( "path/filepath" "github.com/bitrise-io/go-utils/pointers" + "github.com/bitrise-io/stepman/activator/steplib" "github.com/bitrise-io/stepman/cli" "github.com/bitrise-io/stepman/models" "github.com/bitrise-io/stepman/stepid" @@ -23,22 +24,50 @@ func ActivateSteplibRefStep( stepYMLPath := filepath.Join(workDir, "current_step.yml") activationResult := ActivatedStep{ StepYMLPath: stepYMLPath, - OrigStepYMLPath: "", // TODO: temporary during refactors, see definition - WorkDir: activatedStepDir, DidStepLibUpdate: false, } - err := cli.Setup(id.SteplibSource, "", log) + stepInfo, didUpdate, err := prepareStepLibForActivation(log, id, didStepLibUpdateInWorkflow, isOfflineMode) + activationResult.DidStepLibUpdate = didUpdate if err != nil { - return activationResult, fmt.Errorf("setup %s: %s", id.SteplibSource, err) + return activationResult, err } - versionConstraint, err := models.ParseRequiredVersion(id.Version) + err = steplib.ActivateStep(id.SteplibSource, id.IDorURI, stepInfo.Version, activatedStepDir, stepYMLPath, log, isOfflineMode) if err != nil { return activationResult, err } + + // TODO: this is sketchy, we should clean this up, but this pointer originates in the CLI codebase + stepInfoPtr.ID = stepInfo.ID + if stepInfoPtr.Step.Title == nil || *stepInfoPtr.Step.Title == "" { + stepInfoPtr.Step.Title = pointers.NewStringPtr(stepInfo.ID) + } + stepInfoPtr.Version = stepInfo.Version + stepInfoPtr.LatestVersion = stepInfo.LatestVersion + stepInfoPtr.OriginalVersion = stepInfo.OriginalVersion + stepInfoPtr.GroupInfo = stepInfo.GroupInfo + + return activationResult, nil +} + +func prepareStepLibForActivation( + log stepman.Logger, + id stepid.CanonicalID, + didStepLibUpdateInWorkflow bool, + isOfflineMode bool, +) (stepInfo models.StepInfoModel, didUpdate bool, err error) { + err = cli.Setup(id.SteplibSource, "", log) + if err != nil { + return models.StepInfoModel{}, false, fmt.Errorf("setup %s: %s", id.SteplibSource, err) + } + + versionConstraint, err := models.ParseRequiredVersion(id.Version) + if err != nil { + return models.StepInfoModel{}, false, err + } if versionConstraint.VersionLockType == models.InvalidVersionConstraint { - return activationResult, fmt.Errorf("version constraint is invalid: %s %s", id.IDorURI, id.Version) + return models.StepInfoModel{}, false, fmt.Errorf("version constraint is invalid: %s %s", id.IDorURI, id.Version) } if shouldUpdateStepLibForStep(versionConstraint, isOfflineMode, didStepLibUpdateInWorkflow) { @@ -47,13 +76,13 @@ func ActivateSteplibRefStep( if err != nil { log.Warnf("Step version constraint is latest or version locked, but failed to update StepLib, err: %s", err) } else { - activationResult.DidStepLibUpdate = true + didUpdate = true } } - stepInfo, err := cli.QueryStepInfoFromLibrary(id.SteplibSource, id.IDorURI, id.Version, log) + stepInfo, err = cli.QueryStepInfoFromLibrary(id.SteplibSource, id.IDorURI, id.Version, log) if err != nil { - return activationResult, err + return stepInfo, didUpdate, err } if stepInfo.Step.Title == nil || *stepInfo.Step.Title == "" { @@ -61,22 +90,7 @@ func ActivateSteplibRefStep( } stepInfo.OriginalVersion = id.Version - err = cli.Activate(id.SteplibSource, id.IDorURI, stepInfo.Version, activatedStepDir, stepYMLPath, false, log, isOfflineMode) - if err != nil { - return activationResult, err - } - - // TODO: this is sketchy, we should clean this up, but this pointer originates in the CLI codebase - stepInfoPtr.ID = stepInfo.ID - if stepInfoPtr.Step.Title == nil || *stepInfoPtr.Step.Title == "" { - stepInfoPtr.Step.Title = pointers.NewStringPtr(stepInfo.ID) - } - stepInfoPtr.Version = stepInfo.Version - stepInfoPtr.LatestVersion = stepInfo.LatestVersion - stepInfoPtr.OriginalVersion = stepInfo.OriginalVersion - stepInfoPtr.GroupInfo = stepInfo.GroupInfo - - return activationResult, nil + return stepInfo, didUpdate, nil } func shouldUpdateStepLibForStep(constraint models.VersionConstraint, isOfflineMode bool, didStepLibUpdateInWorkflow bool) bool { diff --git a/vendor/github.com/bitrise-io/stepman/cli/commands.go b/vendor/github.com/bitrise-io/stepman/cli/commands.go index bbdd19cd1..a85d072ce 100644 --- a/vendor/github.com/bitrise-io/stepman/cli/commands.go +++ b/vendor/github.com/bitrise-io/stepman/cli/commands.go @@ -66,7 +66,6 @@ var ( flUpdate, }, }, - activateCommand, { Name: "audit", Usage: "Validates Step or Step Collection.", diff --git a/vendor/github.com/bitrise-io/stepman/cli/errors.go b/vendor/github.com/bitrise-io/stepman/cli/errors.go deleted file mode 100644 index 23a9df8b6..000000000 --- a/vendor/github.com/bitrise-io/stepman/cli/errors.go +++ /dev/null @@ -1,5 +0,0 @@ -package cli - -import "fmt" - -var errStepNotAvailableOfflineMode error = fmt.Errorf("step not available in offline mode") diff --git a/vendor/github.com/bitrise-io/stepman/cli/step_list.go b/vendor/github.com/bitrise-io/stepman/cli/step_list.go index f85c550ee..4b3b82f40 100644 --- a/vendor/github.com/bitrise-io/stepman/cli/step_list.go +++ b/vendor/github.com/bitrise-io/stepman/cli/step_list.go @@ -10,6 +10,7 @@ import ( "github.com/bitrise-io/go-utils/log" "github.com/bitrise-io/go-utils/pointers" "github.com/bitrise-io/go-utils/stringutil" + "github.com/bitrise-io/stepman/activator/steplib" "github.com/bitrise-io/stepman/models" "github.com/bitrise-io/stepman/stepman" "github.com/urfave/cli" @@ -62,7 +63,7 @@ func printRawStepList(log stepman.Logger, stepLibURI string, maintaner string, s } if isShort { // print only step IDs and cached versions - cachedVersions := listCachedStepVersion(log, stepLib, stepLibURI, stepID) + cachedVersions := steplib.ListCachedStepVersions(log, stepLib, stepLibURI, stepID) id := fmt.Sprintf("%s (%s)", stepID, stepGroupInfo.Info.Maintainer) fmt.Printf("%s cached versions: %s\n", printInMaxNChars(id, 55), strings.Join(cachedVersions, ", ")) diff --git a/vendor/modules.txt b/vendor/modules.txt index e3388e88d..3ec22e7f2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -72,9 +72,10 @@ github.com/bitrise-io/go-utils/v2/retryhttp # github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef ## explicit; go 1.18 github.com/bitrise-io/goinp/goinp -# github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1 +# github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf ## explicit; go 1.18 github.com/bitrise-io/stepman/activator +github.com/bitrise-io/stepman/activator/steplib github.com/bitrise-io/stepman/cli github.com/bitrise-io/stepman/models github.com/bitrise-io/stepman/preload