From ddd87068e2a34654866c5c6c9a2f7dbf6e1fe354 Mon Sep 17 00:00:00 2001 From: Szabolcs Toth <54896607+tothszabi@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:09:46 +0100 Subject: [PATCH] [CI-1803] Secret env sharing (#884) * Update dependency * Share secret env keys * Change script flags * Move logic one level up --- .../integration/secret_keys_sharing_test.go | 13 +++++++ .../secret_keys_sharing_test_bitrise.yml | 22 +++++++++++ .../secret_keys_sharing_test_secrets.yml | 3 ++ cli/run_util.go | 26 ++++++++++--- go.mod | 1 + go.sum | 9 +++++ tools/tools.go | 8 ++-- .../go-steputils/v2/secretkeys/secretkeys.go | 39 +++++++++++++++++++ vendor/modules.txt | 3 ++ 9 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 _tests/integration/secret_keys_sharing_test.go create mode 100644 _tests/integration/secret_keys_sharing_test_bitrise.yml create mode 100644 _tests/integration/secret_keys_sharing_test_secrets.yml create mode 100644 vendor/github.com/bitrise-io/go-steputils/v2/secretkeys/secretkeys.go diff --git a/_tests/integration/secret_keys_sharing_test.go b/_tests/integration/secret_keys_sharing_test.go new file mode 100644 index 000000000..3d52c2cba --- /dev/null +++ b/_tests/integration/secret_keys_sharing_test.go @@ -0,0 +1,13 @@ +package integration + +import ( + "github.com/bitrise-io/go-utils/command" + "github.com/stretchr/testify/require" + "testing" +) + +func TestSecretSharing(t *testing.T) { + cmd := command.New(binPath(), "run", "secret-sharing", "--config", "secret_keys_sharing_test_bitrise.yml", "--inventory", "secret_keys_sharing_test_secrets.yml") + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + require.NoError(t, err, out) +} diff --git a/_tests/integration/secret_keys_sharing_test_bitrise.yml b/_tests/integration/secret_keys_sharing_test_bitrise.yml new file mode 100644 index 000000000..992ed3765 --- /dev/null +++ b/_tests/integration/secret_keys_sharing_test_bitrise.yml @@ -0,0 +1,22 @@ +format_version: "11" +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git + +workflows: + secret-sharing: + steps: + - script: + title: This does not receive the secret keys + inputs: + - content: |- + #!/bin/env bash + set -e + + # Add an extra secret so we have two from the secrets file and one dynamically created + envman add --key MY_SECRET_ENV --value 'this is a secret value' --sensitive + - script: + inputs: + - content: |- + #!/bin/env bash + set -e + + if [[ "$BITRISE_SECRET_ENV_KEY_LIST" != "SECRET_ENV_ONE,SECRET_ENV_TWO,MY_SECRET_ENV" ]] ; then exit 1; fi diff --git a/_tests/integration/secret_keys_sharing_test_secrets.yml b/_tests/integration/secret_keys_sharing_test_secrets.yml new file mode 100644 index 000000000..90f85de31 --- /dev/null +++ b/_tests/integration/secret_keys_sharing_test_secrets.yml @@ -0,0 +1,3 @@ +envs: +- SECRET_ENV_ONE: first secret value +- SECRET_ENV_TWO: second secret value diff --git a/cli/run_util.go b/cli/run_util.go index e0761a489..49368528e 100644 --- a/cli/run_util.go +++ b/cli/run_util.go @@ -26,6 +26,7 @@ import ( "github.com/bitrise-io/envman/env" envmanEnv "github.com/bitrise-io/envman/env" envmanModels "github.com/bitrise-io/envman/models" + "github.com/bitrise-io/go-steputils/v2/secretkeys" "github.com/bitrise-io/go-utils/colorstring" "github.com/bitrise-io/go-utils/fileutil" "github.com/bitrise-io/go-utils/pathutil" @@ -438,8 +439,11 @@ func (r WorkflowRunner) executeStep( func (r WorkflowRunner) runStep( stepUUID string, - step stepmanModels.StepModel, stepIDData models.StepIDData, stepDir string, - environments []envmanModels.EnvironmentItemModel, secrets []string) (int, []envmanModels.EnvironmentItemModel, error) { + step stepmanModels.StepModel, + stepIDData models.StepIDData, + stepDir string, + environments []envmanModels.EnvironmentItemModel, + secrets []string) (int, []envmanModels.EnvironmentItemModel, error) { log.Debugf("[BITRISE_CLI] - Try running step: %s (%s)", stepIDData.IDorURI, stepIDData.Version) // Check & Install Step Dependencies @@ -761,7 +765,7 @@ func (r WorkflowRunner) activateAndRunSteps( continue } - stepSecrets := tools.GetSecretValues(secrets) + stepSecretKeys, stepSecretValues := tools.GetSecretKeysAndValues(secrets) if configs.IsSecretEnvsFiltering { sensitiveEnvs, err := getSensitiveEnvs(stepDeclaredEnvironments, expandedStepEnvironment) if err != nil { @@ -772,10 +776,12 @@ func (r WorkflowRunner) activateAndRunSteps( continue } - stepSecrets = append(stepSecrets, tools.GetSecretValues(sensitiveEnvs)...) + sensitiveEnvKeys, sensitiveEnvValues := tools.GetSecretKeysAndValues(sensitiveEnvs) + stepSecretKeys = append(stepSecretKeys, sensitiveEnvKeys...) + stepSecretValues = append(stepSecretValues, sensitiveEnvValues...) } - redactedStepInputs, redactedOriginalInputs, err := redactStepInputs(expandedStepEnvironment, mergedStep.Inputs, stepSecrets) + redactedStepInputs, redactedOriginalInputs, err := redactStepInputs(expandedStepEnvironment, mergedStep.Inputs, stepSecretValues) if err != nil { runResultCollector.registerStepRunResults(&buildRunResults, stepExecutionID, stepStartTime, mergedStep, stepInfoPtr, stepIdxPtr, models.StepRunStatusCodePreparationFailed, 1, @@ -790,9 +796,12 @@ func (r WorkflowRunner) activateAndRunSteps( } } + secretKeysEnv := secretEnvKeysEnvironment(stepSecretKeys) + stepDeclaredEnvironments = append(stepDeclaredEnvironments, secretKeysEnv) + tracker.SendStepStartedEvent(stepStartedProperties, prepareAnalyticsStepInfo(mergedStep, stepInfoPtr), redactedInputsWithType, redactedOriginalInputs) - exit, outEnvironments, err := r.runStep(stepExecutionID, mergedStep, stepIDData, stepDir, stepDeclaredEnvironments, stepSecrets) + exit, outEnvironments, err := r.runStep(stepExecutionID, mergedStep, stepIDData, stepDir, stepDeclaredEnvironments, stepSecretValues) if testDirPath != "" { if err := addTestMetadata(testDirPath, models.TestResultStepInfo{Number: idx, Title: *mergedStep.Title, ID: stepIDData.IDorURI, Version: stepIDData.Version}); err != nil { @@ -892,3 +901,8 @@ func addTestMetadata(testDirPath string, testResultStepInfo models.TestResultSte } return nil } + +func secretEnvKeysEnvironment(keys []string) envmanModels.EnvironmentItemModel { + value := secretkeys.NewManager().Format(keys) + return envmanModels.EnvironmentItemModel{secretkeys.EnvKey: value} +} diff --git a/go.mod b/go.mod index c0e025f7a..d4a74d9e4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.17 require ( github.com/bitrise-io/envman v0.0.0-20221010094751-a03ce30a5316 + github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19 github.com/bitrise-io/go-utils v1.0.8 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16 github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4 diff --git a/go.sum b/go.sum index f70602a5c..cef0ecffc 100644 --- a/go.sum +++ b/go.sum @@ -3,11 +3,15 @@ github.com/bitrise-io/colorstring v0.0.0-20180614154802-a8cd70115192 h1:vSHYT6kC github.com/bitrise-io/colorstring v0.0.0-20180614154802-a8cd70115192/go.mod h1:CIHVcxZUvsG99XUJV6JlR7okNsMMGY81jMvPC20W+O0= github.com/bitrise-io/envman v0.0.0-20221010094751-a03ce30a5316 h1:gRQReCvmNxUVvm71SJYpx8d6wz4ln6ChapNkDStco58= github.com/bitrise-io/envman v0.0.0-20221010094751-a03ce30a5316/go.mod h1:L4WQyg88d87Z4dxNwrYEa0Cwd9/W0gSfXsibw30r8Vw= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19 h1:zP9oZRZA9JGPELjBcWSzo/nd8g7apc7EBd7KZHHFdvs= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19/go.mod h1:/ueNOKnsjcUrlt8Ck75WRNspL7E6nAAylvl9oGJtYio= github.com/bitrise-io/go-utils v0.0.0-20200224122728-e212188d99b4/go.mod h1:tTEsKvbz1LbzuN/KpVFHXnLtcAPdEgIdM41s0lL407s= github.com/bitrise-io/go-utils v0.0.0-20210505121718-07411d72e36e/go.mod h1:nhdaDQFvaMny1CugVV6KjK92/q97ENo0RuKSW5I4fbA= +github.com/bitrise-io/go-utils v1.0.1/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils v1.0.3/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils v1.0.8 h1:ekXH6FK5V8UxkyHm2FsQL8yk9Wqd5fjg1NGlD7jK2kc= github.com/bitrise-io/go-utils v1.0.8/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.13/go.mod h1:gZWtM7PLn1VOroa4gN1La/24aRVc0jg5R701jTsPaO8= github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16 h1:y+Yo0d8pYIjZiKhQuPM/Z5FY9/mu+wrWkyQlch8f9Po= github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ= github.com/bitrise-io/goinp v0.0.0-20210504152833-8559b0680ab1/go.mod h1:iRbd8zAXLeNy+0gic0eqNCxXvDGe8ZEY/uYX2CCeAoo= @@ -17,12 +21,15 @@ github.com/bitrise-io/gows v0.0.0-20211005113107-14f65e686b88 h1:HObzyO+BJrTSLqx github.com/bitrise-io/gows v0.0.0-20211005113107-14f65e686b88/go.mod h1:3Cp9ceJ8wHl1Av6oEE2ff1iWaYLliQuD+oaNdyM0NWQ= github.com/bitrise-io/stepman v0.0.0-20221010110437-a88e9a915b58 h1:xrEcHh9p9ZD/tEJLxv3dc/x4gS+GUsr/c1rVnkW8A6w= github.com/bitrise-io/stepman v0.0.0-20221010110437-a88e9a915b58/go.mod h1:4BBUXFxB5UlF/tZHXrNFjAy9yNvjZ7Lf/TeIpMwjsow= +github.com/bmatcuk/doublestar/v4 v4.2.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= @@ -40,6 +47,7 @@ github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -98,6 +106,7 @@ golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/tools/tools.go b/tools/tools.go index a3b495804..3bca2b8d6 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -268,8 +268,9 @@ func EnvmanClear(envStorePth string) error { // ------------------ // --- Utility -// GetSecretValues filters out built in configuration parameters from the secret envs -func GetSecretValues(secrets []envmanModels.EnvironmentItemModel) []string { +// GetSecretKeysAndValues filters out built in configuration parameters from the secret envs +func GetSecretKeysAndValues(secrets []envmanModels.EnvironmentItemModel) ([]string, []string) { + var secretKeys []string var secretValues []string for _, secret := range secrets { key, value, err := secret.GetKeyValuePair() @@ -279,10 +280,11 @@ func GetSecretValues(secrets []envmanModels.EnvironmentItemModel) []string { } continue } + secretKeys = append(secretKeys, key) secretValues = append(secretValues, value) } - return secretValues + return secretKeys, secretValues } // MoveFile ... diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/secretkeys/secretkeys.go b/vendor/github.com/bitrise-io/go-steputils/v2/secretkeys/secretkeys.go new file mode 100644 index 000000000..aa70812d2 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-steputils/v2/secretkeys/secretkeys.go @@ -0,0 +1,39 @@ +package secretkeys + +import ( + "strings" + + "github.com/bitrise-io/go-utils/v2/env" +) + +const ( + // EnvKey ... + EnvKey = "BITRISE_SECRET_ENV_KEY_LIST" + separator = "," +) + +// Manager ... +type Manager interface { + Load(envRepository env.Repository) []string + Format(keys []string) string +} + +type manager struct { +} + +// NewManager ... +func NewManager() Manager { + return manager{} +} + +// Load ... +func (manager) Load(envRepository env.Repository) []string { + value := envRepository.Get(EnvKey) + keys := strings.Split(value, separator) + return keys +} + +// Format ... +func (manager) Format(keys []string) string { + return strings.Join(keys, separator) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ed9218e0e..cad71fcef 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -9,6 +9,9 @@ github.com/bitrise-io/envman/envman github.com/bitrise-io/envman/models github.com/bitrise-io/envman/output github.com/bitrise-io/envman/version +# github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19 +## explicit; go 1.17 +github.com/bitrise-io/go-steputils/v2/secretkeys # github.com/bitrise-io/go-utils v1.0.8 ## explicit; go 1.13 github.com/bitrise-io/go-utils/colorstring