Skip to content

Commit

Permalink
[CI-1803] Secret env sharing (#884)
Browse files Browse the repository at this point in the history
* Update dependency

* Share secret env keys

* Change script flags

* Move logic one level up
  • Loading branch information
tothszabi committed Sep 5, 2023
1 parent d754ded commit ddd8706
Show file tree
Hide file tree
Showing 9 changed files with 115 additions and 9 deletions.
13 changes: 13 additions & 0 deletions _tests/integration/secret_keys_sharing_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package integration

import (
"github.com/bitrise-io/go-utils/command"
"github.com/stretchr/testify/require"
"testing"
)

func TestSecretSharing(t *testing.T) {
cmd := command.New(binPath(), "run", "secret-sharing", "--config", "secret_keys_sharing_test_bitrise.yml", "--inventory", "secret_keys_sharing_test_secrets.yml")
out, err := cmd.RunAndReturnTrimmedCombinedOutput()
require.NoError(t, err, out)
}
22 changes: 22 additions & 0 deletions _tests/integration/secret_keys_sharing_test_bitrise.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
format_version: "11"
default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git

workflows:
secret-sharing:
steps:
- script:
title: This does not receive the secret keys
inputs:
- content: |-
#!/bin/env bash
set -e
# Add an extra secret so we have two from the secrets file and one dynamically created
envman add --key MY_SECRET_ENV --value 'this is a secret value' --sensitive
- script:
inputs:
- content: |-
#!/bin/env bash
set -e
if [[ "$BITRISE_SECRET_ENV_KEY_LIST" != "SECRET_ENV_ONE,SECRET_ENV_TWO,MY_SECRET_ENV" ]] ; then exit 1; fi
3 changes: 3 additions & 0 deletions _tests/integration/secret_keys_sharing_test_secrets.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
envs:
- SECRET_ENV_ONE: first secret value
- SECRET_ENV_TWO: second secret value
26 changes: 20 additions & 6 deletions cli/run_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/bitrise-io/envman/env"
envmanEnv "github.com/bitrise-io/envman/env"
envmanModels "github.com/bitrise-io/envman/models"
"github.com/bitrise-io/go-steputils/v2/secretkeys"
"github.com/bitrise-io/go-utils/colorstring"
"github.com/bitrise-io/go-utils/fileutil"
"github.com/bitrise-io/go-utils/pathutil"
Expand Down Expand Up @@ -438,8 +439,11 @@ func (r WorkflowRunner) executeStep(

func (r WorkflowRunner) runStep(
stepUUID string,
step stepmanModels.StepModel, stepIDData models.StepIDData, stepDir string,
environments []envmanModels.EnvironmentItemModel, secrets []string) (int, []envmanModels.EnvironmentItemModel, error) {
step stepmanModels.StepModel,
stepIDData models.StepIDData,
stepDir string,
environments []envmanModels.EnvironmentItemModel,
secrets []string) (int, []envmanModels.EnvironmentItemModel, error) {
log.Debugf("[BITRISE_CLI] - Try running step: %s (%s)", stepIDData.IDorURI, stepIDData.Version)

// Check & Install Step Dependencies
Expand Down Expand Up @@ -761,7 +765,7 @@ func (r WorkflowRunner) activateAndRunSteps(
continue
}

stepSecrets := tools.GetSecretValues(secrets)
stepSecretKeys, stepSecretValues := tools.GetSecretKeysAndValues(secrets)
if configs.IsSecretEnvsFiltering {
sensitiveEnvs, err := getSensitiveEnvs(stepDeclaredEnvironments, expandedStepEnvironment)
if err != nil {
Expand All @@ -772,10 +776,12 @@ func (r WorkflowRunner) activateAndRunSteps(
continue
}

stepSecrets = append(stepSecrets, tools.GetSecretValues(sensitiveEnvs)...)
sensitiveEnvKeys, sensitiveEnvValues := tools.GetSecretKeysAndValues(sensitiveEnvs)
stepSecretKeys = append(stepSecretKeys, sensitiveEnvKeys...)
stepSecretValues = append(stepSecretValues, sensitiveEnvValues...)
}

redactedStepInputs, redactedOriginalInputs, err := redactStepInputs(expandedStepEnvironment, mergedStep.Inputs, stepSecrets)
redactedStepInputs, redactedOriginalInputs, err := redactStepInputs(expandedStepEnvironment, mergedStep.Inputs, stepSecretValues)
if err != nil {
runResultCollector.registerStepRunResults(&buildRunResults, stepExecutionID, stepStartTime, mergedStep, stepInfoPtr, stepIdxPtr,
models.StepRunStatusCodePreparationFailed, 1,
Expand All @@ -790,9 +796,12 @@ func (r WorkflowRunner) activateAndRunSteps(
}
}

secretKeysEnv := secretEnvKeysEnvironment(stepSecretKeys)
stepDeclaredEnvironments = append(stepDeclaredEnvironments, secretKeysEnv)

tracker.SendStepStartedEvent(stepStartedProperties, prepareAnalyticsStepInfo(mergedStep, stepInfoPtr), redactedInputsWithType, redactedOriginalInputs)

exit, outEnvironments, err := r.runStep(stepExecutionID, mergedStep, stepIDData, stepDir, stepDeclaredEnvironments, stepSecrets)
exit, outEnvironments, err := r.runStep(stepExecutionID, mergedStep, stepIDData, stepDir, stepDeclaredEnvironments, stepSecretValues)

if testDirPath != "" {
if err := addTestMetadata(testDirPath, models.TestResultStepInfo{Number: idx, Title: *mergedStep.Title, ID: stepIDData.IDorURI, Version: stepIDData.Version}); err != nil {
Expand Down Expand Up @@ -892,3 +901,8 @@ func addTestMetadata(testDirPath string, testResultStepInfo models.TestResultSte
}
return nil
}

func secretEnvKeysEnvironment(keys []string) envmanModels.EnvironmentItemModel {
value := secretkeys.NewManager().Format(keys)
return envmanModels.EnvironmentItemModel{secretkeys.EnvKey: value}
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.17

require (
github.com/bitrise-io/envman v0.0.0-20221010094751-a03ce30a5316
github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19
github.com/bitrise-io/go-utils v1.0.8
github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16
github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,15 @@ github.com/bitrise-io/colorstring v0.0.0-20180614154802-a8cd70115192 h1:vSHYT6kC
github.com/bitrise-io/colorstring v0.0.0-20180614154802-a8cd70115192/go.mod h1:CIHVcxZUvsG99XUJV6JlR7okNsMMGY81jMvPC20W+O0=
github.com/bitrise-io/envman v0.0.0-20221010094751-a03ce30a5316 h1:gRQReCvmNxUVvm71SJYpx8d6wz4ln6ChapNkDStco58=
github.com/bitrise-io/envman v0.0.0-20221010094751-a03ce30a5316/go.mod h1:L4WQyg88d87Z4dxNwrYEa0Cwd9/W0gSfXsibw30r8Vw=
github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19 h1:zP9oZRZA9JGPELjBcWSzo/nd8g7apc7EBd7KZHHFdvs=
github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19/go.mod h1:/ueNOKnsjcUrlt8Ck75WRNspL7E6nAAylvl9oGJtYio=
github.com/bitrise-io/go-utils v0.0.0-20200224122728-e212188d99b4/go.mod h1:tTEsKvbz1LbzuN/KpVFHXnLtcAPdEgIdM41s0lL407s=
github.com/bitrise-io/go-utils v0.0.0-20210505121718-07411d72e36e/go.mod h1:nhdaDQFvaMny1CugVV6KjK92/q97ENo0RuKSW5I4fbA=
github.com/bitrise-io/go-utils v1.0.1/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY=
github.com/bitrise-io/go-utils v1.0.3/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY=
github.com/bitrise-io/go-utils v1.0.8 h1:ekXH6FK5V8UxkyHm2FsQL8yk9Wqd5fjg1NGlD7jK2kc=
github.com/bitrise-io/go-utils v1.0.8/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY=
github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.13/go.mod h1:gZWtM7PLn1VOroa4gN1La/24aRVc0jg5R701jTsPaO8=
github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16 h1:y+Yo0d8pYIjZiKhQuPM/Z5FY9/mu+wrWkyQlch8f9Po=
github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ=
github.com/bitrise-io/goinp v0.0.0-20210504152833-8559b0680ab1/go.mod h1:iRbd8zAXLeNy+0gic0eqNCxXvDGe8ZEY/uYX2CCeAoo=
Expand All @@ -17,12 +21,15 @@ github.com/bitrise-io/gows v0.0.0-20211005113107-14f65e686b88 h1:HObzyO+BJrTSLqx
github.com/bitrise-io/gows v0.0.0-20211005113107-14f65e686b88/go.mod h1:3Cp9ceJ8wHl1Av6oEE2ff1iWaYLliQuD+oaNdyM0NWQ=
github.com/bitrise-io/stepman v0.0.0-20221010110437-a88e9a915b58 h1:xrEcHh9p9ZD/tEJLxv3dc/x4gS+GUsr/c1rVnkW8A6w=
github.com/bitrise-io/stepman v0.0.0-20221010110437-a88e9a915b58/go.mod h1:4BBUXFxB5UlF/tZHXrNFjAy9yNvjZ7Lf/TeIpMwjsow=
github.com/bmatcuk/doublestar/v4 v4.2.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI=
github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
Expand All @@ -40,6 +47,7 @@ github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -98,6 +106,7 @@ golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
8 changes: 5 additions & 3 deletions tools/tools.go
Original file line number Diff line number Diff line change
Expand Up @@ -268,8 +268,9 @@ func EnvmanClear(envStorePth string) error {
// ------------------
// --- Utility

// GetSecretValues filters out built in configuration parameters from the secret envs
func GetSecretValues(secrets []envmanModels.EnvironmentItemModel) []string {
// GetSecretKeysAndValues filters out built in configuration parameters from the secret envs
func GetSecretKeysAndValues(secrets []envmanModels.EnvironmentItemModel) ([]string, []string) {
var secretKeys []string
var secretValues []string
for _, secret := range secrets {
key, value, err := secret.GetKeyValuePair()
Expand All @@ -279,10 +280,11 @@ func GetSecretValues(secrets []envmanModels.EnvironmentItemModel) []string {
}
continue
}
secretKeys = append(secretKeys, key)
secretValues = append(secretValues, value)
}

return secretValues
return secretKeys, secretValues
}

// MoveFile ...
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ github.com/bitrise-io/envman/envman
github.com/bitrise-io/envman/models
github.com/bitrise-io/envman/output
github.com/bitrise-io/envman/version
# github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19
## explicit; go 1.17
github.com/bitrise-io/go-steputils/v2/secretkeys
# github.com/bitrise-io/go-utils v1.0.8
## explicit; go 1.13
github.com/bitrise-io/go-utils/colorstring
Expand Down

0 comments on commit ddd8706

Please sign in to comment.