From a8bc11104e5c15ad2ff03d7e4ae14e2e8fbb435b Mon Sep 17 00:00:00 2001 From: gitirabassi Date: Wed, 31 Jul 2019 23:06:11 +0200 Subject: [PATCH 01/16] feat: initial adding of vualt transit backend to sops initial work on integration feat(vault): added cli coomands working for vualt" fix(vault): fixed config with correct tests fix(vault): added vault to keygroup and to keyservice server fixed metadata load --- .gitignore | 2 + cmd/sops/main.go | 62 +++- config/config.go | 21 +- config/config_test.go | 18 + go.mod | 14 + go.sum | 38 ++- keyservice/keyservice.go | 11 + keyservice/keyservice.pb.go | 644 +++++++++++++++++++++++------------- keyservice/keyservice.proto | 8 + keyservice/server.go | 43 +++ stores/stores.go | 58 +++- vault/keysource.go | 241 ++++++++++++++ vault/keysource_test.go | 146 ++++++++ 13 files changed, 1060 insertions(+), 246 deletions(-) create mode 100644 vault/keysource.go create mode 100644 vault/keysource_test.go diff --git a/.gitignore b/.gitignore index ee580fc0f..d7e97440f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ target Cargo.lock vendor/ +coverage.txt +profile.out diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 7561dcecf..0e14f058c 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -33,6 +33,7 @@ import ( "go.mozilla.org/sops/v3/pgp" "go.mozilla.org/sops/v3/stores/dotenv" "go.mozilla.org/sops/v3/stores/json" + "go.mozilla.org/sops/v3/vault" "go.mozilla.org/sops/v3/version" "google.golang.org/grpc" "gopkg.in/urfave/cli.v1" @@ -224,7 +225,7 @@ func main() { }, cli.BoolFlag{ Name: "recursive", - Usage: "If the source path is a directory, publish all its content recursively", + Usage: "If the source path is a directory, publish all its content recursively", }, cli.BoolFlag{ Name: "verbose", @@ -351,6 +352,10 @@ func main() { Name: "azure-kv", Usage: "the Azure Key Vault key URL the new group should contain. Can be specified more than once", }, + cli.StringSliceFlag{ + Name: "vault", + Usage: "the full vault path to the key used to encrypt/decrypt. Make you choose and configure a key with encrption/decryption enabled (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev'). Can be specified more than once", + }, cli.BoolFlag{ Name: "in-place, i", Usage: "write output back to the same file instead of stdout", @@ -368,6 +373,7 @@ func main() { pgpFps := c.StringSlice("pgp") kmsArns := c.StringSlice("kms") gcpKmses := c.StringSlice("gcp-kms") + vaultURIs := c.StringSlice("vault") azkvs := c.StringSlice("azure-kv") var group sops.KeyGroup for _, fp := range pgpFps { @@ -379,6 +385,14 @@ func main() { for _, kms := range gcpKmses { group = append(group, gcpkms.NewMasterKeyFromResourceID(kms)) } + for _, uri := range vaultURIs { + k, err := vault.NewMasterKeyFromURI(uri) + if err != nil { + log.WithError(err).Error("Failed to add key") + continue + } + group = append(group, k) + } for _, url := range azkvs { k, err := azkv.NewMasterKeyFromURL(url) if err != nil { @@ -501,6 +515,11 @@ func main() { Usage: "comma separated list of Azure Key Vault URLs", EnvVar: "SOPS_AZURE_KEYVAULT_URLS", }, + cli.StringFlag{ + Name: "vault", + Usage: "comma separated list of vault's key URI (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev')", + EnvVar: "SOPS_VAULT_URIS", + }, cli.StringFlag{ Name: "pgp, p", Usage: "comma separated list of PGP fingerprints", @@ -550,6 +569,14 @@ func main() { Name: "rm-kms", Usage: "remove the provided comma-separated list of KMS ARNs from the list of master keys on the given file", }, + cli.StringFlag{ + Name: "add-vault", + Usage: "add the provided comma-separated list of Vault's URI key to the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)", + }, + cli.StringFlag{ + Name: "rm-vault", + Usage: "remove the provided comma-separated list of Vault's URI key from the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)", + }, cli.StringFlag{ Name: "add-pgp", Usage: "add the provided comma-separated list of PGP fingerprints to the list of master keys on the given file", @@ -615,8 +642,8 @@ func main() { return toExitError(err) } if _, err := os.Stat(fileName); os.IsNotExist(err) { - if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-azure-kv") != "" || - c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-azure-kv") != "" { + if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-vault") != "" || c.String("add-azure-kv") != "" || + c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-vault") != "" || c.String("rm-azure-kv") != "" { return common.NewExitError("Error: cannot add or remove keys on non-existent files, use `--kms` and `--pgp` instead.", codes.CannotChangeKeysFromNonExistentFile) } if c.Bool("encrypt") || c.Bool("decrypt") || c.Bool("rotate") { @@ -729,6 +756,13 @@ func main() { for _, k := range azureKeys { addMasterKeys = append(addMasterKeys, k) } + vaultKeys, err := vault.NewMasterKeysFromURIs(c.String("add-vault")) + if err != nil { + return err + } + for _, k := range vaultKeys { + addMasterKeys = append(addMasterKeys, k) + } var rmMasterKeys []keys.MasterKey for _, k := range kms.MasterKeysFromArnString(c.String("rm-kms"), kmsEncryptionContext, c.String("aws-profile")) { @@ -747,6 +781,14 @@ func main() { for _, k := range azureKeys { rmMasterKeys = append(rmMasterKeys, k) } + vaultKeys, err = vault.NewMasterKeysFromURIs(c.String("rm-vault")) + if err != nil { + return err + } + for _, k := range vaultKeys { + rmMasterKeys = append(rmMasterKeys, k) + } + output, err = rotate(rotateOpts{ OutputStore: outputStore, InputStore: inputStore, @@ -938,6 +980,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { var pgpKeys []keys.MasterKey var cloudKmsKeys []keys.MasterKey var azkvKeys []keys.MasterKey + var vaultMkKeys []keys.MasterKey kmsEncryptionContext := kms.ParseKMSContext(c.String("encryption-context")) if c.String("encryption-context") != "" && kmsEncryptionContext == nil { return nil, common.NewExitError("Invalid KMS encryption context format", codes.ErrorInvalidKMSEncryptionContextFormat) @@ -961,12 +1004,21 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { azkvKeys = append(azkvKeys, k) } } + if c.String("vault") != "" { + vaultKeys, err := vault.NewMasterKeysFromURIs(c.String("vault")) + if err != nil { + return nil, err + } + for _, k := range vaultKeys { + vaultMkKeys = append(vaultMkKeys, k) + } + } if c.String("pgp") != "" { for _, k := range pgp.MasterKeysFromFingerprintString(c.String("pgp")) { pgpKeys = append(pgpKeys, k) } } - if c.String("kms") == "" && c.String("pgp") == "" && c.String("gcp-kms") == "" && c.String("azure-kv") == "" { + if c.String("kms") == "" && c.String("pgp") == "" && c.String("gcp-kms") == "" && c.String("azure-kv") == "" && c.String("vault") == "" { conf, err := loadConfig(c, file, kmsEncryptionContext) // config file might just not be supplied, without any error if conf == nil { @@ -983,6 +1035,8 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { group = append(group, cloudKmsKeys...) group = append(group, azkvKeys...) group = append(group, pgpKeys...) + group = append(group, vaultMkKeys...) + log.Debugln("Master keys available: ", group) return []sops.KeyGroup{group}, nil } diff --git a/config/config.go b/config/config.go index 62f48913d..bf815b867 100644 --- a/config/config.go +++ b/config/config.go @@ -19,6 +19,7 @@ import ( "go.mozilla.org/sops/v3/logging" "go.mozilla.org/sops/v3/pgp" "go.mozilla.org/sops/v3/publish" + "go.mozilla.org/sops/v3/vault" ) var log *logrus.Logger @@ -69,6 +70,7 @@ type keyGroup struct { KMS []kmsKey GCPKMS []gcpKmsKey `yaml:"gcp_kms"` AzureKV []azureKVKey `yaml:"azure_keyvault"` + Vault []vaultKey `yaml:"vault"` PGP []string } @@ -89,6 +91,12 @@ type azureKVKey struct { Version string `yaml:"version"` } +type vaultKey struct { + VaultAddress string `yaml:"vault_address"` + BackendPath string `yaml:"backend_path"` + KeyName string `yaml:"key_name"` +} + type destinationRule struct { PathRegex string `yaml:"path_regex"` S3Bucket string `yaml:"s3_bucket"` @@ -110,6 +118,7 @@ type creationRule struct { PGP string GCPKMS string `yaml:"gcp_kms"` AzureKeyVault string `yaml:"azure_keyvault"` + Vault string `yaml:"vault_uris"` KeyGroups []keyGroup `yaml:"key_groups"` ShamirThreshold int `yaml:"shamir_threshold"` UnencryptedSuffix string `yaml:"unencrypted_suffix"` @@ -154,6 +163,9 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ for _, k := range group.AzureKV { keyGroup = append(keyGroup, azkv.NewMasterKey(k.VaultURL, k.Key, k.Version)) } + for _, k := range group.Vault { + keyGroup = append(keyGroup, vault.NewMasterKey(k.VaultAddress, k.BackendPath, k.KeyName)) + } groups = append(groups, keyGroup) } } else { @@ -174,6 +186,13 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ for _, k := range azureKeys { keyGroup = append(keyGroup, k) } + vaultKeys, err := vault.NewMasterKeysFromURIs(cRule.Vault) + if err != nil { + return nil, err + } + for _, k := range vaultKeys { + keyGroup = append(keyGroup, k) + } groups = append(groups, keyGroup) } return groups, nil @@ -250,7 +269,7 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio var dest publish.Destination if dRule != nil { if dRule.S3Bucket != "" && dRule.GCSBucket != "" && dRule.VaultPath != "" { - return nil, fmt.Errorf("error loading config: more than one destinations were found in a single destination rule, you can only use one per rule.") + return nil, fmt.Errorf("error loading config: more than one destinations were found in a single destination rule, you can only use one per rule") } if dRule.S3Bucket != "" { dest = publish.NewS3Destination(dRule.S3Bucket, dRule.S3Prefix) diff --git a/config/config_test.go b/config/config_test.go index 6a728a2ac..8ed76f608 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -48,10 +48,12 @@ creation_rules: kms: "1" pgp: "2" gcp_kms: "3" + vault_uris: http://4:8200/v1/4/keys/4 - path_regex: "" kms: foo pgp: bar gcp_kms: baz + vault_uris: http://127.0.1.1/v1/baz/keys/baz `) var sampleConfigWithPath = []byte(` @@ -60,14 +62,17 @@ creation_rules: kms: "1" pgp: "2" gcp_kms: "3" + vault_uris: http://4:8200/v1/4/keys/4 - path_regex: somefilename.yml kms: bilbo pgp: baggins gcp_kms: precious + vault_uris: https://pluto/v1/pluto/keys/pluto - path_regex: "" kms: foo pgp: bar gcp_kms: baz + vault_uris: https://foz:443/v1/foz/keys/foz `) var sampleConfigWithGroups = []byte(` @@ -87,6 +92,10 @@ creation_rules: - vaultUrl: https://foo.vault.azure.net key: foo-key version: fooversion + vault: + - vault_address: https://foo.vault:8200 + backend_path: foo + key_name: foo-key - kms: - arn: baz pgp: @@ -98,6 +107,10 @@ creation_rules: - vaultUrl: https://bar.vault.azure.net key: bar-key version: barversion + vault: + - vault_address: https://baz.vault:8200 + backend_path: baz + key_name: baz-key `) var sampleConfigWithSuffixParameters = []byte(` @@ -135,6 +148,7 @@ creation_rules: - path_regex: foobar* kms: "1" pgp: "2" + vault_uris: "http://vault.com/v1/bug/keys/pr" unencrypted_suffix: _unencrypted encrypted_suffix: _enc `) @@ -195,12 +209,14 @@ func TestLoadConfigFile(t *testing.T) { KMS: "1", PGP: "2", GCPKMS: "3", + Vault: "http://4:8200/v1/4/keys/4", }, { PathRegex: "", KMS: "foo", PGP: "bar", GCPKMS: "baz", + Vault: "http://127.0.1.1/v1/baz/keys/baz", }, }, } @@ -227,6 +243,7 @@ func TestLoadConfigFileWithGroups(t *testing.T) { PGP: []string{"bar"}, GCPKMS: []gcpKmsKey{{ResourceID: "foo"}}, AzureKV: []azureKVKey{{VaultURL: "https://foo.vault.azure.net", Key: "foo-key", Version: "fooversion"}}, + Vault: []vaultKey{{VaultAddress: "https://foo.vault:8200", BackendPath: "foo", KeyName: "foo-key"}}, }, { KMS: []kmsKey{{Arn: "baz"}}, @@ -236,6 +253,7 @@ func TestLoadConfigFileWithGroups(t *testing.T) { {ResourceID: "baz"}, }, AzureKV: []azureKVKey{{VaultURL: "https://bar.vault.azure.net", Key: "bar-key", Version: "barversion"}}, + Vault: []vaultKey{{VaultAddress: "https://baz.vault:8200", BackendPath: "baz", KeyName: "baz-key"}}, }, }, }, diff --git a/go.mod b/go.mod index ed9685ab5..d46a679a4 100644 --- a/go.mod +++ b/go.mod @@ -5,21 +5,34 @@ go 1.13 require ( cloud.google.com/go v0.43.0 github.com/Azure/azure-sdk-for-go v31.2.0+incompatible + github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect github.com/Azure/go-autorest/autorest v0.9.0 github.com/Azure/go-autorest/autorest/azure/auth v0.1.0 github.com/Azure/go-autorest/autorest/to v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect + github.com/Microsoft/go-winio v0.4.14 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/aws/aws-sdk-go v1.23.13 github.com/blang/semver v3.5.1+incompatible + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/fatih/color v1.7.0 github.com/golang/protobuf v1.3.2 + github.com/google/go-cmp v0.3.0 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf + github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/goware/prefixer v0.0.0-20160118172347-395022866408 github.com/hashicorp/vault/api v1.0.4 github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c github.com/lib/pq v1.2.0 github.com/mitchellh/go-wordwrap v1.0.0 github.com/mozilla-services/yaml v0.0.0-20191106225358-5c216288813c + github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/opencontainers/runc v0.1.1 // indirect + github.com/ory/dockertest v3.3.4+incompatible github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.4.2 github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect @@ -32,4 +45,5 @@ require ( google.golang.org/grpc v1.22.1 gopkg.in/ini.v1 v1.44.0 gopkg.in/urfave/cli.v1 v1.20.0 + gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index 593bd7b68..53d761ab4 100644 --- a/go.sum +++ b/go.sum @@ -7,16 +7,12 @@ contrib.go.opencensus.io/exporter/ocagent v0.4.12 h1:jGFvw3l57ViIVEPKKEUXPcLYIXJ contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= github.com/Azure/azure-sdk-for-go v31.2.0+incompatible h1:kZFnTLmdQYNGfakatSivKHUfUnDZhqNdchHD4oIhp5k= github.com/Azure/azure-sdk-for-go v31.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-autorest v13.3.0+incompatible h1:8Ix0VdeOllBx9jEcZ2Wb1uqWUpE1awmJiaHztwaJCPk= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= -github.com/Azure/go-autorest/autorest v0.5.0 h1:Mlm9qy2fpQ9MvfyI41G2Zf5B4CsgjjNbLOWszfK6KrY= -github.com/Azure/go-autorest/autorest v0.5.0/go.mod h1:9HLKlQjVBH6U3oDfsXOeVc56THsLPw1L03yban4xThw= github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.2 h1:6AWuh3uWrsZJcNoCHrCF/+g4aKPCU39kaMO6/qrnK/4= github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= -github.com/Azure/go-autorest/autorest/adal v0.2.0 h1:7IBDu1jgh+ADHXnEYExkV9RE/ztOOlxdACkkPRthGKw= -github.com/Azure/go-autorest/autorest/adal v0.2.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/azure/auth v0.1.0 h1:YgO/vSnJEc76NLw2ecIXvXa8bDWiqf1pOJzARAoZsYU= @@ -27,13 +23,10 @@ github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZt github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/mocks v0.1.0 h1:Kx+AUU2Te+A3JIyYn6Dfs+cFgx5XorQKuIXrZGoq/SI= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/to v0.2.0 h1:nQOZzFCudTh+TvquAtCRjM01VEYx85e9qbwt5ncW4L8= -github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= github.com/Azure/go-autorest/autorest/to v0.3.0 h1:zebkZaadz7+wIQYgC7GXaz3Wb28yKYfVkkBKwc38VF8= github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= -github.com/Azure/go-autorest/autorest/validation v0.1.0 h1:ISSNzGUh+ZSzizJWOWzs8bwpXIePbGLW4z/AmUFGH5A= -github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= github.com/Azure/go-autorest/autorest/validation v0.2.0 h1:15vMO4y76dehZSq7pAaOLQxC6dZYsSrj2GQpflyM/L4= github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= @@ -44,6 +37,10 @@ github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VY github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -57,9 +54,13 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -67,6 +68,10 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -112,6 +117,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= +github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/goware/prefixer v0.0.0-20160118172347-395022866408 h1:Y9iQJfEqnN3/Nce9cOegemcy/9Ai5k3huT6E80F3zaw= github.com/goware/prefixer v0.0.0-20160118172347-395022866408/go.mod h1:PE1ycukgRPJ7bJ9a1fdfQ9j8i/cEcRAoLZzbxYpNB/s= github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJlb8Kqsd41CTE= @@ -190,7 +197,15 @@ github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/ory/dockertest v3.3.4+incompatible h1:VrpM6Gqg7CrPm3bL4Wm1skO+zFWLbh7/Xb5kGEbJRh8= +github.com/ory/dockertest v3.3.4+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -213,6 +228,7 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -354,6 +370,8 @@ gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/keyservice/keyservice.go b/keyservice/keyservice.go index e2827b1ac..49b7c88dc 100644 --- a/keyservice/keyservice.go +++ b/keyservice/keyservice.go @@ -12,6 +12,7 @@ import ( "go.mozilla.org/sops/v3/keys" "go.mozilla.org/sops/v3/kms" "go.mozilla.org/sops/v3/pgp" + "go.mozilla.org/sops/v3/vault" ) // KeyFromMasterKey converts a SOPS internal MasterKey to an RPC Key that can be serialized with Protocol Buffers @@ -33,6 +34,16 @@ func KeyFromMasterKey(mk keys.MasterKey) Key { }, }, } + case *vault.MasterKey: + return Key{ + KeyType: &Key_VaultKey{ + VaultKey: &VaultKey{ + VaultAddress: mk.VaultAddress, + BackendPath: mk.BackendPath, + KeyName: mk.KeyName, + }, + }, + } case *kms.MasterKey: ctx := make(map[string]string) for k, v := range mk.EncryptionContext { diff --git a/keyservice/keyservice.pb.go b/keyservice/keyservice.pb.go index d39c654ab..0da9ba98c 100644 --- a/keyservice/keyservice.pb.go +++ b/keyservice/keyservice.pb.go @@ -1,32 +1,16 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: keyservice/keyservice.proto -/* -Package keyservice is a generated protocol buffer package. - -It is generated from these files: - keyservice/keyservice.proto - -It has these top-level messages: - Key - PgpKey - KmsKey - GcpKmsKey - AzureKeyVaultKey - EncryptRequest - EncryptResponse - DecryptRequest - DecryptResponse -*/ package keyservice -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" - import ( - context "golang.org/x/net/context" + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -38,7 +22,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Key struct { // Types that are valid to be assigned to KeyType: @@ -46,36 +30,72 @@ type Key struct { // *Key_PgpKey // *Key_GcpKmsKey // *Key_AzureKeyvaultKey - KeyType isKey_KeyType `protobuf_oneof:"key_type"` + // *Key_VaultKey + KeyType isKey_KeyType `protobuf_oneof:"key_type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Key) Reset() { *m = Key{} } +func (m *Key) String() string { return proto.CompactTextString(m) } +func (*Key) ProtoMessage() {} +func (*Key) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{0} +} + +func (m *Key) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Key.Unmarshal(m, b) +} +func (m *Key) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Key.Marshal(b, m, deterministic) +} +func (m *Key) XXX_Merge(src proto.Message) { + xxx_messageInfo_Key.Merge(m, src) +} +func (m *Key) XXX_Size() int { + return xxx_messageInfo_Key.Size(m) +} +func (m *Key) XXX_DiscardUnknown() { + xxx_messageInfo_Key.DiscardUnknown(m) } -func (m *Key) Reset() { *m = Key{} } -func (m *Key) String() string { return proto.CompactTextString(m) } -func (*Key) ProtoMessage() {} -func (*Key) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +var xxx_messageInfo_Key proto.InternalMessageInfo type isKey_KeyType interface { isKey_KeyType() } type Key_KmsKey struct { - KmsKey *KmsKey `protobuf:"bytes,1,opt,name=kms_key,json=kmsKey,oneof"` + KmsKey *KmsKey `protobuf:"bytes,1,opt,name=kms_key,json=kmsKey,proto3,oneof"` } + type Key_PgpKey struct { - PgpKey *PgpKey `protobuf:"bytes,2,opt,name=pgp_key,json=pgpKey,oneof"` + PgpKey *PgpKey `protobuf:"bytes,2,opt,name=pgp_key,json=pgpKey,proto3,oneof"` } + type Key_GcpKmsKey struct { - GcpKmsKey *GcpKmsKey `protobuf:"bytes,3,opt,name=gcp_kms_key,json=gcpKmsKey,oneof"` + GcpKmsKey *GcpKmsKey `protobuf:"bytes,3,opt,name=gcp_kms_key,json=gcpKmsKey,proto3,oneof"` } + type Key_AzureKeyvaultKey struct { - AzureKeyvaultKey *AzureKeyVaultKey `protobuf:"bytes,4,opt,name=azure_keyvault_key,json=azureKeyvaultKey,oneof"` + AzureKeyvaultKey *AzureKeyVaultKey `protobuf:"bytes,4,opt,name=azure_keyvault_key,json=azureKeyvaultKey,proto3,oneof"` } -func (*Key_KmsKey) isKey_KeyType() {} -func (*Key_PgpKey) isKey_KeyType() {} -func (*Key_GcpKmsKey) isKey_KeyType() {} +type Key_VaultKey struct { + VaultKey *VaultKey `protobuf:"bytes,5,opt,name=vault_key,json=vaultKey,proto3,oneof"` +} + +func (*Key_KmsKey) isKey_KeyType() {} + +func (*Key_PgpKey) isKey_KeyType() {} + +func (*Key_GcpKmsKey) isKey_KeyType() {} + func (*Key_AzureKeyvaultKey) isKey_KeyType() {} +func (*Key_VaultKey) isKey_KeyType() {} + func (m *Key) GetKeyType() isKey_KeyType { if m != nil { return m.KeyType @@ -111,126 +131,55 @@ func (m *Key) GetAzureKeyvaultKey() *AzureKeyVaultKey { return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Key) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Key_OneofMarshaler, _Key_OneofUnmarshaler, _Key_OneofSizer, []interface{}{ +func (m *Key) GetVaultKey() *VaultKey { + if x, ok := m.GetKeyType().(*Key_VaultKey); ok { + return x.VaultKey + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*Key) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*Key_KmsKey)(nil), (*Key_PgpKey)(nil), (*Key_GcpKmsKey)(nil), (*Key_AzureKeyvaultKey)(nil), + (*Key_VaultKey)(nil), } } -func _Key_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Key) - // key_type - switch x := m.KeyType.(type) { - case *Key_KmsKey: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.KmsKey); err != nil { - return err - } - case *Key_PgpKey: - b.EncodeVarint(2<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.PgpKey); err != nil { - return err - } - case *Key_GcpKmsKey: - b.EncodeVarint(3<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.GcpKmsKey); err != nil { - return err - } - case *Key_AzureKeyvaultKey: - b.EncodeVarint(4<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.AzureKeyvaultKey); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("Key.KeyType has unexpected type %T", x) - } - return nil +type PgpKey struct { + Fingerprint string `protobuf:"bytes,1,opt,name=fingerprint,proto3" json:"fingerprint,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func _Key_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Key) - switch tag { - case 1: // key_type.kms_key - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(KmsKey) - err := b.DecodeMessage(msg) - m.KeyType = &Key_KmsKey{msg} - return true, err - case 2: // key_type.pgp_key - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(PgpKey) - err := b.DecodeMessage(msg) - m.KeyType = &Key_PgpKey{msg} - return true, err - case 3: // key_type.gcp_kms_key - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(GcpKmsKey) - err := b.DecodeMessage(msg) - m.KeyType = &Key_GcpKmsKey{msg} - return true, err - case 4: // key_type.azure_keyvault_key - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(AzureKeyVaultKey) - err := b.DecodeMessage(msg) - m.KeyType = &Key_AzureKeyvaultKey{msg} - return true, err - default: - return false, nil - } -} - -func _Key_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Key) - // key_type - switch x := m.KeyType.(type) { - case *Key_KmsKey: - s := proto.Size(x.KmsKey) - n += proto.SizeVarint(1<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(s)) - n += s - case *Key_PgpKey: - s := proto.Size(x.PgpKey) - n += proto.SizeVarint(2<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(s)) - n += s - case *Key_GcpKmsKey: - s := proto.Size(x.GcpKmsKey) - n += proto.SizeVarint(3<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(s)) - n += s - case *Key_AzureKeyvaultKey: - s := proto.Size(x.AzureKeyvaultKey) - n += proto.SizeVarint(4<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n +func (m *PgpKey) Reset() { *m = PgpKey{} } +func (m *PgpKey) String() string { return proto.CompactTextString(m) } +func (*PgpKey) ProtoMessage() {} +func (*PgpKey) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{1} } -type PgpKey struct { - Fingerprint string `protobuf:"bytes,1,opt,name=fingerprint" json:"fingerprint,omitempty"` +func (m *PgpKey) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PgpKey.Unmarshal(m, b) +} +func (m *PgpKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PgpKey.Marshal(b, m, deterministic) +} +func (m *PgpKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_PgpKey.Merge(m, src) +} +func (m *PgpKey) XXX_Size() int { + return xxx_messageInfo_PgpKey.Size(m) +} +func (m *PgpKey) XXX_DiscardUnknown() { + xxx_messageInfo_PgpKey.DiscardUnknown(m) } -func (m *PgpKey) Reset() { *m = PgpKey{} } -func (m *PgpKey) String() string { return proto.CompactTextString(m) } -func (*PgpKey) ProtoMessage() {} -func (*PgpKey) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +var xxx_messageInfo_PgpKey proto.InternalMessageInfo func (m *PgpKey) GetFingerprint() string { if m != nil { @@ -240,16 +189,39 @@ func (m *PgpKey) GetFingerprint() string { } type KmsKey struct { - Arn string `protobuf:"bytes,1,opt,name=arn" json:"arn,omitempty"` - Role string `protobuf:"bytes,2,opt,name=role" json:"role,omitempty"` - Context map[string]string `protobuf:"bytes,3,rep,name=context" json:"context,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - AwsProfile string `protobuf:"bytes,4,opt,name=aws_profile" json:"aws_profile,omitempty"` + Arn string `protobuf:"bytes,1,opt,name=arn,proto3" json:"arn,omitempty"` + Role string `protobuf:"bytes,2,opt,name=role,proto3" json:"role,omitempty"` + Context map[string]string `protobuf:"bytes,3,rep,name=context,proto3" json:"context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + AwsProfile string `protobuf:"bytes,4,opt,name=aws_profile,json=awsProfile,proto3" json:"aws_profile,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *KmsKey) Reset() { *m = KmsKey{} } +func (m *KmsKey) String() string { return proto.CompactTextString(m) } +func (*KmsKey) ProtoMessage() {} +func (*KmsKey) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{2} } -func (m *KmsKey) Reset() { *m = KmsKey{} } -func (m *KmsKey) String() string { return proto.CompactTextString(m) } -func (*KmsKey) ProtoMessage() {} -func (*KmsKey) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (m *KmsKey) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_KmsKey.Unmarshal(m, b) +} +func (m *KmsKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_KmsKey.Marshal(b, m, deterministic) +} +func (m *KmsKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_KmsKey.Merge(m, src) +} +func (m *KmsKey) XXX_Size() int { + return xxx_messageInfo_KmsKey.Size(m) +} +func (m *KmsKey) XXX_DiscardUnknown() { + xxx_messageInfo_KmsKey.DiscardUnknown(m) +} + +var xxx_messageInfo_KmsKey proto.InternalMessageInfo func (m *KmsKey) GetArn() string { if m != nil { @@ -272,14 +244,44 @@ func (m *KmsKey) GetContext() map[string]string { return nil } +func (m *KmsKey) GetAwsProfile() string { + if m != nil { + return m.AwsProfile + } + return "" +} + type GcpKmsKey struct { - ResourceId string `protobuf:"bytes,1,opt,name=resource_id,json=resourceId" json:"resource_id,omitempty"` + ResourceId string `protobuf:"bytes,1,opt,name=resource_id,json=resourceId,proto3" json:"resource_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GcpKmsKey) Reset() { *m = GcpKmsKey{} } +func (m *GcpKmsKey) String() string { return proto.CompactTextString(m) } +func (*GcpKmsKey) ProtoMessage() {} +func (*GcpKmsKey) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{3} +} + +func (m *GcpKmsKey) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GcpKmsKey.Unmarshal(m, b) +} +func (m *GcpKmsKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GcpKmsKey.Marshal(b, m, deterministic) +} +func (m *GcpKmsKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_GcpKmsKey.Merge(m, src) +} +func (m *GcpKmsKey) XXX_Size() int { + return xxx_messageInfo_GcpKmsKey.Size(m) +} +func (m *GcpKmsKey) XXX_DiscardUnknown() { + xxx_messageInfo_GcpKmsKey.DiscardUnknown(m) } -func (m *GcpKmsKey) Reset() { *m = GcpKmsKey{} } -func (m *GcpKmsKey) String() string { return proto.CompactTextString(m) } -func (*GcpKmsKey) ProtoMessage() {} -func (*GcpKmsKey) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } +var xxx_messageInfo_GcpKmsKey proto.InternalMessageInfo func (m *GcpKmsKey) GetResourceId() string { if m != nil { @@ -288,16 +290,94 @@ func (m *GcpKmsKey) GetResourceId() string { return "" } +type VaultKey struct { + VaultAddress string `protobuf:"bytes,1,opt,name=vault_address,json=vaultAddress,proto3" json:"vault_address,omitempty"` + BackendPath string `protobuf:"bytes,2,opt,name=backend_path,json=backendPath,proto3" json:"backend_path,omitempty"` + KeyName string `protobuf:"bytes,3,opt,name=key_name,json=keyName,proto3" json:"key_name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VaultKey) Reset() { *m = VaultKey{} } +func (m *VaultKey) String() string { return proto.CompactTextString(m) } +func (*VaultKey) ProtoMessage() {} +func (*VaultKey) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{4} +} + +func (m *VaultKey) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VaultKey.Unmarshal(m, b) +} +func (m *VaultKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VaultKey.Marshal(b, m, deterministic) +} +func (m *VaultKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_VaultKey.Merge(m, src) +} +func (m *VaultKey) XXX_Size() int { + return xxx_messageInfo_VaultKey.Size(m) +} +func (m *VaultKey) XXX_DiscardUnknown() { + xxx_messageInfo_VaultKey.DiscardUnknown(m) +} + +var xxx_messageInfo_VaultKey proto.InternalMessageInfo + +func (m *VaultKey) GetVaultAddress() string { + if m != nil { + return m.VaultAddress + } + return "" +} + +func (m *VaultKey) GetBackendPath() string { + if m != nil { + return m.BackendPath + } + return "" +} + +func (m *VaultKey) GetKeyName() string { + if m != nil { + return m.KeyName + } + return "" +} + type AzureKeyVaultKey struct { - VaultUrl string `protobuf:"bytes,1,opt,name=vault_url,json=vaultUrl" json:"vault_url,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` - Version string `protobuf:"bytes,3,opt,name=version" json:"version,omitempty"` + VaultUrl string `protobuf:"bytes,1,opt,name=vault_url,json=vaultUrl,proto3" json:"vault_url,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Version string `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *AzureKeyVaultKey) Reset() { *m = AzureKeyVaultKey{} } +func (m *AzureKeyVaultKey) String() string { return proto.CompactTextString(m) } +func (*AzureKeyVaultKey) ProtoMessage() {} +func (*AzureKeyVaultKey) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{5} +} + +func (m *AzureKeyVaultKey) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_AzureKeyVaultKey.Unmarshal(m, b) +} +func (m *AzureKeyVaultKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_AzureKeyVaultKey.Marshal(b, m, deterministic) +} +func (m *AzureKeyVaultKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_AzureKeyVaultKey.Merge(m, src) +} +func (m *AzureKeyVaultKey) XXX_Size() int { + return xxx_messageInfo_AzureKeyVaultKey.Size(m) +} +func (m *AzureKeyVaultKey) XXX_DiscardUnknown() { + xxx_messageInfo_AzureKeyVaultKey.DiscardUnknown(m) } -func (m *AzureKeyVaultKey) Reset() { *m = AzureKeyVaultKey{} } -func (m *AzureKeyVaultKey) String() string { return proto.CompactTextString(m) } -func (*AzureKeyVaultKey) ProtoMessage() {} -func (*AzureKeyVaultKey) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } +var xxx_messageInfo_AzureKeyVaultKey proto.InternalMessageInfo func (m *AzureKeyVaultKey) GetVaultUrl() string { if m != nil { @@ -321,14 +401,37 @@ func (m *AzureKeyVaultKey) GetVersion() string { } type EncryptRequest struct { - Key *Key `protobuf:"bytes,1,opt,name=key" json:"key,omitempty"` - Plaintext []byte `protobuf:"bytes,2,opt,name=plaintext,proto3" json:"plaintext,omitempty"` + Key *Key `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Plaintext []byte `protobuf:"bytes,2,opt,name=plaintext,proto3" json:"plaintext,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EncryptRequest) Reset() { *m = EncryptRequest{} } +func (m *EncryptRequest) String() string { return proto.CompactTextString(m) } +func (*EncryptRequest) ProtoMessage() {} +func (*EncryptRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{6} +} + +func (m *EncryptRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EncryptRequest.Unmarshal(m, b) +} +func (m *EncryptRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EncryptRequest.Marshal(b, m, deterministic) +} +func (m *EncryptRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_EncryptRequest.Merge(m, src) +} +func (m *EncryptRequest) XXX_Size() int { + return xxx_messageInfo_EncryptRequest.Size(m) +} +func (m *EncryptRequest) XXX_DiscardUnknown() { + xxx_messageInfo_EncryptRequest.DiscardUnknown(m) } -func (m *EncryptRequest) Reset() { *m = EncryptRequest{} } -func (m *EncryptRequest) String() string { return proto.CompactTextString(m) } -func (*EncryptRequest) ProtoMessage() {} -func (*EncryptRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +var xxx_messageInfo_EncryptRequest proto.InternalMessageInfo func (m *EncryptRequest) GetKey() *Key { if m != nil { @@ -345,13 +448,36 @@ func (m *EncryptRequest) GetPlaintext() []byte { } type EncryptResponse struct { - Ciphertext []byte `protobuf:"bytes,1,opt,name=ciphertext,proto3" json:"ciphertext,omitempty"` + Ciphertext []byte `protobuf:"bytes,1,opt,name=ciphertext,proto3" json:"ciphertext,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EncryptResponse) Reset() { *m = EncryptResponse{} } +func (m *EncryptResponse) String() string { return proto.CompactTextString(m) } +func (*EncryptResponse) ProtoMessage() {} +func (*EncryptResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{7} +} + +func (m *EncryptResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EncryptResponse.Unmarshal(m, b) +} +func (m *EncryptResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EncryptResponse.Marshal(b, m, deterministic) +} +func (m *EncryptResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_EncryptResponse.Merge(m, src) +} +func (m *EncryptResponse) XXX_Size() int { + return xxx_messageInfo_EncryptResponse.Size(m) +} +func (m *EncryptResponse) XXX_DiscardUnknown() { + xxx_messageInfo_EncryptResponse.DiscardUnknown(m) } -func (m *EncryptResponse) Reset() { *m = EncryptResponse{} } -func (m *EncryptResponse) String() string { return proto.CompactTextString(m) } -func (*EncryptResponse) ProtoMessage() {} -func (*EncryptResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +var xxx_messageInfo_EncryptResponse proto.InternalMessageInfo func (m *EncryptResponse) GetCiphertext() []byte { if m != nil { @@ -361,14 +487,37 @@ func (m *EncryptResponse) GetCiphertext() []byte { } type DecryptRequest struct { - Key *Key `protobuf:"bytes,1,opt,name=key" json:"key,omitempty"` - Ciphertext []byte `protobuf:"bytes,2,opt,name=ciphertext,proto3" json:"ciphertext,omitempty"` + Key *Key `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Ciphertext []byte `protobuf:"bytes,2,opt,name=ciphertext,proto3" json:"ciphertext,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DecryptRequest) Reset() { *m = DecryptRequest{} } +func (m *DecryptRequest) String() string { return proto.CompactTextString(m) } +func (*DecryptRequest) ProtoMessage() {} +func (*DecryptRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{8} +} + +func (m *DecryptRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DecryptRequest.Unmarshal(m, b) +} +func (m *DecryptRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DecryptRequest.Marshal(b, m, deterministic) +} +func (m *DecryptRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DecryptRequest.Merge(m, src) +} +func (m *DecryptRequest) XXX_Size() int { + return xxx_messageInfo_DecryptRequest.Size(m) +} +func (m *DecryptRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DecryptRequest.DiscardUnknown(m) } -func (m *DecryptRequest) Reset() { *m = DecryptRequest{} } -func (m *DecryptRequest) String() string { return proto.CompactTextString(m) } -func (*DecryptRequest) ProtoMessage() {} -func (*DecryptRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +var xxx_messageInfo_DecryptRequest proto.InternalMessageInfo func (m *DecryptRequest) GetKey() *Key { if m != nil { @@ -385,13 +534,36 @@ func (m *DecryptRequest) GetCiphertext() []byte { } type DecryptResponse struct { - Plaintext []byte `protobuf:"bytes,1,opt,name=plaintext,proto3" json:"plaintext,omitempty"` + Plaintext []byte `protobuf:"bytes,1,opt,name=plaintext,proto3" json:"plaintext,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DecryptResponse) Reset() { *m = DecryptResponse{} } +func (m *DecryptResponse) String() string { return proto.CompactTextString(m) } +func (*DecryptResponse) ProtoMessage() {} +func (*DecryptResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_8c1e2c407c293790, []int{9} +} + +func (m *DecryptResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DecryptResponse.Unmarshal(m, b) +} +func (m *DecryptResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DecryptResponse.Marshal(b, m, deterministic) +} +func (m *DecryptResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DecryptResponse.Merge(m, src) +} +func (m *DecryptResponse) XXX_Size() int { + return xxx_messageInfo_DecryptResponse.Size(m) +} +func (m *DecryptResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DecryptResponse.DiscardUnknown(m) } -func (m *DecryptResponse) Reset() { *m = DecryptResponse{} } -func (m *DecryptResponse) String() string { return proto.CompactTextString(m) } -func (*DecryptResponse) ProtoMessage() {} -func (*DecryptResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +var xxx_messageInfo_DecryptResponse proto.InternalMessageInfo func (m *DecryptResponse) GetPlaintext() []byte { if m != nil { @@ -404,7 +576,9 @@ func init() { proto.RegisterType((*Key)(nil), "Key") proto.RegisterType((*PgpKey)(nil), "PgpKey") proto.RegisterType((*KmsKey)(nil), "KmsKey") + proto.RegisterMapType((map[string]string)(nil), "KmsKey.ContextEntry") proto.RegisterType((*GcpKmsKey)(nil), "GcpKmsKey") + proto.RegisterType((*VaultKey)(nil), "VaultKey") proto.RegisterType((*AzureKeyVaultKey)(nil), "AzureKeyVaultKey") proto.RegisterType((*EncryptRequest)(nil), "EncryptRequest") proto.RegisterType((*EncryptResponse)(nil), "EncryptResponse") @@ -412,6 +586,48 @@ func init() { proto.RegisterType((*DecryptResponse)(nil), "DecryptResponse") } +func init() { proto.RegisterFile("keyservice/keyservice.proto", fileDescriptor_8c1e2c407c293790) } + +var fileDescriptor_8c1e2c407c293790 = []byte{ + // 574 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x5d, 0x6f, 0xd3, 0x30, + 0x14, 0x5d, 0xd6, 0xad, 0x5d, 0x6e, 0xca, 0x5a, 0xac, 0x09, 0x8d, 0x0d, 0xc1, 0x30, 0x2f, 0x13, + 0x9a, 0x3c, 0x51, 0x5e, 0xd0, 0xde, 0x06, 0x0c, 0x86, 0x2a, 0xa1, 0x29, 0x08, 0xde, 0x50, 0xe5, + 0xa5, 0x77, 0x6d, 0x94, 0x34, 0x31, 0x76, 0xd2, 0x61, 0xfe, 0x1a, 0x7f, 0x8b, 0x1f, 0x80, 0xfc, + 0x91, 0x7e, 0xf1, 0xc2, 0xdb, 0xf5, 0xc9, 0x39, 0xe7, 0x5e, 0x9f, 0xeb, 0x16, 0x8e, 0x33, 0xd4, + 0x0a, 0xe5, 0x3c, 0x4d, 0xf0, 0x7c, 0x59, 0x32, 0x21, 0xcb, 0xaa, 0xa4, 0x7f, 0x02, 0x68, 0x0d, + 0x51, 0x13, 0x0a, 0x9d, 0x6c, 0xa6, 0x46, 0x19, 0xea, 0xc3, 0xe0, 0x24, 0x38, 0x8d, 0x06, 0x1d, + 0x36, 0x9c, 0xa9, 0x21, 0xea, 0xeb, 0xad, 0xb8, 0x9d, 0xd9, 0xca, 0x70, 0xc4, 0x44, 0x58, 0xce, + 0xb6, 0xe7, 0xdc, 0x4c, 0x84, 0xe7, 0x08, 0x5b, 0x91, 0x33, 0x88, 0x26, 0x89, 0x18, 0x35, 0x5e, + 0x2d, 0xcb, 0x03, 0xf6, 0x31, 0x11, 0x0b, 0xbb, 0x70, 0xd2, 0x1c, 0xc8, 0x25, 0x10, 0xfe, 0xab, + 0x96, 0x68, 0xb8, 0x73, 0x5e, 0xe7, 0x95, 0x15, 0xed, 0x58, 0xd1, 0x43, 0x76, 0x69, 0x3e, 0x0d, + 0x51, 0x7f, 0x33, 0x5f, 0x9c, 0xb6, 0xcf, 0x3d, 0x36, 0xf7, 0x18, 0x39, 0x85, 0x70, 0xa9, 0xdc, + 0xb5, 0xca, 0x90, 0xad, 0x28, 0xf6, 0x1a, 0xe6, 0x5b, 0x80, 0xbd, 0x0c, 0xf5, 0xa8, 0xd2, 0x02, + 0xe9, 0x4b, 0x68, 0xbb, 0xd1, 0xc9, 0x09, 0x44, 0x77, 0x69, 0x31, 0x41, 0x29, 0x64, 0x5a, 0x54, + 0xf6, 0xf2, 0x61, 0xbc, 0x0a, 0xd1, 0xdf, 0x01, 0xb4, 0xfd, 0xbc, 0x7d, 0x68, 0x71, 0x59, 0x78, + 0x92, 0x29, 0x09, 0x81, 0x1d, 0x59, 0xe6, 0x68, 0x03, 0x09, 0x63, 0x5b, 0x13, 0x06, 0x9d, 0xa4, + 0x2c, 0x2a, 0xfc, 0x59, 0x1d, 0xb6, 0x4e, 0x5a, 0xa7, 0xd1, 0xe0, 0xc0, 0x67, 0xc9, 0xde, 0x39, + 0xf8, 0xaa, 0xa8, 0xa4, 0x8e, 0x1b, 0x12, 0x79, 0x06, 0x11, 0xbf, 0x57, 0x23, 0x21, 0xcb, 0xbb, + 0x34, 0x47, 0x7b, 0xfd, 0x30, 0x06, 0x7e, 0xaf, 0x6e, 0x1c, 0x72, 0x74, 0x01, 0xdd, 0x55, 0xa5, + 0x19, 0xa3, 0x59, 0x54, 0x18, 0x9b, 0x92, 0x1c, 0xc0, 0xee, 0x9c, 0xe7, 0x75, 0x33, 0x87, 0x3b, + 0x5c, 0x6c, 0xbf, 0x09, 0xe8, 0x19, 0x84, 0x8b, 0xf0, 0x4d, 0x27, 0x89, 0xaa, 0xac, 0x65, 0x82, + 0xa3, 0x74, 0xec, 0x0d, 0xa0, 0x81, 0x3e, 0x8d, 0xe9, 0x0c, 0xf6, 0x9a, 0xec, 0xc8, 0x0b, 0x78, + 0xe0, 0x92, 0xe5, 0xe3, 0xb1, 0x44, 0xa5, 0x3c, 0xbd, 0x6b, 0xc1, 0x4b, 0x87, 0x91, 0xe7, 0xd0, + 0xbd, 0xe5, 0x49, 0x86, 0xc5, 0x78, 0x24, 0x78, 0x35, 0xf5, 0xfd, 0x23, 0x8f, 0xdd, 0xf0, 0x6a, + 0x4a, 0x1e, 0xbb, 0xdc, 0x0b, 0x3e, 0x43, 0xfb, 0x1e, 0xc2, 0xb8, 0x93, 0xa1, 0xfe, 0xcc, 0x67, + 0x48, 0xbf, 0x43, 0x7f, 0x73, 0xc9, 0xe4, 0xb8, 0x59, 0x68, 0x2d, 0x73, 0xdf, 0xd2, 0xed, 0xf0, + 0xab, 0xcc, 0x4d, 0xdc, 0xd6, 0xc7, 0xc7, 0x6d, 0x6a, 0x72, 0x08, 0x9d, 0x39, 0x4a, 0x95, 0x96, + 0x45, 0x63, 0xef, 0x8f, 0xf4, 0x03, 0xec, 0x5f, 0x15, 0x89, 0xd4, 0xa2, 0x8a, 0xf1, 0x47, 0x8d, + 0xaa, 0x22, 0x8f, 0x96, 0xc9, 0x45, 0x83, 0x1d, 0x36, 0x44, 0xed, 0xf2, 0x7b, 0x02, 0xa1, 0xc8, + 0x79, 0xea, 0x96, 0x66, 0xcc, 0xbb, 0xf1, 0x12, 0xa0, 0xaf, 0xa0, 0xb7, 0xf0, 0x51, 0xa2, 0x2c, + 0x14, 0x92, 0xa7, 0x00, 0x49, 0x2a, 0xa6, 0x28, 0xad, 0x22, 0xb0, 0x8a, 0x15, 0x84, 0x5e, 0xc3, + 0xfe, 0x7b, 0xfc, 0xaf, 0xd6, 0xeb, 0x4e, 0xdb, 0xff, 0x38, 0x9d, 0x43, 0x6f, 0xe1, 0xe4, 0x9b, + 0xaf, 0x4d, 0x1b, 0x6c, 0x4c, 0x3b, 0xc8, 0x01, 0x86, 0xa8, 0xbf, 0xb8, 0x9f, 0xb9, 0x79, 0x8c, + 0x7e, 0x76, 0xd2, 0x63, 0xeb, 0x69, 0x1c, 0xf5, 0xd9, 0xc6, 0xb5, 0xe8, 0x96, 0xe1, 0xfb, 0x76, + 0xa4, 0xc7, 0xd6, 0xaf, 0x70, 0xd4, 0x67, 0x1b, 0x93, 0xd0, 0xad, 0xdb, 0xb6, 0xfd, 0x1f, 0x79, + 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, 0x9b, 0x65, 0x80, 0xa7, 0x66, 0x04, 0x00, 0x00, +} + // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -420,8 +636,9 @@ var _ grpc.ClientConn // is compatible with the grpc package it is being compiled against. const _ = grpc.SupportPackageIsVersion4 -// Client API for KeyService service - +// KeyServiceClient is the client API for KeyService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type KeyServiceClient interface { Encrypt(ctx context.Context, in *EncryptRequest, opts ...grpc.CallOption) (*EncryptResponse, error) Decrypt(ctx context.Context, in *DecryptRequest, opts ...grpc.CallOption) (*DecryptResponse, error) @@ -437,7 +654,7 @@ func NewKeyServiceClient(cc *grpc.ClientConn) KeyServiceClient { func (c *keyServiceClient) Encrypt(ctx context.Context, in *EncryptRequest, opts ...grpc.CallOption) (*EncryptResponse, error) { out := new(EncryptResponse) - err := grpc.Invoke(ctx, "/KeyService/Encrypt", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/KeyService/Encrypt", in, out, opts...) if err != nil { return nil, err } @@ -446,20 +663,30 @@ func (c *keyServiceClient) Encrypt(ctx context.Context, in *EncryptRequest, opts func (c *keyServiceClient) Decrypt(ctx context.Context, in *DecryptRequest, opts ...grpc.CallOption) (*DecryptResponse, error) { out := new(DecryptResponse) - err := grpc.Invoke(ctx, "/KeyService/Decrypt", in, out, c.cc, opts...) + err := c.cc.Invoke(ctx, "/KeyService/Decrypt", in, out, opts...) if err != nil { return nil, err } return out, nil } -// Server API for KeyService service - +// KeyServiceServer is the server API for KeyService service. type KeyServiceServer interface { Encrypt(context.Context, *EncryptRequest) (*EncryptResponse, error) Decrypt(context.Context, *DecryptRequest) (*DecryptResponse, error) } +// UnimplementedKeyServiceServer can be embedded to have forward compatible implementations. +type UnimplementedKeyServiceServer struct { +} + +func (*UnimplementedKeyServiceServer) Encrypt(ctx context.Context, req *EncryptRequest) (*EncryptResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Encrypt not implemented") +} +func (*UnimplementedKeyServiceServer) Decrypt(ctx context.Context, req *DecryptRequest) (*DecryptResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Decrypt not implemented") +} + func RegisterKeyServiceServer(s *grpc.Server, srv KeyServiceServer) { s.RegisterService(&_KeyService_serviceDesc, srv) } @@ -516,40 +743,3 @@ var _KeyService_serviceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "keyservice/keyservice.proto", } - -func init() { proto.RegisterFile("keyservice/keyservice.proto", fileDescriptor0) } - -var fileDescriptor0 = []byte{ - // 485 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4d, 0x6f, 0xd4, 0x30, - 0x10, 0x6d, 0x36, 0xcb, 0x6e, 0x33, 0xa9, 0xba, 0xc1, 0xaa, 0xd0, 0x6a, 0x8b, 0x60, 0xe5, 0x53, - 0x85, 0x2a, 0x57, 0x2c, 0x17, 0xd4, 0x5b, 0x81, 0x42, 0x51, 0x2e, 0x28, 0x08, 0x6e, 0x68, 0x15, - 0xd2, 0x21, 0x44, 0xc9, 0x26, 0xc6, 0x71, 0x22, 0xcc, 0x4f, 0xe1, 0x27, 0xf1, 0xab, 0x90, 0x1d, - 0x27, 0xfb, 0xc1, 0x85, 0xdb, 0xf8, 0xf9, 0xcd, 0x9b, 0x79, 0xcf, 0x32, 0x9c, 0xe7, 0xa8, 0x6a, - 0x14, 0x6d, 0x96, 0xe0, 0xd5, 0xb6, 0x64, 0x5c, 0x54, 0xb2, 0xa2, 0x7f, 0x1c, 0x70, 0x43, 0x54, - 0x84, 0xc2, 0x34, 0xdf, 0xd4, 0xeb, 0x1c, 0xd5, 0xdc, 0x59, 0x3a, 0x17, 0xfe, 0x6a, 0xca, 0xc2, - 0x4d, 0x1d, 0xa2, 0xba, 0x3b, 0x8a, 0x26, 0xb9, 0xa9, 0x34, 0x87, 0xa7, 0xdc, 0x70, 0x46, 0x96, - 0xf3, 0x21, 0xe5, 0x96, 0xc3, 0x4d, 0x45, 0x2e, 0xc1, 0x4f, 0x13, 0xbe, 0xee, 0xb5, 0x5c, 0xc3, - 0x03, 0xf6, 0x2e, 0xe1, 0x83, 0x9c, 0x97, 0xf6, 0x07, 0x72, 0x03, 0x24, 0xfe, 0xd5, 0x08, 0xd4, - 0xdc, 0x36, 0x6e, 0x0a, 0x69, 0x9a, 0xc6, 0xa6, 0xe9, 0x21, 0xbb, 0xd1, 0x57, 0x21, 0xaa, 0xcf, - 0xfa, 0xa6, 0xeb, 0x0d, 0x62, 0x8b, 0xb5, 0x16, 0x7b, 0x05, 0x70, 0x9c, 0xa3, 0x5a, 0x4b, 0xc5, - 0x91, 0x3e, 0x83, 0x49, 0xb7, 0x10, 0x59, 0x82, 0xff, 0x2d, 0x2b, 0x53, 0x14, 0x5c, 0x64, 0xa5, - 0x34, 0x96, 0xbc, 0x68, 0x17, 0xa2, 0xbf, 0x1d, 0x98, 0xd8, 0x2d, 0x02, 0x70, 0x63, 0x51, 0x5a, - 0x92, 0x2e, 0x09, 0x81, 0xb1, 0xa8, 0x0a, 0x34, 0x36, 0xbd, 0xc8, 0xd4, 0x84, 0xc1, 0x34, 0xa9, - 0x4a, 0x89, 0x3f, 0xe5, 0xdc, 0x5d, 0xba, 0x17, 0xfe, 0xea, 0xcc, 0x26, 0xc4, 0x5e, 0x77, 0xf0, - 0x6d, 0x29, 0x85, 0x8a, 0x7a, 0xd2, 0xe2, 0x1a, 0x4e, 0x76, 0x2f, 0xf4, 0x94, 0x3e, 0x5d, 0x2f, - 0xd2, 0x25, 0x39, 0x83, 0x07, 0x6d, 0x5c, 0x34, 0xfd, 0x98, 0xee, 0x70, 0x3d, 0x7a, 0xe9, 0xd0, - 0x4b, 0xf0, 0x86, 0xc4, 0xc8, 0x53, 0xf0, 0x05, 0xd6, 0x55, 0x23, 0x12, 0x5c, 0x67, 0xf7, 0x56, - 0x00, 0x7a, 0xe8, 0xfd, 0x3d, 0xfd, 0x02, 0xc1, 0x61, 0x54, 0xe4, 0x1c, 0xbc, 0x2e, 0xd0, 0x46, - 0x14, 0xb6, 0xe5, 0xd8, 0x00, 0x9f, 0x44, 0xa1, 0xed, 0x95, 0xf1, 0x66, 0xb0, 0xa7, 0x6b, 0x32, - 0x87, 0x69, 0x8b, 0xa2, 0xce, 0xaa, 0xd2, 0x3c, 0x9a, 0x17, 0xf5, 0x47, 0xfa, 0x16, 0x4e, 0x6f, - 0xcb, 0x44, 0x28, 0x2e, 0x23, 0xfc, 0xd1, 0x60, 0x2d, 0xc9, 0xa3, 0xad, 0x15, 0x7f, 0x35, 0x66, - 0x21, 0xaa, 0xce, 0xd0, 0x63, 0xf0, 0x78, 0x11, 0x67, 0x5d, 0x48, 0x5a, 0xfc, 0x24, 0xda, 0x02, - 0xf4, 0x39, 0xcc, 0x06, 0x9d, 0x9a, 0x57, 0x65, 0x8d, 0xe4, 0x09, 0x40, 0x92, 0xf1, 0xef, 0x28, - 0x4c, 0x87, 0x63, 0x3a, 0x76, 0x10, 0x7a, 0x07, 0xa7, 0x6f, 0xf0, 0xbf, 0x46, 0xef, 0x2b, 0x8d, - 0xfe, 0x51, 0xba, 0x82, 0xd9, 0xa0, 0x64, 0x87, 0xef, 0x6d, 0xeb, 0x1c, 0x6c, 0xbb, 0x2a, 0x00, - 0x42, 0x54, 0x1f, 0xbb, 0xcf, 0xa2, 0x1f, 0xdf, 0xee, 0x4e, 0x66, 0x6c, 0x3f, 0x8d, 0x45, 0xc0, - 0x0e, 0x6c, 0xd1, 0x23, 0xcd, 0xb7, 0xe3, 0xc8, 0x8c, 0xed, 0x5b, 0x58, 0x04, 0xec, 0x60, 0x13, - 0x7a, 0xf4, 0x75, 0x62, 0x7e, 0xe3, 0x8b, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xeb, 0x29, 0x21, - 0x4e, 0xac, 0x03, 0x00, 0x00, -} diff --git a/keyservice/keyservice.proto b/keyservice/keyservice.proto index 58c72bf6d..e0fbb2a79 100644 --- a/keyservice/keyservice.proto +++ b/keyservice/keyservice.proto @@ -6,6 +6,7 @@ message Key { PgpKey pgp_key = 2; GcpKmsKey gcp_kms_key = 3; AzureKeyVaultKey azure_keyvault_key = 4; + VaultKey vault_key = 5; } } @@ -17,12 +18,19 @@ message KmsKey { string arn = 1; string role = 2; map context = 3; + string aws_profile = 4; } message GcpKmsKey { string resource_id = 1; } +message VaultKey { + string vault_address = 1; + string backend_path = 2; + string key_name = 3; +} + message AzureKeyVaultKey { string vault_url = 1; string name = 2; diff --git a/keyservice/server.go b/keyservice/server.go index f9bc3021e..0f3053a57 100644 --- a/keyservice/server.go +++ b/keyservice/server.go @@ -7,6 +7,7 @@ import ( "go.mozilla.org/sops/v3/gcpkms" "go.mozilla.org/sops/v3/kms" "go.mozilla.org/sops/v3/pgp" + "go.mozilla.org/sops/v3/vault" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -61,6 +62,19 @@ func (ks *Server) encryptWithAzureKeyVault(key *AzureKeyVaultKey, plaintext []by return []byte(azkvKey.EncryptedKey), nil } +func (ks *Server) encryptWithVault(key *VaultKey, plaintext []byte) ([]byte, error) { + vaultKey := vault.MasterKey{ + VaultAddress: key.VaultAddress, + BackendPath: key.BackendPath, + KeyName: key.KeyName, + } + err := vaultKey.Encrypt(plaintext) + if err != nil { + return nil, err + } + return []byte(vaultKey.EncryptedKey), nil +} + func (ks *Server) decryptWithPgp(key *PgpKey, ciphertext []byte) ([]byte, error) { pgpKey := pgp.NewMasterKeyFromFingerprint(key.Fingerprint) pgpKey.EncryptedKey = string(ciphertext) @@ -95,6 +109,17 @@ func (ks *Server) decryptWithAzureKeyVault(key *AzureKeyVaultKey, ciphertext []b return []byte(plaintext), err } +func (ks *Server) decryptWithVault(key *VaultKey, ciphertext []byte) ([]byte, error) { + vaultKey := vault.MasterKey{ + VaultAddress: key.VaultAddress, + BackendPath: key.BackendPath, + KeyName: key.KeyName, + } + vaultKey.EncryptedKey = string(ciphertext) + plaintext, err := vaultKey.Decrypt() + return []byte(plaintext), err +} + // Encrypt takes an encrypt request and encrypts the provided plaintext with the provided key, returning the encrypted // result func (ks Server) Encrypt(ctx context.Context, @@ -134,6 +159,14 @@ func (ks Server) Encrypt(ctx context.Context, response = &EncryptResponse{ Ciphertext: ciphertext, } + case *Key_VaultKey: + ciphertext, err := ks.encryptWithVault(k.VaultKey, req.Plaintext) + if err != nil { + return nil, err + } + response = &EncryptResponse{ + Ciphertext: ciphertext, + } case nil: return nil, status.Errorf(codes.NotFound, "Must provide a key") default: @@ -158,6 +191,8 @@ func keyToString(key Key) string { return fmt.Sprintf("GCP KMS key with resource ID %s", k.GcpKmsKey.ResourceId) case *Key_AzureKeyvaultKey: return fmt.Sprintf("Azure Key Vault key with URL %s/keys/%s/%s", k.AzureKeyvaultKey.VaultUrl, k.AzureKeyvaultKey.Name, k.AzureKeyvaultKey.Version) + case *Key_VaultKey: + return fmt.Sprintf("Hashicorp Vault key with URI %s/v1/%s/keys/%s", k.VaultKey.VaultAddress, k.VaultKey.BackendPath, k.VaultKey.KeyName) default: return fmt.Sprintf("Unknown key type") } @@ -218,6 +253,14 @@ func (ks Server) Decrypt(ctx context.Context, response = &DecryptResponse{ Plaintext: plaintext, } + case *Key_VaultKey: + plaintext, err := ks.decryptWithVault(k.VaultKey, req.Ciphertext) + if err != nil { + return nil, err + } + response = &DecryptResponse{ + Plaintext: plaintext, + } case nil: return nil, grpc.Errorf(codes.NotFound, "Must provide a key") default: diff --git a/stores/stores.go b/stores/stores.go index 4b17a2bf8..8c77b775f 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -19,6 +19,7 @@ import ( "go.mozilla.org/sops/v3/gcpkms" "go.mozilla.org/sops/v3/kms" "go.mozilla.org/sops/v3/pgp" + "go.mozilla.org/sops/v3/vault" ) // SopsFile is a struct used by the stores as a helper to unmarshal the SOPS metadata @@ -40,6 +41,7 @@ type Metadata struct { KMSKeys []kmskey `yaml:"kms" json:"kms"` GCPKMSKeys []gcpkmskey `yaml:"gcp_kms" json:"gcp_kms"` AzureKeyVaultKeys []azkvkey `yaml:"azure_kv" json:"azure_kv"` + VaultKeys []vaultkey `yaml:"vault" json:"vault"` LastModified string `yaml:"lastmodified" json:"lastmodified"` MessageAuthenticationCode string `yaml:"mac" json:"mac"` PGPKeys []pgpkey `yaml:"pgp" json:"pgp"` @@ -54,6 +56,7 @@ type keygroup struct { KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"` GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"` AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"` + VaultKeys []vaultkey `yaml:"vault" json:"vault"` } type pgpkey struct { @@ -77,6 +80,14 @@ type gcpkmskey struct { EncryptedDataKey string `yaml:"enc" json:"enc"` } +type vaultkey struct { + VaultAddress string `yaml:"vault_address" json:"vault_address"` + BackendPath string `yaml:"backend_path" json:"backend_path"` + KeyName string `yaml:"keyname" json:"keyname"` + CreatedAt string `yaml:"created_at" json:"created_at"` + EncryptedDataKey string `yaml:"enc" json:"enc"` +} + type azkvkey struct { VaultURL string `yaml:"vault_url" json:"vault_url"` Name string `yaml:"name" json:"name"` @@ -100,6 +111,7 @@ func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata { m.PGPKeys = pgpKeysFromGroup(group) m.KMSKeys = kmsKeysFromGroup(group) m.GCPKMSKeys = gcpkmsKeysFromGroup(group) + m.VaultKeys = vaultKeysFromGroup(group) m.AzureKeyVaultKeys = azkvKeysFromGroup(group) } else { for _, group := range sopsMetadata.KeyGroups { @@ -107,6 +119,7 @@ func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata { KMSKeys: kmsKeysFromGroup(group), PGPKeys: pgpKeysFromGroup(group), GCPKMSKeys: gcpkmsKeysFromGroup(group), + VaultKeys: vaultKeysFromGroup(group), AzureKeyVaultKeys: azkvKeysFromGroup(group), }) } @@ -159,6 +172,22 @@ func gcpkmsKeysFromGroup(group sops.KeyGroup) (keys []gcpkmskey) { return } +func vaultKeysFromGroup(group sops.KeyGroup) (keys []vaultkey) { + for _, key := range group { + switch key := key.(type) { + case *vault.MasterKey: + keys = append(keys, vaultkey{ + VaultAddress: key.VaultAddress, + BackendPath: key.BackendPath, + KeyName: key.KeyName, + CreatedAt: key.CreationDate.Format(time.RFC3339), + EncryptedDataKey: key.EncryptedKey, + }) + } + } + return +} + func azkvKeysFromGroup(group sops.KeyGroup) (keys []azkvkey) { for _, key := range group { switch key := key.(type) { @@ -216,7 +245,7 @@ func (m *Metadata) ToInternal() (sops.Metadata, error) { }, nil } -func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey, gcpKmsKeys []gcpkmskey, azkvKeys []azkvkey) (sops.KeyGroup, error) { +func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey, gcpKmsKeys []gcpkmskey, azkvKeys []azkvkey, vaultKeys []vaultkey) (sops.KeyGroup, error) { var internalGroup sops.KeyGroup for _, kmsKey := range kmsKeys { k, err := kmsKey.toInternal() @@ -239,6 +268,13 @@ func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey, gcpKmsKeys []gcpkmske } internalGroup = append(internalGroup, k) } + for _, vaultKey := range vaultKeys { + k, err := vaultKey.toInternal() + if err != nil { + return nil, err + } + internalGroup = append(internalGroup, k) + } for _, pgpKey := range pgpKeys { k, err := pgpKey.toInternal() if err != nil { @@ -251,8 +287,8 @@ func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey, gcpKmsKeys []gcpkmske func (m *Metadata) internalKeygroups() ([]sops.KeyGroup, error) { var internalGroups []sops.KeyGroup - if len(m.PGPKeys) > 0 || len(m.KMSKeys) > 0 || len(m.GCPKMSKeys) > 0 || len(m.AzureKeyVaultKeys) > 0 { - internalGroup, err := internalGroupFrom(m.KMSKeys, m.PGPKeys, m.GCPKMSKeys, m.AzureKeyVaultKeys) + if len(m.PGPKeys) > 0 || len(m.KMSKeys) > 0 || len(m.GCPKMSKeys) > 0 || len(m.AzureKeyVaultKeys) > 0 || len(m.VaultKeys) > 0 { + internalGroup, err := internalGroupFrom(m.KMSKeys, m.PGPKeys, m.GCPKMSKeys, m.AzureKeyVaultKeys, m.VaultKeys) if err != nil { return nil, err } @@ -260,7 +296,7 @@ func (m *Metadata) internalKeygroups() ([]sops.KeyGroup, error) { return internalGroups, nil } else if len(m.KeyGroups) > 0 { for _, group := range m.KeyGroups { - internalGroup, err := internalGroupFrom(group.KMSKeys, group.PGPKeys, group.GCPKMSKeys, group.AzureKeyVaultKeys) + internalGroup, err := internalGroupFrom(group.KMSKeys, group.PGPKeys, group.GCPKMSKeys, group.AzureKeyVaultKeys, group.VaultKeys) if err != nil { return nil, err } @@ -313,6 +349,20 @@ func (azkvKey *azkvkey) toInternal() (*azkv.MasterKey, error) { }, nil } +func (vaultKey *vaultkey) toInternal() (*vault.MasterKey, error) { + creationDate, err := time.Parse(time.RFC3339, vaultKey.CreatedAt) + if err != nil { + return nil, err + } + return &vault.MasterKey{ + VaultAddress: vaultKey.VaultAddress, + BackendPath: vaultKey.BackendPath, + KeyName: vaultKey.KeyName, + CreationDate: creationDate, + EncryptedKey: vaultKey.EncryptedDataKey, + }, nil +} + func (pgpKey *pgpkey) toInternal() (*pgp.MasterKey, error) { creationDate, err := time.Parse(time.RFC3339, pgpKey.CreatedAt) if err != nil { diff --git a/vault/keysource.go b/vault/keysource.go new file mode 100644 index 000000000..a793f90d7 --- /dev/null +++ b/vault/keysource.go @@ -0,0 +1,241 @@ +package vault + +import ( + "encoding/base64" + "fmt" + "net/url" + "path/filepath" + "strings" + "time" + + "go.mozilla.org/sops/v3/logging" + + "github.com/hashicorp/vault/api" + "github.com/sirupsen/logrus" +) + +var log *logrus.Logger + +func init() { + log = logging.NewLogger("VAULT_TRANSIT") +} + +// MasterKey is a Vault Transit backend path used to encrypt and decrypt sops' data key. +type MasterKey struct { + EncryptedKey string + KeyName string + BackendPath string + VaultAddress string + CreationDate time.Time +} + +// NewMasterKeysFromURIs gets lots of keys from lots of URIs +func NewMasterKeysFromURIs(uris string) ([]*MasterKey, error) { + var keys []*MasterKey + if uris == "" { + return keys, nil + } + uriList := strings.Split(uris, ",") + for _, uri := range uriList { + if uri == "" { + continue + } + key, err := NewMasterKeyFromURI(uri) + if err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, nil +} + +// NewMasterKeyFromURI obtains the vaultAddress the transit backend path and the key name from the full URI of the key +func NewMasterKeyFromURI(uri string) (*MasterKey, error) { + log.Debugln("Called NewMasterKeyFromURI with uri: ", uri) + var key *MasterKey + if uri == "" { + return key, nil + } + u, err := url.Parse(uri) + if err != nil { + return nil, err + } + if u.Scheme == "" { + return nil, fmt.Errorf("missing scheme in vault URL (should be like this: https://vault.example.com:8200/v1/transit/keys/keyName, got: %v", uri) + } + backendPath, keyName, err := getBackendAndKeyFromPath(u.EscapedPath()) + if err != nil { + return nil, err + } + u.Path = "" + return NewMasterKey(u.String(), backendPath, keyName), nil + +} + +func getBackendAndKeyFromPath(path string) (string, string, error) { + path = strings.TrimPrefix(path, "/") + path = strings.TrimSuffix(path, "/") + values := strings.Split(path, "/") + // minimum lenght should be 4 "v1", "transit", "keys", "keyName" + if len(values) < 4 { + return "", "", fmt.Errorf("The path to the key is not long enough: (eg. https://vault.example.com:8200/v1/transit/keys/keyName") + } + log.Debugf("Path: %v \nValues: %v\n", path, values) + if values[0] != "v1" { + return "", "", fmt.Errorf("probably forgot v1 in the URI") + } + if values[len(values)-2] != "keys" { + return "", "", fmt.Errorf("probably forgot 'keys' in the URI") + } + return filepath.Join(values[1 : len(values)-2]...), values[len(values)-1], nil +} + +// NewMasterKey creates a new MasterKey from a vault address, transit backend path and a key name and setting the creation date to the current date +func NewMasterKey(addess, backendPath, keyName string) *MasterKey { + mk := &MasterKey{ + VaultAddress: addess, + BackendPath: backendPath, + KeyName: keyName, + CreationDate: time.Now().UTC(), + } + log.Debugln("Created Vault Master Key: ", mk) + return mk +} + +// EncryptedDataKey returns the encrypted data key this master key holds +func (key *MasterKey) EncryptedDataKey() []byte { + return []byte(key.EncryptedKey) +} + +// SetEncryptedDataKey sets the encrypted data key for this master key +func (key *MasterKey) SetEncryptedDataKey(enc []byte) { + key.EncryptedKey = string(enc) +} + +// Encrypt takes a sops data key, encrypts it with Vault Transit and stores the result in the EncryptedKey field +func (key *MasterKey) Encrypt(dataKey []byte) error { + path := filepath.Join(key.BackendPath, "encrypt", key.KeyName) + cfg := api.DefaultConfig() + cfg.Address = key.VaultAddress + cli, err := api.NewClient(cfg) + if err != nil { + log.WithField("Path", path).Info("Vault connections failed") + return fmt.Errorf("Cannot create Vault Client: %v", err) + } + encoded := base64.StdEncoding.EncodeToString(dataKey) + payload := make(map[string]interface{}) + payload["plaintext"] = encoded + raw, err := cli.Logical().Write(path, payload) + if err != nil { + log.WithField("Path", path).Info("Encryption failed") + return err + } + if raw == nil || raw.Data == nil { + return fmt.Errorf("The transit backend %s is empty", path) + } + encrypted, ok := raw.Data["ciphertext"] + if ok != true { + return fmt.Errorf("there's not encrypted data") + } + encryptedKey, ok := encrypted.(string) + if ok != true { + return fmt.Errorf("the ciphertext cannot be casted to string") + } + key.EncryptedKey = encryptedKey + return nil +} + +// EncryptIfNeeded encrypts the provided sops' data key and encrypts it if it hasn't been encrypted yet +func (key *MasterKey) EncryptIfNeeded(dataKey []byte) error { + if key.EncryptedKey == "" { + return key.Encrypt(dataKey) + } + return nil +} + +// Decrypt decrypts the EncryptedKey field with CGP KMS and returns the result. +func (key *MasterKey) Decrypt() ([]byte, error) { + path := filepath.Join(key.BackendPath, "decrypt", key.KeyName) + cfg := api.DefaultConfig() + cfg.Address = key.VaultAddress + cli, err := api.NewClient(cfg) + if err != nil { + log.WithField("Path", path).Info("Vault connections failed") + return nil, fmt.Errorf("Cannot create Vault Client: %v", err) + } + payload := make(map[string]interface{}) + payload["ciphertext"] = key.EncryptedKey + raw, err := cli.Logical().Write(path, payload) + if err != nil { + log.WithField("Path", path).Info("Encryption failed") + return nil, err + } + if raw == nil || raw.Data == nil { + return nil, fmt.Errorf("The transit backend %s is empty", path) + } + decrypted, ok := raw.Data["plaintext"] + if ok != true { + return nil, fmt.Errorf("there's no decrypted data") + } + dataKey, ok := decrypted.(string) + if ok != true { + return nil, fmt.Errorf("the plaintest cannot be casted to string") + } + result, err := base64.StdEncoding.DecodeString(dataKey) + if err != nil { + return nil, fmt.Errorf("Couldn't decode base64 plaintext") + } + return result, nil +} + +// NeedsRotation returns whether the data key needs to be rotated or not. +// This is simply copied from GCPKMS +// TODO: handle key rotation on vault side +func (key *MasterKey) NeedsRotation() bool { + //TODO: manage rewrapping https://www.vaultproject.io/api/secret/transit/index.html#rewrap-data + return time.Since(key.CreationDate) > (time.Hour * 24 * 30 * 6) +} + +// ToString converts the key to a string representation +func (key *MasterKey) ToString() string { + return fmt.Sprintf("%s/v1/%s/keys/%s", key.VaultAddress, key.BackendPath, key.KeyName) +} + +func (key *MasterKey) createVaultTransitAndKey() error { + cfg := api.DefaultConfig() + cfg.Address = key.VaultAddress + cli, err := api.NewClient(cfg) + if err != nil { + return fmt.Errorf("Cannot create Vault Client: %v", err) + } + err = cli.Sys().Mount(key.BackendPath, &api.MountInput{ + Type: "transit", + Description: "backend transit used by SOPS", + }) + if err != nil { + return err + } + path := filepath.Join(key.BackendPath, "keys", key.KeyName) + payload := make(map[string]interface{}) + payload["type"] = "rsa-4096" + _, err = cli.Logical().Write(path, payload) + if err != nil { + return err + } + _, err = cli.Logical().Read(path) + if err != nil { + return err + } + return nil +} + +// ToMap converts the MasterKey to a map for serialization purposes +func (key MasterKey) ToMap() map[string]interface{} { + out := make(map[string]interface{}) + out["vault_address"] = key.VaultAddress + out["keyname"] = key.KeyName + out["backend_path"] = key.BackendPath + out["enc"] = key.EncryptedKey + out["created_at"] = key.CreationDate.UTC().Format(time.RFC3339) + return out +} diff --git a/vault/keysource_test.go b/vault/keysource_test.go new file mode 100644 index 000000000..2a0ea3c3d --- /dev/null +++ b/vault/keysource_test.go @@ -0,0 +1,146 @@ +package vault + +import ( + "fmt" + logger "log" + "os" + "testing" + "time" + + "github.com/hashicorp/vault/api" + "github.com/ory/dockertest" + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + // uses a sensible default on windows (tcp/http) and linux/osx (socket) + pool, err := dockertest.NewPool("") + if err != nil { + logger.Fatalf("Could not connect to docker: %s", err) + } + + // pulls an image, creates a container based on it and runs it + resource, err := pool.Run("vault", "1.1.3", []string{"VAULT_DEV_ROOT_TOKEN_ID=secret"}) + if err != nil { + logger.Fatalf("Could not start resource: %s", err) + } + + os.Setenv("VAULT_ADDR", fmt.Sprintf("http://127.0.0.1:%v", resource.GetPort("8200/tcp"))) + os.Setenv("VAULT_TOKEN", "secret") + + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + if err := pool.Retry(func() error { + cli, err := api.NewClient(api.DefaultConfig()) + if err != nil { + return fmt.Errorf("Cannot create Vault Client: %v", err) + } + status, err := cli.Sys().InitStatus() + if err != nil { + return err + } + if status != true { + return fmt.Errorf("Vault not ready yet") + } + return nil + }); err != nil { + logger.Fatalf("Could not connect to docker: %s", err) + } + + key := NewMasterKey(fmt.Sprintf("http://127.0.0.1:%v", resource.GetPort("8200/tcp")), "sops", "main") + err = key.createVaultTransitAndKey() + if err != nil { + logger.Fatal(err) + } + code := 0 + if err == nil { + code = m.Run() + } + + // You can't defer this because os.Exit doesn't care for defer + if err := pool.Purge(resource); err != nil { + logger.Fatalf("Could not purge resource: %s", err) + } + + os.Exit(code) +} + +func TestKeyToMap(t *testing.T) { + key := MasterKey{ + CreationDate: time.Date(2016, time.October, 31, 10, 0, 0, 0, time.UTC), + VaultAddress: "http://127.0.0.1:8200", + BackendPath: "foo", + KeyName: "bar", + EncryptedKey: "this is encrypted", + } + assert.Equal(t, map[string]interface{}{ + "vault_address": "http://127.0.0.1:8200", + "backend_path": "foo", + "keyname": "bar", + "enc": "this is encrypted", + "created_at": "2016-10-31T10:00:00Z", + }, key.ToMap()) +} + +func TestEncryptionDecryption(t *testing.T) { + dataKey := []byte("super very Secret Key!!!") + key := MasterKey{ + VaultAddress: os.Getenv("VAULT_ADDR"), + BackendPath: "sops", + KeyName: "main", + } + err := key.Encrypt(dataKey) + if err != nil { + fmt.Println(err) + t.Fail() + return + } + decrypted, err := key.Decrypt() + if err != nil { + fmt.Println(err) + t.Fail() + return + } + assert.Equal(t, dataKey, decrypted) +} + +func TestNewMasterKeyFromURI(t *testing.T) { + uri1 := "https://vault.example.com:8200/v1/transit/keys/keyName" + uri2 := "https://vault.me.com/v1/super42/bestmarket/keys/slig" + uri3 := "http://127.0.0.1:12121/v1/transit/keys/dev" + mk1 := &MasterKey{ + VaultAddress: "https://vault.example.com:8200", + BackendPath: "transit", + KeyName: "keyName", + } + mk2 := &MasterKey{ + VaultAddress: "https://vault.me.com", + BackendPath: "super42/bestmarket", + KeyName: "slig", + } + mk3 := &MasterKey{ + VaultAddress: "http://127.0.0.1:12121", + BackendPath: "transit", + KeyName: "dev", + } + genMk1, err := NewMasterKeyFromURI(uri1) + if err != nil { + log.Errorln(err) + t.Fail() + } + genMk2, err := NewMasterKeyFromURI(uri2) + if err != nil { + log.Errorln(err) + t.Fail() + } + genMk3, err := NewMasterKeyFromURI(uri3) + if err != nil { + log.Errorln(err) + t.Fail() + } + mk1.CreationDate = genMk1.CreationDate + mk2.CreationDate = genMk2.CreationDate + mk3.CreationDate = genMk3.CreationDate + assert.Equal(t, mk1, genMk1) + assert.Equal(t, mk2, genMk2) + assert.Equal(t, mk3, genMk3) +} From 7004d5c3c3fb717196e96629ca0486e8beea3ae9 Mon Sep 17 00:00:00 2001 From: gitirabassi Date: Thu, 8 Aug 2019 12:00:33 +0200 Subject: [PATCH 02/16] feat(docs): added docs in README.md and in command help fix(doc): fix rst formatting" fix(doc): fix rst formatting --- README.rst | 61 ++++++++++++++++++++++++++++++++++++++++++++++++ cmd/sops/main.go | 14 ++++++++--- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 4c597a33f..038fe3e28 100644 --- a/README.rst +++ b/README.rst @@ -290,6 +290,66 @@ And decrypt it using:: $ sops --decrypt test.enc.yaml +Encrypting using Hashicorp Vault +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To understand how to deploy Vault securely this is not the place, We assume you have a instace (or more) of Vault running and you have privileged access to it. + +To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) + +.. code:: bash + + $ docker run -d -p8200:8200 vault:1.2.0 server -dev -dev-root-token-id=toor + + +.. code:: bash + + $ # Subsitute this with the address of the vault running + $ export VAULT_ADDR=http://127.0.0.1:8200 + + $ # this may not be necessary in case you previously used `vault login` for production use + $ export VAULT_TOKEN=toor + + $ # to check if Vault started correctly and configured + $ vault status + Key Value + --- ----- + Seal Type shamir + Initialized true + Sealed false + Total Shares 1 + Threshold 1 + Version 1.2.0 + Cluster Name vault-cluster-618cc902 + Cluster ID e532e461-e8f0-1352-8a41-fc7c11096908 + HA Enabled false + + $ # We need to enable a transit engine if not already done (I prefer to create a transit engine specifically for sops, in which I can have multiple keys with various permission levels) + $ vault secrets enable -path=sops transit + Success! Enabled the transit secrets engine at: sops/ + + $ # Then we need to create one or more keys + $ vault write sops/keys/firstkey type=rsa-4096 + Success! Data written to: sops/keys/firstkey + + $ vault write sops/keys/secondkey type=rsa-2048 + Success! Data written to: sops/keys/secondkey + + $ vault write sops/keys/thirdkey type=chacha20-poly1305 + Success! Data written to: sops/keys/thirdkey + + $ sops --vault $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml + + $ cat <.sops.yaml + creation_rules: + - path_regex: \.dev\.yaml$ + vault_uris: "$VAULT_ADDR/v1/sops/keys/secondkey" + - \.prod\.yaml$ + vault_uris: "$VAULT_ADDR/v1/sops/keys/thirdkey" + EOF + + $ sops --verbose -e prod/raw.yaml > prod/encrypted.yaml + Adding and removing keys ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -546,6 +606,7 @@ can manage the three sets of configurations for the three types of files: - path_regex: \.prod\.yaml$ kms: 'arn:aws:kms:us-west-2:361527076523:key/5052f06a-5d3f-489e-b86c-57201e06f31e+arn:aws:iam::361527076523:role/hiera-sops-prod,arn:aws:kms:eu-central-1:361527076523:key/cb1fab90-8d17-42a1-a9d8-334968904f94+arn:aws:iam::361527076523:role/hiera-sops-prod' pgp: 'FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4' + hc_vault_uri: "http://localhost:8200/v1/sops/keys/thirdkey" # gcp files using GCP KMS - path_regex: \.gcp\.yaml$ diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 0e14f058c..78d1db438 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -79,6 +79,14 @@ func main() { (you need to setup google application default credentials. See https://developers.google.com/identity/protocols/application-default-credentials) + + To encrypt or decrypt a document with HashiCorp Vault's Transit Secret Engine, specify the + Vault key URI name in the --vault flag or in the SOPS_VAULT_URIS (eg. https://vault.example.org:8200/v1/transit/keys/dev + where 'https://vault.example.org:8200' is the vault server, 'transit' the backendPath, and 'dev' is the name of the key ) + environment variable. + (you need to enable the Transit Secrets Engine in Vault. See + https://www.vaultproject.io/docs/secrets/transit/index.html) + To encrypt or decrypt a document with Azure Key Vault, specify the Azure Key Vault key URL in the --azure-kv flag or in the SOPS_AZURE_KEYVAULT_URL environment variable. @@ -92,11 +100,11 @@ func main() { To use multiple KMS or PGP keys, separate them by commas. For example: $ sops -p "10F2...0A, 85D...B3F21" file.yaml - The -p, -k, --gcp-kms and --azure-kv flags are only used to encrypt new documents. Editing + The -p, -k, --gcp-kms, --vault and --azure-kv flags are only used to encrypt new documents. Editing or decrypting existing documents can be done with "sops file" or "sops -d file" respectively. The KMS and PGP keys listed in the encrypted documents are used then. To manage master keys in existing documents, use - the "add-{kms,pgp,gcp-kms,azure-kv}" and "rm-{kms,pgp,gcp-kms,azure-kv}" flags. + the "add-{kms,pgp,gcp-kms,azure-kv,vault}" and "rm-{kms,pgp,gcp-kms,azure-kv,vault}" flags. To use a different GPG binary than the one in your PATH, set SOPS_GPG_EXEC. To use a GPG key server other than gpg.mozilla.org, set SOPS_GPG_KEYSERVER. @@ -1036,7 +1044,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { group = append(group, azkvKeys...) group = append(group, pgpKeys...) group = append(group, vaultMkKeys...) - log.Debugln("Master keys available: ", group) + log.Infof("Master keys available: %+v", group) return []sops.KeyGroup{group}, nil } From 37d68f1106f15766ba0b595552c47aa6a92874e5 Mon Sep 17 00:00:00 2001 From: gitirabassi Date: Wed, 4 Sep 2019 09:22:39 +0200 Subject: [PATCH 03/16] fix(vault): addressed typos and fixes from autrilla feat(cli): moved vault to hc-vault naming --- README.rst | 10 ++-- cmd/sops/main.go | 40 ++++++++-------- config/config.go | 4 +- config/config_test.go | 16 +++---- go.mod | 1 + stores/stores.go | 4 +- vault/keysource.go | 100 ++++++++++++++++++++++++++-------------- vault/keysource_test.go | 3 +- 8 files changed, 105 insertions(+), 73 deletions(-) diff --git a/README.rst b/README.rst index 038fe3e28..a9b16d84c 100644 --- a/README.rst +++ b/README.rst @@ -293,7 +293,7 @@ And decrypt it using:: Encrypting using Hashicorp Vault ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To understand how to deploy Vault securely this is not the place, We assume you have a instace (or more) of Vault running and you have privileged access to it. +To understand how to deploy Vault securely this is not the place, we assume you have a instance (or more) of Vault running and you have privileged access to it. To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) @@ -338,14 +338,14 @@ To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) $ vault write sops/keys/thirdkey type=chacha20-poly1305 Success! Data written to: sops/keys/thirdkey - $ sops --vault $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml + $ sops --hc-vault $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml $ cat <.sops.yaml creation_rules: - path_regex: \.dev\.yaml$ - vault_uris: "$VAULT_ADDR/v1/sops/keys/secondkey" + hc_vault_uris: "$VAULT_ADDR/v1/sops/keys/secondkey" - \.prod\.yaml$ - vault_uris: "$VAULT_ADDR/v1/sops/keys/thirdkey" + hc_vault_uris: "$VAULT_ADDR/v1/sops/keys/thirdkey" EOF $ sops --verbose -e prod/raw.yaml > prod/encrypted.yaml @@ -606,7 +606,7 @@ can manage the three sets of configurations for the three types of files: - path_regex: \.prod\.yaml$ kms: 'arn:aws:kms:us-west-2:361527076523:key/5052f06a-5d3f-489e-b86c-57201e06f31e+arn:aws:iam::361527076523:role/hiera-sops-prod,arn:aws:kms:eu-central-1:361527076523:key/cb1fab90-8d17-42a1-a9d8-334968904f94+arn:aws:iam::361527076523:role/hiera-sops-prod' pgp: 'FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4' - hc_vault_uri: "http://localhost:8200/v1/sops/keys/thirdkey" + hc_vault_uris: "http://localhost:8200/v1/sops/keys/thirdkey" # gcp files using GCP KMS - path_regex: \.gcp\.yaml$ diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 78d1db438..fb62d3936 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -81,7 +81,7 @@ func main() { To encrypt or decrypt a document with HashiCorp Vault's Transit Secret Engine, specify the - Vault key URI name in the --vault flag or in the SOPS_VAULT_URIS (eg. https://vault.example.org:8200/v1/transit/keys/dev + Vault key URI name in the --hc-vault flag or in the SOPS_VAULT_URIS (eg. https://vault.example.org:8200/v1/transit/keys/dev where 'https://vault.example.org:8200' is the vault server, 'transit' the backendPath, and 'dev' is the name of the key ) environment variable. (you need to enable the Transit Secrets Engine in Vault. See @@ -100,11 +100,11 @@ func main() { To use multiple KMS or PGP keys, separate them by commas. For example: $ sops -p "10F2...0A, 85D...B3F21" file.yaml - The -p, -k, --gcp-kms, --vault and --azure-kv flags are only used to encrypt new documents. Editing + The -p, -k, --gcp-kms, --hc-vault and --azure-kv flags are only used to encrypt new documents. Editing or decrypting existing documents can be done with "sops file" or "sops -d file" respectively. The KMS and PGP keys listed in the encrypted documents are used then. To manage master keys in existing documents, use - the "add-{kms,pgp,gcp-kms,azure-kv,vault}" and "rm-{kms,pgp,gcp-kms,azure-kv,vault}" flags. + the "add-{kms,pgp,gcp-kms,azure-kv,hc-vault}" and "rm-{kms,pgp,gcp-kms,azure-kv,hc-vault}" flags. To use a different GPG binary than the one in your PATH, set SOPS_GPG_EXEC. To use a GPG key server other than gpg.mozilla.org, set SOPS_GPG_KEYSERVER. @@ -361,7 +361,7 @@ func main() { Usage: "the Azure Key Vault key URL the new group should contain. Can be specified more than once", }, cli.StringSliceFlag{ - Name: "vault", + Name: "hc-vault", Usage: "the full vault path to the key used to encrypt/decrypt. Make you choose and configure a key with encrption/decryption enabled (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev'). Can be specified more than once", }, cli.BoolFlag{ @@ -381,7 +381,7 @@ func main() { pgpFps := c.StringSlice("pgp") kmsArns := c.StringSlice("kms") gcpKmses := c.StringSlice("gcp-kms") - vaultURIs := c.StringSlice("vault") + vaultURIs := c.StringSlice("hc-vault") azkvs := c.StringSlice("azure-kv") var group sops.KeyGroup for _, fp := range pgpFps { @@ -524,7 +524,7 @@ func main() { EnvVar: "SOPS_AZURE_KEYVAULT_URLS", }, cli.StringFlag{ - Name: "vault", + Name: "hc-vault", Usage: "comma separated list of vault's key URI (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev')", EnvVar: "SOPS_VAULT_URIS", }, @@ -578,11 +578,11 @@ func main() { Usage: "remove the provided comma-separated list of KMS ARNs from the list of master keys on the given file", }, cli.StringFlag{ - Name: "add-vault", + Name: "add-hc-vault", Usage: "add the provided comma-separated list of Vault's URI key to the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)", }, cli.StringFlag{ - Name: "rm-vault", + Name: "rm-hc-vault", Usage: "remove the provided comma-separated list of Vault's URI key from the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)", }, cli.StringFlag{ @@ -650,8 +650,8 @@ func main() { return toExitError(err) } if _, err := os.Stat(fileName); os.IsNotExist(err) { - if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-vault") != "" || c.String("add-azure-kv") != "" || - c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-vault") != "" || c.String("rm-azure-kv") != "" { + if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-hc-vault") != "" || c.String("add-azure-kv") != "" || + c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-hc-vault") != "" || c.String("rm-azure-kv") != "" { return common.NewExitError("Error: cannot add or remove keys on non-existent files, use `--kms` and `--pgp` instead.", codes.CannotChangeKeysFromNonExistentFile) } if c.Bool("encrypt") || c.Bool("decrypt") || c.Bool("rotate") { @@ -764,11 +764,11 @@ func main() { for _, k := range azureKeys { addMasterKeys = append(addMasterKeys, k) } - vaultKeys, err := vault.NewMasterKeysFromURIs(c.String("add-vault")) + hcVaultKeys, err := vault.NewMasterKeysFromURIs(c.String("add-hc-vault")) if err != nil { return err } - for _, k := range vaultKeys { + for _, k := range hcVaultKeys { addMasterKeys = append(addMasterKeys, k) } @@ -789,11 +789,11 @@ func main() { for _, k := range azureKeys { rmMasterKeys = append(rmMasterKeys, k) } - vaultKeys, err = vault.NewMasterKeysFromURIs(c.String("rm-vault")) + hcVaultKeys, err = vault.NewMasterKeysFromURIs(c.String("rm-hc-vault")) if err != nil { return err } - for _, k := range vaultKeys { + for _, k := range hcVaultKeys { rmMasterKeys = append(rmMasterKeys, k) } @@ -988,7 +988,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { var pgpKeys []keys.MasterKey var cloudKmsKeys []keys.MasterKey var azkvKeys []keys.MasterKey - var vaultMkKeys []keys.MasterKey + var hcVaultMkKeys []keys.MasterKey kmsEncryptionContext := kms.ParseKMSContext(c.String("encryption-context")) if c.String("encryption-context") != "" && kmsEncryptionContext == nil { return nil, common.NewExitError("Invalid KMS encryption context format", codes.ErrorInvalidKMSEncryptionContextFormat) @@ -1012,13 +1012,13 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { azkvKeys = append(azkvKeys, k) } } - if c.String("vault") != "" { - vaultKeys, err := vault.NewMasterKeysFromURIs(c.String("vault")) + if c.String("hc-vault") != "" { + hcVaultKeys, err := vault.NewMasterKeysFromURIs(c.String("vault")) if err != nil { return nil, err } - for _, k := range vaultKeys { - vaultMkKeys = append(vaultMkKeys, k) + for _, k := range hcVaultKeys { + hcVaultMkKeys = append(hcVaultMkKeys, k) } } if c.String("pgp") != "" { @@ -1043,7 +1043,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { group = append(group, cloudKmsKeys...) group = append(group, azkvKeys...) group = append(group, pgpKeys...) - group = append(group, vaultMkKeys...) + group = append(group, hcVaultMkKeys...) log.Infof("Master keys available: %+v", group) return []sops.KeyGroup{group}, nil } diff --git a/config/config.go b/config/config.go index bf815b867..47d4407ea 100644 --- a/config/config.go +++ b/config/config.go @@ -70,7 +70,7 @@ type keyGroup struct { KMS []kmsKey GCPKMS []gcpKmsKey `yaml:"gcp_kms"` AzureKV []azureKVKey `yaml:"azure_keyvault"` - Vault []vaultKey `yaml:"vault"` + Vault []vaultKey `yaml:"hc_vault"` PGP []string } @@ -118,7 +118,7 @@ type creationRule struct { PGP string GCPKMS string `yaml:"gcp_kms"` AzureKeyVault string `yaml:"azure_keyvault"` - Vault string `yaml:"vault_uris"` + Vault string `yaml:"hc_vault_uris"` KeyGroups []keyGroup `yaml:"key_groups"` ShamirThreshold int `yaml:"shamir_threshold"` UnencryptedSuffix string `yaml:"unencrypted_suffix"` diff --git a/config/config_test.go b/config/config_test.go index 8ed76f608..704c48a2c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -48,12 +48,12 @@ creation_rules: kms: "1" pgp: "2" gcp_kms: "3" - vault_uris: http://4:8200/v1/4/keys/4 + hc_vault_uris: http://4:8200/v1/4/keys/4 - path_regex: "" kms: foo pgp: bar gcp_kms: baz - vault_uris: http://127.0.1.1/v1/baz/keys/baz + hc_vault_uris: http://127.0.1.1/v1/baz/keys/baz `) var sampleConfigWithPath = []byte(` @@ -62,17 +62,17 @@ creation_rules: kms: "1" pgp: "2" gcp_kms: "3" - vault_uris: http://4:8200/v1/4/keys/4 + hc_vault_uris: http://4:8200/v1/4/keys/4 - path_regex: somefilename.yml kms: bilbo pgp: baggins gcp_kms: precious - vault_uris: https://pluto/v1/pluto/keys/pluto + hc_vault_uris: https://pluto/v1/pluto/keys/pluto - path_regex: "" kms: foo pgp: bar gcp_kms: baz - vault_uris: https://foz:443/v1/foz/keys/foz + hc_vault_uris: https://foz:443/v1/foz/keys/foz `) var sampleConfigWithGroups = []byte(` @@ -92,7 +92,7 @@ creation_rules: - vaultUrl: https://foo.vault.azure.net key: foo-key version: fooversion - vault: + hc_vault: - vault_address: https://foo.vault:8200 backend_path: foo key_name: foo-key @@ -107,7 +107,7 @@ creation_rules: - vaultUrl: https://bar.vault.azure.net key: bar-key version: barversion - vault: + hc_vault: - vault_address: https://baz.vault:8200 backend_path: baz key_name: baz-key @@ -148,7 +148,7 @@ creation_rules: - path_regex: foobar* kms: "1" pgp: "2" - vault_uris: "http://vault.com/v1/bug/keys/pr" + hc_vault_uris "http://vault.com/v1/bug/keys/pr" unencrypted_suffix: _unencrypted encrypted_suffix: _enc `) diff --git a/go.mod b/go.mod index d46a679a4..dbb950df5 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/hashicorp/vault/api v1.0.4 github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c github.com/lib/pq v1.2.0 + github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-wordwrap v1.0.0 github.com/mozilla-services/yaml v0.0.0-20191106225358-5c216288813c github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/stores/stores.go b/stores/stores.go index 8c77b775f..dd2bdd94e 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -41,7 +41,7 @@ type Metadata struct { KMSKeys []kmskey `yaml:"kms" json:"kms"` GCPKMSKeys []gcpkmskey `yaml:"gcp_kms" json:"gcp_kms"` AzureKeyVaultKeys []azkvkey `yaml:"azure_kv" json:"azure_kv"` - VaultKeys []vaultkey `yaml:"vault" json:"vault"` + VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault"` LastModified string `yaml:"lastmodified" json:"lastmodified"` MessageAuthenticationCode string `yaml:"mac" json:"mac"` PGPKeys []pgpkey `yaml:"pgp" json:"pgp"` @@ -56,7 +56,7 @@ type keygroup struct { KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"` GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"` AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"` - VaultKeys []vaultkey `yaml:"vault" json:"vault"` + VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault"` } type pgpkey struct { diff --git a/vault/keysource.go b/vault/keysource.go index a793f90d7..49cb1e8c3 100644 --- a/vault/keysource.go +++ b/vault/keysource.go @@ -1,19 +1,24 @@ package vault import ( + "bytes" "encoding/base64" "fmt" + "io" "net/url" + "os" + "path" "path/filepath" "strings" "time" - "go.mozilla.org/sops/v3/logging" + "github.com/hashicorp/vault/api" + homedir "github.com/mitchellh/go-homedir" "github.com/sirupsen/logrus" + "go.mozilla.org/sops/v3/logging" ) - var log *logrus.Logger func init() { @@ -72,22 +77,22 @@ func NewMasterKeyFromURI(uri string) (*MasterKey, error) { } -func getBackendAndKeyFromPath(path string) (string, string, error) { - path = strings.TrimPrefix(path, "/") - path = strings.TrimSuffix(path, "/") - values := strings.Split(path, "/") - // minimum lenght should be 4 "v1", "transit", "keys", "keyName" +func getBackendAndKeyFromPath(fullPath string) (string, string, error) { + fullPath = strings.TrimPrefix(fullPath, "/") + fullPath = strings.TrimSuffix(fullPath, "/") + values := strings.Split(fullPath, "/") + // minimum length should be 4 "v1", "transit", "keys", "keyName" if len(values) < 4 { return "", "", fmt.Errorf("The path to the key is not long enough: (eg. https://vault.example.com:8200/v1/transit/keys/keyName") } - log.Debugf("Path: %v \nValues: %v\n", path, values) + log.Debugf("Path: %v \nValues: %v\n", fullPath, values) if values[0] != "v1" { return "", "", fmt.Errorf("probably forgot v1 in the URI") } if values[len(values)-2] != "keys" { return "", "", fmt.Errorf("probably forgot 'keys' in the URI") } - return filepath.Join(values[1 : len(values)-2]...), values[len(values)-1], nil + return path.Join(values[1 : len(values)-2]...), values[len(values)-1], nil } // NewMasterKey creates a new MasterKey from a vault address, transit backend path and a key name and setting the creation date to the current date @@ -112,33 +117,62 @@ func (key *MasterKey) SetEncryptedDataKey(enc []byte) { key.EncryptedKey = string(enc) } -// Encrypt takes a sops data key, encrypts it with Vault Transit and stores the result in the EncryptedKey field -func (key *MasterKey) Encrypt(dataKey []byte) error { - path := filepath.Join(key.BackendPath, "encrypt", key.KeyName) +func vaultClient(address string) (*api.Client, error) { cfg := api.DefaultConfig() - cfg.Address = key.VaultAddress + cfg.Address = address cli, err := api.NewClient(cfg) if err != nil { - log.WithField("Path", path).Info("Vault connections failed") - return fmt.Errorf("Cannot create Vault Client: %v", err) + return nil, fmt.Errorf("Cannot create Vault Client: %v", err) + } + if cli.Token() != "" { + return cli, nil + } + homePath, err := homedir.Dir() + if err != nil { + panic(fmt.Sprintf("error getting user's home directory: %v", err)) + } + tokenPath := filepath.Join(homePath, ".vault-token") + f, err := os.Open(tokenPath) + if os.IsNotExist(err) { + return cli, nil + } + if err != nil { + return nil, err + } + defer f.Close() + + buf := bytes.NewBuffer(nil) + if _, err := io.Copy(buf, f); err != nil { + return nil, err + } + cli.SetToken(strings.TrimSpace(buf.String())) + return cli, nil +} + +// Encrypt takes a sops data key, encrypts it with Vault Transit and stores the result in the EncryptedKey field +func (key *MasterKey) Encrypt(dataKey []byte) error { + fullPath := path.Join(key.BackendPath, "encrypt", key.KeyName) + cli, err := vaultClient(key.VaultAddress) + if err != nil { + return err } encoded := base64.StdEncoding.EncodeToString(dataKey) payload := make(map[string]interface{}) payload["plaintext"] = encoded - raw, err := cli.Logical().Write(path, payload) + raw, err := cli.Logical().Write(fullPath, payload) if err != nil { - log.WithField("Path", path).Info("Encryption failed") + log.WithField("Path", fullPath).Info("Encryption failed") return err } if raw == nil || raw.Data == nil { - return fmt.Errorf("The transit backend %s is empty", path) + return fmt.Errorf("The transit backend %s is empty", fullPath) } encrypted, ok := raw.Data["ciphertext"] - if ok != true { + if !ok { return fmt.Errorf("there's not encrypted data") } encryptedKey, ok := encrypted.(string) - if ok != true { + if !ok { return fmt.Errorf("the ciphertext cannot be casted to string") } key.EncryptedKey = encryptedKey @@ -153,25 +187,22 @@ func (key *MasterKey) EncryptIfNeeded(dataKey []byte) error { return nil } -// Decrypt decrypts the EncryptedKey field with CGP KMS and returns the result. +// Decrypt decrypts the EncryptedKey field with Vault Transit and returns the result. func (key *MasterKey) Decrypt() ([]byte, error) { - path := filepath.Join(key.BackendPath, "decrypt", key.KeyName) - cfg := api.DefaultConfig() - cfg.Address = key.VaultAddress - cli, err := api.NewClient(cfg) + fullPath := path.Join(key.BackendPath, "decrypt", key.KeyName) + cli, err := vaultClient(key.VaultAddress) if err != nil { - log.WithField("Path", path).Info("Vault connections failed") - return nil, fmt.Errorf("Cannot create Vault Client: %v", err) + return nil, err } payload := make(map[string]interface{}) payload["ciphertext"] = key.EncryptedKey - raw, err := cli.Logical().Write(path, payload) + raw, err := cli.Logical().Write(fullPath, payload) if err != nil { - log.WithField("Path", path).Info("Encryption failed") + log.WithField("Path", fullPath).Info("Encryption failed") return nil, err } if raw == nil || raw.Data == nil { - return nil, fmt.Errorf("The transit backend %s is empty", path) + return nil, fmt.Errorf("The transit backend %s is empty", fullPath) } decrypted, ok := raw.Data["plaintext"] if ok != true { @@ -202,9 +233,10 @@ func (key *MasterKey) ToString() string { } func (key *MasterKey) createVaultTransitAndKey() error { - cfg := api.DefaultConfig() - cfg.Address = key.VaultAddress - cli, err := api.NewClient(cfg) + cli, err := vaultClient(key.VaultAddress) + if err != nil { + return err + } if err != nil { return fmt.Errorf("Cannot create Vault Client: %v", err) } @@ -215,7 +247,7 @@ func (key *MasterKey) createVaultTransitAndKey() error { if err != nil { return err } - path := filepath.Join(key.BackendPath, "keys", key.KeyName) + path := path.Join(key.BackendPath, "keys", key.KeyName) payload := make(map[string]interface{}) payload["type"] = "rsa-4096" _, err = cli.Logical().Write(path, payload) diff --git a/vault/keysource_test.go b/vault/keysource_test.go index 2a0ea3c3d..5de02abf9 100644 --- a/vault/keysource_test.go +++ b/vault/keysource_test.go @@ -20,14 +20,13 @@ func TestMain(m *testing.M) { } // pulls an image, creates a container based on it and runs it - resource, err := pool.Run("vault", "1.1.3", []string{"VAULT_DEV_ROOT_TOKEN_ID=secret"}) + resource, err := pool.Run("vault", "1.2.2", []string{"VAULT_DEV_ROOT_TOKEN_ID=secret"}) if err != nil { logger.Fatalf("Could not start resource: %s", err) } os.Setenv("VAULT_ADDR", fmt.Sprintf("http://127.0.0.1:%v", resource.GetPort("8200/tcp"))) os.Setenv("VAULT_TOKEN", "secret") - // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { cli, err := api.NewClient(api.DefaultConfig()) From b5a892835e9819e57bfe7f5f59addc3c648a84ba Mon Sep 17 00:00:00 2001 From: gitirabassi Date: Wed, 4 Sep 2019 10:08:51 +0200 Subject: [PATCH 04/16] fix(test): typo while rebasing --- config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config_test.go b/config/config_test.go index 704c48a2c..a1e41fe01 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -148,7 +148,7 @@ creation_rules: - path_regex: foobar* kms: "1" pgp: "2" - hc_vault_uris "http://vault.com/v1/bug/keys/pr" + hc_vault_uris: "https://vault.com/v1/bug/keys/pr" unencrypted_suffix: _unencrypted encrypted_suffix: _enc `) From 925636116261c845c66d36e47e9376c7ce9591d2 Mon Sep 17 00:00:00 2001 From: ldue Date: Wed, 5 Feb 2020 19:44:27 +0100 Subject: [PATCH 05/16] fix typos and imporve error messages for vault kms --- README.rst | 6 +++--- vault/keysource.go | 20 ++++++------------- vault_example.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 17 deletions(-) create mode 100644 vault_example.yml diff --git a/README.rst b/README.rst index a9b16d84c..687de1dc0 100644 --- a/README.rst +++ b/README.rst @@ -324,11 +324,11 @@ To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) Cluster ID e532e461-e8f0-1352-8a41-fc7c11096908 HA Enabled false - $ # We need to enable a transit engine if not already done (I prefer to create a transit engine specifically for sops, in which I can have multiple keys with various permission levels) + $ # It is required to enable a transit engine if not already done (It is suggested to create a transit engine specifically for sops, in which it is possible to have multiple keys with various permission levels) $ vault secrets enable -path=sops transit - Success! Enabled the transit secrets engine at: sops/ + Success! Enabled thˇe transit secrets engine at: sops/ - $ # Then we need to create one or more keys + $ # Then create one or more keys $ vault write sops/keys/firstkey type=rsa-4096 Success! Data written to: sops/keys/firstkey diff --git a/vault/keysource.go b/vault/keysource.go index 49cb1e8c3..893c57630 100644 --- a/vault/keysource.go +++ b/vault/keysource.go @@ -9,16 +9,16 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "time" - - "github.com/hashicorp/vault/api" homedir "github.com/mitchellh/go-homedir" "github.com/sirupsen/logrus" "go.mozilla.org/sops/v3/logging" ) + var log *logrus.Logger func init() { @@ -66,7 +66,7 @@ func NewMasterKeyFromURI(uri string) (*MasterKey, error) { return nil, err } if u.Scheme == "" { - return nil, fmt.Errorf("missing scheme in vault URL (should be like this: https://vault.example.com:8200/v1/transit/keys/keyName, got: %v", uri) + return nil, fmt.Errorf("missing scheme in vault URL (should be like this: https://vault.example.com:8200/v1/transit/keys/keyName), got: %v", uri) } backendPath, keyName, err := getBackendAndKeyFromPath(u.EscapedPath()) if err != nil { @@ -78,20 +78,12 @@ func NewMasterKeyFromURI(uri string) (*MasterKey, error) { } func getBackendAndKeyFromPath(fullPath string) (string, string, error) { + if re := regexp.MustCompile(`https://*/v[\d]+/.*/.*/.*`); re.Match([]byte(fullPath)) { + return "", "", fmt.Errorf("Vault path does not seem to be formatted correctly: (eg. https://vault.example.com:8200/v1/transit/keys/keyName)") + } fullPath = strings.TrimPrefix(fullPath, "/") fullPath = strings.TrimSuffix(fullPath, "/") values := strings.Split(fullPath, "/") - // minimum length should be 4 "v1", "transit", "keys", "keyName" - if len(values) < 4 { - return "", "", fmt.Errorf("The path to the key is not long enough: (eg. https://vault.example.com:8200/v1/transit/keys/keyName") - } - log.Debugf("Path: %v \nValues: %v\n", fullPath, values) - if values[0] != "v1" { - return "", "", fmt.Errorf("probably forgot v1 in the URI") - } - if values[len(values)-2] != "keys" { - return "", "", fmt.Errorf("probably forgot 'keys' in the URI") - } return path.Join(values[1 : len(values)-2]...), values[len(values)-1], nil } diff --git a/vault_example.yml b/vault_example.yml new file mode 100644 index 000000000..c9a9f8b33 --- /dev/null +++ b/vault_example.yml @@ -0,0 +1,48 @@ +hello: ENC[AES256_GCM,data:1DWfYHFHwafRZSOSuMy8rHvF3Da+9NFjcwPdFPmepZl2CKHL76L38CC5H2NzIQ==,iv:AhSDXStvMXp6ulPwFWW8+izJo6l5Zem9Xh6+I7/GKys=,tag:+nkbrSzLFm1VXiggb5GknA==,type:str] +example_key: ENC[AES256_GCM,data:DuMgy5C7ogPbVnj/AQ==,iv:1HVbU5Au93VZBLZeptcVwNokRRzjEPuQDuotMjd2XGg=,tag:XLXfZBMir549CB63cPXWRg==,type:str] +#ENC[AES256_GCM,data:Im8AtphMHRR1J/mJUAkK+w==,iv:Q0SbH5WWK1RocH2Ey9dm31r0HErnpX3O9SadovEbkyM=,tag:e5C9Z1lxGRqyjJ1egK5j+A==,type:comment] +example_array: +- ENC[AES256_GCM,data:afrijaaN1uwHthqGwiM=,iv:s6uHtm+pvWn41dgHc+CjpOEeRJNU8j/XO1a94+Tq014=,tag:1ZYd5E5kVYr4uzbrJUIcdg==,type:str] +- ENC[AES256_GCM,data:v0EwR5O95NqWiE9JTdE=,iv:kjngmi9ovbo/I2fq+4qosmXPEEbnhGbUrlWs5QjfSW8=,tag:hEe8Pg9Jh7VO2gS0Xs3/dA==,type:str] +- ENC[AES256_GCM,data:ozYm/g==,iv:XlJn1kpL4einNppyepr5Cf6GyqQ+UGBINUweTo8Bv0g=,tag:i4aSUTa+5TWizRfDPfisgg==,type:str] +example_number: ENC[AES256_GCM,data:usW8VsUKCodC,iv:rV+wC5EhQkrKOEP1A7fRvspd5Ppi1fAAD7hO/4RPyNs=,tag:O+U01BGeSLas7c27smDY5w==,type:float] +example_booleans: +- ENC[AES256_GCM,data:qvuTxg==,iv:aXdvSBMZazOv3njuo7nOISH7zehTL996xeA2TEiNZ1o=,tag:rNUfPPcoTOnA4zDEvjNxdQ==,type:bool] +- ENC[AES256_GCM,data:nD776fA=,iv:bHginidE6+txuTqMkq3WWfX7zqunVrTQaxRb1t4Pur8=,tag:cdXibaSD2iky2MpWYHa0ow==,type:bool] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + lastmodified: '2020-02-05T18:40:22Z' + mac: ENC[AES256_GCM,data:/umyrmP04uBAWGEo/S//XZbJRqiu+EwEus9aalwRkbDTqKIh6/DlVEEd4rehTtV2fNaoGn7YGMU/xpAD6YVXfe5MRIPvcIZtO0tRqJUyyhBzjdJ+3OTn33hIe41YzRLttv5dYYpPMwCUFKlc9FpgCYRCB5S3gV/FpeVhLa4vX84=,iv:RyN17tAExRbINcysGxWVHnofdFP/wsZNCJGGOd3riMw=,tag:datuK7eC/MBNHe75m2BQYA==,type:str] + pgp: + - created_at: '2020-02-05T18:40:03Z' + enc: |- + -----BEGIN PGP MESSAGE----- + + wcBMAyUpShfNkFB/AQgAbQcnbA+jcSICvLXcMQxHQAmUQMuzqlVrJtbIBs5Ozzaa + xhH80ZEEe4TWiTBbR5rp03xNFAwZxvEkHYiTmHzXciXLEFb94ffsF/oHEwrWcjt7 + EVOBA0Zpg1wg19X1J74J1VgKRa0Sv56eucN29Gt5BtdbPFOVbDEsIzA8wJ717CsM + 5KGptg7WM5r5AKUdHp9p5GlY+kqPJEs7C0O1CBsqFKh99JSS7Ly8PO4p1dmKHSt8 + N7xeBjlIc8OmBvTTeDqoRNY3mDsWO3EmbTXjc5pwO0NGe9D6rusmiszzZi15vyoH + rCglG6nme+XTjcEbEJy9I8EV5cTW139B9VIp79tGidLgAeSN/9ckLYjoh2Jj3MrZ + 4Hue4fJg4A7g1OFeD+D/4gMzFIPghOVMRZUXyyUgI9/TFzMBLzxr94rfLwUScQd6 + MDeQzQINnODg5HZAKmAE+gTZMgiZCJxDgdjiggzUg+GNEQA= + =QwHF + -----END PGP MESSAGE----- + fp: FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4 + - created_at: '2020-02-05T18:40:03Z' + enc: | + -----BEGIN PGP MESSAGE----- + + hIwDXFUltYFwV4MBA/9Vx8OisRoOxtVCwTQ2WUHJFy0JecQB/XownJR1uBPfvV/T + RJRnIkmoJzgEHTtGv3Tapy7+byBeaoJ9idEmOH8+JzAu8Enmd9PXGI/2jpm6Od+U + gfe7no6wuj1URiTsfHpOBZG7fFfz226GXkACqJj/gCU+pcZqxrzyEbxtbGba/dJe + AfcMYol++hNwd37dyuJRhon99Mu1dB35okJ3wGsOmqAyzfe0mwx1yMCOysOnwu3e + 8rgiMOJAvxjOaZquXyULCQsmIv/QuQzTL79x1L7rSrRl4vg/C3P7uS2kpCPScA== + =J4KQ + -----END PGP MESSAGE----- + fp: D7229043384BCC60326C6FB9D8720D957C3D3074 + unencrypted_suffix: _unencrypted + version: 3.5.0 From 5350855c5b697d9edb9594ce4f143c26291994f0 Mon Sep 17 00:00:00 2001 From: ldue Date: Wed, 5 Feb 2020 20:08:19 +0100 Subject: [PATCH 06/16] rename package from vault to hcvault --- cmd/sops/main.go | 10 +++++----- config/config.go | 6 +++--- go.sum | 4 ++++ {vault => hcvault}/keysource.go | 2 +- {vault => hcvault}/keysource_test.go | 2 +- keyservice/keyservice.go | 8 ++++---- keyservice/server.go | 6 +++--- stores/stores.go | 8 ++++---- 8 files changed, 25 insertions(+), 21 deletions(-) rename {vault => hcvault}/keysource.go (99%) rename {vault => hcvault}/keysource_test.go (99%) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index fb62d3936..d1599e123 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -26,6 +26,7 @@ import ( "go.mozilla.org/sops/v3/cmd/sops/subcommand/updatekeys" "go.mozilla.org/sops/v3/config" "go.mozilla.org/sops/v3/gcpkms" + "go.mozilla.org/sops/v3/hcvault" "go.mozilla.org/sops/v3/keys" "go.mozilla.org/sops/v3/keyservice" "go.mozilla.org/sops/v3/kms" @@ -33,7 +34,6 @@ import ( "go.mozilla.org/sops/v3/pgp" "go.mozilla.org/sops/v3/stores/dotenv" "go.mozilla.org/sops/v3/stores/json" - "go.mozilla.org/sops/v3/vault" "go.mozilla.org/sops/v3/version" "google.golang.org/grpc" "gopkg.in/urfave/cli.v1" @@ -394,7 +394,7 @@ func main() { group = append(group, gcpkms.NewMasterKeyFromResourceID(kms)) } for _, uri := range vaultURIs { - k, err := vault.NewMasterKeyFromURI(uri) + k, err := hcvault.NewMasterKeyFromURI(uri) if err != nil { log.WithError(err).Error("Failed to add key") continue @@ -764,7 +764,7 @@ func main() { for _, k := range azureKeys { addMasterKeys = append(addMasterKeys, k) } - hcVaultKeys, err := vault.NewMasterKeysFromURIs(c.String("add-hc-vault")) + hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("add-hc-vault")) if err != nil { return err } @@ -789,7 +789,7 @@ func main() { for _, k := range azureKeys { rmMasterKeys = append(rmMasterKeys, k) } - hcVaultKeys, err = vault.NewMasterKeysFromURIs(c.String("rm-hc-vault")) + hcVaultKeys, err = hcvault.NewMasterKeysFromURIs(c.String("rm-hc-vault")) if err != nil { return err } @@ -1013,7 +1013,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { } } if c.String("hc-vault") != "" { - hcVaultKeys, err := vault.NewMasterKeysFromURIs(c.String("vault")) + hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("vault")) if err != nil { return nil, err } diff --git a/config/config.go b/config/config.go index 47d4407ea..bf872abd9 100644 --- a/config/config.go +++ b/config/config.go @@ -15,11 +15,11 @@ import ( "go.mozilla.org/sops/v3" "go.mozilla.org/sops/v3/azkv" "go.mozilla.org/sops/v3/gcpkms" + "go.mozilla.org/sops/v3/hcvault" "go.mozilla.org/sops/v3/kms" "go.mozilla.org/sops/v3/logging" "go.mozilla.org/sops/v3/pgp" "go.mozilla.org/sops/v3/publish" - "go.mozilla.org/sops/v3/vault" ) var log *logrus.Logger @@ -164,7 +164,7 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ keyGroup = append(keyGroup, azkv.NewMasterKey(k.VaultURL, k.Key, k.Version)) } for _, k := range group.Vault { - keyGroup = append(keyGroup, vault.NewMasterKey(k.VaultAddress, k.BackendPath, k.KeyName)) + keyGroup = append(keyGroup, hcvault.NewMasterKey(k.VaultAddress, k.BackendPath, k.KeyName)) } groups = append(groups, keyGroup) } @@ -186,7 +186,7 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ for _, k := range azureKeys { keyGroup = append(keyGroup, k) } - vaultKeys, err := vault.NewMasterKeysFromURIs(cRule.Vault) + vaultKeys, err := hcvault.NewMasterKeysFromURIs(cRule.Vault) if err != nil { return nil, err } diff --git a/go.sum b/go.sum index 53d761ab4..937da94cc 100644 --- a/go.sum +++ b/go.sum @@ -43,7 +43,9 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -219,6 +221,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -354,6 +357,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.22.1 h1:/7cs52RnTJmD43s3uxzlq2U7nqVTd/37viQwMrMNlOM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/vault/keysource.go b/hcvault/keysource.go similarity index 99% rename from vault/keysource.go rename to hcvault/keysource.go index 893c57630..9d02f6d10 100644 --- a/vault/keysource.go +++ b/hcvault/keysource.go @@ -1,4 +1,4 @@ -package vault +package hcvault import ( "bytes" diff --git a/vault/keysource_test.go b/hcvault/keysource_test.go similarity index 99% rename from vault/keysource_test.go rename to hcvault/keysource_test.go index 5de02abf9..44c750eb2 100644 --- a/vault/keysource_test.go +++ b/hcvault/keysource_test.go @@ -1,4 +1,4 @@ -package vault +package hcvault import ( "fmt" diff --git a/keyservice/keyservice.go b/keyservice/keyservice.go index 49b7c88dc..f10019b23 100644 --- a/keyservice/keyservice.go +++ b/keyservice/keyservice.go @@ -9,10 +9,10 @@ import ( "go.mozilla.org/sops/v3/azkv" "go.mozilla.org/sops/v3/gcpkms" + "go.mozilla.org/sops/v3/hcvault" "go.mozilla.org/sops/v3/keys" "go.mozilla.org/sops/v3/kms" "go.mozilla.org/sops/v3/pgp" - "go.mozilla.org/sops/v3/vault" ) // KeyFromMasterKey converts a SOPS internal MasterKey to an RPC Key that can be serialized with Protocol Buffers @@ -34,13 +34,13 @@ func KeyFromMasterKey(mk keys.MasterKey) Key { }, }, } - case *vault.MasterKey: + case *hcvault.MasterKey: return Key{ KeyType: &Key_VaultKey{ VaultKey: &VaultKey{ VaultAddress: mk.VaultAddress, - BackendPath: mk.BackendPath, - KeyName: mk.KeyName, + BackendPath: mk.BackendPath, + KeyName: mk.KeyName, }, }, } diff --git a/keyservice/server.go b/keyservice/server.go index 0f3053a57..0981707c2 100644 --- a/keyservice/server.go +++ b/keyservice/server.go @@ -5,9 +5,9 @@ import ( "go.mozilla.org/sops/v3/azkv" "go.mozilla.org/sops/v3/gcpkms" + "go.mozilla.org/sops/v3/hcvault" "go.mozilla.org/sops/v3/kms" "go.mozilla.org/sops/v3/pgp" - "go.mozilla.org/sops/v3/vault" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -63,7 +63,7 @@ func (ks *Server) encryptWithAzureKeyVault(key *AzureKeyVaultKey, plaintext []by } func (ks *Server) encryptWithVault(key *VaultKey, plaintext []byte) ([]byte, error) { - vaultKey := vault.MasterKey{ + vaultKey := hcvault.MasterKey{ VaultAddress: key.VaultAddress, BackendPath: key.BackendPath, KeyName: key.KeyName, @@ -110,7 +110,7 @@ func (ks *Server) decryptWithAzureKeyVault(key *AzureKeyVaultKey, ciphertext []b } func (ks *Server) decryptWithVault(key *VaultKey, ciphertext []byte) ([]byte, error) { - vaultKey := vault.MasterKey{ + vaultKey := hcvault.MasterKey{ VaultAddress: key.VaultAddress, BackendPath: key.BackendPath, KeyName: key.KeyName, diff --git a/stores/stores.go b/stores/stores.go index dd2bdd94e..1980e5ee4 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -17,9 +17,9 @@ import ( "go.mozilla.org/sops/v3" "go.mozilla.org/sops/v3/azkv" "go.mozilla.org/sops/v3/gcpkms" + "go.mozilla.org/sops/v3/hcvault" "go.mozilla.org/sops/v3/kms" "go.mozilla.org/sops/v3/pgp" - "go.mozilla.org/sops/v3/vault" ) // SopsFile is a struct used by the stores as a helper to unmarshal the SOPS metadata @@ -175,7 +175,7 @@ func gcpkmsKeysFromGroup(group sops.KeyGroup) (keys []gcpkmskey) { func vaultKeysFromGroup(group sops.KeyGroup) (keys []vaultkey) { for _, key := range group { switch key := key.(type) { - case *vault.MasterKey: + case *hcvault.MasterKey: keys = append(keys, vaultkey{ VaultAddress: key.VaultAddress, BackendPath: key.BackendPath, @@ -349,12 +349,12 @@ func (azkvKey *azkvkey) toInternal() (*azkv.MasterKey, error) { }, nil } -func (vaultKey *vaultkey) toInternal() (*vault.MasterKey, error) { +func (vaultKey *vaultkey) toInternal() (*hcvault.MasterKey, error) { creationDate, err := time.Parse(time.RFC3339, vaultKey.CreatedAt) if err != nil { return nil, err } - return &vault.MasterKey{ + return &hcvault.MasterKey{ VaultAddress: vaultKey.VaultAddress, BackendPath: vaultKey.BackendPath, KeyName: vaultKey.KeyName, From b68f1f27089f33320b972bbd5eea27afb3db415d Mon Sep 17 00:00:00 2001 From: ldue Date: Mon, 10 Feb 2020 09:33:57 +0100 Subject: [PATCH 07/16] refactor vault keysource url validation --- hcvault/keysource.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/hcvault/keysource.go b/hcvault/keysource.go index 9d02f6d10..262696667 100644 --- a/hcvault/keysource.go +++ b/hcvault/keysource.go @@ -68,7 +68,7 @@ func NewMasterKeyFromURI(uri string) (*MasterKey, error) { if u.Scheme == "" { return nil, fmt.Errorf("missing scheme in vault URL (should be like this: https://vault.example.com:8200/v1/transit/keys/keyName), got: %v", uri) } - backendPath, keyName, err := getBackendAndKeyFromPath(u.EscapedPath()) + backendPath, keyName, err := getBackendAndKeyFromPath(u.RequestURI()) if err != nil { return nil, err } @@ -77,14 +77,23 @@ func NewMasterKeyFromURI(uri string) (*MasterKey, error) { } -func getBackendAndKeyFromPath(fullPath string) (string, string, error) { - if re := regexp.MustCompile(`https://*/v[\d]+/.*/.*/.*`); re.Match([]byte(fullPath)) { +func getBackendAndKeyFromPath(fullPath string) (backendPath, keyName string, err error) { + // Running vault behind a reverse proxy with longer urls seems not to be supported + // by the vault client api so we have a separate Error for that here. + if re := regexp.MustCompile(`/[^/]+/v[\d]+/[^/]+/[^/]+/[^/]+`); re.Match([]byte(fullPath)) { + return "", "", fmt.Errorf("Running Vault with an prefixed url is not supported! (Format has to be like https://vault.example.com:8200/v1/transit/keys/keyName)") + } else if re := regexp.MustCompile(`/v[\d]+/[^/]+/[^/]+/[^/]+`); re.Match([]byte(fullPath)) == false { return "", "", fmt.Errorf("Vault path does not seem to be formatted correctly: (eg. https://vault.example.com:8200/v1/transit/keys/keyName)") } fullPath = strings.TrimPrefix(fullPath, "/") fullPath = strings.TrimSuffix(fullPath, "/") - values := strings.Split(fullPath, "/") - return path.Join(values[1 : len(values)-2]...), values[len(values)-1], nil + + dirs := strings.Split(fullPath, "/") + + keyName = dirs[len(dirs)-1] + backendPath = path.Join(dirs[1 : len(dirs)-2]...) + err = nil + return } // NewMasterKey creates a new MasterKey from a vault address, transit backend path and a key name and setting the creation date to the current date From adf1b2314afafff8440e19a5f33c37cdc2f7dc56 Mon Sep 17 00:00:00 2001 From: ldue Date: Mon, 10 Feb 2020 09:34:34 +0100 Subject: [PATCH 08/16] add negative test cases for vault keysource --- hcvault/keysource_test.go | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/hcvault/keysource_test.go b/hcvault/keysource_test.go index 44c750eb2..6e69ce8de 100644 --- a/hcvault/keysource_test.go +++ b/hcvault/keysource_test.go @@ -106,6 +106,7 @@ func TestNewMasterKeyFromURI(t *testing.T) { uri1 := "https://vault.example.com:8200/v1/transit/keys/keyName" uri2 := "https://vault.me.com/v1/super42/bestmarket/keys/slig" uri3 := "http://127.0.0.1:12121/v1/transit/keys/dev" + mk1 := &MasterKey{ VaultAddress: "https://vault.example.com:8200", BackendPath: "transit", @@ -126,20 +127,42 @@ func TestNewMasterKeyFromURI(t *testing.T) { log.Errorln(err) t.Fail() } + genMk2, err := NewMasterKeyFromURI(uri2) if err != nil { log.Errorln(err) t.Fail() } + genMk3, err := NewMasterKeyFromURI(uri3) if err != nil { log.Errorln(err) t.Fail() } - mk1.CreationDate = genMk1.CreationDate - mk2.CreationDate = genMk2.CreationDate - mk3.CreationDate = genMk3.CreationDate - assert.Equal(t, mk1, genMk1) - assert.Equal(t, mk2, genMk2) - assert.Equal(t, mk3, genMk3) + + if assert.NotNil(t, genMk1) { + mk1.CreationDate = genMk1.CreationDate + assert.Equal(t, mk1, genMk1) + } + if assert.NotNil(t, genMk2) { + mk2.CreationDate = genMk2.CreationDate + assert.Equal(t, mk2, genMk2) + } + if assert.NotNil(t, genMk3) { + mk3.CreationDate = genMk3.CreationDate + assert.Equal(t, mk3, genMk3) + } + + badURIs := []string{ + "vault.me/keys/dev/mykey", + "http://127.0.0.1:12121/v1/keys/dev", + "tcp://127.0.0.1:12121/v1/keys/dev", + } + for _, uri := range badURIs { + if _, err = NewMasterKeyFromURI(uri); err == nil { + log.Errorf("Should be a invalid uri: %s", uri) + t.Fail() + } + } + } From 086b91fd6d4eb85c6f793710ce7eb5a1b7424d18 Mon Sep 17 00:00:00 2001 From: ldue Date: Wed, 26 Feb 2020 14:12:24 +0100 Subject: [PATCH 09/16] add hc vault transit config option via objects additional to URIs --- README.rst | 6 +++--- cmd/sops/main.go | 22 +++++++++++----------- config/config.go | 8 ++++++-- config/config_test.go | 29 +++++++++++++++++++++++++---- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 687de1dc0..1f1ec2f32 100644 --- a/README.rst +++ b/README.rst @@ -338,14 +338,14 @@ To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) $ vault write sops/keys/thirdkey type=chacha20-poly1305 Success! Data written to: sops/keys/thirdkey - $ sops --hc-vault $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml + $ sops --hc-vault-transit $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml $ cat <.sops.yaml creation_rules: - path_regex: \.dev\.yaml$ - hc_vault_uris: "$VAULT_ADDR/v1/sops/keys/secondkey" + hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/secondkey" - \.prod\.yaml$ - hc_vault_uris: "$VAULT_ADDR/v1/sops/keys/thirdkey" + hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/thirdkey" EOF $ sops --verbose -e prod/raw.yaml > prod/encrypted.yaml diff --git a/cmd/sops/main.go b/cmd/sops/main.go index d1599e123..1d62fab82 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -81,7 +81,7 @@ func main() { To encrypt or decrypt a document with HashiCorp Vault's Transit Secret Engine, specify the - Vault key URI name in the --hc-vault flag or in the SOPS_VAULT_URIS (eg. https://vault.example.org:8200/v1/transit/keys/dev + Vault key URI name in the --hc-vault-transit flag or in the SOPS_VAULT_URIS (eg. https://vault.example.org:8200/v1/transit/keys/dev where 'https://vault.example.org:8200' is the vault server, 'transit' the backendPath, and 'dev' is the name of the key ) environment variable. (you need to enable the Transit Secrets Engine in Vault. See @@ -100,11 +100,11 @@ func main() { To use multiple KMS or PGP keys, separate them by commas. For example: $ sops -p "10F2...0A, 85D...B3F21" file.yaml - The -p, -k, --gcp-kms, --hc-vault and --azure-kv flags are only used to encrypt new documents. Editing + The -p, -k, --gcp-kms, --hc-vault-transit and --azure-kv flags are only used to encrypt new documents. Editing or decrypting existing documents can be done with "sops file" or "sops -d file" respectively. The KMS and PGP keys listed in the encrypted documents are used then. To manage master keys in existing documents, use - the "add-{kms,pgp,gcp-kms,azure-kv,hc-vault}" and "rm-{kms,pgp,gcp-kms,azure-kv,hc-vault}" flags. + the "add-{kms,pgp,gcp-kms,azure-kv,hc-vault-transit}" and "rm-{kms,pgp,gcp-kms,azure-kv,hc-vault-transit}" flags. To use a different GPG binary than the one in your PATH, set SOPS_GPG_EXEC. To use a GPG key server other than gpg.mozilla.org, set SOPS_GPG_KEYSERVER. @@ -524,7 +524,7 @@ func main() { EnvVar: "SOPS_AZURE_KEYVAULT_URLS", }, cli.StringFlag{ - Name: "hc-vault", + Name: "hc-vault--transit", Usage: "comma separated list of vault's key URI (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev')", EnvVar: "SOPS_VAULT_URIS", }, @@ -578,11 +578,11 @@ func main() { Usage: "remove the provided comma-separated list of KMS ARNs from the list of master keys on the given file", }, cli.StringFlag{ - Name: "add-hc-vault", + Name: "add-hc-vault-transit", Usage: "add the provided comma-separated list of Vault's URI key to the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)", }, cli.StringFlag{ - Name: "rm-hc-vault", + Name: "rm-hc-vault-transit", Usage: "remove the provided comma-separated list of Vault's URI key from the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)", }, cli.StringFlag{ @@ -650,8 +650,8 @@ func main() { return toExitError(err) } if _, err := os.Stat(fileName); os.IsNotExist(err) { - if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-hc-vault") != "" || c.String("add-azure-kv") != "" || - c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-hc-vault") != "" || c.String("rm-azure-kv") != "" { + if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-hc-vault-transit") != "" || c.String("add-azure-kv") != "" || + c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-hc-vault-transit") != "" || c.String("rm-azure-kv") != "" { return common.NewExitError("Error: cannot add or remove keys on non-existent files, use `--kms` and `--pgp` instead.", codes.CannotChangeKeysFromNonExistentFile) } if c.Bool("encrypt") || c.Bool("decrypt") || c.Bool("rotate") { @@ -764,7 +764,7 @@ func main() { for _, k := range azureKeys { addMasterKeys = append(addMasterKeys, k) } - hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("add-hc-vault")) + hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("add-hc-vault-transit")) if err != nil { return err } @@ -789,7 +789,7 @@ func main() { for _, k := range azureKeys { rmMasterKeys = append(rmMasterKeys, k) } - hcVaultKeys, err = hcvault.NewMasterKeysFromURIs(c.String("rm-hc-vault")) + hcVaultKeys, err = hcvault.NewMasterKeysFromURIs(c.String("rm-hc-vault-transit")) if err != nil { return err } @@ -1012,7 +1012,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { azkvKeys = append(azkvKeys, k) } } - if c.String("hc-vault") != "" { + if c.String("hc-vault-transit") != "" { hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("vault")) if err != nil { return nil, err diff --git a/config/config.go b/config/config.go index bf872abd9..e13583aa6 100644 --- a/config/config.go +++ b/config/config.go @@ -118,7 +118,8 @@ type creationRule struct { PGP string GCPKMS string `yaml:"gcp_kms"` AzureKeyVault string `yaml:"azure_keyvault"` - Vault string `yaml:"hc_vault_uris"` + VaultURI string `yaml:"hc_vault_transit_uri"` + Vault []vaultKey `yaml:"hc_vault_transit"` KeyGroups []keyGroup `yaml:"key_groups"` ShamirThreshold int `yaml:"shamir_threshold"` UnencryptedSuffix string `yaml:"unencrypted_suffix"` @@ -186,13 +187,16 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ for _, k := range azureKeys { keyGroup = append(keyGroup, k) } - vaultKeys, err := hcvault.NewMasterKeysFromURIs(cRule.Vault) + vaultKeys, err := hcvault.NewMasterKeysFromURIs(cRule.VaultURI) if err != nil { return nil, err } for _, k := range vaultKeys { keyGroup = append(keyGroup, k) } + for _, k := range cRule.Vault { + keyGroup = append(keyGroup, hcvault.NewMasterKey(k.VaultAddress, k.BackendPath, k.KeyName)) + } groups = append(groups, keyGroup) } return groups, nil diff --git a/config/config_test.go b/config/config_test.go index a1e41fe01..61f753674 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -48,12 +48,20 @@ creation_rules: kms: "1" pgp: "2" gcp_kms: "3" - hc_vault_uris: http://4:8200/v1/4/keys/4 + hc_vault_transit_uri: http://4:8200/v1/4/keys/4 - path_regex: "" kms: foo pgp: bar gcp_kms: baz - hc_vault_uris: http://127.0.1.1/v1/baz/keys/baz + hc_vault_transit_uri: http://127.0.1.1/v1/baz/keys/baz + - path_regex: "" + kms: one + pgp: two + gcp_kms: three + hc_vault_transit: + - vault_address: http://127.0.1.1 + backend_path: keys + key_name: baz `) var sampleConfigWithPath = []byte(` @@ -209,14 +217,27 @@ func TestLoadConfigFile(t *testing.T) { KMS: "1", PGP: "2", GCPKMS: "3", - Vault: "http://4:8200/v1/4/keys/4", + VaultURI: "http://4:8200/v1/4/keys/4", }, { PathRegex: "", KMS: "foo", PGP: "bar", GCPKMS: "baz", - Vault: "http://127.0.1.1/v1/baz/keys/baz", + VaultURI: "http://127.0.1.1/v1/baz/keys/baz", + }, + { + PathRegex: "", + KMS: "one", + PGP: "two", + GCPKMS: "three", + Vault: []vaultKey{ + vaultKey{ + VaultAddress: "http://127.0.1.1", + BackendPath: "keys", + KeyName: "baz", + }, + }, }, }, } From d6885b309c9c77d383ef1e3d1c4d726659930d14 Mon Sep 17 00:00:00 2001 From: ldue Date: Wed, 26 Feb 2020 14:12:39 +0100 Subject: [PATCH 10/16] remove vault_example.yml --- vault_example.yml | 48 ----------------------------------------------- 1 file changed, 48 deletions(-) delete mode 100644 vault_example.yml diff --git a/vault_example.yml b/vault_example.yml deleted file mode 100644 index c9a9f8b33..000000000 --- a/vault_example.yml +++ /dev/null @@ -1,48 +0,0 @@ -hello: ENC[AES256_GCM,data:1DWfYHFHwafRZSOSuMy8rHvF3Da+9NFjcwPdFPmepZl2CKHL76L38CC5H2NzIQ==,iv:AhSDXStvMXp6ulPwFWW8+izJo6l5Zem9Xh6+I7/GKys=,tag:+nkbrSzLFm1VXiggb5GknA==,type:str] -example_key: ENC[AES256_GCM,data:DuMgy5C7ogPbVnj/AQ==,iv:1HVbU5Au93VZBLZeptcVwNokRRzjEPuQDuotMjd2XGg=,tag:XLXfZBMir549CB63cPXWRg==,type:str] -#ENC[AES256_GCM,data:Im8AtphMHRR1J/mJUAkK+w==,iv:Q0SbH5WWK1RocH2Ey9dm31r0HErnpX3O9SadovEbkyM=,tag:e5C9Z1lxGRqyjJ1egK5j+A==,type:comment] -example_array: -- ENC[AES256_GCM,data:afrijaaN1uwHthqGwiM=,iv:s6uHtm+pvWn41dgHc+CjpOEeRJNU8j/XO1a94+Tq014=,tag:1ZYd5E5kVYr4uzbrJUIcdg==,type:str] -- ENC[AES256_GCM,data:v0EwR5O95NqWiE9JTdE=,iv:kjngmi9ovbo/I2fq+4qosmXPEEbnhGbUrlWs5QjfSW8=,tag:hEe8Pg9Jh7VO2gS0Xs3/dA==,type:str] -- ENC[AES256_GCM,data:ozYm/g==,iv:XlJn1kpL4einNppyepr5Cf6GyqQ+UGBINUweTo8Bv0g=,tag:i4aSUTa+5TWizRfDPfisgg==,type:str] -example_number: ENC[AES256_GCM,data:usW8VsUKCodC,iv:rV+wC5EhQkrKOEP1A7fRvspd5Ppi1fAAD7hO/4RPyNs=,tag:O+U01BGeSLas7c27smDY5w==,type:float] -example_booleans: -- ENC[AES256_GCM,data:qvuTxg==,iv:aXdvSBMZazOv3njuo7nOISH7zehTL996xeA2TEiNZ1o=,tag:rNUfPPcoTOnA4zDEvjNxdQ==,type:bool] -- ENC[AES256_GCM,data:nD776fA=,iv:bHginidE6+txuTqMkq3WWfX7zqunVrTQaxRb1t4Pur8=,tag:cdXibaSD2iky2MpWYHa0ow==,type:bool] -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - lastmodified: '2020-02-05T18:40:22Z' - mac: ENC[AES256_GCM,data:/umyrmP04uBAWGEo/S//XZbJRqiu+EwEus9aalwRkbDTqKIh6/DlVEEd4rehTtV2fNaoGn7YGMU/xpAD6YVXfe5MRIPvcIZtO0tRqJUyyhBzjdJ+3OTn33hIe41YzRLttv5dYYpPMwCUFKlc9FpgCYRCB5S3gV/FpeVhLa4vX84=,iv:RyN17tAExRbINcysGxWVHnofdFP/wsZNCJGGOd3riMw=,tag:datuK7eC/MBNHe75m2BQYA==,type:str] - pgp: - - created_at: '2020-02-05T18:40:03Z' - enc: |- - -----BEGIN PGP MESSAGE----- - - wcBMAyUpShfNkFB/AQgAbQcnbA+jcSICvLXcMQxHQAmUQMuzqlVrJtbIBs5Ozzaa - xhH80ZEEe4TWiTBbR5rp03xNFAwZxvEkHYiTmHzXciXLEFb94ffsF/oHEwrWcjt7 - EVOBA0Zpg1wg19X1J74J1VgKRa0Sv56eucN29Gt5BtdbPFOVbDEsIzA8wJ717CsM - 5KGptg7WM5r5AKUdHp9p5GlY+kqPJEs7C0O1CBsqFKh99JSS7Ly8PO4p1dmKHSt8 - N7xeBjlIc8OmBvTTeDqoRNY3mDsWO3EmbTXjc5pwO0NGe9D6rusmiszzZi15vyoH - rCglG6nme+XTjcEbEJy9I8EV5cTW139B9VIp79tGidLgAeSN/9ckLYjoh2Jj3MrZ - 4Hue4fJg4A7g1OFeD+D/4gMzFIPghOVMRZUXyyUgI9/TFzMBLzxr94rfLwUScQd6 - MDeQzQINnODg5HZAKmAE+gTZMgiZCJxDgdjiggzUg+GNEQA= - =QwHF - -----END PGP MESSAGE----- - fp: FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4 - - created_at: '2020-02-05T18:40:03Z' - enc: | - -----BEGIN PGP MESSAGE----- - - hIwDXFUltYFwV4MBA/9Vx8OisRoOxtVCwTQ2WUHJFy0JecQB/XownJR1uBPfvV/T - RJRnIkmoJzgEHTtGv3Tapy7+byBeaoJ9idEmOH8+JzAu8Enmd9PXGI/2jpm6Od+U - gfe7no6wuj1URiTsfHpOBZG7fFfz226GXkACqJj/gCU+pcZqxrzyEbxtbGba/dJe - AfcMYol++hNwd37dyuJRhon99Mu1dB35okJ3wGsOmqAyzfe0mwx1yMCOysOnwu3e - 8rgiMOJAvxjOaZquXyULCQsmIv/QuQzTL79x1L7rSrRl4vg/C3P7uS2kpCPScA== - =J4KQ - -----END PGP MESSAGE----- - fp: D7229043384BCC60326C6FB9D8720D957C3D3074 - unencrypted_suffix: _unencrypted - version: 3.5.0 From 3372156cf67641115aff2e3dbdabb38c18a2b4e0 Mon Sep 17 00:00:00 2001 From: ldue Date: Wed, 26 Feb 2020 14:17:16 +0100 Subject: [PATCH 11/16] streamline key name to snake case --- hcvault/keysource.go | 2 +- hcvault/keysource_test.go | 2 +- stores/stores.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hcvault/keysource.go b/hcvault/keysource.go index 262696667..382405128 100644 --- a/hcvault/keysource.go +++ b/hcvault/keysource.go @@ -266,7 +266,7 @@ func (key *MasterKey) createVaultTransitAndKey() error { func (key MasterKey) ToMap() map[string]interface{} { out := make(map[string]interface{}) out["vault_address"] = key.VaultAddress - out["keyname"] = key.KeyName + out["key_name"] = key.KeyName out["backend_path"] = key.BackendPath out["enc"] = key.EncryptedKey out["created_at"] = key.CreationDate.UTC().Format(time.RFC3339) diff --git a/hcvault/keysource_test.go b/hcvault/keysource_test.go index 6e69ce8de..9f12748fc 100644 --- a/hcvault/keysource_test.go +++ b/hcvault/keysource_test.go @@ -74,7 +74,7 @@ func TestKeyToMap(t *testing.T) { assert.Equal(t, map[string]interface{}{ "vault_address": "http://127.0.0.1:8200", "backend_path": "foo", - "keyname": "bar", + "key_name": "bar", "enc": "this is encrypted", "created_at": "2016-10-31T10:00:00Z", }, key.ToMap()) diff --git a/stores/stores.go b/stores/stores.go index 1980e5ee4..07d78779b 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -83,7 +83,7 @@ type gcpkmskey struct { type vaultkey struct { VaultAddress string `yaml:"vault_address" json:"vault_address"` BackendPath string `yaml:"backend_path" json:"backend_path"` - KeyName string `yaml:"keyname" json:"keyname"` + KeyName string `yaml:"key_name" json:"key_name"` CreatedAt string `yaml:"created_at" json:"created_at"` EncryptedDataKey string `yaml:"enc" json:"enc"` } From 201f6615a0c79e82eb2d676a293b1e88e1eddf91 Mon Sep 17 00:00:00 2001 From: ldue Date: Wed, 26 Feb 2020 14:32:41 +0100 Subject: [PATCH 12/16] rename `BackendPath` to `EnginePath` for hc vault --- cmd/sops/main.go | 2 +- config/config.go | 6 +++--- config/config_test.go | 12 ++++++------ hcvault/keysource.go | 26 +++++++++++++------------- hcvault/keysource_test.go | 12 ++++++------ keyservice/keyservice.go | 2 +- keyservice/keyservice.pb.go | 9 +++++---- keyservice/keyservice.proto | 2 +- keyservice/server.go | 6 +++--- stores/stores.go | 6 +++--- 10 files changed, 42 insertions(+), 41 deletions(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 1d62fab82..fecb7e61c 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -82,7 +82,7 @@ func main() { To encrypt or decrypt a document with HashiCorp Vault's Transit Secret Engine, specify the Vault key URI name in the --hc-vault-transit flag or in the SOPS_VAULT_URIS (eg. https://vault.example.org:8200/v1/transit/keys/dev - where 'https://vault.example.org:8200' is the vault server, 'transit' the backendPath, and 'dev' is the name of the key ) + where 'https://vault.example.org:8200' is the vault server, 'transit' the enginePath, and 'dev' is the name of the key ) environment variable. (you need to enable the Transit Secrets Engine in Vault. See https://www.vaultproject.io/docs/secrets/transit/index.html) diff --git a/config/config.go b/config/config.go index e13583aa6..93f10ec73 100644 --- a/config/config.go +++ b/config/config.go @@ -93,7 +93,7 @@ type azureKVKey struct { type vaultKey struct { VaultAddress string `yaml:"vault_address"` - BackendPath string `yaml:"backend_path"` + EnginePath string `yaml:"engine_path"` KeyName string `yaml:"key_name"` } @@ -165,7 +165,7 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ keyGroup = append(keyGroup, azkv.NewMasterKey(k.VaultURL, k.Key, k.Version)) } for _, k := range group.Vault { - keyGroup = append(keyGroup, hcvault.NewMasterKey(k.VaultAddress, k.BackendPath, k.KeyName)) + keyGroup = append(keyGroup, hcvault.NewMasterKey(k.VaultAddress, k.EnginePath, k.KeyName)) } groups = append(groups, keyGroup) } @@ -195,7 +195,7 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ keyGroup = append(keyGroup, k) } for _, k := range cRule.Vault { - keyGroup = append(keyGroup, hcvault.NewMasterKey(k.VaultAddress, k.BackendPath, k.KeyName)) + keyGroup = append(keyGroup, hcvault.NewMasterKey(k.VaultAddress, k.EnginePath, k.KeyName)) } groups = append(groups, keyGroup) } diff --git a/config/config_test.go b/config/config_test.go index 61f753674..8f56810d1 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -60,7 +60,7 @@ creation_rules: gcp_kms: three hc_vault_transit: - vault_address: http://127.0.1.1 - backend_path: keys + engine_path: keys key_name: baz `) @@ -102,7 +102,7 @@ creation_rules: version: fooversion hc_vault: - vault_address: https://foo.vault:8200 - backend_path: foo + engine_path: foo key_name: foo-key - kms: - arn: baz @@ -117,7 +117,7 @@ creation_rules: version: barversion hc_vault: - vault_address: https://baz.vault:8200 - backend_path: baz + engine_path: baz key_name: baz-key `) @@ -234,7 +234,7 @@ func TestLoadConfigFile(t *testing.T) { Vault: []vaultKey{ vaultKey{ VaultAddress: "http://127.0.1.1", - BackendPath: "keys", + EnginePath: "keys", KeyName: "baz", }, }, @@ -264,7 +264,7 @@ func TestLoadConfigFileWithGroups(t *testing.T) { PGP: []string{"bar"}, GCPKMS: []gcpKmsKey{{ResourceID: "foo"}}, AzureKV: []azureKVKey{{VaultURL: "https://foo.vault.azure.net", Key: "foo-key", Version: "fooversion"}}, - Vault: []vaultKey{{VaultAddress: "https://foo.vault:8200", BackendPath: "foo", KeyName: "foo-key"}}, + Vault: []vaultKey{{VaultAddress: "https://foo.vault:8200", EnginePath: "foo", KeyName: "foo-key"}}, }, { KMS: []kmsKey{{Arn: "baz"}}, @@ -274,7 +274,7 @@ func TestLoadConfigFileWithGroups(t *testing.T) { {ResourceID: "baz"}, }, AzureKV: []azureKVKey{{VaultURL: "https://bar.vault.azure.net", Key: "bar-key", Version: "barversion"}}, - Vault: []vaultKey{{VaultAddress: "https://baz.vault:8200", BackendPath: "baz", KeyName: "baz-key"}}, + Vault: []vaultKey{{VaultAddress: "https://baz.vault:8200", EnginePath: "baz", KeyName: "baz-key"}}, }, }, }, diff --git a/hcvault/keysource.go b/hcvault/keysource.go index 382405128..1e38720c8 100644 --- a/hcvault/keysource.go +++ b/hcvault/keysource.go @@ -29,7 +29,7 @@ func init() { type MasterKey struct { EncryptedKey string KeyName string - BackendPath string + EnginePath string VaultAddress string CreationDate time.Time } @@ -68,16 +68,16 @@ func NewMasterKeyFromURI(uri string) (*MasterKey, error) { if u.Scheme == "" { return nil, fmt.Errorf("missing scheme in vault URL (should be like this: https://vault.example.com:8200/v1/transit/keys/keyName), got: %v", uri) } - backendPath, keyName, err := getBackendAndKeyFromPath(u.RequestURI()) + enginePath, keyName, err := getBackendAndKeyFromPath(u.RequestURI()) if err != nil { return nil, err } u.Path = "" - return NewMasterKey(u.String(), backendPath, keyName), nil + return NewMasterKey(u.String(), enginePath, keyName), nil } -func getBackendAndKeyFromPath(fullPath string) (backendPath, keyName string, err error) { +func getBackendAndKeyFromPath(fullPath string) (enginePath, keyName string, err error) { // Running vault behind a reverse proxy with longer urls seems not to be supported // by the vault client api so we have a separate Error for that here. if re := regexp.MustCompile(`/[^/]+/v[\d]+/[^/]+/[^/]+/[^/]+`); re.Match([]byte(fullPath)) { @@ -91,16 +91,16 @@ func getBackendAndKeyFromPath(fullPath string) (backendPath, keyName string, err dirs := strings.Split(fullPath, "/") keyName = dirs[len(dirs)-1] - backendPath = path.Join(dirs[1 : len(dirs)-2]...) + enginePath = path.Join(dirs[1 : len(dirs)-2]...) err = nil return } // NewMasterKey creates a new MasterKey from a vault address, transit backend path and a key name and setting the creation date to the current date -func NewMasterKey(addess, backendPath, keyName string) *MasterKey { +func NewMasterKey(addess, enginePath, keyName string) *MasterKey { mk := &MasterKey{ VaultAddress: addess, - BackendPath: backendPath, + EnginePath: enginePath, KeyName: keyName, CreationDate: time.Now().UTC(), } @@ -152,7 +152,7 @@ func vaultClient(address string) (*api.Client, error) { // Encrypt takes a sops data key, encrypts it with Vault Transit and stores the result in the EncryptedKey field func (key *MasterKey) Encrypt(dataKey []byte) error { - fullPath := path.Join(key.BackendPath, "encrypt", key.KeyName) + fullPath := path.Join(key.EnginePath, "encrypt", key.KeyName) cli, err := vaultClient(key.VaultAddress) if err != nil { return err @@ -190,7 +190,7 @@ func (key *MasterKey) EncryptIfNeeded(dataKey []byte) error { // Decrypt decrypts the EncryptedKey field with Vault Transit and returns the result. func (key *MasterKey) Decrypt() ([]byte, error) { - fullPath := path.Join(key.BackendPath, "decrypt", key.KeyName) + fullPath := path.Join(key.EnginePath, "decrypt", key.KeyName) cli, err := vaultClient(key.VaultAddress) if err != nil { return nil, err @@ -230,7 +230,7 @@ func (key *MasterKey) NeedsRotation() bool { // ToString converts the key to a string representation func (key *MasterKey) ToString() string { - return fmt.Sprintf("%s/v1/%s/keys/%s", key.VaultAddress, key.BackendPath, key.KeyName) + return fmt.Sprintf("%s/v1/%s/keys/%s", key.VaultAddress, key.EnginePath, key.KeyName) } func (key *MasterKey) createVaultTransitAndKey() error { @@ -241,14 +241,14 @@ func (key *MasterKey) createVaultTransitAndKey() error { if err != nil { return fmt.Errorf("Cannot create Vault Client: %v", err) } - err = cli.Sys().Mount(key.BackendPath, &api.MountInput{ + err = cli.Sys().Mount(key.EnginePath, &api.MountInput{ Type: "transit", Description: "backend transit used by SOPS", }) if err != nil { return err } - path := path.Join(key.BackendPath, "keys", key.KeyName) + path := path.Join(key.EnginePath, "keys", key.KeyName) payload := make(map[string]interface{}) payload["type"] = "rsa-4096" _, err = cli.Logical().Write(path, payload) @@ -267,7 +267,7 @@ func (key MasterKey) ToMap() map[string]interface{} { out := make(map[string]interface{}) out["vault_address"] = key.VaultAddress out["key_name"] = key.KeyName - out["backend_path"] = key.BackendPath + out["engine_path"] = key.EnginePath out["enc"] = key.EncryptedKey out["created_at"] = key.CreationDate.UTC().Format(time.RFC3339) return out diff --git a/hcvault/keysource_test.go b/hcvault/keysource_test.go index 9f12748fc..f8682af66 100644 --- a/hcvault/keysource_test.go +++ b/hcvault/keysource_test.go @@ -67,13 +67,13 @@ func TestKeyToMap(t *testing.T) { key := MasterKey{ CreationDate: time.Date(2016, time.October, 31, 10, 0, 0, 0, time.UTC), VaultAddress: "http://127.0.0.1:8200", - BackendPath: "foo", + EnginePath: "foo", KeyName: "bar", EncryptedKey: "this is encrypted", } assert.Equal(t, map[string]interface{}{ "vault_address": "http://127.0.0.1:8200", - "backend_path": "foo", + "engine_path": "foo", "key_name": "bar", "enc": "this is encrypted", "created_at": "2016-10-31T10:00:00Z", @@ -84,7 +84,7 @@ func TestEncryptionDecryption(t *testing.T) { dataKey := []byte("super very Secret Key!!!") key := MasterKey{ VaultAddress: os.Getenv("VAULT_ADDR"), - BackendPath: "sops", + EnginePath: "sops", KeyName: "main", } err := key.Encrypt(dataKey) @@ -109,17 +109,17 @@ func TestNewMasterKeyFromURI(t *testing.T) { mk1 := &MasterKey{ VaultAddress: "https://vault.example.com:8200", - BackendPath: "transit", + EnginePath: "transit", KeyName: "keyName", } mk2 := &MasterKey{ VaultAddress: "https://vault.me.com", - BackendPath: "super42/bestmarket", + EnginePath: "super42/bestmarket", KeyName: "slig", } mk3 := &MasterKey{ VaultAddress: "http://127.0.0.1:12121", - BackendPath: "transit", + EnginePath: "transit", KeyName: "dev", } genMk1, err := NewMasterKeyFromURI(uri1) diff --git a/keyservice/keyservice.go b/keyservice/keyservice.go index f10019b23..f67732ff8 100644 --- a/keyservice/keyservice.go +++ b/keyservice/keyservice.go @@ -39,7 +39,7 @@ func KeyFromMasterKey(mk keys.MasterKey) Key { KeyType: &Key_VaultKey{ VaultKey: &VaultKey{ VaultAddress: mk.VaultAddress, - BackendPath: mk.BackendPath, + EnginePath: mk.EnginePath, KeyName: mk.KeyName, }, }, diff --git a/keyservice/keyservice.pb.go b/keyservice/keyservice.pb.go index 0da9ba98c..433baabb3 100644 --- a/keyservice/keyservice.pb.go +++ b/keyservice/keyservice.pb.go @@ -6,11 +6,12 @@ package keyservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -292,7 +293,7 @@ func (m *GcpKmsKey) GetResourceId() string { type VaultKey struct { VaultAddress string `protobuf:"bytes,1,opt,name=vault_address,json=vaultAddress,proto3" json:"vault_address,omitempty"` - BackendPath string `protobuf:"bytes,2,opt,name=backend_path,json=backendPath,proto3" json:"backend_path,omitempty"` + EnginePath string `protobuf:"bytes,2,opt,name=engine_path,json=enginePath,proto3" json:"engine_path,omitempty"` KeyName string `protobuf:"bytes,3,opt,name=key_name,json=keyName,proto3" json:"key_name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -331,9 +332,9 @@ func (m *VaultKey) GetVaultAddress() string { return "" } -func (m *VaultKey) GetBackendPath() string { +func (m *VaultKey) GetEnginePath() string { if m != nil { - return m.BackendPath + return m.EnginePath } return "" } diff --git a/keyservice/keyservice.proto b/keyservice/keyservice.proto index e0fbb2a79..a6f5e436f 100644 --- a/keyservice/keyservice.proto +++ b/keyservice/keyservice.proto @@ -27,7 +27,7 @@ message GcpKmsKey { message VaultKey { string vault_address = 1; - string backend_path = 2; + string engine_path = 2; string key_name = 3; } diff --git a/keyservice/server.go b/keyservice/server.go index 0981707c2..d2734bd97 100644 --- a/keyservice/server.go +++ b/keyservice/server.go @@ -65,7 +65,7 @@ func (ks *Server) encryptWithAzureKeyVault(key *AzureKeyVaultKey, plaintext []by func (ks *Server) encryptWithVault(key *VaultKey, plaintext []byte) ([]byte, error) { vaultKey := hcvault.MasterKey{ VaultAddress: key.VaultAddress, - BackendPath: key.BackendPath, + EnginePath: key.EnginePath, KeyName: key.KeyName, } err := vaultKey.Encrypt(plaintext) @@ -112,7 +112,7 @@ func (ks *Server) decryptWithAzureKeyVault(key *AzureKeyVaultKey, ciphertext []b func (ks *Server) decryptWithVault(key *VaultKey, ciphertext []byte) ([]byte, error) { vaultKey := hcvault.MasterKey{ VaultAddress: key.VaultAddress, - BackendPath: key.BackendPath, + EnginePath: key.EnginePath, KeyName: key.KeyName, } vaultKey.EncryptedKey = string(ciphertext) @@ -192,7 +192,7 @@ func keyToString(key Key) string { case *Key_AzureKeyvaultKey: return fmt.Sprintf("Azure Key Vault key with URL %s/keys/%s/%s", k.AzureKeyvaultKey.VaultUrl, k.AzureKeyvaultKey.Name, k.AzureKeyvaultKey.Version) case *Key_VaultKey: - return fmt.Sprintf("Hashicorp Vault key with URI %s/v1/%s/keys/%s", k.VaultKey.VaultAddress, k.VaultKey.BackendPath, k.VaultKey.KeyName) + return fmt.Sprintf("Hashicorp Vault key with URI %s/v1/%s/keys/%s", k.VaultKey.VaultAddress, k.VaultKey.EnginePath, k.VaultKey.KeyName) default: return fmt.Sprintf("Unknown key type") } diff --git a/stores/stores.go b/stores/stores.go index 07d78779b..323a91673 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -82,7 +82,7 @@ type gcpkmskey struct { type vaultkey struct { VaultAddress string `yaml:"vault_address" json:"vault_address"` - BackendPath string `yaml:"backend_path" json:"backend_path"` + EnginePath string `yaml:"engine_path" json:"engine_path"` KeyName string `yaml:"key_name" json:"key_name"` CreatedAt string `yaml:"created_at" json:"created_at"` EncryptedDataKey string `yaml:"enc" json:"enc"` @@ -178,7 +178,7 @@ func vaultKeysFromGroup(group sops.KeyGroup) (keys []vaultkey) { case *hcvault.MasterKey: keys = append(keys, vaultkey{ VaultAddress: key.VaultAddress, - BackendPath: key.BackendPath, + EnginePath: key.EnginePath, KeyName: key.KeyName, CreatedAt: key.CreationDate.Format(time.RFC3339), EncryptedDataKey: key.EncryptedKey, @@ -356,7 +356,7 @@ func (vaultKey *vaultkey) toInternal() (*hcvault.MasterKey, error) { } return &hcvault.MasterKey{ VaultAddress: vaultKey.VaultAddress, - BackendPath: vaultKey.BackendPath, + EnginePath: vaultKey.EnginePath, KeyName: vaultKey.KeyName, CreationDate: creationDate, EncryptedKey: vaultKey.EncryptedDataKey, From ff1b1fd1851995b9b2ebd3bcac1f1dcc2a93060a Mon Sep 17 00:00:00 2001 From: vnzongzna Date: Mon, 20 Apr 2020 00:16:54 +0530 Subject: [PATCH 13/16] correction in hc-vault-transit commands Signed-off-by: vnzongzna --- cmd/sops/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index fecb7e61c..0163de1d9 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -524,7 +524,7 @@ func main() { EnvVar: "SOPS_AZURE_KEYVAULT_URLS", }, cli.StringFlag{ - Name: "hc-vault--transit", + Name: "hc-vault-transit", Usage: "comma separated list of vault's key URI (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev')", EnvVar: "SOPS_VAULT_URIS", }, @@ -1013,7 +1013,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { } } if c.String("hc-vault-transit") != "" { - hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("vault")) + hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("hc-vault-transit")) if err != nil { return nil, err } @@ -1026,7 +1026,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { pgpKeys = append(pgpKeys, k) } } - if c.String("kms") == "" && c.String("pgp") == "" && c.String("gcp-kms") == "" && c.String("azure-kv") == "" && c.String("vault") == "" { + if c.String("kms") == "" && c.String("pgp") == "" && c.String("gcp-kms") == "" && c.String("azure-kv") == "" && c.String("hc-vault-transit") == "" { conf, err := loadConfig(c, file, kmsEncryptionContext) // config file might just not be supplied, without any error if conf == nil { From 331970dc7da7b1207398af2504c0f588bf701410 Mon Sep 17 00:00:00 2001 From: gitirabassi Date: Mon, 20 Apr 2020 01:12:56 +0530 Subject: [PATCH 14/16] resolving conflict Signed-off-by: vnzongzna --- stores/dotenv/store.go | 5 ++--- stores/dotenv/store_test.go | 5 ----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/stores/dotenv/store.go b/stores/dotenv/store.go index 8add8a097..89cc2e69b 100644 --- a/stores/dotenv/store.go +++ b/stores/dotenv/store.go @@ -81,7 +81,7 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { } branch = append(branch, sops.TreeItem{ Key: string(line[:pos]), - Value: strings.Replace(string(line[pos+1:]), "\\n", "\n", -1), + Value: string(line[pos+1:]), }) } } @@ -119,8 +119,7 @@ func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) { if comment, ok := item.Key.(sops.Comment); ok { line = fmt.Sprintf("#%s\n", comment.Value) } else { - value := strings.Replace(item.Value.(string), "\n", "\\n", -1) - line = fmt.Sprintf("%s=%s\n", item.Key, value) + line = fmt.Sprintf("%s=%s\n", item.Key, item.Value) } buffer.WriteString(line) } diff --git a/stores/dotenv/store_test.go b/stores/dotenv/store_test.go index f4bd2cc85..5c7c3beed 100644 --- a/stores/dotenv/store_test.go +++ b/stores/dotenv/store_test.go @@ -13,7 +13,6 @@ VAR1=val1 VAR2=val2 #comment VAR3_unencrypted=val3 -VAR4=val4\nval4 `, "\n")) var BRANCH = sops.TreeBranch{ @@ -33,10 +32,6 @@ var BRANCH = sops.TreeBranch{ Key: "VAR3_unencrypted", Value: "val3", }, - sops.TreeItem{ - Key: "VAR4", - Value: "val4\nval4", - }, } func TestLoadPlainFile(t *testing.T) { From c18356423c5f912a1aac41624297ff1b516943dc Mon Sep 17 00:00:00 2001 From: Vaibhav Kaushik Date: Thu, 23 Apr 2020 14:00:45 +0530 Subject: [PATCH 15/16] Apply suggestions from code review Co-Authored-By: Adrian Utrilla --- README.rst | 12 ++++++------ cmd/sops/main.go | 2 +- hcvault/keysource.go | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 1f1ec2f32..489cc4925 100644 --- a/README.rst +++ b/README.rst @@ -293,7 +293,7 @@ And decrypt it using:: Encrypting using Hashicorp Vault ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To understand how to deploy Vault securely this is not the place, we assume you have a instance (or more) of Vault running and you have privileged access to it. +We assume you have an instance (or more) of Vault running and you have privileged access to it. For instructions on how to deploy a secure instance of Vault, refer to Hashicorp's official documentation. To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) @@ -304,13 +304,13 @@ To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) .. code:: bash - $ # Subsitute this with the address of the vault running + $ # Substitute this with the address Vault is running on $ export VAULT_ADDR=http://127.0.0.1:8200 $ # this may not be necessary in case you previously used `vault login` for production use $ export VAULT_TOKEN=toor - $ # to check if Vault started correctly and configured + $ # to check if Vault started and is configured correctly $ vault status Key Value --- ----- @@ -326,7 +326,7 @@ To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) $ # It is required to enable a transit engine if not already done (It is suggested to create a transit engine specifically for sops, in which it is possible to have multiple keys with various permission levels) $ vault secrets enable -path=sops transit - Success! Enabled thˇe transit secrets engine at: sops/ + Success! Enabled the transit secrets engine at: sops/ $ # Then create one or more keys $ vault write sops/keys/firstkey type=rsa-4096 @@ -340,11 +340,11 @@ To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) $ sops --hc-vault-transit $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml - $ cat <.sops.yaml + $ cat < .sops.yaml creation_rules: - path_regex: \.dev\.yaml$ hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/secondkey" - - \.prod\.yaml$ + - path_regex: \.prod\.yaml$ hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/thirdkey" EOF diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 0163de1d9..454a6f78a 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -81,7 +81,7 @@ func main() { To encrypt or decrypt a document with HashiCorp Vault's Transit Secret Engine, specify the - Vault key URI name in the --hc-vault-transit flag or in the SOPS_VAULT_URIS (eg. https://vault.example.org:8200/v1/transit/keys/dev + Vault key URI name in the --hc-vault-transit flag or in the SOPS_VAULT_URIS environment variable (eg. https://vault.example.org:8200/v1/transit/keys/dev where 'https://vault.example.org:8200' is the vault server, 'transit' the enginePath, and 'dev' is the name of the key ) environment variable. (you need to enable the Transit Secrets Engine in Vault. See diff --git a/hcvault/keysource.go b/hcvault/keysource.go index 1e38720c8..4f4b5124c 100644 --- a/hcvault/keysource.go +++ b/hcvault/keysource.go @@ -81,9 +81,9 @@ func getBackendAndKeyFromPath(fullPath string) (enginePath, keyName string, err // Running vault behind a reverse proxy with longer urls seems not to be supported // by the vault client api so we have a separate Error for that here. if re := regexp.MustCompile(`/[^/]+/v[\d]+/[^/]+/[^/]+/[^/]+`); re.Match([]byte(fullPath)) { - return "", "", fmt.Errorf("Running Vault with an prefixed url is not supported! (Format has to be like https://vault.example.com:8200/v1/transit/keys/keyName)") + return "", "", fmt.Errorf("running Vault with a prefixed url is not supported! (Format has to be like https://vault.example.com:8200/v1/transit/keys/keyName)") } else if re := regexp.MustCompile(`/v[\d]+/[^/]+/[^/]+/[^/]+`); re.Match([]byte(fullPath)) == false { - return "", "", fmt.Errorf("Vault path does not seem to be formatted correctly: (eg. https://vault.example.com:8200/v1/transit/keys/keyName)") + return "", "", fmt.Errorf("vault path does not seem to be formatted correctly: (eg. https://vault.example.com:8200/v1/transit/keys/keyName)") } fullPath = strings.TrimPrefix(fullPath, "/") fullPath = strings.TrimSuffix(fullPath, "/") From 05ad5e55e959eec111db8bd2cc90a2fd938b6c27 Mon Sep 17 00:00:00 2001 From: vnzongzna Date: Wed, 29 Apr 2020 00:56:24 +0530 Subject: [PATCH 16/16] allowing only hc_vault_transit_uri as input Co-Authored-By: gitirabassi Co-Authored-By: ldue Signed-off-by: vnzongzna --- cmd/sops/main.go | 6 +++--- config/config.go | 18 ++++++------------ config/config_test.go | 33 ++++----------------------------- stores/dotenv/store.go | 5 +++-- stores/dotenv/store_test.go | 5 +++++ 5 files changed, 21 insertions(+), 46 deletions(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 454a6f78a..bafb56ef6 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -361,7 +361,7 @@ func main() { Usage: "the Azure Key Vault key URL the new group should contain. Can be specified more than once", }, cli.StringSliceFlag{ - Name: "hc-vault", + Name: "hc-vault-transit", Usage: "the full vault path to the key used to encrypt/decrypt. Make you choose and configure a key with encrption/decryption enabled (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev'). Can be specified more than once", }, cli.BoolFlag{ @@ -381,7 +381,7 @@ func main() { pgpFps := c.StringSlice("pgp") kmsArns := c.StringSlice("kms") gcpKmses := c.StringSlice("gcp-kms") - vaultURIs := c.StringSlice("hc-vault") + vaultURIs := c.StringSlice("hc-vault-transit") azkvs := c.StringSlice("azure-kv") var group sops.KeyGroup for _, fp := range pgpFps { @@ -1044,7 +1044,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { group = append(group, azkvKeys...) group = append(group, pgpKeys...) group = append(group, hcVaultMkKeys...) - log.Infof("Master keys available: %+v", group) + log.Debugf("Master keys available: %+v", group) return []sops.KeyGroup{group}, nil } diff --git a/config/config.go b/config/config.go index 93f10ec73..bdfc930b5 100644 --- a/config/config.go +++ b/config/config.go @@ -70,7 +70,7 @@ type keyGroup struct { KMS []kmsKey GCPKMS []gcpKmsKey `yaml:"gcp_kms"` AzureKV []azureKVKey `yaml:"azure_keyvault"` - Vault []vaultKey `yaml:"hc_vault"` + Vault []string `yaml:"hc_vault"` PGP []string } @@ -91,12 +91,6 @@ type azureKVKey struct { Version string `yaml:"version"` } -type vaultKey struct { - VaultAddress string `yaml:"vault_address"` - EnginePath string `yaml:"engine_path"` - KeyName string `yaml:"key_name"` -} - type destinationRule struct { PathRegex string `yaml:"path_regex"` S3Bucket string `yaml:"s3_bucket"` @@ -119,7 +113,6 @@ type creationRule struct { GCPKMS string `yaml:"gcp_kms"` AzureKeyVault string `yaml:"azure_keyvault"` VaultURI string `yaml:"hc_vault_transit_uri"` - Vault []vaultKey `yaml:"hc_vault_transit"` KeyGroups []keyGroup `yaml:"key_groups"` ShamirThreshold int `yaml:"shamir_threshold"` UnencryptedSuffix string `yaml:"unencrypted_suffix"` @@ -165,7 +158,11 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ keyGroup = append(keyGroup, azkv.NewMasterKey(k.VaultURL, k.Key, k.Version)) } for _, k := range group.Vault { - keyGroup = append(keyGroup, hcvault.NewMasterKey(k.VaultAddress, k.EnginePath, k.KeyName)) + if masterKey, err := hcvault.NewMasterKeyFromURI(k); err == nil { + keyGroup = append(keyGroup, masterKey) + } else { + return nil, err + } } groups = append(groups, keyGroup) } @@ -194,9 +191,6 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ for _, k := range vaultKeys { keyGroup = append(keyGroup, k) } - for _, k := range cRule.Vault { - keyGroup = append(keyGroup, hcvault.NewMasterKey(k.VaultAddress, k.EnginePath, k.KeyName)) - } groups = append(groups, keyGroup) } return groups, nil diff --git a/config/config_test.go b/config/config_test.go index 8f56810d1..e3def53a1 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -54,14 +54,6 @@ creation_rules: pgp: bar gcp_kms: baz hc_vault_transit_uri: http://127.0.1.1/v1/baz/keys/baz - - path_regex: "" - kms: one - pgp: two - gcp_kms: three - hc_vault_transit: - - vault_address: http://127.0.1.1 - engine_path: keys - key_name: baz `) var sampleConfigWithPath = []byte(` @@ -101,9 +93,7 @@ creation_rules: key: foo-key version: fooversion hc_vault: - - vault_address: https://foo.vault:8200 - engine_path: foo - key_name: foo-key + - 'https://foo.vault:8200/v1/foo/keys/foo-key' - kms: - arn: baz pgp: @@ -116,9 +106,7 @@ creation_rules: key: bar-key version: barversion hc_vault: - - vault_address: https://baz.vault:8200 - engine_path: baz - key_name: baz-key + - 'https://baz.vault:8200/v1/baz/keys/baz-key' `) var sampleConfigWithSuffixParameters = []byte(` @@ -226,19 +214,6 @@ func TestLoadConfigFile(t *testing.T) { GCPKMS: "baz", VaultURI: "http://127.0.1.1/v1/baz/keys/baz", }, - { - PathRegex: "", - KMS: "one", - PGP: "two", - GCPKMS: "three", - Vault: []vaultKey{ - vaultKey{ - VaultAddress: "http://127.0.1.1", - EnginePath: "keys", - KeyName: "baz", - }, - }, - }, }, } @@ -264,7 +239,7 @@ func TestLoadConfigFileWithGroups(t *testing.T) { PGP: []string{"bar"}, GCPKMS: []gcpKmsKey{{ResourceID: "foo"}}, AzureKV: []azureKVKey{{VaultURL: "https://foo.vault.azure.net", Key: "foo-key", Version: "fooversion"}}, - Vault: []vaultKey{{VaultAddress: "https://foo.vault:8200", EnginePath: "foo", KeyName: "foo-key"}}, + Vault: []string{"https://foo.vault:8200/v1/foo/keys/foo-key"}, }, { KMS: []kmsKey{{Arn: "baz"}}, @@ -274,7 +249,7 @@ func TestLoadConfigFileWithGroups(t *testing.T) { {ResourceID: "baz"}, }, AzureKV: []azureKVKey{{VaultURL: "https://bar.vault.azure.net", Key: "bar-key", Version: "barversion"}}, - Vault: []vaultKey{{VaultAddress: "https://baz.vault:8200", EnginePath: "baz", KeyName: "baz-key"}}, + Vault: []string{"https://baz.vault:8200/v1/baz/keys/baz-key"}, }, }, }, diff --git a/stores/dotenv/store.go b/stores/dotenv/store.go index 89cc2e69b..8add8a097 100644 --- a/stores/dotenv/store.go +++ b/stores/dotenv/store.go @@ -81,7 +81,7 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { } branch = append(branch, sops.TreeItem{ Key: string(line[:pos]), - Value: string(line[pos+1:]), + Value: strings.Replace(string(line[pos+1:]), "\\n", "\n", -1), }) } } @@ -119,7 +119,8 @@ func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) { if comment, ok := item.Key.(sops.Comment); ok { line = fmt.Sprintf("#%s\n", comment.Value) } else { - line = fmt.Sprintf("%s=%s\n", item.Key, item.Value) + value := strings.Replace(item.Value.(string), "\n", "\\n", -1) + line = fmt.Sprintf("%s=%s\n", item.Key, value) } buffer.WriteString(line) } diff --git a/stores/dotenv/store_test.go b/stores/dotenv/store_test.go index 5c7c3beed..f4bd2cc85 100644 --- a/stores/dotenv/store_test.go +++ b/stores/dotenv/store_test.go @@ -13,6 +13,7 @@ VAR1=val1 VAR2=val2 #comment VAR3_unencrypted=val3 +VAR4=val4\nval4 `, "\n")) var BRANCH = sops.TreeBranch{ @@ -32,6 +33,10 @@ var BRANCH = sops.TreeBranch{ Key: "VAR3_unencrypted", Value: "val3", }, + sops.TreeItem{ + Key: "VAR4", + Value: "val4\nval4", + }, } func TestLoadPlainFile(t *testing.T) {