adds e2e test scenarios for help and scan command (#564)

* test policies and iac files

* 1. tests for help
2. initials tests for scan command

* fix failing tests

* 1. k8s tests
2. remove unused iac files and golden outputs

* added tests for:
1. rule filtering
2. resource skipping
3. remote types

* refactor code

* added copyright to all e2e test and helper files

* incorporate review comments

* add copyright in tools.go

pkg/iac-providers/providers.go

@@ -71,6 +71,7 @@ func SupportedIacVersions() []string {
 		for k := range versions {
 			versionSlice = append(versionSlice, string(k))
 		}
+		sort.Strings(versionSlice)
 		versionString := strings.Join(versionSlice, ", ")
 		iacVersions = append(iacVersions, fmt.Sprintf("%s: %s", string(iac), versionString))
 	}

pkg/results/types.go

@@ -46,7 +46,7 @@ type ViolationStore struct {
 
 // ScanSummary will hold the default scan summary data
 type ScanSummary struct {
-	ResourcePath         string `json:"file/folder" yaml:"file/folder" xml:"file/folder,attr"`
+	ResourcePath         string `json:"file/folder" yaml:"file/folder" xml:"file_folder,attr"`
 	IacType              string `json:"iac_type" yaml:"iac_type" xml:"iac_type,attr"`
 	Timestamp            string `json:"scanned_at" yaml:"scanned_at" xml:"scanned_at,attr"`
 	ShowViolationDetails bool   `json:"-" yaml:"-" xml:"-"`

pkg/writer/xml_test.go

@@ -17,7 +17,7 @@ const (
   <skipped_violations>
     <violation rule_name="s3EnforceUserACL" description="S3 bucket Access is allowed to all AWS Account Users." rule_id="AWS.S3Bucket.DS.High.1043" severity="HIGH" category="S3" resource_name="bucket" resource_type="aws_s3_bucket" file="modules/m1/main.tf" line="20"></violation>
   </skipped_violations>
-  <scan_summary file/folder="test" iac_type="terraform" scanned_at="2020-12-12 11:21:29.902796 +0000 UTC" policies_validated="566" violated_policies="1" low="0" medium="0" high="1"></scan_summary>
+  <scan_summary file_folder="test" iac_type="terraform" scanned_at="2020-12-12 11:21:29.902796 +0000 UTC" policies_validated="566" violated_policies="1" low="0" medium="0" high="1"></scan_summary>
 </results>
 	`
 )

scripts/run-e2e.sh

@@ -6,4 +6,4 @@ set -o pipefail
 
 export TERRASCAN_BIN_PATH=${PWD}/bin/terrascan
 
-go test -v ./test/...
+go test -p 1 -v ./test/...

test/e2e/help/golden/help_command.txt

@@ -0,0 +1,23 @@
+Terrascan
+
+Detect compliance and security violations across Infrastructure as Code to mitigate risk before provisioning cloud native infrastructure.
+For more information, please visit https://docs.accurics.com
+
+Usage:
+  terrascan [command]
+
+Available Commands:
+  help        Help about any command
+  init        Initializes Terrascan and clones policies from the Terrascan GitHub repository.
+  scan        Detect compliance and security violations across Infrastructure as Code.
+  server      Run Terrascan as an API server
+  version     Terrascan version
+
+Flags:
+  -c, --config-path string   config file path
+  -h, --help                 help for terrascan
+  -l, --log-level string     log level (debug, info, warn, error, panic, fatal) (default "info")
+  -x, --log-type string      log output type (console, json) (default "console")
+  -o, --output string        output type (human, json, yaml, xml, junit-xml) (default "human")
+
+Use "terrascan [command] --help" for more information about a command.

test/e2e/help/golden/help_flag.txt

@@ -0,0 +1,21 @@
+Terrascan
+
+Detect compliance and security violations across Infrastructure as Code to mitigate risk before provisioning cloud native infrastructure.
+For more information, please visit https://docs.accurics.com
+
+Usage:
+  terrascan [command]
+
+Available Commands:
+  init        Initializes Terrascan and clones policies from the Terrascan GitHub repository.
+  scan        Detect compliance and security violations across Infrastructure as Code.
+  server      Run Terrascan as an API server
+  version     Terrascan version
+
+Flags:
+  -c, --config-path string   config file path
+  -l, --log-level string     log level (debug, info, warn, error, panic, fatal) (default "info")
+  -x, --log-type string      log output type (console, json) (default "console")
+  -o, --output string        output type (human, json, yaml, xml, junit-xml) (default "human")
+
+Use "terrascan [command] --help" for more information about a command.

test/e2e/help/golden/help_init.txt

@@ -0,0 +1,15 @@
+Terrascan
+
+Initializes Terrascan and clones policies from the Terrascan GitHub repository.
+
+Usage:
+  terrascan init [flags]
+
+Flags:
+  -h, --help   help for init
+
+Global Flags:
+  -c, --config-path string   config file path
+  -l, --log-level string     log level (debug, info, warn, error, panic, fatal) (default "info")
+  -x, --log-type string      log output type (console, json) (default "console")
+  -o, --output string        output type (human, json, yaml, xml, junit-xml) (default "human")

test/e2e/help/golden/help_scan.txt

@@ -0,0 +1,29 @@
+Terrascan
+
+Detect compliance and security violations across Infrastructure as Code to mitigate risk before provisioning cloud native infrastructure.
+
+Usage:
+  terrascan scan [flags]
+
+Flags:
+      --config-only               will output resource config (should only be used for debugging purposes)
+  -h, --help                      help for scan
+  -d, --iac-dir string            path to a directory containing one or more IaC files (default ".")
+  -f, --iac-file string           path to a single IaC file
+  -i, --iac-type string           iac type (helm, k8s, kustomize, terraform)
+      --iac-version string        iac version (helm: v3, k8s: v1, kustomize: v3, terraform: v12, v13, v14)
+  -p, --policy-path stringArray   policy path directory
+  -t, --policy-type strings       policy type (all, aws, azure, gcp, github, k8s) (default [all])
+  -r, --remote-type string        type of remote backend (git, s3, gcs, http, terraform-registry)
+  -u, --remote-url string         url pointing to remote IaC repository
+      --scan-rules strings        one or more rules to scan (example: --scan-rules="ruleID1,ruleID2")
+      --severity string           minimum severity level of the policy violations to be reported by terrascan
+      --skip-rules strings        one or more rules to skip while scanning (example: --skip-rules="ruleID1,ruleID2")
+      --use-colors string         color output (auto, t, f) (default "auto")
+  -v, --verbose                   will show violations with details (applicable for default output)
+
+Global Flags:
+  -c, --config-path string   config file path
+  -l, --log-level string     log level (debug, info, warn, error, panic, fatal) (default "info")
+  -x, --log-type string      log output type (console, json) (default "console")
+  -o, --output string        output type (human, json, yaml, xml, junit-xml) (default "human")

test/e2e/help/golden/help_server.txt

@@ -0,0 +1,15 @@
+Terrascan
+
+Run Terrascan as an API server that inspects incoming IaC (Infrastructure-as-Code) files and returns the scan results.
+
+Usage:
+  terrascan server [flags]
+
+Flags:
+  -h, --help   help for server
+
+Global Flags:
+  -c, --config-path string   config file path
+  -l, --log-level string     log level (debug, info, warn, error, panic, fatal) (default "info")
+  -x, --log-type string      log output type (console, json) (default "console")
+  -o, --output string        output type (human, json, yaml, xml, junit-xml) (default "human")

test/e2e/help/golden/help_unsupported_command.txt

@@ -0,0 +1,18 @@
+Unknown help topic [`test`]
+Usage:
+  terrascan [command]
+
+Available Commands:
+  help        Help about any command
+  init        Initializes Terrascan and clones policies from the Terrascan GitHub repository.
+  scan        Detect compliance and security violations across Infrastructure as Code.
+  server      Run Terrascan as an API server
+  version     Terrascan version
+
+Flags:
+  -c, --config-path string   config file path
+  -l, --log-level string     log level (debug, info, warn, error, panic, fatal) (default "info")
+  -x, --log-type string      log output type (console, json) (default "console")
+  -o, --output string        output type (human, json, yaml, xml, junit-xml) (default "human")
+
+Use "terrascan [command] --help" for more information about a command.

test/e2e/help/golden/help_version.txt

@@ -0,0 +1,15 @@
+Terrascan
+
+Displays the version of this Terrascan binary
+
+Usage:
+  terrascan version [flags]
+
+Flags:
+  -h, --help   help for version
+
+Global Flags:
+  -c, --config-path string   config file path
+  -l, --log-level string     log level (debug, info, warn, error, panic, fatal) (default "info")
+  -x, --log-type string      log output type (console, json) (default "console")
+  -o, --output string        output type (human, json, yaml, xml, junit-xml) (default "human")

test/e2e/help/golden/incorrect_command.txt

@@ -0,0 +1,2 @@
+Error: unknown command "test" for "terrascan"
+Run 'terrascan --help' for usage.

test/e2e/help/golden/no_command.txt

@@ -0,0 +1,23 @@
+Terrascan
+
+Detect compliance and security violations across Infrastructure as Code to mitigate risk before provisioning cloud native infrastructure.
+For more information, please visit https://docs.accurics.com
+
+Usage:
+  terrascan [command]
+
+Available Commands:
+  help        Help about any command
+  init        Initializes Terrascan and clones policies from the Terrascan GitHub repository.
+  scan        Detect compliance and security violations across Infrastructure as Code.
+  server      Run Terrascan as an API server
+  version     Terrascan version
+
+Flags:
+  -c, --config-path string   config file path
+  -h, --help                 help for terrascan
+  -l, --log-level string     log level (debug, info, warn, error, panic, fatal) (default "info")
+  -x, --log-type string      log output type (console, json) (default "console")
+  -o, --output string        output type (human, json, yaml, xml, junit-xml) (default "human")
+
+Use "terrascan [command] --help" for more information about a command.

test/e2e/help/help_suite_test.go

@@ -0,0 +1,28 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+package help_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestHelp(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Help Suite")
+}

test/e2e/help/help_test.go

@@ -0,0 +1,117 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+package help_test
+
+import (
+	"io"
+
+	helpUtils "github.com/accurics/terrascan/test/e2e/help"
+	"github.com/accurics/terrascan/test/helper"
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/gomega/gbytes"
+	"github.com/onsi/gomega/gexec"
+)
+
+var (
+	helpCommand string = "help"
+)
+
+var _ = Describe("Help", func() {
+
+	var session *gexec.Session
+	var terrascanBinaryPath string
+
+	var outWriter, errWriter io.Writer
+
+	BeforeSuite(func() {
+		terrascanBinaryPath = helper.GetTerrascanBinaryPath()
+	})
+
+	BeforeEach(func() {
+		outWriter = gbytes.NewBuffer()
+		errWriter = gbytes.NewBuffer()
+	})
+
+	AfterEach(func() {
+		outWriter = nil
+		errWriter = nil
+	})
+
+	Describe("terrascan is run without any command", func() {
+		It("should print all supported commands and exit with status code 0", func() {
+			session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter)
+			helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_command.txt", true)
+		})
+	})
+
+	Describe("terrascan is run -h flag", func() {
+		It("should print all supported commands and exit with status code 0", func() {
+			session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "-h")
+			helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_flag.txt", true)
+		})
+	})
+
+	Describe("terrascan is run with an unkonwn command", func() {
+		It("should exit with status code 1 and display a error message", func() {
+			session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "test")
+			helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeOne, "golden/incorrect_command.txt", false)
+		})
+	})
+
+	Describe("help is run", func() {
+		Context("with no arguments", func() {
+			It("should print the terrascan help and exit with status code 0", func() {
+				session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand)
+				helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_command.txt", true)
+			})
+		})
+
+		Context("for init command", func() {
+			It("should print help for init and exit with status code 0", func() {
+				session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "init")
+				helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_init.txt", true)
+			})
+		})
+
+		Context("for scan command", func() {
+			It("should print help for init and exit with status code 0", func() {
+				session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "scan")
+				helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_scan.txt", true)
+			})
+		})
+
+		Context("for server command", func() {
+			It("should print help for init and exit with status code 0", func() {
+				session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "server")
+				helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_server.txt", true)
+			})
+		})
+
+		Context("for version command", func() {
+			It("should print help for init and exit with status code 0", func() {
+				session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "version")
+				helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_version.txt", true)
+			})
+		})
+
+		Context("for an unkonwn command", func() {
+			It("should display that help topic is not available for entered command and exit with status code 0", func() {
+				session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "test")
+				helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_unsupported_command.txt", false)
+			})
+		})
+	})
+})

test/e2e/help/help_utils.go

@@ -0,0 +1,33 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package help
+
+import (
+	"path/filepath"
+
+	"github.com/accurics/terrascan/test/helper"
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+)
+
+// ValidateExitCodeAndOutput validates the exit code and output of the command
+func ValidateExitCodeAndOutput(session *gexec.Session, exitCode int, relFilePath string, isStdOut bool) {
+	gomega.Eventually(session).Should(gexec.Exit(exitCode))
+	goldenFileAbsPath, err := filepath.Abs(relFilePath)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	helper.CompareActualWithGolden(session, goldenFileAbsPath, isStdOut)
+}