[FAB-6252] Cleanup Makefile

This patch reorganizes the Makefile variables and
allows overriding of tools being used.

The softhsm2 image build target is simplified.
Support for Go tags is added.
SW integration tests no longer need softhsm image.

Change-Id: I58705f12b7bcbcc893e8534705cfcf5bed60dec0
Signed-off-by: Troy Ronda <troy@troyronda.com>

.gitignore

@@ -12,4 +12,4 @@ vendor/
 
 
 # Files auto-generated by docker-compose
-test/fixtures/tls/fabricca/server/*.pem
+test/fixtures/tls/fabricca/certs/server/ca.org*.example.com-cert.pem

Makefile

@@ -13,35 +13,75 @@
 # checks: runs all check conditions (license, spelling, linting)
 # clean: stops docker conatainers used for integration testing
 # mock-gen: generate mocks needed for testing (using mockgen)
-# channel-artifacts: generates the channel configuration transactions and blocks used by tests
+# channel-config-gen: generates the channel configuration transactions and blocks used by tests
 # populate: populates generated files (not included in git) - currently only vendor
 # populate-vendor: populate the vendor directory based on the lock
-# populate-clean: cleans up populated files (might become part of clean eventually)
+# populate-clean: cleans up populated files (might become part of clean eventually) 
 # thirdparty-pin: pulls (and patches) pinned dependencies into the project under internal
 #
+#
+# Instructions to generate .tx files used for creating channels:
+# Download the configtxgen binary for your OS from (it is located in the .tar.gz file):
+# https://nexus.hyperledger.org/content/repositories/releases/org/hyperledger/fabric/hyperledger-fabric
+# Sample command: $ path/to/configtxgen -profile TwoOrgsChannel -outputCreateChannelTx testchannel.tx -channelID testchannel
+# More Docs: http://hyperledger-fabric.readthedocs.io/en/latest/configtxgen.html
+#
+
+# Tool commands
+GO_CMD             ?= go
+GO_DEP_CMD         ?= dep
+DOCKER_CMD         ?= docker
+DOCKER_COMPOSE_CMD ?= docker-compose
 
+# Build flags
+ARCH         := $(shell uname -m)
+GO_LDFLAGS   ?= -ldflags=-s
+EXPERIMENTAL ?= true  # includes experimental features in the tests
+BRANCHFAB    ?= false # requires testing against fabric with cherry picks from gerrit
 
-export ARCH=$(shell uname -m)
-export LDFLAGS=-ldflags=-s
-export DOCKER_NS=hyperledger
-export DOCKER_TAG=$(ARCH)-0.3.1
-export GO_DEP_COMMIT=v0.3.0 # the version of dep that will be installed by depend-install (or in the CI)
+ifeq ($(EXPERIMENTAL),true)
+GO_TAGS += experimental
+endif
+
+ifeq ($(BRANCHFAB),true)
+GO_TAGS += branchfab
+endif
 
 # Upstream fabric patching
-THIRDPARTY_FABRIC_CA_BRANCH=release
-THIRDPARTY_FABRIC_CA_COMMIT=v1.0.1
-THIRDPARTY_FABRIC_BRANCH=master
-THIRDPARTY_FABRIC_COMMIT=a657db28a0ff53ed512bd6f4ac4786a0f4ca709c
+THIRDPARTY_FABRIC_CA_BRANCH ?= release
+THIRDPARTY_FABRIC_CA_COMMIT ?= v1.0.2
+THIRDPARTY_FABRIC_BRANCH    ?= master
+THIRDPARTY_FABRIC_COMMIT    ?= a657db28a0ff53ed512bd6f4ac4786a0f4ca709c
+
+# Tool versions
+GO_DEP_COMMIT        := v0.3.0 # the version of dep that will be installed by depend-install (or in the CI)
+FABRIC_TOOLS_VERSION ?= 1.0.1
+FABRIC_BASE_VERSION  ?= 0.3.1
+
+# Fabric Base Docker Image
+FABRIC_BASE_IMAGE   ?= hyperledger/fabric-baseimage
+FABRIC_BASE_TAG     ?= $(ARCH)-$(FABRIC_BASE_VERSION)
+
+# Fabric Tools Docker Image
+FABRIC_TOOLS_IMAGE  ?= hyperledger/fabric-tools
+FABRIC_TOOLS_TAG    ?= $(ARCH)-$(FABRIC_TOOLS_VERSION)
 
 # Local variables used by makefile
 PACKAGE_NAME=github.com/hyperledger/fabric-sdk-go
-FABRIC_TOOLS_RELEASE=1.0.1
 
 # Detect CI
 ifdef JENKINS_URL
-export FABRIC_SDKGO_DEPEND_INSTALL=true
+export FABRIC_SDKGO_DEPEND_INSTALL := true
 endif
 
+# Global environment exported for scripts
+export GO_CMD
+export GO_DEP_CMD
+export ARCH
+export GO_LDFLAGS
+export GO_DEP_COMMIT
+export GO_TAGS
+
 all: checks unit-test integration-test
 
 depend:
@@ -62,60 +102,56 @@ lint: populate
 spelling:
 	@test/scripts/check_spelling.sh
 
-edit-docker:
-	@cd ./test/fixtures && sed -i.bak -e 's/_NS_/$(DOCKER_NS)/g' Dockerfile\
-	&& sed -i.bak -e 's/_TAG_/$(DOCKER_TAG)/g'  Dockerfile\
-	&& rm -rf Dockerfile.bak
-
 build-softhsm2-image:
-	 @cd ./test/fixtures && docker build --no-cache -q  -t "softhsm2-image" . \
-
-restore-docker-file:
-	@cd ./test/fixtures && sed -i.bak -e 's/$(DOCKER_NS)/_NS_/g' Dockerfile\
-	&& sed -i.bak -e 's/$(DOCKER_TAG)/_TAG_/g'  Dockerfile\
-	&& rm -rf Dockerfile.bak
+	 @$(DOCKER_CMD) build --no-cache -q -t "softhsm2-image" \
+		--build-arg FABRIC_BASE_IMAGE=$(FABRIC_BASE_IMAGE) \
+		--build-arg FABRIC_BASE_TAG=$(FABRIC_BASE_TAG) \
+		./test/fixtures/softhsm2
 
 unit-test: checks depend populate
 	@test/scripts/unit.sh
 
 unit-tests: unit-test
 
-integration-tests: clean depend populate edit-docker build-softhsm2-image restore-docker-file
-	@cd ./test/fixtures && docker-compose -f docker-compose.yaml -f docker-compose-integration-test.yaml up --force-recreate --abort-on-container-exit
-	@test/scripts/check_status.sh "-f ./test/fixtures/docker-compose.yaml -f ./test/fixtures/docker-compose-integration-test.yaml"
+integration-tests-nopkcs11: clean depend populate
+	@cd ./test/fixtures && $(DOCKER_COMPOSE_CMD) -f docker-compose.yaml -f docker-compose-nopkcs11-test.yaml up --force-recreate --abort-on-container-exit
+	@cd test/fixtures && ../scripts/check_status.sh "-f ./docker-compose.yaml -f ./docker-compose-nopkcs11-test.yaml"
 
-integration-tests-pkcs11: clean depend populate edit-docker build-softhsm2-image restore-docker-file
-	@cd ./test/fixtures && docker-compose -f docker-compose.yaml -f docker-compose-pkcs11-test.yaml up --force-recreate --abort-on-container-exit
-	@test/scripts/check_status.sh "-f ./test/fixtures/docker-compose.yaml -f ./test/fixtures/docker-compose-pkcs11-test.yaml"
+integration-tests-pkcs11: clean depend populate build-softhsm2-image
+	@cd ./test/fixtures && $(DOCKER_COMPOSE_CMD) -f docker-compose.yaml -f docker-compose-pkcs11-test.yaml up --force-recreate --abort-on-container-exit
+	@cd test/fixtures && ../scripts/check_status.sh "-f ./docker-compose.yaml -f ./docker-compose-pkcs11-test.yaml"
 
-integration-test: integration-tests integration-tests-pkcs11
+integration-test: integration-tests-nopkcs11 integration-tests-pkcs11
 
 mock-gen:
-	mockgen -build_flags '$(LDFLAGS)' github.com/hyperledger/fabric-sdk-go/api/apitxn ProposalProcessor | sed "s/github.com\/hyperledger\/fabric-sdk-go\/vendor\///g"  > api/apitxn/mocks/mockapitxn.gen.go
-	mockgen -build_flags '$(LDFLAGS)' github.com/hyperledger/fabric-sdk-go/api/apiconfig Config | sed "s/github.com\/hyperledger\/fabric-sdk-go\/vendor\///g"  > api/apiconfig/mocks/mockconfig.gen.go
-	mockgen -build_flags '$(LDFLAGS)' github.com/hyperledger/fabric-sdk-go/api/apifabca FabricCAClient | sed "s/github.com\/hyperledger\/fabric-sdk-go\/vendor\///g"  > api/apifabca/mocks/mockfabriccaclient.gen.go
+	mockgen -build_flags '$(GO_LDFLAGS)' github.com/hyperledger/fabric-sdk-go/api/apitxn ProposalProcessor | sed "s/github.com\/hyperledger\/fabric-sdk-go\/vendor\///g"  > api/apitxn/mocks/mockapitxn.gen.go
+	mockgen -build_flags '$(GO_LDFLAGS)' github.com/hyperledger/fabric-sdk-go/api/apiconfig Config | sed "s/github.com\/hyperledger\/fabric-sdk-go\/vendor\///g"  > api/apiconfig/mocks/mockconfig.gen.go
+	mockgen -build_flags '$(GO_LDFLAGS)' github.com/hyperledger/fabric-sdk-go/api/apifabca FabricCAClient | sed "s/github.com\/hyperledger\/fabric-sdk-go\/vendor\///g"  > api/apifabca/mocks/mockfabriccaclient.gen.go
 
-channel-artifacts:
-	@echo "Generating test channel configuration transactions and blocks"
-	@docker run -i \
+channel-config-gen:
+	@echo "Generating test channel configuration transactions and blocks ..."
+	@$(DOCKER_CMD) run -i \
 		-v $(abspath .):/opt/gopath/src/$(PACKAGE_NAME) \
-		hyperledger/fabric-tools:$(ARCH)-$(FABRIC_TOOLS_RELEASE) \
+		$(FABRIC_TOOLS_IMAGE):$(FABRIC_TOOLS_TAG) \
 		/bin/bash -c "/opt/gopath/src/${PACKAGE_NAME}/test/scripts/generate_channeltx.sh"
 
 thirdparty-pin:
-	UPSTREAM_COMMIT=$(THIRDPARTY_FABRIC_COMMIT) UPSTREAM_BRANCH=$(THIRDPARTY_FABRIC_BRANCH) scripts/third_party_pins/fabric/apply_upstream.sh
-	UPSTREAM_COMMIT=$(THIRDPARTY_FABRIC_CA_COMMIT) UPSTREAM_BRANCH=$(THIRDPARTY_FABRIC_CA_BRANCH) scripts/third_party_pins/fabric-ca/apply_upstream.sh
+	@echo "Pinning third party packages ..."
+	@UPSTREAM_COMMIT=$(THIRDPARTY_FABRIC_COMMIT) UPSTREAM_BRANCH=$(THIRDPARTY_FABRIC_BRANCH) scripts/third_party_pins/fabric/apply_upstream.sh
+	@UPSTREAM_COMMIT=$(THIRDPARTY_FABRIC_CA_COMMIT) UPSTREAM_BRANCH=$(THIRDPARTY_FABRIC_CA_BRANCH) scripts/third_party_pins/fabric-ca/apply_upstream.sh
 
 populate: populate-vendor
 
 populate-vendor:
 	@echo "Populating vendor ..."
-	@dep ensure -vendor-only
+	@$(GO_DEP_CMD) ensure -vendor-only
 
 populate-clean:
 	rm -Rf vendor
 
 clean:
+	$(GO_CMD) clean
 	rm -Rf /tmp/enroll_user /tmp/msp /tmp/keyvaluestore
 	rm -f integration-report.xml report.xml
-	cd test/fixtures && docker-compose -f docker-compose.yaml -f docker-compose-integration-test.yaml -f docker-compose-pkcs11-test.yaml down
+	rm -f test/fixtures/tls/fabricca/certs/server/ca.org*.example.com-cert.pem
+	cd test/fixtures && $(DOCKER_COMPOSE_CMD) -f docker-compose.yaml -f docker-compose-nopkcs11-test.yaml -f docker-compose-pkcs11-test.yaml down

scripts/third_party_pins/fabric-ca/apply_fabric_ca_client_utils.sh

@@ -11,6 +11,8 @@
 
 IMPORT_SUBSTS=($IMPORT_SUBSTS)
 
+GOIMPORTS_CMD=goimports
+
 declare -a PKGS=(
     "api"
     "lib"
@@ -66,7 +68,7 @@ do
     do
         sed -i '' -e $subst $TMP_PROJECT_PATH/${i}
     done
-    goimports -w $TMP_PROJECT_PATH/${i}
+    $GOIMPORTS_CMD -w $TMP_PROJECT_PATH/${i}
 done
 
 # Copy patched project into internal paths

scripts/third_party_pins/fabric/apply_fabric_bccsp.sh

@@ -11,6 +11,8 @@
 
 IMPORT_SUBSTS=($IMPORT_SUBSTS)
 
+GOIMPORTS_CMD=goimports
+
 declare -a PKGS=(
     "bccsp"
     "bccsp/factory"
@@ -87,7 +89,7 @@ do
     do
         sed -i '' -e $subst $TMP_PROJECT_PATH/${i}
     done
-    goimports -w $TMP_PROJECT_PATH/${i}
+    $GOIMPORTS_CMD -w $TMP_PROJECT_PATH/${i}
 done
 
 # Copy patched project into internal paths

scripts/third_party_pins/fabric/apply_fabric_client_utils.sh

@@ -11,6 +11,8 @@
 
 IMPORT_SUBSTS=($IMPORT_SUBSTS)
 
+GOIMPORTS_CMD=goimports
+
 declare -a PKGS=(
     "common/crypto"
     "common/errors"
@@ -87,7 +89,7 @@ do
     do
         sed -i '' -e $subst $TMP_PROJECT_PATH/${i}
     done
-    goimports -w $TMP_PROJECT_PATH/${i}
+    $GOIMPORTS_CMD -w $TMP_PROJECT_PATH/${i}
 done
 
 # Copy patched project into internal paths

scripts/third_party_pins/fabric/apply_fabric_external_utils.sh

@@ -11,6 +11,8 @@
 
 IMPORT_SUBSTS=($IMPORT_SUBSTS)
 
+GOIMPORTS_CMD=goimports
+
 declare -a PKGS=(
     "common/flogging"
 )
@@ -37,7 +39,7 @@ do
     do
         sed -i '' -e $subst $TMP_PROJECT_PATH/${i}
     done
-    goimports -w $TMP_PROJECT_PATH/${i}
+    $GOIMPORTS_CMD -w $TMP_PROJECT_PATH/${i}
 done
 
 # Copy patched project into internal paths

scripts/third_party_pins/fabric/apply_fabric_protos.sh

@@ -11,6 +11,8 @@
 
 IMPORT_SUBSTS=($IMPORT_SUBSTS)
 
+GOIMPORTS_CMD=goimports
+
 declare -a PKGS=(
     "protos/common"
     "protos/peer"
@@ -58,7 +60,7 @@ do
     do
         sed -i '' -e $subst $TMP_PROJECT_PATH/${i}
     done
-    goimports -w $TMP_PROJECT_PATH/${i}
+    $GOIMPORTS_CMD -w $TMP_PROJECT_PATH/${i}
 done
 
 # Copy patched project into internal paths

scripts/third_party_pins/fabric/apply_fabric_protos_internal.sh

@@ -11,6 +11,8 @@
 
 IMPORT_SUBSTS=($IMPORT_SUBSTS)
 
+GOIMPORTS_CMD=goimports
+
 declare -a PKGS=(
     "protos/utils"
 
@@ -49,7 +51,7 @@ do
     do
         sed -i '' -e $subst $TMP_PROJECT_PATH/${i}
     done
-    goimports -w $TMP_PROJECT_PATH/${i}
+    $GOIMPORTS_CMD -w $TMP_PROJECT_PATH/${i}
 done
 
 # Copy patched project into internal paths

test/fixtures/.env

@@ -16,9 +16,11 @@ FABRIC_ORDERER_FIXTURE_TAG=1.0.1
 FABRIC_PEER_FIXTURE_TAG=1.0.1
 FABRIC_COUCHDB_FIXTURE_TAG=1.0.1
 FABRIC_BUILDER_FIXTURE_TAG=1.0.1
+FABRIC_BASEOS_FIXTURE_TAG=0.3.1
 
 FABRIC_CA_FIXTURE_IMAGE=hyperledger/fabric-ca
 FABRIC_ORDERER_FIXTURE_IMAGE=hyperledger/fabric-orderer
 FABRIC_PEER_FIXTURE_IMAGE=hyperledger/fabric-peer
 FABRIC_COUCHDB_FIXTURE_IMAGE=hyperledger/fabric-couchdb
 FABRIC_BUILDER_FIXTURE_IMAGE=hyperledger/fabric-ccenv
+FABRIC_BASEOS_FIXTURE_IMAGE=hyperledger/fabric-baseimage