[FAB-9717] stabilising integration-tests-local

- cleaned integration test configs, moved local overrides
 to 'override' sub directory.
- Fixed EndpointFilter issue with recursive URL lookup
- Added SearchPeerConfigByURL util
- Fixed entity manager issues in endpointconfig


Change-Id: Iaeb854074e3cd5f9861f5255c4111e8d5da2c269
Signed-off-by: Sudesh Shetty <sudesh.shetty@securekey.com>

pkg/client/common/filter/endpoint.go

@@ -10,6 +10,7 @@ package filter
 import (
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/context"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/fab"
+	"github.com/hyperledger/fabric-sdk-go/pkg/fab/comm"
 )
 
 // EndpointType represents endpoint type
@@ -48,7 +49,7 @@ type EndpointFilter struct {
 // Accept returns false if this peer is to be excluded from the target list
 func (f *EndpointFilter) Accept(peer fab.Peer) bool {
 
-	peerConfig, err := f.ctx.EndpointConfig().PeerConfig(peer.URL())
+	peerConfig, err := comm.SearchPeerConfigFromURL(f.ctx.EndpointConfig(), peer.URL())
 	if err != nil {
 		return true
 	}

pkg/fab/comm/network.go

@@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0
 package comm
 
 import (
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/errors/status"
 	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/fab"
 	"github.com/pkg/errors"
 )
@@ -39,3 +40,33 @@ func NetworkPeerConfigFromURL(cfg fab.EndpointConfig, url string) (*fab.NetworkP
 
 	return &np, nil
 }
+
+// SearchPeerConfigFromURL searches for the peer configuration based on a URL.
+func SearchPeerConfigFromURL(cfg fab.EndpointConfig, url string) (*fab.PeerConfig, error) {
+	peerCfg, err := cfg.PeerConfig(url)
+
+	if peerCfg != nil {
+		return peerCfg, nil
+	}
+
+	if err != nil {
+		s, ok := status.FromError(err)
+		if !ok || s.Code != status.NoMatchingPeerEntity.ToInt32() {
+			return nil, errors.Wrapf(err, "unable to get peer config from [%s]", url)
+		}
+	}
+
+	//May be url is already matched, search through all network peers
+	networkPeers, err := cfg.NetworkPeers()
+	if err != nil {
+		return nil, errors.WithMessage(err, "unable to load network peer config")
+	}
+
+	for _, peer := range networkPeers {
+		if peer.URL == url {
+			return &peer.PeerConfig, nil
+		}
+	}
+
+	return nil, errors.Errorf("unable to get peerconfig for given url : %s", url)
+}

pkg/fab/comm/network_test.go

@@ -9,12 +9,15 @@ package comm
 import (
 	"testing"
 
+	"github.com/hyperledger/fabric-sdk-go/pkg/common/providers/core"
 	"github.com/hyperledger/fabric-sdk-go/pkg/core/config"
 	fabImpl "github.com/hyperledger/fabric-sdk-go/pkg/fab"
 	"github.com/stretchr/testify/assert"
 )
 
 const configTestFilePath = "../../core/config/testdata/config_test.yaml"
+const entityMatcherTestFilePath = "../../core/config/testdata/config_test_entity_matchers.yaml"
+const localOverrideEntityMatcher = "../../../test/fixtures/config/overrides/local_entity_matchers.yaml"
 
 func TestNetworkPeerConfigFromURL(t *testing.T) {
 	configBackend, err := config.FromFile(configTestFilePath)()
@@ -40,3 +43,48 @@ func TestNetworkPeerConfigFromURL(t *testing.T) {
 	assert.Equal(t, "peer0.org1.example.com:7051", np.URL, "wrong URL")
 	assert.Equal(t, "Org1MSP", np.MSPID, "wrong MSP")
 }
+
+func TestSearchPeerConfigFromURL(t *testing.T) {
+	configBackend1, err := config.FromFile(localOverrideEntityMatcher)()
+	if err != nil {
+		t.Fatalf("Unexpected error reading config backend: %v", err)
+	}
+
+	configBackend2, err := config.FromFile(entityMatcherTestFilePath)()
+	if err != nil {
+		t.Fatalf("Unexpected error reading config backend: %v", err)
+	}
+
+	//override entitymatcher
+	backends := append([]core.ConfigBackend{}, configBackend1...)
+	backends = append(backends, configBackend2...)
+
+	sampleConfig, err := fabImpl.ConfigFromBackend(backends...)
+	if err != nil {
+		t.Fatalf("Unexpected error reading config: %v", err)
+	}
+
+	peer0Org1, err := sampleConfig.PeerConfig("peer0.org1.example.com")
+	assert.Nil(t, err, "supposed to get no error")
+
+	//Positive scenario,
+	// peerconfig should be found using matched URL
+	testURL := "localhost:7051"
+	peerConfig, err := SearchPeerConfigFromURL(sampleConfig, testURL)
+	assert.Nil(t, err, "supposed to get no error")
+	assert.NotNil(t, peerConfig, "supposed to get valid peerConfig by url :%s", testURL)
+	assert.Equal(t, testURL, peerConfig.URL)
+	assert.Nil(t, err, "supposed to get no error")
+	assert.Equal(t, peer0Org1.EventURL, peerConfig.EventURL)
+
+	// peerconfig should be found using actual URL
+	testURL2 := "peer0.org1.example.com:7051"
+	peerConfig, err = SearchPeerConfigFromURL(sampleConfig, testURL2)
+
+	assert.Nil(t, err, "supposed to get no error")
+	assert.NotNil(t, peerConfig, "supposed to get valid peerConfig by url :%s", testURL2)
+	assert.Equal(t, testURL, peerConfig.URL)
+	assert.Nil(t, err, "supposed to get no error")
+	assert.Equal(t, peer0Org1.EventURL, peerConfig.EventURL)
+
+}

pkg/fab/endpointconfig.go

@@ -255,7 +255,10 @@ func (c *EndpointConfig) PeerConfig(nameOrURL string) (*fab.PeerConfig, error) {
 	} else {
 		for _, staticPeerConfig := range networkConfig.Peers {
 			if strings.EqualFold(staticPeerConfig.URL, nameOrURL) {
-				matchPeerConfig = &staticPeerConfig
+				matchPeerConfig = c.tryMatchingPeerConfig(networkConfig, nameOrURL)
+				if matchPeerConfig == nil {
+					matchPeerConfig = &staticPeerConfig
+				}
 				break
 			}
 		}

pkg/fab/events/endpoint/endpointdiscovery.go

@@ -100,7 +100,7 @@ func (s *discoveryService) GetPeers() ([]fab.Peer, error) {
 		} else {
 			peerConfig, err = s.ctx.EndpointConfig().PeerConfig(peer.URL())
 			if err != nil {
-				errStatus, ok := err.(*status.Status)
+				errStatus, ok := status.FromError(err)
 				if ok && errStatus.Code == status.NoMatchingPeerEntity.ToInt32() {
 					logger.Debugf("no peer found for URL :%s", peer.URL())
 					continue

test/fixtures/config/overrides/local_orderers_peers.yaml

@@ -0,0 +1,65 @@
+#
+# Copyright SecureKey Technologies Inc. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# The network connection profile provides client applications the information about the target
+# blockchain network that are necessary for the applications to interact with it. These are all
+# knowledge that must be acquired from out-of-band sources. This file provides such a source.
+#
+
+#
+# List of orderers to send transaction and channel create/update requests to. For the time
+# being only one orderer is needed. If more than one is defined, which one get used by the
+# SDK is implementation specific. Consult each SDK's documentation for its handling of orderers.
+#
+orderers:
+  local.orderer.example.com:
+    url: orderer.example.com:7050
+
+    # these are standard properties defined by the gRPC library
+    # they will be passed in as-is to gRPC client constructor
+    grpcOptions:
+      ssl-target-name-override: orderer.example.com
+      # These parameters should be set in coordination with the keepalive policy on the server,
+      # as incompatible settings can result in closing of connection.
+      # When duration of the 'keep-alive-time' is set to 0 or less the keep alive client parameters are disabled
+      keep-alive-time: 0s
+      keep-alive-timeout: 20s
+      keep-alive-permit: false
+      fail-fast: false
+      # allow-insecure will be taken into consideration if address has no protocol defined, if true then grpc or else grpcs
+      allow-insecure: false
+
+    tlsCACerts:
+      # Certificate location absolute path
+      path: ${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/${CRYPTOCONFIG_FIXTURES_PATH}/ordererOrganizations/example.com/tlsca/tlsca.example.com-cert.pem
+
+#
+# List of peers to send various requests to, including endorsement, query
+# and event listener registration.
+#
+peers:
+  local.peer0.org1.example.com:
+    # this URL is used to send endorsement and query requests
+    url: peer0.org1.example.com:7051
+    # eventUrl is only needed when using eventhub (default is delivery service)
+    eventUrl: peer0.org1.example.com:7053
+
+    #TODO to be moved to high level, common for all grpc connections
+    grpcOptions:
+      ssl-target-name-override: peer0.org1.example.com
+      # These parameters should be set in coordination with the keepalive policy on the server,
+      # as incompatible settings can result in closing of connection.
+      # When duration of the 'keep-alive-time' is set to 0 or less the keep alive client parameters are disabled
+      keep-alive-time: 0s
+      keep-alive-timeout: 20s
+      keep-alive-permit: false
+      fail-fast: false
+      # allow-insecure will be taken into consideration if address has no protocol defined, if true then grpc or else grpcs
+      allow-insecure: false
+
+    tlsCACerts:
+      # Certificate location absolute path
+      path: ${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/${CRYPTOCONFIG_FIXTURES_PATH}/peerOrganizations/org1.example.com/tlsca/tlsca.org1.example.com-cert.pem

test/integration/e2e/end_to_end.go

@@ -41,7 +41,7 @@ func Run(t *testing.T, configOpt core.ConfigProvider, sdkOpts ...fabsdk.Option)
 
 	if integration.IsLocal() {
 		//If it is a local test then add entity mapping to config backend to parse URLs
-		configOpt = integration.AddLocalEntityMapping(configOpt)
+		configOpt = integration.AddLocalEntityMapping(configOpt, integration.LocalOrdererPeersCAsConfig)
 	}
 
 	sdk, err := fabsdk.New(configOpt, sdkOpts...)

test/integration/e2e/end_to_end_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/hyperledger/fabric-sdk-go/pkg/core/config"
-	"github.com/hyperledger/fabric-sdk-go/test/integration"
 )
 
 func TestE2E(t *testing.T) {
@@ -19,5 +18,5 @@ func TestE2E(t *testing.T) {
 	Run(t, config.FromFile("../../fixtures/config/config_test.yaml"))
 
 	//Using setup done set above by end to end test, run below test with new config which has no orderer config inside
-	runWithNoOrdererConfig(t, integration.ConfigNoOrdererBackend)
+	runWithNoOrdererConfig(t, config.FromFile("../../fixtures/config/config_test_no_orderer.yaml"))
 }

test/integration/e2e/no_orderer_config_test.go

@@ -21,6 +21,11 @@ import (
 // runWithNoOrdererConfig enables chclient scenarios using config and sdk options provided
 func runWithNoOrdererConfig(t *testing.T, configOpt core.ConfigProvider, sdkOpts ...fabsdk.Option) {
 
+	if integration.IsLocal() {
+		//If it is a local test then add entity mapping to config backend to parse URLs
+		configOpt = integration.AddLocalEntityMapping(configOpt, integration.LocalOrdererPeersConfig)
+	}
+
 	sdk, err := fabsdk.New(configOpt, sdkOpts...)
 	if err != nil {
 		t.Fatalf("Failed to create new SDK: %s", err)

test/integration/env.go

@@ -15,27 +15,26 @@ import (
 )
 
 const (
-	configPath           = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/test/fixtures/config/config_test.yaml"
-	configPathNoOrderer  = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/test/fixtures/config/config_test_no_orderer.yaml"
-	entityMangerLocal    = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/test//fixtures/config/local_entity_matchers.yaml"
-	localOrdererPeersCAs = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/test/fixtures/config/local_orderers_peers_ca.yaml"
+	configPath        = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/test/fixtures/config/config_test.yaml"
+	entityMangerLocal = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/test//fixtures/config/overrides/local_entity_matchers.yaml"
+	//LocalOrdererPeersCAsConfig config file to override on local test having only peers, orderers and CA entity entries
+	LocalOrdererPeersCAsConfig = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/test/fixtures/config/overrides/local_orderers_peers_ca.yaml"
+	//LocalOrdererPeersConfig config file to override on local test having only peers and orderers entity entries
+	LocalOrdererPeersConfig = "${GOPATH}/src/github.com/hyperledger/fabric-sdk-go/test/fixtures/config/overrides/local_orderers_peers.yaml"
 )
 
 // ConfigBackend contains config backend for integration tests
-var ConfigBackend = fetchConfigBackend(configPath)
+var ConfigBackend = fetchConfigBackend(configPath, LocalOrdererPeersCAsConfig)
 
-// ConfigNoOrdererBackend contains config backend for no orderer integration tests
-var ConfigNoOrdererBackend = fetchConfigBackend(configPathNoOrderer)
-
-func fetchConfigBackend(configPath string) core.ConfigProvider {
+func fetchConfigBackend(configPath string, localOverride string) core.ConfigProvider {
 	configProvider := config.FromFile(pathvar.Subst(configPath))
 
 	args := os.Args[1:]
 	for _, arg := range args {
 		//If testlocal is enabled, then update config backend to run 'local' test
 		if arg == "testLocal=true" {
 			return func() ([]core.ConfigBackend, error) {
-				return appendLocalEntityMappingBackend(configProvider)
+				return appendLocalEntityMappingBackend(configProvider, localOverride)
 			}
 		}
 	}
@@ -56,14 +55,14 @@ func IsLocal() bool {
 
 //AddLocalEntityMapping adds local test entity mapping to config backend
 // and returns updated config provider
-func AddLocalEntityMapping(configProvider core.ConfigProvider) core.ConfigProvider {
+func AddLocalEntityMapping(configProvider core.ConfigProvider, configOverridePath string) core.ConfigProvider {
 	return func() ([]core.ConfigBackend, error) {
-		return appendLocalEntityMappingBackend(configProvider)
+		return appendLocalEntityMappingBackend(configProvider, configOverridePath)
 	}
 }
 
 //appendLocalEntityMappingBackend appends entity matcher backend to given config provider
-func appendLocalEntityMappingBackend(configProvider core.ConfigProvider) ([]core.ConfigBackend, error) {
+func appendLocalEntityMappingBackend(configProvider core.ConfigProvider, configOverridePath string) ([]core.ConfigBackend, error) {
 	//Current backend
 	currentBackends, err := configProvider()
 	if err != nil {
@@ -77,8 +76,8 @@ func appendLocalEntityMappingBackend(configProvider core.ConfigProvider) ([]core
 		return nil, err
 	}
 
-	//Local orderer, peer, CA config backend
-	configProvider = config.FromFile(pathvar.Subst(localOrdererPeersCAs))
+	//Local orderer/peer/CA config overrides
+	configProvider = config.FromFile(pathvar.Subst(configOverridePath))
 	localBackends, err := configProvider()
 	if err != nil {
 		return nil, err

test/integration/sdk/channel_client_test.go

@@ -309,8 +309,14 @@ func TestNoEndpoints(t *testing.T) {
 
 	// Using shared SDK instance to increase test speed.
 	testSetup := mainTestSetup
+	configProvider := config.FromFile("../../fixtures/config/config_test_endpoints.yaml")
 
-	sdk, err := fabsdk.New(config.FromFile("../../fixtures/config/config_test_endpoints.yaml"))
+	if integration.IsLocal() {
+		//If it is a local test then add entity mapping to config backend to parse URLs
+		configProvider = integration.AddLocalEntityMapping(configProvider, integration.LocalOrdererPeersConfig)
+	}
+
+	sdk, err := fabsdk.New(configProvider)
 	if err != nil {
 		t.Fatalf("Failed to create new SDK: %s", err)
 	}
@@ -328,13 +334,13 @@ func TestNoEndpoints(t *testing.T) {
 	_, err = chClient.Query(channel.Request{ChaincodeID: mainChaincodeID, Fcn: "invoke", Args: integration.ExampleCCQueryArgs()},
 		channel.WithRetry(retry.DefaultChannelOpts))
 	if err == nil || !strings.Contains(err.Error(), "targets were not provided") {
-		t.Fatalf("Should have failed due to no chaincode query peers")
+		t.Fatal("Should have failed due to no chaincode query peers")
 	}
 
 	// Test execute transaction: since peer has been disabled for endorsement this transaction should fail
 	_, err = chClient.Execute(channel.Request{ChaincodeID: mainChaincodeID, Fcn: "invoke", Args: integration.ExampleCCTxArgs()},
 		channel.WithRetry(retry.DefaultChannelOpts))
 	if err == nil || !strings.Contains(err.Error(), "targets were not provided") {
-		t.Fatalf("Should have failed due to no endorsing peers")
+		t.Fatal("Should have failed due to no endorsing peers")
 	}
 }