[FABG-809] Proper handling of 403 code

Change-Id: I80aeb1088041692e39fd99455f1b8784475980c8
Signed-off-by: Bob Stasyszyn <Bob.Stasyszyn@securekey.com>

pkg/fab/events/client/client.go

@@ -363,13 +363,7 @@ func (c *Client) mustSetConnectionState(newState ConnectionState) {
 
 func (c *Client) monitorConnection() {
 	logger.Debug("Monitoring connection")
-	for {
-		event, ok := <-c.connEvent
-		if !ok {
-			logger.Debugln("Connection has closed.")
-			break
-		}
-
+	for event := range c.connEvent {
 		if c.Stopped() {
 			logger.Debugln("Event client has been stopped.")
 			break
@@ -382,6 +376,12 @@ func (c *Client) monitorConnection() {
 		} else if c.reconn {
 			logger.Warnf("Event client has disconnected. Details: %s", event.Err)
 			if c.setConnectionState(Connected, Disconnected) {
+				if event.Err.IsFatal() {
+					logger.Warnf("Reconnect is not possible due to fatal error. Terminating: %s", event.Err)
+					go c.Close()
+					break
+				}
+
 				logger.Warn("Attempting to reconnect...")
 				go c.reconnect()
 			} else if c.setConnectionState(Connecting, Disconnected) {

pkg/fab/events/client/client_test.go

@@ -645,7 +645,7 @@ func TestReconnect(t *testing.T) {
 		t.Parallel()
 		testConnect(t, 3, mockconn.ConnectedOutcome,
 			mockconn.NewConnectResults(
-				mockconn.NewConnectResult(mockconn.ThirdAttempt, mockconn.SucceedResult),
+				mockconn.NewConnectResult(mockconn.ThirdAttempt, clientmocks.ConnFactory),
 			),
 		)
 	})
@@ -665,10 +665,10 @@ func TestReconnect(t *testing.T) {
 	//     -> should fail to reconnect on the first and second attempt but succeed on the third attempt
 	t.Run("#3", func(t *testing.T) {
 		t.Parallel()
-		testReconnect(t, true, 3, mockconn.ReconnectedOutcome,
+		testReconnect(t, true, 3, mockconn.ReconnectedOutcome, newDisconnectedEvent(),
 			mockconn.NewConnectResults(
-				mockconn.NewConnectResult(mockconn.FirstAttempt, mockconn.SucceedResult),
-				mockconn.NewConnectResult(mockconn.FourthAttempt, mockconn.SucceedResult),
+				mockconn.NewConnectResult(mockconn.FirstAttempt, clientmocks.ConnFactory),
+				mockconn.NewConnectResult(mockconn.FourthAttempt, clientmocks.ConnFactory),
 			),
 		)
 	})
@@ -679,9 +679,9 @@ func TestReconnect(t *testing.T) {
 	//     -> should fail to reconnect after two attempts and then close
 	t.Run("#4", func(t *testing.T) {
 		t.Parallel()
-		testReconnect(t, true, 2, mockconn.ClosedOutcome,
+		testReconnect(t, true, 2, mockconn.ClosedOutcome, newDisconnectedEvent(),
 			mockconn.NewConnectResults(
-				mockconn.NewConnectResult(mockconn.FirstAttempt, mockconn.SucceedResult),
+				mockconn.NewConnectResult(mockconn.FirstAttempt, clientmocks.ConnFactory),
 			),
 		)
 	})
@@ -692,9 +692,22 @@ func TestReconnect(t *testing.T) {
 	//     -> should fail and not attempt to reconnect and then close
 	t.Run("#5", func(t *testing.T) {
 		t.Parallel()
-		testReconnect(t, false, 0, mockconn.ClosedOutcome,
+		testReconnect(t, false, 0, mockconn.ClosedOutcome, newDisconnectedEvent(),
 			mockconn.NewConnectResults(
-				mockconn.NewConnectResult(mockconn.FirstAttempt, mockconn.SucceedResult),
+				mockconn.NewConnectResult(mockconn.FirstAttempt, clientmocks.ConnFactory),
+			),
+		)
+	})
+
+	// (1) Connect
+	//     -> should succeed to connect on the first attempt
+	// (2) Disconnect with fatal error
+	//     -> should fail and not attempt to reconnect and then close
+	t.Run("#6", func(t *testing.T) {
+		t.Parallel()
+		testReconnect(t, true, 0, mockconn.ClosedOutcome, newFatalDisconnectedEvent(),
+			mockconn.NewConnectResults(
+				mockconn.NewConnectResult(mockconn.FirstAttempt, clientmocks.ConnFactory),
 			),
 		)
 	})
@@ -720,8 +733,8 @@ func TestReconnectRegistration(t *testing.T) {
 		testReconnectRegistration(
 			t, mockconn.ExpectFiveBlocks, mockconn.ExpectThreeCC,
 			mockconn.NewConnectResults(
-				mockconn.NewConnectResult(mockconn.FirstAttempt, mockconn.SucceedResult),
-				mockconn.NewConnectResult(mockconn.SecondAttempt, mockconn.SucceedResult)),
+				mockconn.NewConnectResult(mockconn.FirstAttempt, clientmocks.ConnFactory),
+				mockconn.NewConnectResult(mockconn.SecondAttempt, clientmocks.ConnFactory)),
 		)
 	})
 }
@@ -1051,7 +1064,7 @@ func testConnect(t *testing.T, maxConnectAttempts uint, expectedOutcome mockconn
 	}
 }
 
-func testReconnect(t *testing.T, reconnect bool, maxReconnectAttempts uint, expectedOutcome mockconn.Outcome, connAttemptResult mockconn.ConnectAttemptResults) {
+func testReconnect(t *testing.T, reconnect bool, maxReconnectAttempts uint, expectedOutcome mockconn.Outcome, event esdispatcher.Event, connAttemptResult mockconn.ConnectAttemptResults) {
 	cp := mockconn.NewProviderFactory()
 
 	connectch := make(chan *dispatcher.ConnectionEvent)
@@ -1089,7 +1102,7 @@ func testReconnect(t *testing.T, reconnect bool, maxReconnectAttempts uint, expe
 	go listenConnection(connectch, outcomech)
 
 	// Test automatic reconnect handling
-	cp.Connection().ProduceEvent(dispatcher.NewDisconnectedEvent(errors.New("testing reconnect handling")))
+	cp.Connection().ProduceEvent(event)
 
 	var outcome mockconn.Outcome
 
@@ -1708,3 +1721,11 @@ func testTransferRegistrations(t *testing.T, transferFunc transferFunc) {
 
 	eventClient2.Unregister(breg)
 }
+
+func newDisconnectedEvent() esdispatcher.Event {
+	return dispatcher.NewDisconnectedEvent(errors.New("testing reconnect handling"))
+}
+
+func newFatalDisconnectedEvent() esdispatcher.Event {
+	return dispatcher.NewFatalDisconnectedEvent(errors.New("testing reconnect handling"))
+}

pkg/fab/events/client/dispatcher/events.go

@@ -34,14 +34,40 @@ func NewConnectedEvent() *ConnectedEvent {
 	return &ConnectedEvent{}
 }
 
+// DisconnectedError is the error that is associated with the disconnect.
+type DisconnectedError interface {
+	error
+
+	// IsFatal returns true if the error is fatal, meaning that a reconnect attempt would not succeed
+	IsFatal() bool
+}
+
+type disconnectedError struct {
+	cause error
+	fatal bool
+}
+
+func (e *disconnectedError) Error() string {
+	return e.cause.Error()
+}
+
+func (e *disconnectedError) IsFatal() bool {
+	return e.fatal
+}
+
 // DisconnectedEvent indicates that the client has disconnected from the server
 type DisconnectedEvent struct {
-	Err error
+	Err DisconnectedError
 }
 
 // NewDisconnectedEvent creates a new DisconnectedEvent
-func NewDisconnectedEvent(err error) *DisconnectedEvent {
-	return &DisconnectedEvent{Err: err}
+func NewDisconnectedEvent(cause error) *DisconnectedEvent {
+	return &DisconnectedEvent{Err: &disconnectedError{cause: cause}}
+}
+
+// NewFatalDisconnectedEvent creates a new DisconnectedEvent which indicates that a reconnect is not possible
+func NewFatalDisconnectedEvent(cause error) *DisconnectedEvent {
+	return &DisconnectedEvent{Err: &disconnectedError{cause: cause, fatal: true}}
 }
 
 // ConnectEvent is a request to connect to the server
@@ -72,10 +98,10 @@ func NewDisconnectEvent(errch chan<- error) *DisconnectEvent {
 // the disconnect error.
 type ConnectionEvent struct {
 	Connected bool
-	Err       error
+	Err       DisconnectedError
 }
 
 // NewConnectionEvent returns a new ConnectionEvent
-func NewConnectionEvent(connected bool, err error) *ConnectionEvent {
+func NewConnectionEvent(connected bool, err DisconnectedError) *ConnectionEvent {
 	return &ConnectionEvent{Connected: connected, Err: err}
 }

pkg/fab/events/client/mocks/mockconnection.go

@@ -59,12 +59,17 @@ const (
 	ClosedOutcome Outcome = "closed"
 	// TimedOutOutcome means that the client timed out
 	TimedOutOutcome Outcome = "timeout"
-	// ConnectedOutcome means that the client connect
+	// ConnectedOutcome means that the client connected
 	ConnectedOutcome Outcome = "connected"
 	// ErrorOutcome means that the operation resulted in an error
 	ErrorOutcome Outcome = "error"
 )
 
+// ConnFactory creates mock connections
+var ConnFactory = func(opts ...Opt) Connection {
+	return NewMockConnection(opts...)
+}
+
 // Connection extends Connection and adds functions
 // to allow simulating certain situations
 type Connection interface {
@@ -219,41 +224,32 @@ func (cp *ProviderFactory) Provider(conn Connection) api.ConnectionProvider {
 // to return a connection, what authorization to give the connection, etc.
 func (cp *ProviderFactory) FlakeyProvider(connAttemptResults ConnectAttemptResults, opts ...Opt) api.ConnectionProvider {
 	var connectAttempt Attempt
-	return func(context.Client, fab.ChannelCfg, fab.Peer) (api.Connection, error) {
+	return func(ctx context.Client, cfg fab.ChannelCfg, peer fab.Peer) (api.Connection, error) {
 		connectAttempt++
 
-		_, ok := connAttemptResults[connectAttempt]
+		result, ok := connAttemptResults[connectAttempt]
 		if !ok {
 			return nil, errors.New("simulating failed connection attempt")
 		}
 
 		cp.mtx.Lock()
 		defer cp.mtx.Unlock()
 
-		copts := &Opts{}
-		for _, opt := range opts {
-			opt(copts)
-		}
-		factory := copts.Factory
-		if factory == nil {
-			cp.connection = NewMockConnection(opts...)
-		} else {
-			cp.connection = factory(opts...)
-		}
+		cp.connection = result.ConnFactory(opts...)
 
 		return cp.connection, nil
 	}
 }
 
-// ConnectResult contains the data to use for the N'th connection attempt
+// ConnectResult contains the connection factory to use for the N'th connection attempt
 type ConnectResult struct {
-	Attempt Attempt
-	Result  Result
+	Attempt     Attempt
+	ConnFactory ConnectionFactory
 }
 
 // NewConnectResult returns a new ConnectResult
-func NewConnectResult(attempt Attempt, result Result) ConnectResult {
-	return ConnectResult{Attempt: attempt, Result: result}
+func NewConnectResult(attempt Attempt, connFactory ConnectionFactory) ConnectResult {
+	return ConnectResult{Attempt: attempt, ConnFactory: connFactory}
 }
 
 // ConnectAttemptResults maps a connection attempt to a connection result
@@ -323,10 +319,3 @@ func WithResults(funcResults ...*OperationResult) Opt {
 		}
 	}
 }
-
-// WithFactory specifies the connection factory for creating new mock connections
-func WithFactory(factory ConnectionFactory) Opt {
-	return func(opts *Opts) {
-		opts.Factory = factory
-	}
-}