Merge pull request #15268 from lavacat/main-before-follower-send-fail…

…point tests: trigger raftBeforeFollowerSend failpoint in linearizability test
etcd-io · Feb 9, 2023 · d0575e4 · d0575e4
2 parents 2eadb0a + 4b91ae2
commit d0575e4
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 12 deletions.
diff --git a/tests/linearizability/failpoints.go b/tests/linearizability/failpoints.go
@@ -55,7 +55,7 @@ var (
 	RaftBeforeLeaderSendPanic                Failpoint = goPanicFailpoint{"raftBeforeLeaderSend", nil, Leader}
 	BlackholePeerNetwork                     Failpoint = blackholePeerNetworkFailpoint{duration: time.Second}
 	DelayPeerNetwork                         Failpoint = delayPeerNetworkFailpoint{duration: time.Second, baseLatency: 75 * time.Millisecond, randomizedLatency: 50 * time.Millisecond}
-	RandomFailpoint                          Failpoint = randomFailpoint{[]Failpoint{
+	oneNodeClusterFailpoints                           = []Failpoint{
 		KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic,
 		RaftAfterSavePanic, DefragBeforeCopyPanic, DefragBeforeRenamePanic,
 		BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
@@ -67,17 +67,18 @@ var (
 		RaftBeforeLeaderSendPanic,
 		BlackholePeerNetwork,
 		DelayPeerNetwork,
-	}}
-	RaftBeforeApplySnapPanic Failpoint = goPanicFailpoint{"raftBeforeApplySnap", triggerBlackholeUntilSnapshot, Follower}
-	RaftAfterApplySnapPanic  Failpoint = goPanicFailpoint{"raftAfterApplySnap", triggerBlackholeUntilSnapshot, Follower}
-	RaftAfterWALReleasePanic Failpoint = goPanicFailpoint{"raftAfterWALRelease", triggerBlackholeUntilSnapshot, Follower}
-	RaftBeforeSaveSnapPanic  Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackholeUntilSnapshot, Follower}
-	RaftAfterSaveSnapPanic   Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackholeUntilSnapshot, Follower}
-	RandomSnapshotFailpoint  Failpoint = randomFailpoint{[]Failpoint{
+	}
+	RandomOneNodeClusterFailpoint   Failpoint = randomFailpoint{oneNodeClusterFailpoints}
+	RaftBeforeFollowerSendPanic     Failpoint = goPanicFailpoint{"raftBeforeFollowerSend", nil, Follower}
+	RandomMultiNodeClusterFailpoint Failpoint = randomFailpoint{append(oneNodeClusterFailpoints, RaftBeforeFollowerSendPanic)}
+	RaftBeforeApplySnapPanic        Failpoint = goPanicFailpoint{"raftBeforeApplySnap", triggerBlackholeUntilSnapshot, Follower}
+	RaftAfterApplySnapPanic         Failpoint = goPanicFailpoint{"raftAfterApplySnap", triggerBlackholeUntilSnapshot, Follower}
+	RaftAfterWALReleasePanic        Failpoint = goPanicFailpoint{"raftAfterWALRelease", triggerBlackholeUntilSnapshot, Follower}
+	RaftBeforeSaveSnapPanic         Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackholeUntilSnapshot, Follower}
+	RaftAfterSaveSnapPanic          Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackholeUntilSnapshot, Follower}
+	RandomSnapshotFailpoint         Failpoint = randomFailpoint{[]Failpoint{
 		RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic,
 	}}
-	// TODO: Figure out how to reliably trigger below failpoints and add them to RandomFailpoint
-	raftBeforeFollowerSendPanic Failpoint = goPanicFailpoint{"raftBeforeFollowerSend", nil, AnyMember}
 )
 
 type Failpoint interface {

diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go
@@ -96,7 +96,7 @@ func TestLinearizability(t *testing.T) {
 	for _, traffic := range trafficList {
 		scenarios = append(scenarios, scenario{
 			name:      "ClusterOfSize1/" + traffic.name,
-			failpoint: RandomFailpoint,
+			failpoint: RandomOneNodeClusterFailpoint,
 			traffic:   &traffic,
 			config: *e2e.NewConfig(
 				e2e.WithClusterSize(1),
@@ -108,7 +108,7 @@ func TestLinearizability(t *testing.T) {
 		})
 		scenarios = append(scenarios, scenario{
 			name:      "ClusterOfSize3/" + traffic.name,
-			failpoint: RandomFailpoint,
+			failpoint: RandomMultiNodeClusterFailpoint,
 			traffic:   &traffic,
 			config: *e2e.NewConfig(
 				e2e.WithSnapshotCount(100),