feat: improve relayer picking (#1301)

# Related Github tickets

- Closes #2193

# Background

At the moment we always pick the same validator to relay messages. We
need to improve this mechanism to pick different validators, especially
if the one selected is failing to relay messages. With this PR:
* We will pick randomly from the top 5 validators.
* On every failed message delivery, the success rate of a validator
drops to the base score of 0.5 if it's above 0.5.

# Testing completed

- [x] test coverage exists or has been added/updated
- [x] tested in a private testnet

# Breaking changes

- [x] I have checked my code for breaking changes
- [x] If there are breaking changes, there is a supporting migration.

tests/integration/metrix/keeper/keeper_intergration_test.go

@@ -592,9 +592,12 @@ var _ = Describe("update relay metrics", func() {
 			Expect(metrics[1].ExecutionTime).To(Equal(math.NewInt(349)))
 			Expect(metrics[2].ExecutionTime).To(Equal(math.NewInt(399)))
 
+			// First validator always succeeds
 			Expect(metrics[0].SuccessRate).To(Equal(palomath.LegacyDecFromFloat64(1)))
+			// Second validator always fails
 			Expect(metrics[1].SuccessRate).To(Equal(palomath.LegacyDecFromFloat64(0)))
-			Expect(metrics[2].SuccessRate).To(Equal(palomath.LegacyDecFromFloat64(.9)))
+			// Third validator fails the last message (j = 100)
+			Expect(metrics[2].SuccessRate).To(Equal(palomath.LegacyDecFromFloat64(.5)))
 		})
 	})
 })

x/evm/keeper/msg_assigner.go

@@ -17,6 +17,9 @@ import (
 	valsettypes "github.com/palomachain/paloma/x/valset/types"
 )
 
+// topValidatorPoolSize is the number of validators considered when picking
+const topValidatorPoolSize = 5
+
 type msgAssigner struct {
 	ValsetKeeper   types.ValsetKeeper
 	metrixKeeper   types.MetrixKeeper
@@ -73,7 +76,12 @@ func (ma msgAssigner) PickValidatorForMessage(ctx context.Context, weights *type
 		return "", errors.New("no assignable validators for message")
 	}
 
-	winner := assignableValidators[0].address
+	ts := sdk.UnwrapSDKContext(ctx).BlockTime().Unix()
+
+	// Use block timestamp to pick a random validator from the top validators
+	winnerIdx := ts % int64(min(len(assignableValidators), topValidatorPoolSize))
+
+	winner := assignableValidators[winnerIdx].address
 	ma.scores = removeWinnerFromSnapshot(ma.scores, winner)
 	return winner, nil
 }