RFC: prov/verbs: add param for setting RDMA CM ToS

By default, if the RDMA CM application does not explicitly
set a ToS, then the system default is used. This system default
can be changed via the default_roce_tos configfs param, but this
is global to the system and is somewhat cumbersome to deal with.

This change introduces a new environment param: FI_VERBS_TOS

This param can be used to explicitly set the ToS via the
rdma_set_option API. If the ToS is set, then the system default
is ignored and the new value is used instead.

This allows for multiple concurrent workloads to use different
ToS values.

Valid range is -1 through 255. If unset or set to -1, then
the call to rdma_set_option is omitted, thus preserving existing
behavior.

Not supported/tested on Windows.

Signed-off-by: Jacob Moroni <jmoroni@google.com>

prov/verbs/src/verbs_domain_xrc.c

@@ -67,6 +67,10 @@ static int vrb_create_ini_qp(struct vrb_xrc_ep *ep)
 			   "XRC INI QP rdma_create_qp_ex failed %d\n", -ret);
 		return ret;
 	}
+
+	if (vrb_rdma_set_tos(ep->base_ep.id))
+		VRB_WARN_ERRNO(FI_LOG_EP_CTRL, "vrb_rdma_set_tos");
+
 	return FI_SUCCESS;
 #else /* VERBS_HAVE_XRC */
 	return -FI_ENOSYS;
@@ -400,6 +404,9 @@ int vrb_ep_create_tgt_qp(struct vrb_xrc_ep *ep, uint32_t tgt_qpn)
 	}
 	ep->tgt_ibv_qp = ep->tgt_id->qp;
 
+	if (vrb_rdma_set_tos(ep->tgt_id))
+		VRB_WARN_ERRNO(FI_LOG_EP_CTRL, "vrb_rdma_set_tos");
+
 	return FI_SUCCESS;
 #else /* VERBS_HAVE_XRC */
 	return -FI_ENOSYS;

prov/verbs/src/verbs_ep.c

@@ -1063,6 +1063,9 @@ static int vrb_ep_enable(struct fid_ep *ep_fid)
 			/* Allow shared XRC INI QP not controlled by RDMA CM
 			 * to share same post functions as RC QP. */
 			ep->ibv_qp = ep->id->qp;
+
+			if (vrb_rdma_set_tos(ep->id))
+				VRB_WARN_ERRNO(FI_LOG_EP_CTRL, "vrb_rdma_set_tos");
 		}
 		break;
 	case FI_EP_DGRAM:

prov/verbs/src/verbs_eq.c

@@ -893,6 +893,9 @@ vrb_eq_addr_resolved_event(struct vrb_ep *ep)
 		/* Allow shared XRC INI QP not controlled by RDMA CM
 		 * to share same post functions as RC QP. */
 		ep->ibv_qp = ep->id->qp;
+
+		if (vrb_rdma_set_tos(ep->id))
+			VRB_WARN_ERRNO(FI_LOG_EP_CTRL, "vrb_rdma_set_tos");
 	}
 
 	assert(ep->ibv_qp);

prov/verbs/src/verbs_init.c

@@ -45,6 +45,7 @@ static const char *local_node = "localhost";
 #define VERBS_DEFAULT_MIN_RNR_TIMER 12
 
 struct vrb_gl_data vrb_gl_data = {
+	.tos			= VERBS_TOS_UNSET,
 	.def_tx_size		= 384,
 	.def_rx_size		= 384,
 	.def_tx_iov_limit	= 4,
@@ -637,6 +638,14 @@ static int vrb_get_param_str(const char *param_name,
 int vrb_read_params(void)
 {
 	/* Common parameters */
+	if (vrb_get_param_int("tos", "RDMA CM ToS value. If unset or -1, then "
+			      "the ToS will not be explicitly set and the system "
+			      "default will be used. Valid range is -1 through 255.",
+			      &vrb_gl_data.tos) ||
+	    (vrb_gl_data.tos < -1 || vrb_gl_data.tos > 255)) {
+		VRB_WARN(FI_LOG_CORE, "Invalid value of ToS\n");
+		return -FI_EINVAL;
+	}
 	if (vrb_get_param_int("tx_size", "Default maximum tx context size",
 			      &vrb_gl_data.def_tx_size) ||
 	    (vrb_gl_data.def_tx_size < 0)) {

prov/verbs/src/verbs_ofi.h

@@ -161,6 +161,8 @@
 #define VERBS_ANY_DOMAIN "verbs_any_domain"
 #define VERBS_ANY_FABRIC "verbs_any_fabric"
 
+#define VERBS_TOS_UNSET	(-1)
+
 #ifdef HAVE_FABRIC_PROFILE
 struct vrb_profile;
 typedef struct vrb_profile    vrb_profile_t;
@@ -176,6 +178,7 @@ extern ofi_mutex_t vrb_init_mutex;
 extern struct dlist_entry verbs_devs;
 
 extern struct vrb_gl_data {
+	int	tos;
 	int	def_tx_size;
 	int	def_rx_size;
 	int	def_tx_iov_limit;
@@ -1050,6 +1053,16 @@ vrb_free_recv_wr(struct vrb_progress *progress, struct vrb_recv_wr *wr)
 	ofi_buf_free(wr);
 }
 
+static inline int vrb_rdma_set_tos(struct rdma_cm_id *id)
+{
+	if (vrb_gl_data.tos == VERBS_TOS_UNSET)
+		return 0;
+
+	uint8_t tos = vrb_gl_data.tos;
+	return rdma_set_option(id, RDMA_OPTION_ID, RDMA_OPTION_ID_TOS, &tos,
+			       sizeof(tos));
+}
+
 int vrb_ep_ops_open(struct fid *fid, const char *name,
                      uint64_t flags, void **ops, void *context);
 

prov/verbs/src/windows/verbs_nd_rdma.c

@@ -240,6 +240,13 @@ int rdma_destroy_id(struct rdma_cm_id *id)
 	return 0;
 }
 
+int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
+		    void *optval, size_t optlen)
+{
+	errno = ENOSYS;
+	return -1;
+}
+
 int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
 {
 	struct nd_cm_id *id_nd;