Support multiple RC implementations, and add tests for that.

config/m4/mpi.m4

@@ -31,6 +31,7 @@ AS_IF([test "x$with_mpi" == xyes],
         AS_IF([test -z "$MPIRUN"], AC_MSG_ERROR([--with-mpi was requested but MPI was not found in the PATH in $mpi_path]),[:])
         ],[:])
 
+AS_IF([test -n "$MPIRUN"], AC_DEFINE([HAVE_MPI], [1], [MPI support]))
 AM_CONDITIONAL([HAVE_MPI],    [test -n "$MPIRUN"])
 AM_CONDITIONAL([HAVE_MPICC],  [test -n "$MPICC"])
 AM_CONDITIONAL([HAVE_MPIRUN], [test -n "$MPIRUN"])

src/ucs/type/component.h

@@ -20,7 +20,6 @@
 typedef ucs_status_t (*ucs_component_init_cb_t)(void *base_ptr);
 typedef void (*ucs_component_cleanup_cb_t)(void *base_ptr);
 typedef struct ucs_component {
-    const char                  *name;
     ucs_component_init_cb_t     init;
     ucs_component_cleanup_cb_t  cleanup;
     size_t                      size;
@@ -60,9 +59,8 @@ typedef struct ucs_component {
     \
     size_t ucs_##_name##_component_offset; \
     \
-    void UCS_F_CTOR UCS_PP_APPEND_UNIQUE_ID(ucs_component_##_name##_register)() { \
+    static void UCS_F_CTOR UCS_PP_APPEND_UNIQUE_ID(ucs_component_##_name##_register)() { \
         static ucs_component_t comp = { \
-            #_name, \
             (ucs_component_init_cb_t)_init, \
             (ucs_component_cleanup_cb_t)_cleanup, \
             _size}; \

src/uct/Makefile.am

@@ -40,11 +40,11 @@ libuct_la_SOURCES += \
 	ib/base/ib_iface.c
 
 if HAVE_MLX5_HW
-noinst_HEADERS += \
-	ib/mlx5/ib_mlx5.h
-
 libuct_la_SOURCES += \
 	ib/mlx5/ib_mlx5.c
+
+noinst_HEADERS += \
+	ib/mlx5/ib_mlx5.h
 endif
 
 
@@ -56,12 +56,12 @@ noinst_HEADERS += \
 
 libuct_la_SOURCES += \
 	ib/rc/rc_ep.c \
-	ib/rc/rc_iface.c
+	ib/rc/rc_iface.c \
+	ib/rc/rc_verbs.c
 
 if HAVE_MLX5_HW
 libuct_la_SOURCES += \
-	ib/rc/rc_mlx5.c \
-	ib/rc/rc_mlx5.h
+	ib/rc/rc_mlx5.c
 endif
 
 endif

src/uct/ib/rc/rc_iface.c

@@ -63,10 +63,23 @@ void uct_rc_iface_remove_ep(uct_rc_iface_t *iface, uct_rc_ep_t *ep)
     sglib_hashed_uct_rc_ep_t_delete(iface->eps, ep);
 }
 
+ucs_status_t uct_rc_iface_flush(uct_iface_h tl_iface, uct_req_h *req_p,
+                                uct_completion_cb_t cb)
+{
+    uct_rc_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_iface_t);
+
+    if (iface->tx.outstanding > 0) {
+        return UCS_ERR_WOULD_BLOCK;
+    }
+
+    return UCS_OK;
+}
 
-static UCS_CLASS_INIT_FUNC(uct_rc_iface_t, uct_context_h context, const char *dev_name)
+static UCS_CLASS_INIT_FUNC(uct_rc_iface_t, uct_iface_ops_t *ops,
+                           uct_context_h context, const char *dev_name)
 {
-    UCS_CLASS_CALL_SUPER_INIT(context, dev_name);
+    UCS_CLASS_CALL_SUPER_INIT(ops, context, dev_name);
+    self->tx.outstanding = 0;
     return UCS_OK;
 }
 

src/uct/ib/rc/rc_iface.h

@@ -16,7 +16,13 @@
 
 struct uct_rc_iface {
     uct_ib_iface_t           super;
+
+    struct {
+        unsigned             outstanding;
+    } tx;
+
     uct_rc_ep_t              *eps[UCT_RC_QP_HASH_SIZE];
+
 };
 
 typedef struct uct_rc_iface_config {
@@ -35,4 +41,7 @@ uct_rc_ep_t *uct_rc_iface_lookup_ep(uct_rc_iface_t *iface, unsigned qp_num);
 void uct_rc_iface_add_ep(uct_rc_iface_t *iface, uct_rc_ep_t *ep);
 void uct_rc_iface_remove_ep(uct_rc_iface_t *iface, uct_rc_ep_t *ep);
 
+ucs_status_t uct_rc_iface_flush(uct_iface_h tl_iface, uct_req_h *req_p,
+                                uct_completion_cb_t cb);
+
 #endif

src/uct/ib/rc/rc_mlx5.c

@@ -6,7 +6,6 @@
 */
 
 #include "rc_iface.h"
-#include "rc_mlx5.h"
 
 #include <uct/api/uct.h>
 #include <uct/ib/base/ib_context.h>
@@ -20,6 +19,35 @@
 #include <arpa/inet.h> /* For htonl */
 
 
+typedef struct {
+    uct_rc_ep_t      super;
+    unsigned         qpn_ds;
+
+    struct {
+        unsigned       sw_pi;
+        unsigned       max_pi;
+        void           *seg;
+        void           *bf_reg;
+        unsigned long  bf_size;
+        uint32_t       *dbrec;
+        void           *qstart;
+        void           *qend;
+    } tx;
+} uct_rc_mlx5_ep_t;
+
+typedef struct {
+    uct_rc_iface_t     super;
+    struct {
+        void           *cq_buf;
+        unsigned       cq_ci;
+        unsigned       cq_length;
+    } tx;
+} uct_rc_mlx5_iface_t;
+
+typedef struct uct_rc_mlx5_iface_config {
+    uct_rc_iface_config_t  super;
+} uct_rc_mlx5_iface_config_t;
+
 typedef struct {
     struct mlx5_wqe_ctrl_seg     ctrl;
     struct mlx5_wqe_raddr_seg    raddr;
@@ -88,7 +116,7 @@ static ucs_status_t uct_rc_mlx5_ep_put_short(uct_ep_h tl_ep, void *buffer,
     uct_ib_mlx5_wqe_rc_rdma_inl_seg_t *seg = ep->tx.seg;
     unsigned wqe_size;
     uint32_t sw_pi_16_n;
-    uint64_t *src, *dst, *end;
+    uint64_t *src, *dst;
     unsigned sw_pi, i;
 
     sw_pi = ep->tx.sw_pi;
@@ -110,10 +138,10 @@ static ucs_status_t uct_rc_mlx5_ep_put_short(uct_ep_h tl_ep, void *buffer,
     /* Data */
     UCS_STATIC_ASSERT(seg + 1 == ((void*)seg + 32 + 4));
     seg->inl.byte_count          = htonl(length | MLX5_INLINE_SEG);
-    memcpy(seg + 1, buffer, length);
+    __builtin_memcpy(seg + 1, buffer, length);
 
     /* Write doorbell record */
-    ucs_compiler_fence();
+    ucs_compiler_fence(); /* Put memory store fence here too, to prevent WC being flushed after DBrec */
     *ep->tx.dbrec = sw_pi_16_n;
 
     /* Make sure that doorbell record is written before ringing the doorbell */
@@ -122,27 +150,27 @@ static ucs_status_t uct_rc_mlx5_ep_put_short(uct_ep_h tl_ep, void *buffer,
     /* Set up copy pointers */
     dst = ep->tx.bf_reg;
     src = (void*)seg;
-    end = (void*)seg + sizeof(*seg) + length;
 
     /* BF copy */
-    do {
-        for (i = 0; i < MLX5_SEND_WQE_BB / sizeof(*dst); ++i) {
-            *dst++ = *src++;
-        }
-    } while (src < end);
+    for (i = 0; i < MLX5_SEND_WQE_BB / sizeof(*dst); ++i) {
+        *dst++ = *src++;
+    }
+
+    /* We don't want the compiler to reorder instructions and hurt our latency */
+    ucs_compiler_fence();
 
     /* Flip BF register */
     ep->tx.bf_reg = (void*) ((uintptr_t) ep->tx.bf_reg ^ ep->tx.bf_size);
 
-    /* Completion counters */
-    ++ep->tx.sw_pi;
-    ++ucs_derived_of(ep->super.super.iface, uct_rc_mlx5_iface_t)->tx.outstanding;
-
     /* Advance queue pointer */
     if (ucs_unlikely((ep->tx.seg += MLX5_SEND_WQE_BB) >= ep->tx.qend)) {
         ep->tx.seg = ep->tx.qstart;
     }
 
+    /* Completion counters */
+    ++ep->tx.sw_pi;
+    ++ucs_derived_of(ep->super.super.iface, uct_rc_iface_t)->tx.outstanding;
+
     return UCS_OK;
 }
 
@@ -158,8 +186,9 @@ static void uct_rc_mlx5_iface_progress(void *arg)
     if (uct_ib_mlx5_cqe_hw_owned(cqe, index, iface->tx.cq_length)) {
         return; /* CQ is empty */
     }
+
     iface->tx.cq_ci = index + 1;
-    --iface->tx.outstanding;
+    --iface->super.tx.outstanding;
 
     ucs_memory_cpu_load_fence();
 
@@ -170,18 +199,6 @@ static void uct_rc_mlx5_iface_progress(void *arg)
     ++ep->tx.max_pi;
 }
 
-static ucs_status_t uct_rc_mlx5_iface_flush(uct_iface_h tl_iface, uct_req_h *req_p,
-                                            uct_completion_cb_t cb)
-{
-    uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_mlx5_iface_t);
-
-    if (iface->tx.outstanding > 0) {
-        return UCS_ERR_WOULD_BLOCK;
-    }
-
-    return UCS_OK;
-}
-
 static ucs_status_t uct_rc_mlx5_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr)
 {
     uct_rc_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_iface_t);
@@ -196,7 +213,7 @@ static void UCS_CLASS_DELETE_FUNC_NAME(uct_rc_mlx5_iface_t)(uct_iface_t*);
 uct_iface_ops_t uct_rc_mlx5_iface_ops = {
     .iface_close         = UCS_CLASS_DELETE_FUNC_NAME(uct_rc_mlx5_iface_t),
     .iface_get_address   = uct_rc_iface_get_address,
-    .iface_flush         = uct_rc_mlx5_iface_flush,
+    .iface_flush         = uct_rc_iface_flush,
     .ep_get_address      = uct_rc_ep_get_address,
     .ep_connect_to_iface = NULL,
     .ep_connect_to_ep    = uct_rc_ep_connect_to_ep,
@@ -230,7 +247,6 @@ static UCS_CLASS_INIT_FUNC(uct_rc_mlx5_iface_t, uct_context_h context,
     self->tx.cq_buf      = cq_info.buf;
     self->tx.cq_ci       = 0;
     self->tx.cq_length   = cq_info.cqe_cnt;
-    self->tx.outstanding = 0;
 
     ucs_notifier_chain_add(&context->progress_chain, uct_rc_mlx5_iface_progress,
                            self);
@@ -243,7 +259,7 @@ static UCS_CLASS_CLEANUP_FUNC(uct_rc_mlx5_iface_t)
     ucs_notifier_chain_remove(&context->progress_chain, uct_rc_mlx5_iface_progress, self);
 }
 
-UCS_CLASS_DEFINE(uct_rc_mlx5_iface_t, uct_ib_iface_t);
+UCS_CLASS_DEFINE(uct_rc_mlx5_iface_t, uct_rc_iface_t);
 static UCS_CLASS_DEFINE_NEW_FUNC(uct_rc_mlx5_iface_t, uct_iface_t, uct_context_h,
                                  const char*, uct_iface_config_t*);
 static UCS_CLASS_DEFINE_DELETE_FUNC(uct_rc_mlx5_iface_t, uct_iface_t);