UCP: Non-contiguous registration send/recv support for RC and DC (#5/6)

src/ucp/api/ucp.h

@@ -240,9 +240,9 @@ enum ucp_worker_attr_field {
 enum ucp_dt_type {
     UCP_DATATYPE_CONTIG   = 0,      /**< Contiguous datatype */
     UCP_DATATYPE_IOV      = 2,      /**< Scatter-gather list with multiple pointers */
-    UCP_DATATYPE_IOV_R    = 3,      /**< Same as IOV, but reusable */
-    UCP_DATATYPE_STRIDE   = 4,      /**< Interleaving a pointers to strided data */
-    UCP_DATATYPE_STRIDE_R = 5,      /**< Strided datatype */
+    //UCP_DATATYPE_IOV_R    = 3,      /**< Same as IOV, but reusable */
+    //UCP_DATATYPE_STRIDE   = 4,      /**< Interleaving a pointers to strided data */
+    //UCP_DATATYPE_STRIDE_R = 5,      /**< Strided datatype */
     UCP_DATATYPE_GENERIC  = 7,      /**< Generic datatype with
                                          user-defined pack/unpack routines */
     UCP_DATATYPE_SHIFT    = 3,      /**< Number of bits defining

src/ucp/core/ucp_request.c

@@ -214,6 +214,14 @@ UCS_PROFILE_FUNC(ucs_status_t, ucp_request_memory_reg,
             }
         }
         state->dt.iov.memh = memh;
+
+        /* If non-contiguous bind is not supported - use the existing mapping */
+        uct_md_attr = &context->tl_mds[mdi].attr;
+        if (!(uct_md_attr->cap.flags & UCT_MD_FLAG_REG_NC)) {
+            break;
+        }
+
+        status = ucp_dt_reusable_create(ep, buffer, length, datatype, state);
         break;
 
     default:

src/uct/ib/base/ib_md.c

@@ -170,6 +170,14 @@ static ucs_status_t uct_ib_md_query(uct_md_h uct_md, uct_md_attr_t *md_attr)
         md_attr->cap.flags |= UCT_MD_FLAG_ALLOC;
     }
 
+    if (IBV_EXP_HAVE_UMR(&md->dev.dev_attr)) {
+        md_attr->cap.flags |= UCT_MD_FLAG_REG_NC;
+    }
+
+    if (IBV_EXP_HAVE_UMR(&md->dev.dev_attr)) {
+        md_attr->cap.flags |= UCT_MD_FLAG_REG_NC;
+    }
+
     md_attr->reg_cost      = md->reg_cost;
     md_attr->local_cpus    = md->dev.local_cpus;
     return UCS_OK;
@@ -207,6 +215,9 @@ static ucs_status_t uct_ib_md_umr_qp_create(uct_ib_md_t *md)
     ucs_status_t status = UCS_ERR_IO_ERROR;
 
     ibdev = &md->dev;
+    if (!IBV_EXP_HAVE_UMR(&ibdev->dev_attr)) {
+        return UCS_ERR_UNSUPPORTED;
+    }
 
     /* TODO: fix port selection. It looks like active port should be used */
     port_num = ibdev->first_port;