support hns RoCE

Author: shizhibao <shizhibao@huawei.com> Date: Tue Oct 27 20:33:31 2020 +0800
shizhibao · Oct 29, 2020 · edeff5c · edeff5c
1 parent 64ac740
commit edeff5c
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 11 deletions.
diff --git a/src/uct/ib/base/ib_device.h b/src/uct/ib/base/ib_device.h
@@ -202,6 +202,21 @@ ucs_status_t uct_ib_device_select_gid_index(uct_ib_device_t *dev,
 const char *uct_ib_device_name(uct_ib_device_t *dev);
 
 
+/**
+ * @return true if device name begins with "hns".
+ */
+static inline int uct_ib_device_is_hns(struct ibv_device *device)
+{
+#if HAVE_HNS_ROCE
+#define UCT_IB_DEVICE_HNS "hns"
+#define UCT_IB_DEVICE_HNS_LEN 3
+    return !strncmp(ibv_get_device_name(device), UCT_IB_DEVICE_HNS, UCT_IB_DEVICE_HNS_LEN);
+#else
+    return 0;
+#endif
+}
+
+
 /**
  * @return whether the port is InfiniBand
  */

diff --git a/src/uct/ib/base/ib_iface.c b/src/uct/ib/base/ib_iface.c
@@ -584,15 +584,20 @@ ucs_status_t uct_ib_verbs_create_cq(struct ibv_context *context, int cqe,
 #if HAVE_DECL_IBV_CREATE_CQ_ATTR_IGNORE_OVERRUN
     struct ibv_cq_init_attr_ex cq_attr = {};
 
-    cq_attr.cqe = cqe;
-    cq_attr.channel = channel;
-    cq_attr.comp_vector = comp_vector;
-    if (ignore_overrun) {
-        cq_attr.comp_mask = IBV_CQ_INIT_ATTR_MASK_FLAGS;
-        cq_attr.flags = IBV_CREATE_CQ_ATTR_IGNORE_OVERRUN;
-    }
+    if (uct_ib_device_is_hns(context->device)) {
+        *inl = 0;
+        cq = ibv_create_cq(context, cqe, NULL, channel, comp_vector);
+    } else {
+        cq_attr.cqe = cqe;
+        cq_attr.channel = channel;
+        cq_attr.comp_vector = comp_vector;
+        if (ignore_overrun) {
+            cq_attr.comp_mask = IBV_CQ_INIT_ATTR_MASK_FLAGS;
+            cq_attr.flags = IBV_CREATE_CQ_ATTR_IGNORE_OVERRUN;
+        }
 
-    cq = ibv_cq_ex_to_cq(ibv_create_cq_ex(context, &cq_attr));
+        cq = ibv_cq_ex_to_cq(ibv_create_cq_ex(context, &cq_attr));
+    }
     if (!cq && (errno == ENOSYS))
 #endif
     {

diff --git a/src/uct/ib/base/ib_md.c b/src/uct/ib/base/ib_md.c
@@ -1937,7 +1937,12 @@ static ucs_status_t uct_ib_verbs_md_open(struct ibv_device *ibv_device,
 #if HAVE_DECL_IBV_EXP_QUERY_DEVICE
     ret = ibv_exp_query_device(dev->ibv_context, &dev->dev_attr);
 #elif HAVE_DECL_IBV_QUERY_DEVICE_EX
-    ret = ibv_query_device_ex(dev->ibv_context, NULL, &dev->dev_attr);
+    if (uct_ib_device_is_hns(ibv_device)) {
+        memset(&dev->dev_attr, 0, sizeof(dev->dev_attr));
+        ret = ibv_query_device(dev->ibv_context, &dev->dev_attr.orig_attr);
+    } else {
+        ret = ibv_query_device_ex(dev->ibv_context, NULL, &dev->dev_attr);
+    }
 #else
     ret = ibv_query_device(dev->ibv_context, &dev->dev_attr);
 #endif

diff --git a/src/uct/ib/configure.m4 b/src/uct/ib/configure.m4
@@ -368,9 +368,9 @@ AS_IF([test "x$with_ib" = "xyes"],
            ])
 
        # Hns RoCE support
-       AC_CHECK_FILE(/usr/lib64/libibverbs/libhns-rdmav17.so,
+       AC_CHECK_FILE(/usr/lib64/libibverbs/libhns-rdmav25.so,
        [
-            AC_CHECK_FILE(/usr/lib/modules/$(uname --release)/updates/drivers/infiniband/hw/hns/hns-roce.ko,
+            AC_CHECK_FILE(/usr/lib/modules/$(uname --release)/updates/drivers/infiniband/hw/hns/hns-roce-cae.ko,
             [
                 AC_CHECK_FILE(/usr/lib/modules/$(uname --release)/updates/drivers/infiniband/hw/hns/hns-roce-hw-v2.ko,
                 [