From 2636504544342c42edb755d2d02a180ccc42b273 Mon Sep 17 00:00:00 2001 From: Artemy Kovalyov Date: Mon, 4 Mar 2019 12:06:52 +0200 Subject: [PATCH 1/2] UCT/IB/MLX5: Fasten DC support check --- src/uct/ib/mlx5/ib_mlx5_dv.c | 55 ++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5_dv.c b/src/uct/ib/mlx5/ib_mlx5_dv.c index cd1652c1565..d322bfc6278 100644 --- a/src/uct/ib/mlx5/ib_mlx5_dv.c +++ b/src/uct/ib/mlx5/ib_mlx5_dv.c @@ -26,12 +26,16 @@ ucs_status_t uct_ib_mlx5dv_init_obj(uct_ib_mlx5dv_t *obj, uint64_t type) static ucs_status_t uct_ib_mlx5_device_init(uct_ib_device_t *dev) { struct ibv_context *ctx = dev->ibv_context; + struct ibv_srq_init_attr srq_attr = {}; struct ibv_qp_init_attr_ex qp_attr = {}; struct mlx5dv_qp_init_attr dv_attr = {}; + struct ibv_qp_attr attr = {}; ucs_status_t status = UCS_OK; + struct ibv_srq *srq; struct ibv_pd *pd; struct ibv_cq *cq; struct ibv_qp *qp; + int ret; if (!(uct_ib_device_spec(dev)->flags & UCT_IB_DEVICE_FLAG_MLX5_PRM)) { return UCS_OK; @@ -50,24 +54,63 @@ static ucs_status_t uct_ib_mlx5_device_init(uct_ib_device_t *dev) goto err_cq; } + srq_attr.attr.max_sge = 1; + srq_attr.attr.max_wr = 1; + srq = ibv_create_srq(pd, &srq_attr); + if (srq == NULL) { + ucs_error("ibv_create_cq() failed: %m"); + status = UCS_ERR_IO_ERROR; + goto err_srq; + } + qp_attr.send_cq = cq; qp_attr.recv_cq = cq; - qp_attr.cap.max_send_wr = 1; - qp_attr.cap.max_send_sge = 1; qp_attr.qp_type = IBV_QPT_DRIVER; qp_attr.comp_mask = IBV_QP_INIT_ATTR_PD; qp_attr.pd = pd; + qp_attr.srq = srq; dv_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_DC; - dv_attr.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI; + dv_attr.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCT; + dv_attr.dc_init_attr.dct_access_key = UCT_IB_KEY; - /* create DCI qp successful means DC is supported */ + /* create DCT qp successful means DC is supported */ qp = mlx5dv_create_qp(ctx, &qp_attr, &dv_attr); - if (qp) { - ibv_destroy_qp(qp); + if (qp == NULL) { + goto err_qp; + } + + attr.qp_state = IBV_QPS_INIT; + attr.port_num = 1; + attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | + IBV_ACCESS_REMOTE_READ | + IBV_ACCESS_REMOTE_ATOMIC; + ret = ibv_modify_qp(qp, &attr, IBV_QP_STATE | + IBV_QP_PKEY_INDEX | + IBV_QP_PORT | + IBV_QP_ACCESS_FLAGS); + if (ret != 0) { + goto err; + } + + attr.qp_state = IBV_QPS_RTR; + attr.path_mtu = IBV_MTU_256; + attr.ah_attr.port_num = 1; + + ret = ibv_modify_qp(qp, &attr, IBV_QP_STATE | + IBV_QP_MIN_RNR_TIMER | + IBV_QP_AV | + IBV_QP_PATH_MTU); + + if (ret == 0) { dev->flags |= UCT_IB_DEVICE_FLAG_DC; } +err: + ibv_destroy_qp(qp); +err_qp: + ibv_destroy_srq(srq); +err_srq: ibv_destroy_cq(cq); err_cq: ibv_dealloc_pd(pd); From 777efd4734516cd08a4fce02c990ed86e4a170c7 Mon Sep 17 00:00:00 2001 From: Artemy Kovalyov Date: Thu, 7 Mar 2019 16:20:37 +0200 Subject: [PATCH 2/2] UCT/IB/MLX5: Fasten DC support check - review fixes 1 --- src/uct/ib/mlx5/ib_mlx5_dv.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/uct/ib/mlx5/ib_mlx5_dv.c b/src/uct/ib/mlx5/ib_mlx5_dv.c index d322bfc6278..9fc78ba9519 100644 --- a/src/uct/ib/mlx5/ib_mlx5_dv.c +++ b/src/uct/ib/mlx5/ib_mlx5_dv.c @@ -54,21 +54,21 @@ static ucs_status_t uct_ib_mlx5_device_init(uct_ib_device_t *dev) goto err_cq; } - srq_attr.attr.max_sge = 1; - srq_attr.attr.max_wr = 1; + srq_attr.attr.max_sge = 1; + srq_attr.attr.max_wr = 1; srq = ibv_create_srq(pd, &srq_attr); if (srq == NULL) { - ucs_error("ibv_create_cq() failed: %m"); + ucs_error("ibv_create_srq() failed: %m"); status = UCS_ERR_IO_ERROR; goto err_srq; } - qp_attr.send_cq = cq; - qp_attr.recv_cq = cq; - qp_attr.qp_type = IBV_QPT_DRIVER; - qp_attr.comp_mask = IBV_QP_INIT_ATTR_PD; - qp_attr.pd = pd; - qp_attr.srq = srq; + qp_attr.send_cq = cq; + qp_attr.recv_cq = cq; + qp_attr.qp_type = IBV_QPT_DRIVER; + qp_attr.comp_mask = IBV_QP_INIT_ATTR_PD; + qp_attr.pd = pd; + qp_attr.srq = srq; dv_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_DC; dv_attr.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCT; @@ -80,11 +80,11 @@ static ucs_status_t uct_ib_mlx5_device_init(uct_ib_device_t *dev) goto err_qp; } - attr.qp_state = IBV_QPS_INIT; - attr.port_num = 1; - attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | - IBV_ACCESS_REMOTE_READ | - IBV_ACCESS_REMOTE_ATOMIC; + attr.qp_state = IBV_QPS_INIT; + attr.port_num = 1; + attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | + IBV_ACCESS_REMOTE_READ | + IBV_ACCESS_REMOTE_ATOMIC; ret = ibv_modify_qp(qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | @@ -93,8 +93,8 @@ static ucs_status_t uct_ib_mlx5_device_init(uct_ib_device_t *dev) goto err; } - attr.qp_state = IBV_QPS_RTR; - attr.path_mtu = IBV_MTU_256; + attr.qp_state = IBV_QPS_RTR; + attr.path_mtu = IBV_MTU_256; attr.ah_attr.port_num = 1; ret = ibv_modify_qp(qp, &attr, IBV_QP_STATE |