From 6f0e8d03b79ea85c5a52d55c759e7a0ccc8486af Mon Sep 17 00:00:00 2001 From: Alina Sklarevich Date: Tue, 6 Mar 2018 19:16:07 +0200 Subject: [PATCH 1/2] IB/DC: new vendor_part_id for ConnectX-5 and enable dc for a Generic HCA + Added the vendor_part_id=41682 to the list of usable IB HCAs. + Don't disqualify the dc transport if its version_flag is not UCT_DC_IFACE_ADDR_DC_V1 or UCT_DC_IFACE_ADDR_DC_V2. This could be the case for a "Generic HCA" which does support dc. (cherry picked from commit 03046ae931b7bb7f78dda4ceef5ebe0592678bd3) --- src/uct/ib/base/ib_device.c | 3 +++ src/uct/ib/dc/base/dc_iface.c | 14 ++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/uct/ib/base/ib_device.c b/src/uct/ib/base/ib_device.c index e4d75a8e20f..4d71c33b009 100644 --- a/src/uct/ib/base/ib_device.c +++ b/src/uct/ib/base/ib_device.c @@ -54,6 +54,9 @@ static uct_ib_device_spec_t uct_ib_builtin_device_specs[] = { {0x02c9, 4120, "ConnectX-5", UCT_IB_DEVICE_FLAG_MELLANOX | UCT_IB_DEVICE_FLAG_MLX5_PRM | UCT_IB_DEVICE_FLAG_DC_V2, 39}, + {0x02c9, 41682, "ConnectX-5", + UCT_IB_DEVICE_FLAG_MELLANOX | UCT_IB_DEVICE_FLAG_MLX5_PRM | + UCT_IB_DEVICE_FLAG_DC_V2, 37}, {0, 0, "Generic HCA", 0, 0} }; diff --git a/src/uct/ib/dc/base/dc_iface.c b/src/uct/ib/dc/base/dc_iface.c index cc52a6308e8..a427bc1a9eb 100644 --- a/src/uct/ib/dc/base/dc_iface.c +++ b/src/uct/ib/dc/base/dc_iface.c @@ -219,8 +219,7 @@ void uct_dc_iface_set_quota(uct_dc_iface_t *iface, uct_dc_iface_config_t *config ucs_min(iface->super.config.tx_qp_len, config->quota); } -static ucs_status_t uct_dc_iface_init_version(uct_dc_iface_t *iface, - uct_md_h md) +static void uct_dc_iface_init_version(uct_dc_iface_t *iface, uct_md_h md) { uct_ib_device_t *dev; unsigned ver; @@ -230,16 +229,13 @@ static ucs_status_t uct_dc_iface_init_version(uct_dc_iface_t *iface, if (ver & UCT_IB_DEVICE_FLAG_DC_V2) { iface->version_flag = UCT_DC_IFACE_ADDR_DC_V2; - return UCS_OK; } if (ver & UCT_IB_DEVICE_FLAG_DC_V1) { iface->version_flag = UCT_DC_IFACE_ADDR_DC_V1; - return UCS_OK; } iface->version_flag = 0; - return UCS_ERR_UNSUPPORTED; } UCS_CLASS_INIT_FUNC(uct_dc_iface_t, uct_dc_iface_ops_t *ops, uct_md_h md, @@ -265,10 +261,7 @@ UCS_CLASS_INIT_FUNC(uct_dc_iface_t, uct_dc_iface_ops_t *ops, uct_md_h md, return UCS_ERR_INVALID_PARAM; } - status = uct_dc_iface_init_version(self, md); - if (status != UCS_OK) { - return status; - } + uct_dc_iface_init_version(self, md); self->tx.ndci = config->ndci; self->tx.policy = config->tx_policy; @@ -358,10 +351,11 @@ int uct_dc_iface_is_reachable(const uct_iface_h tl_iface, uct_dc_iface_t UCS_V_UNUSED *iface = ucs_derived_of(tl_iface, uct_dc_iface_t); uct_dc_iface_addr_t *addr = (uct_dc_iface_addr_t *)iface_addr; + uint8_t dc_ver_mask = UCT_DC_IFACE_ADDR_DC_V1 | UCT_DC_IFACE_ADDR_DC_V2; ucs_assert_always(iface_addr != NULL); - return (addr->flags & iface->version_flag) && + return ((addr->flags & dc_ver_mask) == (iface->version_flag & dc_ver_mask)) && (UCT_DC_IFACE_ADDR_TM_ENABLED(addr) == UCT_RC_IFACE_TM_ENABLED(&iface->super)) && uct_ib_iface_is_reachable(tl_iface, dev_addr, iface_addr); From ae744b8b321fa10e7f88baca1581821ba9dca5e1 Mon Sep 17 00:00:00 2001 From: Alina Sklarevich Date: Wed, 7 Mar 2018 19:31:15 +0200 Subject: [PATCH 2/2] DC: enable dc for a Generic HCA - apply CR comments. (cherry picked from commit 4bed082d790a534d6029985b3497aaa952d22d74) --- src/uct/ib/dc/base/dc_iface.c | 8 ++++---- src/uct/ib/dc/base/dc_iface.h | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/uct/ib/dc/base/dc_iface.c b/src/uct/ib/dc/base/dc_iface.c index a427bc1a9eb..50bd0c0b1d2 100644 --- a/src/uct/ib/dc/base/dc_iface.c +++ b/src/uct/ib/dc/base/dc_iface.c @@ -226,6 +226,9 @@ static void uct_dc_iface_init_version(uct_dc_iface_t *iface, uct_md_h md) dev = &ucs_derived_of(md, uct_ib_md_t)->dev; ver = uct_ib_device_spec(dev)->flags & UCT_IB_DEVICE_FLAG_DC; + ucs_assert(ver != UCT_IB_DEVICE_FLAG_DC); + + iface->version_flag = 0; if (ver & UCT_IB_DEVICE_FLAG_DC_V2) { iface->version_flag = UCT_DC_IFACE_ADDR_DC_V2; @@ -234,8 +237,6 @@ static void uct_dc_iface_init_version(uct_dc_iface_t *iface, uct_md_h md) if (ver & UCT_IB_DEVICE_FLAG_DC_V1) { iface->version_flag = UCT_DC_IFACE_ADDR_DC_V1; } - - iface->version_flag = 0; } UCS_CLASS_INIT_FUNC(uct_dc_iface_t, uct_dc_iface_ops_t *ops, uct_md_h md, @@ -351,11 +352,10 @@ int uct_dc_iface_is_reachable(const uct_iface_h tl_iface, uct_dc_iface_t UCS_V_UNUSED *iface = ucs_derived_of(tl_iface, uct_dc_iface_t); uct_dc_iface_addr_t *addr = (uct_dc_iface_addr_t *)iface_addr; - uint8_t dc_ver_mask = UCT_DC_IFACE_ADDR_DC_V1 | UCT_DC_IFACE_ADDR_DC_V2; ucs_assert_always(iface_addr != NULL); - return ((addr->flags & dc_ver_mask) == (iface->version_flag & dc_ver_mask)) && + return ((addr->flags & UCT_DC_IFACE_ADDR_DC_VERS) == iface->version_flag) && (UCT_DC_IFACE_ADDR_TM_ENABLED(addr) == UCT_RC_IFACE_TM_ENABLED(&iface->super)) && uct_ib_iface_is_reachable(tl_iface, dev_addr, iface_addr); diff --git a/src/uct/ib/dc/base/dc_iface.h b/src/uct/ib/dc/base/dc_iface.h index fd909d4508b..4959449ca07 100644 --- a/src/uct/ib/dc/base/dc_iface.h +++ b/src/uct/ib/dc/base/dc_iface.h @@ -23,9 +23,11 @@ typedef struct uct_dc_iface uct_dc_iface_t; typedef enum { - UCT_DC_IFACE_ADDR_HW_TM = UCS_BIT(0), - UCT_DC_IFACE_ADDR_DC_V1 = UCS_BIT(1), - UCT_DC_IFACE_ADDR_DC_V2 = UCS_BIT(2) + UCT_DC_IFACE_ADDR_HW_TM = UCS_BIT(0), + UCT_DC_IFACE_ADDR_DC_V1 = UCS_BIT(1), + UCT_DC_IFACE_ADDR_DC_V2 = UCS_BIT(2), + UCT_DC_IFACE_ADDR_DC_VERS = UCT_DC_IFACE_ADDR_DC_V1 | + UCT_DC_IFACE_ADDR_DC_V2 } uct_dc_iface_addr_flags_t;