Skip to content

Commit

Permalink
Merge pull request #7731 from dmitrygx/topic/uct/ib_pkey_roce_1_12
Browse files Browse the repository at this point in the history 
UCT/IB: Choose PKEY index 0 for RoCE devices [v1.12]
  • Loading branch information
@dmitrygx
dmitrygx authored Dec 1, 2021
2 parents d0208cd + 656d82d commit c42f020
Show file tree
Hide file tree
Showing 7 changed files with 64 additions and 43 deletions.
1 change: 1 addition & 0 deletions src/uct/ib/base/ib_device.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
#define UCT_IB_MAX_MESSAGE_SIZE (2UL << 30) /* Maximal IB message size */
#define UCT_IB_PKEY_PARTITION_MASK 0x7fff /* IB partition number mask */
#define UCT_IB_PKEY_MEMBERSHIP_MASK 0x8000 /* Full/send-only member */
#define UCT_IB_PKEY_DEFAULT 0xffff /* Default PKEY */
#define UCT_IB_DEV_MAX_PORTS 2
#define UCT_IB_FABRIC_TIME_MAX 32
#define UCT_IB_INVALID_RKEY 0xffffffffu
Expand Down
64 changes: 35 additions & 29 deletions src/uct/ib/base/ib_iface.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -786,13 +786,20 @@ void uct_ib_iface_fill_ah_attr_from_addr(uct_ib_iface_t *iface,
static ucs_status_t uct_ib_iface_init_pkey(uct_ib_iface_t *iface,
const uct_ib_iface_config_t *config)
{
uct_ib_device_t *dev = uct_ib_iface_device(iface);
uint16_t pkey_tbl_len = uct_ib_iface_port_attr(iface)->pkey_tbl_len;
int pkey_found = 0;
uint16_t lim_pkey = UCT_IB_ADDRESS_INVALID_PKEY;
uint16_t lim_pkey_index = UINT16_MAX;
uct_ib_device_t *dev = uct_ib_iface_device(iface);
uint16_t pkey_tbl_len = uct_ib_iface_port_attr(iface)->pkey_tbl_len;
int UCS_V_UNUSED pkey_found = 0;
uint16_t lim_pkey = UCT_IB_ADDRESS_INVALID_PKEY;
uint16_t lim_pkey_index = UINT16_MAX;
uint16_t pkey_index, port_pkey, pkey;

if (uct_ib_iface_is_roce(iface)) {
/* RoCE: use PKEY index 0, which contains the default PKEY: 0xffff */
iface->pkey_index = 0;
iface->pkey = UCT_IB_PKEY_DEFAULT;
goto out_pkey_found;
}

if ((config->pkey != UCS_HEXUNITS_AUTO) &&
(config->pkey > UCT_IB_PKEY_PARTITION_MASK)) {
ucs_error("requested pkey 0x%x is invalid, should be in the range 0..0x%x",
Expand Down Expand Up @@ -821,43 +828,42 @@ static ucs_status_t uct_ib_iface_init_pkey(uct_ib_iface_t *iface,
if ((config->pkey == UCS_HEXUNITS_AUTO) ||
/* take only the lower 15 bits for the comparison */
((pkey & UCT_IB_PKEY_PARTITION_MASK) == config->pkey)) {
if (!(pkey & UCT_IB_PKEY_MEMBERSHIP_MASK) &&
if (pkey & UCT_IB_PKEY_MEMBERSHIP_MASK) {
iface->pkey_index = pkey_index;
iface->pkey = pkey;
pkey_found = 1;
goto out_pkey_found;
} else if (lim_pkey == UCT_IB_ADDRESS_INVALID_PKEY) {
/* limited PKEY has not yet been found */
(lim_pkey == UCT_IB_ADDRESS_INVALID_PKEY)) {
lim_pkey_index = pkey_index;
lim_pkey = pkey;
continue;
}

iface->pkey_index = pkey_index;
iface->pkey = pkey;
pkey_found = 1;
break;
}
}

if (!pkey_found) {
if (lim_pkey == UCT_IB_ADDRESS_INVALID_PKEY) {
/* PKEY neither with full nor with limited membership was found */
if (config->pkey == UCS_HEXUNITS_AUTO) {
ucs_error("there is no valid pkey to use on "
UCT_IB_IFACE_FMT, UCT_IB_IFACE_ARG(iface));
} else {
ucs_error("unable to find specified pkey 0x%x on "UCT_IB_IFACE_FMT,
config->pkey, UCT_IB_IFACE_ARG(iface));
}
ucs_assert(!pkey_found);

return UCS_ERR_NO_ELEM;
if (lim_pkey == UCT_IB_ADDRESS_INVALID_PKEY) {
/* PKEY neither with full nor with limited membership was found */
if (config->pkey == UCS_HEXUNITS_AUTO) {
ucs_error("there is no valid pkey to use on " UCT_IB_IFACE_FMT,
UCT_IB_IFACE_ARG(iface));
} else {
ucs_assert(lim_pkey_index != UINT16_MAX);
iface->pkey_index = lim_pkey_index;
iface->pkey = lim_pkey;
ucs_error("unable to find specified pkey 0x%x on "UCT_IB_IFACE_FMT,
config->pkey, UCT_IB_IFACE_ARG(iface));
}

return UCS_ERR_NO_ELEM;
}

ucs_assertv(lim_pkey_index < pkey_tbl_len, "lim_pkey_index=%u"
" pkey_tbl_len=%u", lim_pkey_index, pkey_tbl_len);
iface->pkey_index = lim_pkey_index;
iface->pkey = lim_pkey;

out_pkey_found:
ucs_debug("using pkey[%d] 0x%x on "UCT_IB_IFACE_FMT, iface->pkey_index,
iface->pkey, UCT_IB_IFACE_ARG(iface));

return UCS_OK;
}

Expand Down Expand Up @@ -1666,7 +1672,7 @@ ucs_status_t uct_ib_iface_query(uct_ib_iface_t *iface, size_t xport_hdr_len,
encoding = 64.0/66.0;
break;
default:
ucs_error("Invalid active_speed on %s:%d: %d",
ucs_error("Invalid active_speed on " UCT_IB_IFACE_FMT ": %d",
UCT_IB_IFACE_ARG(iface), active_speed);
return UCS_ERR_IO_ERROR;
}
Expand Down
7 changes: 5 additions & 2 deletions src/uct/ib/base/ib_iface.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,9 +553,12 @@ uint8_t uct_ib_iface_config_select_sl(const uct_ib_iface_config_t *ib_config);


#define UCT_IB_IFACE_FMT \
"%s:%d"
"%s:%d/%s"
#define UCT_IB_IFACE_ARG(_iface) \
uct_ib_device_name(uct_ib_iface_device(_iface)), (_iface)->config.port_num
uct_ib_device_name(uct_ib_iface_device(_iface)), \
(_iface)->config.port_num, \
uct_ib_iface_is_roce(_iface) ? "RoCE" : "IB"



#define UCT_IB_IFACE_VERBS_COMPLETION_ERR(_type, _iface, _i, _wc) \
Expand Down
15 changes: 12 additions & 3 deletions src/uct/ib/dc/dc_mlx5_devx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ ucs_status_t uct_dc_mlx5_iface_devx_create_dct(uct_dc_mlx5_iface_t *iface,
UCT_IB_MLX5DV_SET(dctc, dctc, cs_res, uct_ib_mlx5_qpc_cs_res(
iface->super.super.super.config.max_inl_cqe[UCT_IB_DIR_RX], 1));
UCT_IB_MLX5DV_SET(dctc, dctc, atomic_mode, UCT_IB_MLX5_ATOMIC_MODE);
UCT_IB_MLX5DV_SET(dctc, dctc, pkey_index, iface->super.super.super.pkey_index);
if (!uct_ib_iface_is_roce(&iface->super.super.super)) {
UCT_IB_MLX5DV_SET(dctc, dctc, pkey_index,
iface->super.super.super.pkey_index);
}
UCT_IB_MLX5DV_SET(dctc, dctc, port, iface->super.super.super.config.port_num);

UCT_IB_MLX5DV_SET(dctc, dctc, min_rnr_nak, iface->super.super.config.min_rnr_timer);
Expand Down Expand Up @@ -108,7 +111,10 @@ ucs_status_t uct_dc_mlx5_iface_devx_dci_connect(uct_dc_mlx5_iface_t *iface,
qpc = UCT_IB_MLX5DV_ADDR_OF(rst2init_qp_in, in_2init, qpc);
UCT_IB_MLX5DV_SET(qpc, qpc, pm_state, UCT_IB_MLX5_QPC_PM_STATE_MIGRATED);
UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.vhca_port_num, iface->super.super.super.config.port_num);
UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.pkey_index, iface->super.super.super.pkey_index);
if (!uct_ib_iface_is_roce(&iface->super.super.super)) {
UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.pkey_index,
iface->super.super.super.pkey_index);
}

status = uct_ib_mlx5_devx_modify_qp(qp, in_2init, sizeof(in_2init),
out_2init, sizeof(out_2init));
Expand Down Expand Up @@ -166,7 +172,10 @@ ucs_status_t uct_dc_mlx5_iface_devx_set_srq_dc_params(uct_dc_mlx5_iface_t *iface
char out[UCT_IB_MLX5DV_ST_SZ_BYTES(set_xrq_dc_params_entry_out)] = {};
int ret;

UCT_IB_MLX5DV_SET(set_xrq_dc_params_entry_in, in, pkey_table_index, iface->super.super.super.pkey_index);
if (!uct_ib_iface_is_roce(&iface->super.super.super)) {
UCT_IB_MLX5DV_SET(set_xrq_dc_params_entry_in, in, pkey_table_index,
iface->super.super.super.pkey_index);
}
UCT_IB_MLX5DV_SET(set_xrq_dc_params_entry_in, in, mtu, iface->super.super.super.config.path_mtu);
UCT_IB_MLX5DV_SET(set_xrq_dc_params_entry_in, in, sl, iface->super.super.super.config.sl);
UCT_IB_MLX5DV_SET(set_xrq_dc_params_entry_in, in, reverse_sl, iface->super.super.super.config.sl);
Expand Down
5 changes: 4 additions & 1 deletion src/uct/ib/mlx5/dv/ib_mlx5_dv.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,10 @@ ucs_status_t uct_ib_mlx5_devx_create_qp(uct_ib_iface_t *iface,
UCT_IB_MLX5DV_SET(rst2init_qp_in, in_2init, qpn, qp->qp_num);
UCT_IB_MLX5DV_SET(qpc, qpc, pm_state, UCT_IB_MLX5_QPC_PM_STATE_MIGRATED);
UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.vhca_port_num, attr->super.port);
UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.pkey_index, iface->pkey_index);
if (!uct_ib_iface_is_roce(iface)) {
UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.pkey_index,
iface->pkey_index);
}
UCT_IB_MLX5DV_SET(qpc, qpc, rwe, true);

ret = mlx5dv_devx_obj_modify(qp->devx.obj, in_2init, sizeof(in_2init),
Expand Down
9 changes: 5 additions & 4 deletions src/uct/ib/mlx5/ib_mlx5.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -917,6 +917,7 @@ uct_ib_mlx5_iface_select_sl(uct_ib_iface_t *iface,
#if HAVE_DEVX
uct_ib_mlx5_md_t *md = ucs_derived_of(iface->super.md, uct_ib_mlx5_md_t);
#endif
const char *dev_name = uct_ib_device_name(uct_ib_iface_device(iface));
uint16_t ooo_sl_mask = 0;
ucs_status_t status;

Expand All @@ -926,8 +927,8 @@ uct_ib_mlx5_iface_select_sl(uct_ib_iface_t *iface,
iface->config.port_num)) {
/* Ethernet priority for RoCE devices can't be selected regardless
* AR support requested by user, pass empty ooo_sl_mask */
return uct_ib_mlx5_select_sl(ib_config, UCS_NO, 0, 1,
UCT_IB_IFACE_ARG(iface),
return uct_ib_mlx5_select_sl(ib_config, UCS_NO, 0, 1, dev_name,
iface->config.port_num,
&iface->config.sl);
}

Expand All @@ -942,8 +943,8 @@ uct_ib_mlx5_iface_select_sl(uct_ib_iface_t *iface,
#endif

return uct_ib_mlx5_select_sl(ib_config, ib_mlx5_config->ar_enable,
ooo_sl_mask, status == UCS_OK,
UCT_IB_IFACE_ARG(iface),
ooo_sl_mask, status == UCS_OK, dev_name,
iface->config.port_num,
&iface->config.sl);
}

Expand Down
6 changes: 2 additions & 4 deletions src/uct/ib/mlx5/ib_mlx5_log.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,11 +160,9 @@ ucs_status_t uct_ib_mlx5_completion_with_err(uct_ib_iface_t *iface,
}

ucs_log(log_level,
"%s on " UCT_IB_IFACE_FMT
"/%s (synd 0x%x vend 0x%x hw_synd %d/%d)\n"
"%s on " UCT_IB_IFACE_FMT " (synd 0x%x vend 0x%x hw_synd %d/%d)\n"
"%s QP 0x%x wqe[%d]: %s %s",
err_info, UCT_IB_IFACE_ARG(iface),
uct_ib_iface_is_roce(iface) ? "RoCE" : "IB", ecqe->syndrome,
err_info, UCT_IB_IFACE_ARG(iface), ecqe->syndrome,
ecqe->vendor_err_synd, ecqe->hw_synd_type >> 4, ecqe->hw_err_synd,
uct_ib_qp_type_str(iface->config.qp_type), qp_num, pi, wqe_info,
peer_info);
Expand Down

0 comments on commit c42f020

Please sign in to comment.