From eebad3b8b3cdc7c07fdf873049e7ade5a19056c8 Mon Sep 17 00:00:00 2001 From: Sergey Lebedev Date: Wed, 8 Nov 2023 17:00:11 +0000 Subject: [PATCH] BUILD: fix regcache alignment --- config/m4/ucx.m4 | 5 +++++ .../tl/mlx5/mcast/tl_mlx5_mcast_rcache.c | 6 ++---- src/components/tl/mlx5/tl_mlx5_rcache.c | 2 -- src/components/tl/sharp/tl_sharp_context.c | 2 -- src/utils/ucc_rcache.h | 19 ++++++++++++++++++- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/config/m4/ucx.m4 b/config/m4/ucx.m4 index b3a3b871c3..14d64be785 100644 --- a/config/m4/ucx.m4 +++ b/config/m4/ucx.m4 @@ -113,6 +113,11 @@ AS_IF([test "x$ucx_checked" != "xyes"],[ [AC_DEFINE([UCS_HAVE_CONFIG_GLOBAL_LIST_ENTRY_FLAGS], [1], [flags for config table])], [], [#include ]) + + AC_CHECK_MEMBER(ucs_rcache_region_t.alignment, + [AC_DEFINE([UCS_HAVE_RCACHE_REGION_ALIGNMENT], [1], [flags for ucs_rcache_get])], + [], + [#include ]) ], [ AS_IF([test "x$with_ucx" != "xguess"], diff --git a/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c b/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c index c67a2d3179..75c62ac81f 100644 --- a/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c +++ b/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c @@ -19,7 +19,7 @@ static ucs_status_t ucc_tl_mlx5_mcast_coll_reg_mr(ucc_tl_mlx5_mcast_coll_context tl_error(ctx->lib, "failed to register MR"); return UCS_ERR_NO_MEMORY; } - + return UCS_OK; } @@ -33,7 +33,7 @@ static ucc_status_t ucc_tl_mlx5_mcast_coll_dereg_mr(ucc_tl_mlx5_mcast_coll_conte } tl_debug(ctx->lib, "external memory deregister: mr %p", mr); - + if (ibv_dereg_mr(mr)) { tl_error(ctx->lib, "couldn't destroy mr %p", mr); return UCC_ERR_NO_RESOURCE; @@ -140,12 +140,10 @@ ucc_status_t ucc_tl_mlx5_mcast_setup_rcache(ucc_tl_mlx5_mcast_coll_context_t *ct { ucc_rcache_params_t rcache_params; - rcache_params.alignment = 64; rcache_params.ucm_event_priority = 1000; rcache_params.max_regions = ULONG_MAX; rcache_params.max_size = SIZE_MAX; rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_mcast_rcache_region_t); - rcache_params.max_alignment = ucc_get_page_size(); rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED | UCM_EVENT_MEM_TYPE_FREE; rcache_params.context = ctx; diff --git a/src/components/tl/mlx5/tl_mlx5_rcache.c b/src/components/tl/mlx5/tl_mlx5_rcache.c index 1414c82d15..d6f2aa47d8 100644 --- a/src/components/tl/mlx5/tl_mlx5_rcache.c +++ b/src/components/tl/mlx5/tl_mlx5_rcache.c @@ -63,8 +63,6 @@ ucc_status_t tl_mlx5_rcache_create(ucc_tl_mlx5_context_t *ctx) ucc_rcache_params_t rcache_params; rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_rcache_region_t); - rcache_params.alignment = UCS_PGT_ADDR_ALIGN; - rcache_params.max_alignment = ucc_get_page_size(); rcache_params.ucm_event_priority = 1000; rcache_params.context = (void *)ctx; rcache_params.ops = &ucc_rcache_ops; diff --git a/src/components/tl/sharp/tl_sharp_context.c b/src/components/tl/sharp/tl_sharp_context.c index 6e0477680a..a0fbe61b79 100644 --- a/src/components/tl/sharp/tl_sharp_context.c +++ b/src/components/tl/sharp/tl_sharp_context.c @@ -269,12 +269,10 @@ ucc_status_t ucc_tl_sharp_rcache_create(struct sharp_coll_context *context, { ucc_rcache_params_t rcache_params; - rcache_params.alignment = 64; rcache_params.ucm_event_priority = 1000; rcache_params.max_regions = ULONG_MAX; rcache_params.max_size = SIZE_MAX; rcache_params.region_struct_size = sizeof(ucc_tl_sharp_rcache_region_t); - rcache_params.max_alignment = ucc_get_page_size(); rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED | UCM_EVENT_MEM_TYPE_FREE; rcache_params.context = context; diff --git a/src/utils/ucc_rcache.h b/src/utils/ucc_rcache.h index dd1d6298e6..46993caacb 100644 --- a/src/utils/ucc_rcache.h +++ b/src/utils/ucc_rcache.h @@ -8,6 +8,7 @@ #include #include +#include //TODO: handle external events #define ucc_rcache_t ucs_rcache_t @@ -25,8 +26,17 @@ static inline ucc_status_t ucc_rcache_create(const ucc_rcache_params_t *params, const char *name, ucc_rcache_t **rcache_p) { +#ifndef UCS_HAVE_RCACHE_REGION_ALIGNMENT + ucc_rcache_params_t params_dup = *params; + params_dup.alignment = UCS_PGT_ADDR_ALIGN; + params_dup.max_alignment = ucc_get_page_size(); + + return ucs_status_to_ucc_status(ucs_rcache_create( + ¶ms_dup, name, NULL, rcache_p)); +#else return ucs_status_to_ucc_status(ucs_rcache_create( - params, name, NULL, rcache_p)); + params, name, NULL, rcache_p)); +#endif } /* [arg] parameter allows passing additional information from mem_reg callabck. @@ -36,9 +46,16 @@ static inline ucc_status_t ucc_rcache_get(ucc_rcache_t *rcache, void *address, size_t length, void *arg, ucc_rcache_region_t **region_p) { +#ifdef UCS_HAVE_RCACHE_REGION_ALIGNMENT return ucs_status_to_ucc_status(ucs_rcache_get( rcache, address, length, + ucc_get_page_size(), PROT_READ | PROT_WRITE, arg, region_p)); +#else + return ucs_status_to_ucc_status(ucs_rcache_get( + rcache, address, length, + PROT_READ | PROT_WRITE, arg, region_p)); +#endif } #endif