Skip to content

Commit

Permalink
BUILD: fix regcache alignment (openucx#877)
Browse files Browse the repository at this point in the history
  • Loading branch information
Sergei-Lebedev authored and janjust committed Jan 31, 2024
1 parent 54958ab commit dcf8716
Show file tree
Hide file tree
Showing 5 changed files with 25 additions and 9 deletions.
5 changes: 5 additions & 0 deletions config/m4/ucx.m4
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,11 @@ AS_IF([test "x$ucx_checked" != "xyes"],[
[AC_DEFINE([UCS_HAVE_CONFIG_GLOBAL_LIST_ENTRY_FLAGS], [1], [flags for config table])],
[],
[#include <ucs/config/parser.h>])
AC_CHECK_MEMBER(ucs_rcache_region_t.alignment,
[AC_DEFINE([UCS_HAVE_RCACHE_REGION_ALIGNMENT], [1], [flags for ucs_rcache_get])],
[],
[#include <ucs/memory/rcache.h>])
],
[
AS_IF([test "x$with_ucx" != "xguess"],
Expand Down
6 changes: 2 additions & 4 deletions src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ static ucs_status_t ucc_tl_mlx5_mcast_coll_reg_mr(ucc_tl_mlx5_mcast_coll_context
tl_error(ctx->lib, "failed to register MR");
return UCS_ERR_NO_MEMORY;
}

return UCS_OK;
}

Expand All @@ -33,7 +33,7 @@ static ucc_status_t ucc_tl_mlx5_mcast_coll_dereg_mr(ucc_tl_mlx5_mcast_coll_conte
}

tl_debug(ctx->lib, "external memory deregister: mr %p", mr);

if (ibv_dereg_mr(mr)) {
tl_error(ctx->lib, "couldn't destroy mr %p", mr);
return UCC_ERR_NO_RESOURCE;
Expand Down Expand Up @@ -140,12 +140,10 @@ ucc_status_t ucc_tl_mlx5_mcast_setup_rcache(ucc_tl_mlx5_mcast_coll_context_t *ct
{
ucc_rcache_params_t rcache_params;

rcache_params.alignment = 64;
rcache_params.ucm_event_priority = 1000;
rcache_params.max_regions = ULONG_MAX;
rcache_params.max_size = SIZE_MAX;
rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_mcast_rcache_region_t);
rcache_params.max_alignment = ucc_get_page_size();
rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED |
UCM_EVENT_MEM_TYPE_FREE;
rcache_params.context = ctx;
Expand Down
2 changes: 0 additions & 2 deletions src/components/tl/mlx5/tl_mlx5_rcache.c
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ ucc_status_t tl_mlx5_rcache_create(ucc_tl_mlx5_context_t *ctx)
ucc_rcache_params_t rcache_params;

rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_rcache_region_t);
rcache_params.alignment = UCS_PGT_ADDR_ALIGN;
rcache_params.max_alignment = ucc_get_page_size();
rcache_params.ucm_event_priority = 1000;
rcache_params.context = (void *)ctx;
rcache_params.ops = &ucc_rcache_ops;
Expand Down
2 changes: 0 additions & 2 deletions src/components/tl/sharp/tl_sharp_context.c
Original file line number Diff line number Diff line change
Expand Up @@ -269,12 +269,10 @@ ucc_status_t ucc_tl_sharp_rcache_create(struct sharp_coll_context *context,
{
ucc_rcache_params_t rcache_params;

rcache_params.alignment = 64;
rcache_params.ucm_event_priority = 1000;
rcache_params.max_regions = ULONG_MAX;
rcache_params.max_size = SIZE_MAX;
rcache_params.region_struct_size = sizeof(ucc_tl_sharp_rcache_region_t);
rcache_params.max_alignment = ucc_get_page_size();
rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED |
UCM_EVENT_MEM_TYPE_FREE;
rcache_params.context = context;
Expand Down
19 changes: 18 additions & 1 deletion src/utils/ucc_rcache.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

#include <ucs/memory/rcache.h>
#include <ucm/api/ucm.h>
#include <utils/ucc_sys.h>

//TODO: handle external events
#define ucc_rcache_t ucs_rcache_t
Expand All @@ -25,8 +26,17 @@ static inline ucc_status_t
ucc_rcache_create(const ucc_rcache_params_t *params,
const char *name, ucc_rcache_t **rcache_p)
{
#ifndef UCS_HAVE_RCACHE_REGION_ALIGNMENT
ucc_rcache_params_t params_dup = *params;
params_dup.alignment = UCS_PGT_ADDR_ALIGN;
params_dup.max_alignment = ucc_get_page_size();

return ucs_status_to_ucc_status(ucs_rcache_create(
&params_dup, name, NULL, rcache_p));
#else
return ucs_status_to_ucc_status(ucs_rcache_create(
params, name, NULL, rcache_p));
params, name, NULL, rcache_p));
#endif
}

/* [arg] parameter allows passing additional information from mem_reg callabck.
Expand All @@ -36,9 +46,16 @@ static inline ucc_status_t
ucc_rcache_get(ucc_rcache_t *rcache, void *address, size_t length,
void *arg, ucc_rcache_region_t **region_p)
{
#ifdef UCS_HAVE_RCACHE_REGION_ALIGNMENT
return ucs_status_to_ucc_status(ucs_rcache_get(
rcache, address, length,
ucc_get_page_size(),
PROT_READ | PROT_WRITE, arg, region_p));
#else
return ucs_status_to_ucc_status(ucs_rcache_get(
rcache, address, length,
PROT_READ | PROT_WRITE, arg, region_p));
#endif
}

#endif

0 comments on commit dcf8716

Please sign in to comment.