diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 44540d84a76..8f41f433ba9 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -98,7 +98,7 @@ #endif #include "ompi/runtime/ompi_cr.h" -#if defined(MEMORY_LINUX_PTMALLOC2) && MEMORY_LINUX_PTMALLOC2 +#if MEMORY_LINUX_HAVE_MALLOC_HOOK_SUPPORT #include "opal/mca/memory/linux/memory_linux.h" /* So this sucks, but with OPAL in its own library that is brought in implicity from libmpi, there are times when the malloc initialize @@ -106,7 +106,7 @@ from here, since any MPI code is going to call MPI_Init... */ OPAL_DECLSPEC void (*__malloc_initialize_hook) (void) = opal_memory_linux_malloc_init_hook; -#endif +#endif /* MEMORY_LINUX_HAVE_MALLOC_HOOK_SUPPORT */ /* This is required for the boundaries of the hash tables used to store * the F90 types returned by the MPI_Type_create_f90_XXX functions. diff --git a/opal/mca/btl/openib/btl_openib.h b/opal/mca/btl/openib/btl_openib.h index c0a2fbfda9a..9ccd972e87d 100644 --- a/opal/mca/btl/openib/btl_openib.h +++ b/opal/mca/btl/openib/btl_openib.h @@ -300,11 +300,6 @@ struct mca_btl_openib_component_t { #if BTL_OPENIB_FAILOVER_ENABLED int verbose_failover; #endif -#if BTL_OPENIB_MALLOC_HOOKS_ENABLED - int use_memalign; - size_t memalign_threshold; - void* (*previous_malloc_hook)(size_t __size, const void*); -#endif #if OPAL_CUDA_SUPPORT bool cuda_async_send; bool cuda_async_recv; diff --git a/opal/mca/btl/openib/btl_openib_component.c b/opal/mca/btl/openib/btl_openib_component.c index 77957d9d275..766ceefb94a 100644 --- a/opal/mca/btl/openib/btl_openib_component.c +++ b/opal/mca/btl/openib/btl_openib_component.c @@ -42,21 +42,6 @@ #include #include #include -#if BTL_OPENIB_MALLOC_HOOKS_ENABLED -/* - * The include of malloc.h below breaks abstractions in OMPI (by - * directly including a header file from another component), but has - * been ruled "ok" because the openib component is only supported on - * Linux. - * - * The malloc hooks in newer glibc were deprecated, including stock - * malloc.h causes compilation warnings. Instead, we use the internal - * linux component malloc.h which does not cause these warnings. - * Internally, OMPI uses the built-in ptmalloc from the linux memory - * component anyway. - */ -#include "opal/mca/memory/linux/malloc.h" -#endif #include "opal/mca/event/event.h" #include "opal/align.h" @@ -123,7 +108,6 @@ static void btl_openib_handle_incoming_completion(mca_btl_base_module_t* btl, * Local variables */ static mca_btl_openib_device_t *receive_queues_device = NULL; -static bool malloc_hook_set = false; static int num_devices_intentionally_ignored = 0; mca_btl_openib_component_t mca_btl_openib_component = { @@ -147,30 +131,6 @@ mca_btl_openib_component_t mca_btl_openib_component = { } }; -#if BTL_OPENIB_MALLOC_HOOKS_ENABLED -/* This is a memory allocator hook. The purpose of this is to make - * every malloc aligned since this speeds up IB HCA work. - * There two basic cases here: - * - * 1. Memory manager for Open MPI is enabled. Then memalign below will - * be overridden by __memalign_hook which is set to - * opal_memory_linux_memalign_hook. Thus, _malloc_hook is going to - * use opal_memory_linux_memalign_hook. - * - * 2. No memory manager support. The memalign below is just regular glibc - * memalign which will be called through __malloc_hook instead of malloc. - */ -static void *btl_openib_malloc_hook(size_t sz, const void* caller) -{ - if (sz < mca_btl_openib_component.memalign_threshold && - malloc_hook_set) { - return mca_btl_openib_component.previous_malloc_hook(sz, caller); - } else { - return memalign(mca_btl_openib_component.use_memalign, sz); - } -} -#endif - static int btl_openib_component_register(void) { int ret; @@ -257,16 +217,6 @@ static int btl_openib_component_close(void) free(mca_btl_openib_component.default_recv_qps); } -#if BTL_OPENIB_MALLOC_HOOKS_ENABLED - /* Must check to see whether the malloc hook was set before - assigning it back because ompi_info will call _register() and - then _close() (which won't set the hook) */ - if (malloc_hook_set) { - __malloc_hook = mca_btl_openib_component.previous_malloc_hook; - malloc_hook_set = false; - } -#endif - /* close memory registration debugging output */ opal_output_close (mca_btl_openib_component.memory_registration_verbose); @@ -2547,20 +2497,6 @@ btl_openib_component_init(int *num_btl_modules, *num_btl_modules = 0; num_devs = 0; -#if BTL_OPENIB_MALLOC_HOOKS_ENABLED - /* If we got this far, then setup the memory alloc hook (because - we're most likely going to be using this component). The hook - is to be set up as early as possible in this function since we - want most of the allocated resources be aligned.*/ - if (mca_btl_openib_component.use_memalign > 0 && - (opal_mem_hooks_support_level() & - (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_CHUNK_SUPPORT)) != 0) { - mca_btl_openib_component.previous_malloc_hook = __malloc_hook; - __malloc_hook = btl_openib_malloc_hook; - malloc_hook_set = true; - } -#endif - /* Per https://svn.open-mpi.org/trac/ompi/ticket/1305, check to see if $sysfsdir/class/infiniband exists. If it does not, assume that the RDMA hardware drivers are not loaded, and @@ -2960,13 +2896,6 @@ btl_openib_component_init(int *num_btl_modules, mca_btl_openib_component.ib_num_btls = 0; btl_openib_modex_send(); -#if BTL_OPENIB_MALLOC_HOOKS_ENABLED - /*Unset malloc hook since the component won't start*/ - if (malloc_hook_set) { - __malloc_hook = mca_btl_openib_component.previous_malloc_hook; - malloc_hook_set = false; - } -#endif if (NULL != btls) { free(btls); } diff --git a/opal/mca/btl/openib/btl_openib_mca.c b/opal/mca/btl/openib/btl_openib_mca.c index 75d8c1a9337..c8d4b6aba3f 100644 --- a/opal/mca/btl/openib/btl_openib_mca.c +++ b/opal/mca/btl/openib/btl_openib_mca.c @@ -703,26 +703,19 @@ int btl_openib_register_mca_params(void) 0, &mca_btl_openib_component.gid_index, REGINT_GE_ZERO)); -#if BTL_OPENIB_MALLOC_HOOKS_ENABLED - CHECK(reg_int("memalign", NULL, - "[64 | 32 | 0] - Enable (64bit or 32bit)/Disable(0) memory" - "alignment for all malloc calls if btl openib is used.", - 32, &mca_btl_openib_component.use_memalign, - REGINT_GE_ZERO)); +#if MEMORY_LINUX_MALLOC_ALIGN_ENABLED + ret = mca_base_var_find ("opal", "memory", "linux", "memalign"); + if (0 <= ret) { + (void) mca_base_var_register_synonym(ret, "ompi", "btl", "openib", "memalign", + MCA_BASE_VAR_SYN_FLAG_DEPRECATED); + } - mca_btl_openib_component.memalign_threshold = - mca_btl_openib_module.super.btl_eager_limit; - tmp = mca_base_component_var_register(&mca_btl_openib_component.super.btl_version, - "memalign_threshold", - "Allocating memory more than btl_openib_memalign_threshhold" - "bytes will automatically be algined to the value of btl_openib_memalign bytes." - "memalign_threshhold defaults to the same value as mca_btl_openib_eager_limit.", - MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, - OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, - &mca_btl_openib_component.memalign_threshold); - if (0 > tmp) ret = tmp; -#endif + ret = mca_base_var_find ("opal", "memory", "linux", "memalign_threshold"); + if (0 <= ret) { + (void) mca_base_var_register_synonym(ret, "ompi", "btl", "openib", "memalign_threshold", + MCA_BASE_VAR_SYN_FLAG_DEPRECATED); + } +#endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */ /* Register any MCA params for the connect pseudo-components */ if (OPAL_SUCCESS == ret) { @@ -823,16 +816,5 @@ int btl_openib_verify_mca_params (void) } #endif -#if BTL_OPENIB_MALLOC_HOOKS_ENABLED - if (mca_btl_openib_component.use_memalign != 32 - && mca_btl_openib_component.use_memalign != 64 - && mca_btl_openib_component.use_memalign != 0){ - opal_show_help("help-mpi-btl-openib.txt", "invalid mca param value", - true, "Wrong btl_openib_memalign parameter value. Allowed values: 64, 32, 0.", - "btl_openib_memalign is reset to 32"); - mca_btl_openib_component.use_memalign = 32; - } -#endif - return OPAL_SUCCESS; } diff --git a/opal/mca/btl/openib/configure.m4 b/opal/mca/btl/openib/configure.m4 index 1d07ae81cb4..9c3d9025c6d 100644 --- a/opal/mca/btl/openib/configure.m4 +++ b/opal/mca/btl/openib/configure.m4 @@ -120,28 +120,6 @@ AC_DEFUN([MCA_opal_btl_openib_CONFIG],[ [enable openib BTL failover]) AM_CONDITIONAL([MCA_btl_openib_enable_failover], [test "x$btl_openib_failover_enabled" = "x1"]) - # Check for __malloc_hook availability - AC_ARG_ENABLE(btl-openib-malloc-alignment, - AC_HELP_STRING([--enable-btl-openib-malloc-alignment], [Enable support for allocated memory alignment. Default: enabled if supported, disabled otherwise.])) - - btl_openib_malloc_hooks_enabled=0 - AS_IF([test "$enable_btl_openib_malloc_alignment" != "no"], - [AC_CHECK_HEADER([malloc.h], - [AC_CHECK_FUNC([__malloc_hook], - [AC_CHECK_FUNC([__realloc_hook], - [AC_CHECK_FUNC([__free_hook], - [btl_openib_malloc_hooks_enabled=1])])])])]) - - AS_IF([test "$enable_btl_openib_malloc_alignment" = "yes" && test "$btl_openib_malloc_hooks_enabled" = "0"], - [AC_MSG_ERROR([openib malloc alignment is requested but __malloc_hook is not available])]) - AC_MSG_CHECKING([whether the openib BTL will use malloc hooks]) - AS_IF([test "$btl_openib_malloc_hooks_enabled" = "0"], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([yes])]) - - AC_DEFINE_UNQUOTED(BTL_OPENIB_MALLOC_HOOKS_ENABLED, [$btl_openib_malloc_hooks_enabled], - [Whether the openib BTL malloc hooks are enabled]) - # make sure that CUDA-aware checks have been done AC_REQUIRE([OPAL_CHECK_CUDA]) diff --git a/opal/mca/memory/linux/configure.m4 b/opal/mca/memory/linux/configure.m4 index 93525b9ca9c..e44786e1eb3 100644 --- a/opal/mca/memory/linux/configure.m4 +++ b/opal/mca/memory/linux/configure.m4 @@ -63,6 +63,42 @@ AC_DEFUN([MCA_opal_memory_linux_CONFIG],[ [memory_linux_ptmalloc2_happy=no memory_linux_ummu_happy=no])]) + + ###################################################################### + # if memory hook available + ###################################################################### + memory_hook_found=1 + AS_IF([test "$memory_hook_found" -eq 1], + [memory_hook_found=0 AC_CHECK_HEADER([malloc.h], + [AC_CHECK_FUNC([__malloc_initialize_hook], + [AC_CHECK_FUNC([__malloc_hook], + [AC_CHECK_FUNC([__realloc_hook], + [AC_CHECK_FUNC([__free_hook], + [memory_hook_found=1])])])])])]) + AC_MSG_CHECKING([whether the system can use malloc hooks]) + AS_IF([test "$memory_hook_found" = "0"], + [AC_MSG_RESULT([no])], + [AC_MSG_RESULT([yes])]) + AC_DEFINE_UNQUOTED([MEMORY_LINUX_HAVE_MALLOC_HOOK_SUPPORT], [$memory_hook_found], + [Whether the system has Memory Allocation Hooks]) + + AC_ARG_ENABLE(memory-linux-malloc-alignment, + AC_HELP_STRING([--enable-memory-linux-malloc-alignment], [Enable support for allocated memory alignment. Default: enabled if supported, disabled otherwise.])) + + malloc_align_enabled=0 + AS_IF([test "$enable_memory_linux_malloc_alignment" != "no"], + [malloc_align_enabled=$memory_hook_found]) + + AS_IF([test "$enable_memory_linux_malloc_alignment" = "yes" && test "$malloc_align_enabled" = "0"], + [AC_MSG_ERROR([memory linux malloc alignment is requested but __malloc_hook is not available])]) + AC_MSG_CHECKING([whether the memory linux will use malloc alignment]) + AS_IF([test "$malloc_align_enabled" = "0"], + [AC_MSG_RESULT([no])], + [AC_MSG_RESULT([yes])]) + + AC_DEFINE_UNQUOTED(MEMORY_LINUX_MALLOC_ALIGN_ENABLED, [$malloc_align_enabled], + [Whether the memory linux malloc alignment is enabled]) + ###################################################################### # ptmalloc2 ###################################################################### diff --git a/opal/mca/memory/linux/help-opal-memory-linux.txt b/opal/mca/memory/linux/help-opal-memory-linux.txt index b3327c7356a..4e16124dc0c 100644 --- a/opal/mca/memory/linux/help-opal-memory-linux.txt +++ b/opal/mca/memory/linux/help-opal-memory-linux.txt @@ -27,3 +27,10 @@ alternate memory hook manager *may* be used instead (if available). Local host: %s UMMU device: %s Error: %s (%d) +# +[invalid mca param value] +WARNING: An invalid MCA parameter value was found for memory/linux +component. + + Problem: %s + Resolution: %s diff --git a/opal/mca/memory/linux/hooks.c b/opal/mca/memory/linux/hooks.c index 02fffb6b4ca..910d8e6e202 100644 --- a/opal/mca/memory/linux/hooks.c +++ b/opal/mca/memory/linux/hooks.c @@ -33,6 +33,7 @@ #include "opal/mca/mca.h" #include "opal/mca/memory/memory.h" #include "opal/constants.h" +#include "opal/memoryhooks/memory.h" #include "opal/mca/memory/linux/memory_linux.h" @@ -734,7 +735,10 @@ static check_result_t check(const char *name) } } -/* OMPI's init function */ + +/* This function is called on loading libmpi in case system has Memory Allocation Hooks + * (see ompi/runtime/ompi_mpi_init.c for details) + */ void opal_memory_linux_malloc_init_hook(void) { check_result_t r1, lp, lpp; diff --git a/opal/mca/memory/linux/memory_linux.h b/opal/mca/memory/linux/memory_linux.h index 24cb303c1f8..f1616c0d6a6 100644 --- a/opal/mca/memory/linux/memory_linux.h +++ b/opal/mca/memory/linux/memory_linux.h @@ -31,6 +31,11 @@ typedef struct opal_memory_linux_component_t { int ummunotify_fd; #endif +#if MEMORY_LINUX_MALLOC_ALIGN_ENABLED + int use_memalign; + size_t memalign_threshold; +#endif + #if MEMORY_LINUX_PTMALLOC2 /* Ptmalloc2-specific data. Note that these variables are all marked as volatile. * This is needed because of what may be a buggy optimization in the GCC 4.9.2 @@ -64,13 +69,16 @@ int opal_memory_linux_ummunotify_close(void); /* memory_linux_ptmalloc2.c */ int opal_memory_linux_ptmalloc2_open(void); int opal_memory_linux_ptmalloc2_close(void); -OPAL_DECLSPEC void opal_memory_linux_malloc_init_hook(void); /* memory_linux_munmap.c */ OPAL_DECLSPEC int opal_memory_linux_free_ptmalloc2_munmap(void *start, size_t length, int from_alloc); OPAL_DECLSPEC int munmap(void* addr, size_t len); #endif /* !MEMORY_LINUX_PTMALLOC2 */ +#if MEMORY_LINUX_HAVE_MALLOC_HOOK_SUPPORT +OPAL_DECLSPEC void opal_memory_linux_malloc_init_hook(void); +#endif /* MEMORY_LINUX_HAVE_MALLOC_HOOK_SUPPORT */ + END_C_DECLS #endif diff --git a/opal/mca/memory/linux/memory_linux_component.c b/opal/mca/memory/linux/memory_linux_component.c index ad14eb98511..a33094cfa7d 100644 --- a/opal/mca/memory/linux/memory_linux_component.c +++ b/opal/mca/memory/linux/memory_linux_component.c @@ -39,12 +39,17 @@ #include "opal_config.h" +#if HAVE_MALLOC_H +#include +#endif + #include "opal/constants.h" #include "opal/mca/base/mca_base_var.h" #include "opal/mca/memory/memory.h" #include "opal/mca/memory/base/empty.h" #include "opal/memoryhooks/memory.h" #include "opal/util/output.h" +#include "opal/util/show_help.h" #include "opal/mca/memory/linux/memory_linux.h" #undef opal_memory_changed @@ -100,6 +105,26 @@ opal_memory_linux_component_t mca_memory_linux_component = { static bool ptmalloc2_available = MEMORY_LINUX_PTMALLOC2; static bool ummunotify_available = MEMORY_LINUX_UMMUNOTIFY; +#if MEMORY_LINUX_MALLOC_ALIGN_ENABLED + +static void *(*prev_malloc_hook)(size_t, const void *); + +/* This is a memory allocator hook. The purpose of this is to make + * every malloc aligned. + * There two basic cases here: + * + * 1. Memory manager for Open MPI is enabled. Then memalign below will + * be overridden by __memalign_hook which is set to + * opal_memory_linux_memalign_hook. Thus, _malloc_hook is going to + * use opal_memory_linux_memalign_hook. + * + * 2. No memory manager support. The memalign below is just regular glibc + * memalign which will be called through __malloc_hook instead of malloc. + */ +static void *_opal_memory_linux_malloc_align_hook(size_t sz, const void* caller); +#endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */ + + /* * Register MCA params */ @@ -162,6 +187,53 @@ static int linux_register(void) OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY, &opal_memory_linux_disable); + if (0 > ret) { + return ret; + } + +#if MEMORY_LINUX_MALLOC_ALIGN_ENABLED + mca_memory_linux_component.use_memalign = 0; + ret = mca_base_component_var_register(&mca_memory_linux_component.super.memoryc_version, + "memalign", + "[64 | 32 | 0] - Enable (64bit or 32bit)/Disable(0) memory" + "alignment for all malloc calls (default: 0).", + MCA_BASE_VAR_TYPE_INT, + NULL, + 0, + 0, + OPAL_INFO_LVL_5, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_memory_linux_component.use_memalign); + if (0 > ret) { + return ret; + } + + mca_memory_linux_component.memalign_threshold = 12288; + ret = mca_base_component_var_register(&mca_memory_linux_component.super.memoryc_version, + "memalign_threshold", + "Allocating memory more than memory_linux_memalign_threshold" + "bytes will automatically be aligned to the value of memory_linux_memalign bytes." + "(default: 12288)", + MCA_BASE_VAR_TYPE_SIZE_T, + NULL, + 0, + 0, + OPAL_INFO_LVL_5, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_memory_linux_component.memalign_threshold); + if (0 > ret) { + return ret; + } + + if (mca_memory_linux_component.use_memalign != 32 + && mca_memory_linux_component.use_memalign != 64 + && mca_memory_linux_component.use_memalign != 0){ + opal_show_help("help-opal-memory-linux.txt", "invalid mca param value", + true, "Wrong memalign parameter value. Allowed values: 64, 32, 0.", + "memory_linux_memalign is reset to 32"); + mca_memory_linux_component.use_memalign = 32; + } +#endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */ return (0 > ret) ? ret : OPAL_SUCCESS; } @@ -188,7 +260,7 @@ static int linux_open(void) if (mca_memory_linux_component.verbose_level >= 10) { opal_output(0, "memory:linux: ummunotify successfully initialized; we'll use that"); } - return OPAL_SUCCESS; + goto done; } if (mca_memory_linux_component.verbose_level >= 10) { opal_output(0, "memory:linux: ummunotify failed to initialize"); @@ -206,7 +278,7 @@ static int linux_open(void) if (mca_memory_linux_component.verbose_level >= 10) { opal_output(0, "memory:linux: ptmalloc2 successfully initialized; we'll use that"); } - return OPAL_SUCCESS; + goto done; } if (mca_memory_linux_component.verbose_level >= 10) { opal_output(0, "memory:linux: ptmalloc2 failed to initialize"); @@ -222,12 +294,36 @@ static int linux_open(void) opal_output(0, "memory:linux: no memory hooks available in this process"); } return OPAL_ERR_NOT_AVAILABLE; + +done: + +#if MEMORY_LINUX_MALLOC_ALIGN_ENABLED + /* save original call */ + prev_malloc_hook = NULL; + + if (mca_memory_linux_component.use_memalign > 0 && + (opal_mem_hooks_support_level() & + (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_CHUNK_SUPPORT)) != 0) { + prev_malloc_hook = __malloc_hook; + __malloc_hook = _opal_memory_linux_malloc_align_hook; + } +#endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */ + + return OPAL_SUCCESS; } static int linux_close(void) { int v = mca_memory_linux_component.verbose_level; +#if MEMORY_LINUX_MALLOC_ALIGN_ENABLED + /* restore original call */ + if (prev_malloc_hook) { + __malloc_hook = prev_malloc_hook; + prev_malloc_hook = NULL; + } +#endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */ + #if MEMORY_LINUX_UMMUNOTIFY if (ummunotify_opened) { if (v >= 10) { @@ -249,3 +345,14 @@ static int linux_close(void) return OPAL_SUCCESS; } + +#if MEMORY_LINUX_MALLOC_ALIGN_ENABLED +static void *_opal_memory_linux_malloc_align_hook(size_t sz, const void* caller) +{ + if (sz < mca_memory_linux_component.memalign_threshold) { + return prev_malloc_hook(sz, caller); + } else { + return memalign(mca_memory_linux_component.use_memalign, sz); + } +} +#endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */