diff --git a/config/m4/ucs.m4 b/config/m4/ucs.m4 index 6722a70d5eb..a65bcac5890 100644 --- a/config/m4/ucs.m4 +++ b/config/m4/ucs.m4 @@ -115,24 +115,6 @@ AS_IF([test "x$enable_tuning" = xyes], AM_CONDITIONAL([HAVE_TUNING],[test "x$HAVE_TUNING" = "xyes"]) -# -# Enable memory tracking -# -AC_ARG_ENABLE([memtrack], - AS_HELP_STRING([--enable-memtrack], - [Enable memory tracking, useful for profiling, default: NO]), - [], - [enable_memtrack=no]) - -AS_IF([test "x$enable_memtrack" = xyes], - [AS_MESSAGE([enabling memory tracking]) - AC_DEFINE([ENABLE_MEMTRACK], [1], [Enable memory tracking]) - HAVE_MEMTRACK=yes], - [:] - ) -AM_CONDITIONAL([HAVE_MEMTRACK],[test "x$HAVE_MEMTRACK" = "xyes"]) - - # # Disable logging levels below INFO # diff --git a/configure.ac b/configure.ac index d0b6472c382..82a89db328e 100644 --- a/configure.ac +++ b/configure.ac @@ -170,7 +170,6 @@ AS_IF([test "x$with_docs_only" = xyes], AM_CONDITIONAL([HAVE_GTEST], [false]) AM_CONDITIONAL([HAVE_STATS], [false]) AM_CONDITIONAL([HAVE_TUNING], [false]) - AM_CONDITIONAL([HAVE_MEMTRACK], [false]) AM_CONDITIONAL([HAVE_IB], [false]) AM_CONDITIONAL([HAVE_MLX5_HW], [false]) AM_CONDITIONAL([HAVE_MLX5_HW_UD], [false]) diff --git a/contrib/configure-devel b/contrib/configure-devel index 7525e93d7da..f7ce860a4d2 100755 --- a/contrib/configure-devel +++ b/contrib/configure-devel @@ -18,7 +18,6 @@ $basedir/../configure \ --enable-profiling \ --enable-frame-pointer \ --enable-stats \ - --enable-memtrack \ --enable-fault-injection \ --enable-debug-data \ --enable-mt \ diff --git a/contrib/configure-prof b/contrib/configure-prof index 14a306dbb3b..9e18f3c48aa 100755 --- a/contrib/configure-prof +++ b/contrib/configure-prof @@ -20,5 +20,4 @@ $basedir/../configure \ --enable-profiling \ --enable-frame-pointer \ --enable-stats \ - --enable-memtrack \ "$@" diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index 67c5bea304e..cdfde3c2827 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/ucp/core/ucp_mm.c b/src/ucp/core/ucp_mm.c index de012b2192b..c1b5b39b8ca 100644 --- a/src/ucp/core/ucp_mm.c +++ b/src/ucp/core/ucp_mm.c @@ -13,7 +13,7 @@ #include "ucp_worker.h" #include -#include +#include #include #include #include diff --git a/src/ucp/dt/dt_generic.c b/src/ucp/dt/dt_generic.c index cf38142710b..fa5131c8286 100644 --- a/src/ucp/dt/dt_generic.c +++ b/src/ucp/dt/dt_generic.c @@ -11,7 +11,7 @@ #include "dt_generic.h" #include -#include +#include ucs_status_t ucp_dt_create_generic(const ucp_generic_dt_ops_t *ops, void *context, diff --git a/src/ucs/Makefile.am b/src/ucs/Makefile.am index b61a4df4bb2..3b347689677 100644 --- a/src/ucs/Makefile.am +++ b/src/ucs/Makefile.am @@ -34,6 +34,7 @@ nobase_dist_libucs_la_HEADERS = \ datastruct/string_buffer.h \ datastruct/string_set.h \ debug/log_def.h \ + debug/memtrack.h \ memory/rcache.h \ memory/memory_type.h \ memory/memtype_cache.h \ @@ -86,7 +87,7 @@ noinst_HEADERS = \ debug/assert.h \ debug/debug.h \ debug/log.h \ - debug/memtrack.h \ + debug/memtrack_int.h \ memory/numa.h \ memory/rcache_int.h \ profile/profile.h \ diff --git a/src/ucs/config/global_opts.c b/src/ucs/config/global_opts.c index 6d8af3b73d7..b8dc65ed428 100644 --- a/src/ucs/config/global_opts.c +++ b/src/ucs/config/global_opts.c @@ -184,7 +184,6 @@ static ucs_config_field_t ucs_global_opts_table[] = { #endif -#ifdef ENABLE_MEMTRACK {"MEMTRACK_DEST", "", "Destination to output memory tracking report to. If the value is empty,\n" "results are not reported. Possible values are:\n" @@ -197,7 +196,6 @@ static ucs_config_field_t ucs_global_opts_table[] = { "Memory limit allocated by memtrack. In case if limit is reached then\n" "memtrack report is generated and process is terminated.\n", ucs_offsetof(ucs_global_opts_t, memtrack_limit), UCS_CONFIG_TYPE_MEMUNITS}, -#endif {"PROFILE_MODE", "", "Profile collection modes. If none is specified, profiling is disabled.\n" diff --git a/src/ucs/datastruct/mpmc.c b/src/ucs/datastruct/mpmc.c index 611ea2642dc..f7e8cfeb987 100644 --- a/src/ucs/datastruct/mpmc.c +++ b/src/ucs/datastruct/mpmc.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include ucs_status_t ucs_mpmc_queue_init(ucs_mpmc_queue_t *mpmc, uint32_t length) diff --git a/src/ucs/datastruct/pgtable.c b/src/ucs/datastruct/pgtable.c index cb9dd8b1614..8965c923031 100644 --- a/src/ucs/datastruct/pgtable.c +++ b/src/ucs/datastruct/pgtable.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/ucs/datastruct/ptr_array.c b/src/ucs/datastruct/ptr_array.c index 3c5a0d59994..708d94179f9 100644 --- a/src/ucs/datastruct/ptr_array.c +++ b/src/ucs/datastruct/ptr_array.c @@ -83,6 +83,7 @@ static void ucs_ptr_array_clear(ucs_ptr_array_t *ptr_array) ptr_array->start = NULL; ptr_array->size = 0; ptr_array->freelist = UCS_PTR_ARRAY_SENTINEL; + ptr_array->name = NULL; } void ucs_ptr_array_init(ucs_ptr_array_t *ptr_array, uint32_t init_placeholder, @@ -90,9 +91,6 @@ void ucs_ptr_array_init(ucs_ptr_array_t *ptr_array, uint32_t init_placeholder, { ptr_array->init_placeholder = init_placeholder; ucs_ptr_array_clear(ptr_array); -#ifdef ENABLE_MEMTRACK - ucs_snprintf_zero(ptr_array->name, sizeof(ptr_array->name), "%s", name); -#endif } void ucs_ptr_array_cleanup(ucs_ptr_array_t *ptr_array) @@ -115,7 +113,7 @@ void ucs_ptr_array_cleanup(ucs_ptr_array_t *ptr_array) ucs_ptr_array_clear(ptr_array); } -static void ucs_ptr_array_grow(ucs_ptr_array_t *ptr_array UCS_MEMTRACK_ARG) +static void ucs_ptr_array_grow(ucs_ptr_array_t *ptr_array) { ucs_ptr_array_elem_t *new_array; unsigned curr_size, new_size; @@ -129,7 +127,8 @@ static void ucs_ptr_array_grow(ucs_ptr_array_t *ptr_array UCS_MEMTRACK_ARG) } /* Allocate new array */ - new_array = ucs_malloc(new_size * sizeof(ucs_ptr_array_elem_t) UCS_MEMTRACK_VAL); + new_array = ucs_malloc(new_size * sizeof(ucs_ptr_array_elem_t), + ptr_array->name); ucs_assert_always(new_array != NULL); memcpy(new_array, ptr_array->start, curr_size * sizeof(ucs_ptr_array_elem_t)); @@ -168,7 +167,7 @@ unsigned ucs_ptr_array_insert(ucs_ptr_array_t *ptr_array, void *value, ucs_assert_always(((uintptr_t)value & UCS_PTR_ARRAY_FLAG_FREE) == 0); if (ptr_array->freelist == UCS_PTR_ARRAY_SENTINEL) { - ucs_ptr_array_grow(ptr_array UCS_MEMTRACK_NAME(ptr_array->name)); + ucs_ptr_array_grow(ptr_array); } /* Get the first item on the free list */ diff --git a/src/ucs/datastruct/ptr_array.h b/src/ucs/datastruct/ptr_array.h index b1d87a4c30f..e4f7961f245 100644 --- a/src/ucs/datastruct/ptr_array.h +++ b/src/ucs/datastruct/ptr_array.h @@ -8,7 +8,7 @@ #define PTR_ARRAY_H_ #include -#include +#include /* @@ -35,9 +35,7 @@ typedef struct ucs_ptr_array { ucs_ptr_array_elem_t *start; unsigned freelist; unsigned size; -#ifdef ENABLE_MEMTRACK - char name[64]; -#endif + const char *name; } ucs_ptr_array_t; diff --git a/src/ucs/datastruct/strided_alloc.c b/src/ucs/datastruct/strided_alloc.c index a688e12c318..d036bdc0d02 100644 --- a/src/ucs/datastruct/strided_alloc.c +++ b/src/ucs/datastruct/strided_alloc.c @@ -11,7 +11,7 @@ #include "queue.h" #include -#include +#include #include #include @@ -33,8 +33,8 @@ struct ucs_strided_alloc_elem { }; static ucs_strided_alloc_chunk_t * -ucs_strided_alloc_chunk_alloc(ucs_strided_alloc_t *sa, size_t chunk_size - UCS_MEMTRACK_ARG) +ucs_strided_alloc_chunk_alloc(ucs_strided_alloc_t *sa, size_t chunk_size, + const char* alloc_name) { ucs_status_t status; size_t size; @@ -42,7 +42,7 @@ ucs_strided_alloc_chunk_alloc(ucs_strided_alloc_t *sa, size_t chunk_size size = chunk_size; ptr = NULL; - status = ucs_mmap_alloc(&size, &ptr, 0 UCS_MEMTRACK_VAL); + status = ucs_mmap_alloc(&size, &ptr, 0, alloc_name); if (status != UCS_OK) { ucs_error("failed to allocate a chunk of %zu bytes", chunk_size); return NULL; @@ -76,7 +76,7 @@ static void ucs_strided_alloc_calc(ucs_strided_alloc_t *sa, size_t *chunk_size, } static ucs_status_t -ucs_strided_alloc_grow(ucs_strided_alloc_t *sa UCS_MEMTRACK_ARG) +ucs_strided_alloc_grow(ucs_strided_alloc_t *sa, const char* alloc_name) { size_t chunk_size, elems_per_chunk; ucs_strided_alloc_chunk_t *chunk; @@ -86,7 +86,7 @@ ucs_strided_alloc_grow(ucs_strided_alloc_t *sa UCS_MEMTRACK_ARG) ucs_strided_alloc_calc(sa, &chunk_size, &elems_per_chunk); - chunk = ucs_strided_alloc_chunk_alloc(sa, chunk_size UCS_MEMTRACK_VAL); + chunk = ucs_strided_alloc_chunk_alloc(sa, chunk_size, alloc_name); if (chunk == NULL) { return UCS_ERR_NO_MEMORY; } @@ -146,7 +146,7 @@ void* ucs_strided_alloc_get(ucs_strided_alloc_t *sa, const char *alloc_name) unsigned i; if (sa->freelist == NULL) { - status = ucs_strided_alloc_grow(sa UCS_MEMTRACK_VAL); + status = ucs_strided_alloc_grow(sa, alloc_name); if (status != UCS_OK) { return NULL; } diff --git a/src/ucs/datastruct/string_buffer.c b/src/ucs/datastruct/string_buffer.c index 3eceaacf2bc..77f52dc6c6c 100644 --- a/src/ucs/datastruct/string_buffer.c +++ b/src/ucs/datastruct/string_buffer.c @@ -12,7 +12,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/ucs/datastruct/string_set.c b/src/ucs/datastruct/string_set.c index f8153aceb49..52bdc24f0ae 100644 --- a/src/ucs/datastruct/string_set.c +++ b/src/ucs/datastruct/string_set.c @@ -12,7 +12,7 @@ #include #include -#include +#include #define UCS_STRING_SET_ALLOC_NAME "string_set" diff --git a/src/ucs/debug/debug.c b/src/ucs/debug/debug.c index c36635884c0..cd3c5fb78ce 100644 --- a/src/ucs/debug/debug.c +++ b/src/ucs/debug/debug.c @@ -382,8 +382,8 @@ ucs_status_t ucs_debug_backtrace_create(backtrace_h *bckt, int strip) ucs_status_t status; *bckt = NULL; - status = ucs_mmap_alloc(&size, (void**)bckt, 0 - UCS_MEMTRACK_NAME("debug backtrace object")); + status = ucs_mmap_alloc(&size, (void**)bckt, 0, + "debug backtrace object"); if (status != UCS_OK) { return status; } @@ -589,8 +589,8 @@ ucs_status_t ucs_debug_backtrace_create(backtrace_h *bckt, int strip) ucs_status_t status; *bckt = NULL; - status = ucs_mmap_alloc(&size, (void**)bckt, 0 - UCS_MEMTRACK_NAME("debug backtrace object")); + status = ucs_mmap_alloc(&size, (void**)bckt, 0, + "debug backtrace object"); if (status != UCS_OK) { return status; } diff --git a/src/ucs/debug/memtrack.c b/src/ucs/debug/memtrack.c index 5399919a2fc..59512bc2fbb 100644 --- a/src/ucs/debug/memtrack.c +++ b/src/ucs/debug/memtrack.c @@ -22,8 +22,6 @@ #endif -#ifdef ENABLE_MEMTRACK - #define UCS_MEMTRACK_FORMAT_STRING ("%22s: size: %9lu / %9lu\tcount: %9u / %9u\n") @@ -172,7 +170,8 @@ static void ucs_memtrack_generate_report() } } -void ucs_memtrack_allocated(void *ptr, size_t size, const char *name) +static UCS_F_NOINLINE void +ucs_memtrack_do_allocated(void *ptr, size_t size, const char *name) { ucs_memtrack_entry_t *entry; khiter_t iter; @@ -185,7 +184,7 @@ void ucs_memtrack_allocated(void *ptr, size_t size, const char *name) ucs_assert(!ucs_check_if_align_pow2((uintptr_t)ptr, UCX_ALLOC_ALIGN)); #endif - if ((ptr == NULL) || !ucs_memtrack_is_enabled()) { + if (ptr == NULL) { return; } @@ -224,13 +223,23 @@ void ucs_memtrack_allocated(void *ptr, size_t size, const char *name) pthread_mutex_unlock(&ucs_memtrack_context.lock); } -void ucs_memtrack_releasing(void* ptr) +static UCS_F_ALWAYS_INLINE void +ucs_memtrack_allocated_internal(void *ptr, size_t size, const char *name) +{ + if (!ucs_memtrack_is_enabled()) { + return; + } + + ucs_memtrack_do_allocated(ptr, size, name); +} + +static UCS_F_NOINLINE void ucs_memtrack_do_releasing(void *ptr) { ucs_memtrack_entry_t *entry; khiter_t iter; size_t size; - if ((ptr == NULL) || !ucs_memtrack_is_enabled()) { + if (ptr == NULL) { return; } @@ -238,8 +247,11 @@ void ucs_memtrack_releasing(void* ptr) iter = kh_get(ucs_memtrack_ptr_hash, &ucs_memtrack_context.ptrs, (uintptr_t)ptr); if (iter == kh_end(&ucs_memtrack_context.ptrs)) { + /* workaround for coverity - print debug message from unlocked + * memtrack */ + pthread_mutex_unlock(&ucs_memtrack_context.lock); ucs_debug("address %p not found in memtrack ptr hash", ptr); - goto out_unlock; + return; } /* remote pointer from hash */ @@ -251,29 +263,37 @@ void ucs_memtrack_releasing(void* ptr) ucs_memtrack_entry_update(entry, -size); ucs_memtrack_entry_update(&ucs_memtrack_context.total, -size); -out_unlock: pthread_mutex_unlock(&ucs_memtrack_context.lock); } +static UCS_F_ALWAYS_INLINE void ucs_memtrack_releasing_internal(void *ptr) +{ + if (!ucs_memtrack_is_enabled()) { + return; + } + + ucs_memtrack_do_releasing(ptr); +} + void *ucs_malloc(size_t size, const char *name) { void *ptr = malloc(size); - ucs_memtrack_allocated(ptr, size, name); + ucs_memtrack_allocated_internal(ptr, size, name); return ptr; } void *ucs_calloc(size_t nmemb, size_t size, const char *name) { void *ptr = calloc(nmemb, size); - ucs_memtrack_allocated(ptr, nmemb * size, name); + ucs_memtrack_allocated_internal(ptr, nmemb * size, name); return ptr; } void *ucs_realloc(void *ptr, size_t size, const char *name) { - ucs_memtrack_releasing(ptr); + ucs_memtrack_releasing_internal(ptr); ptr = realloc(ptr, size); - ucs_memtrack_allocated(ptr, size, name); + ucs_memtrack_allocated_internal(ptr, size, name); return ptr; } @@ -287,14 +307,14 @@ int ucs_posix_memalign(void **ptr, size_t boundary, size_t size, const char *nam #error "Port me" #endif if (ret == 0) { - ucs_memtrack_allocated(*ptr, size, name); + ucs_memtrack_allocated_internal(*ptr, size, name); } return ret; } void ucs_free(void *ptr) { - ucs_memtrack_releasing(ptr); + ucs_memtrack_releasing_internal(ptr); free(ptr); } @@ -303,28 +323,28 @@ void *ucs_mmap(void *addr, size_t length, int prot, int flags, int fd, { void *ptr = mmap(addr, length, prot, flags, fd, offset); if (ptr != MAP_FAILED) { - ucs_memtrack_allocated(ptr, length, name); + ucs_memtrack_allocated_internal(ptr, length, name); } return ptr; } int ucs_munmap(void *addr, size_t length) { - ucs_memtrack_releasing(addr); + ucs_memtrack_releasing_internal(addr); return munmap(addr, length); } char *ucs_strdup(const char *src, const char *name) { char *str = strdup(src); - ucs_memtrack_allocated(str, strlen(str) + 1, name); + ucs_memtrack_allocated_internal(str, strlen(str) + 1, name); return str; } char *ucs_strndup(const char *src, size_t n, const char *name) { char *str = strndup(src, n); - ucs_memtrack_allocated(str, strlen(str) + 1, name); + ucs_memtrack_allocated_internal(str, strlen(str) + 1, name); return str; } @@ -403,4 +423,12 @@ int ucs_memtrack_is_enabled() return ucs_memtrack_context.enabled; } -#endif +void ucs_memtrack_allocated(void *ptr, size_t size, const char *name) +{ + ucs_memtrack_allocated_internal(ptr, size, name); +} + +void ucs_memtrack_releasing(void *ptr) +{ + ucs_memtrack_releasing_internal(ptr); +} diff --git a/src/ucs/debug/memtrack.h b/src/ucs/debug/memtrack.h index 38e48ba1716..1560c893e05 100644 --- a/src/ucs/debug/memtrack.h +++ b/src/ucs/debug/memtrack.h @@ -7,80 +7,14 @@ #ifndef UCS_MEMTRACK_H_ #define UCS_MEMTRACK_H_ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - #include -#include -#include +#include BEGIN_C_DECLS /** @file memtrack.h */ -enum { - UCS_MEMTRACK_STAT_ALLOCATION_COUNT, - UCS_MEMTRACK_STAT_ALLOCATION_SIZE, - UCS_MEMTRACK_STAT_LAST -}; - - -/** - * Allocation site entry - */ -typedef struct ucs_memtrack_entry { - size_t size; /* currently allocated total size */ - size_t peak_size; /* peak allocated total size */ - unsigned count; /* number of currently allocated blocks */ - unsigned peak_count; /* peak number of allocated blocks */ - char name[0]; /* allocation name */ -} ucs_memtrack_entry_t; - - - -#ifdef ENABLE_MEMTRACK - -#define UCS_MEMTRACK_ARG , const char* alloc_name -#define UCS_MEMTRACK_VAL , alloc_name -#define UCS_MEMTRACK_VAL_ALWAYS alloc_name -#define UCS_MEMTRACK_NAME(_n) , _n - - -/** - * Start tracking memory (or increment reference count). - */ -void ucs_memtrack_init(); - - -/** - * Stop tracking memory (or decrement reference count). - */ -void ucs_memtrack_cleanup(); - - -/* - * Check if memtrack is enabled at the moment. - */ -int ucs_memtrack_is_enabled(); - - -/** - * Print a summary of memory tracked so far. - * - * @param output Stream to direct output to. - */ -void ucs_memtrack_dump(FILE* output); - - -/** - * Calculates the total of buffers currently tracked. - * - * @param total Entry (pre-allocated) to place results in. - */ -void ucs_memtrack_total(ucs_memtrack_entry_t* total); - /** * Track custom allocation. Need to be called after custom allocation returns. @@ -94,53 +28,6 @@ void ucs_memtrack_allocated(void *ptr, size_t size, const char *name); */ void ucs_memtrack_releasing(void *ptr); - -/* - * Memory allocation replacements. Their interface is the same as the originals, - * except the additional parameter which specifies the allocation name. - */ -void *ucs_malloc(size_t size, const char *name); -void *ucs_calloc(size_t nmemb, size_t size, const char *name); -void *ucs_realloc(void *ptr, size_t size, const char *name); -int ucs_posix_memalign(void **ptr, size_t boundary, size_t size, - const char *name); -void ucs_free(void *ptr); -void *ucs_mmap(void *addr, size_t length, int prot, int flags, int fd, - off_t offset, const char *name); -int ucs_munmap(void *addr, size_t length); -char *ucs_strdup(const char *src, const char *name); -char *ucs_strndup(const char *src, size_t n, const char *name); - -#else - -#define UCS_MEMTRACK_ARG -#define UCS_MEMTRACK_VAL -#define UCS_MEMTRACK_VAL_ALWAYS "" -#define UCS_MEMTRACK_NAME(_n) - -#define ucs_memtrack_init() UCS_EMPTY_STATEMENT -#define ucs_memtrack_cleanup() UCS_EMPTY_STATEMENT -#define ucs_memtrack_is_enabled() 0 -#define ucs_memtrack_dump(_output) UCS_EMPTY_STATEMENT -#define ucs_memtrack_total(_total) ucs_memtrack_total_init(_total) - -#define ucs_memtrack_allocated(_ptr, _sz, ...) UCS_EMPTY_STATEMENT -#define ucs_memtrack_releasing(_ptr) UCS_EMPTY_STATEMENT - -#define ucs_malloc(_s, ...) malloc(_s) -#define ucs_calloc(_n, _s, ...) calloc(_n, _s) -#define ucs_realloc(_p, _s, ...) realloc(_p, _s) -#if HAVE_POSIX_MEMALIGN -#define ucs_posix_memalign(_pp, _b, _s, ...) posix_memalign(_pp, _b, _s) -#endif -#define ucs_free(_p) free(_p) -#define ucs_mmap(_a, _l, _p, _fl, _fd, _o, ...) mmap(_a, _l, _p, _fl, _fd, _o) -#define ucs_munmap(_a, _l) munmap(_a, _l) -#define ucs_strdup(_src, ...) strdup(_src) -#define ucs_strndup(_src, _n, ...) strndup(_src, _n) - -#endif /* ENABLE_MEMTRACK */ - END_C_DECLS #endif diff --git a/src/ucs/debug/memtrack_int.h b/src/ucs/debug/memtrack_int.h new file mode 100644 index 00000000000..b1fe9d4bfcb --- /dev/null +++ b/src/ucs/debug/memtrack_int.h @@ -0,0 +1,96 @@ +/** +* Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED. +* Copyright (C) UT-Battelle, LLC. 2014. ALL RIGHTS RESERVED. +* See file LICENSE for terms. +*/ + +#ifndef UCS_MEMTRACK_INT_H_ +#define UCS_MEMTRACK_INT_H_ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include +#include +#include + + +BEGIN_C_DECLS + +/** @file memtrack_int.h */ + +enum { + UCS_MEMTRACK_STAT_ALLOCATION_COUNT, + UCS_MEMTRACK_STAT_ALLOCATION_SIZE, + UCS_MEMTRACK_STAT_LAST +}; + + +/** + * Allocation site entry + */ +typedef struct ucs_memtrack_entry { + size_t size; /* currently allocated total size */ + size_t peak_size; /* peak allocated total size */ + unsigned count; /* number of currently allocated blocks */ + unsigned peak_count; /* peak number of allocated blocks */ + char name[0]; /* allocation name */ +} ucs_memtrack_entry_t; + + + +/** + * Start tracking memory (or increment reference count). + */ +void ucs_memtrack_init(); + + +/** + * Stop tracking memory (or decrement reference count). + */ +void ucs_memtrack_cleanup(); + + +/* + * Check if memtrack is enabled at the moment. + */ +int ucs_memtrack_is_enabled(); + + +/** + * Print a summary of memory tracked so far. + * + * @param output Stream to direct output to. + */ +void ucs_memtrack_dump(FILE* output); + + +/** + * Calculates the total of buffers currently tracked. + * + * @param total Entry (pre-allocated) to place results in. + */ +void ucs_memtrack_total(ucs_memtrack_entry_t* total); + + +/* + * Memory allocation replacements. Their interface is the same as the originals, + * except the additional parameter which specifies the allocation name. + */ +void *ucs_malloc(size_t size, const char *name); +void *ucs_calloc(size_t nmemb, size_t size, const char *name); +void *ucs_realloc(void *ptr, size_t size, const char *name); +int ucs_posix_memalign(void **ptr, size_t boundary, size_t size, + const char *name); +void ucs_free(void *ptr); +void *ucs_mmap(void *addr, size_t length, int prot, int flags, int fd, + off_t offset, const char *name); +int ucs_munmap(void *addr, size_t length); +char *ucs_strdup(const char *src, const char *name); +char *ucs_strndup(const char *src, size_t n, const char *name); + +END_C_DECLS + +#endif diff --git a/src/ucs/memory/memtype_cache.c b/src/ucs/memory/memtype_cache.c index a0e105760cb..9c391b606c1 100644 --- a/src/ucs/memory/memtype_cache.c +++ b/src/ucs/memory/memtype_cache.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/ucs/memory/numa.h b/src/ucs/memory/numa.h index c26b414d0ec..4fc7edf5d0d 100644 --- a/src/ucs/memory/numa.h +++ b/src/ucs/memory/numa.h @@ -11,7 +11,7 @@ # include "config.h" #endif -#include +#include #if HAVE_NUMA #include diff --git a/src/ucs/memory/rcache.c b/src/ucs/memory/rcache.c index f8b5bbb0758..2f9638e25f9 100644 --- a/src/ucs/memory/rcache.c +++ b/src/ucs/memory/rcache.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/ucs/sys/event_set.c b/src/ucs/sys/event_set.c index 06de682f978..3e122d5957c 100644 --- a/src/ucs/sys/event_set.c +++ b/src/ucs/sys/event_set.c @@ -10,7 +10,7 @@ #include "event_set.h" -#include +#include #include #include #include diff --git a/src/ucs/sys/init.c b/src/ucs/sys/init.c index 93c42d5a9cc..2d8dbf702dc 100644 --- a/src/ucs/sys/init.c +++ b/src/ucs/sys/init.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/ucs/sys/module.c b/src/ucs/sys/module.c index ee6b0dd5946..f0430b71907 100644 --- a/src/ucs/sys/module.c +++ b/src/ucs/sys/module.c @@ -15,7 +15,7 @@ #include "module.h" #include -#include +#include #include #include #include diff --git a/src/ucs/sys/sys.c b/src/ucs/sys/sys.c index 5fbc2bf8a8c..4b5f7a59cb3 100644 --- a/src/ucs/sys/sys.c +++ b/src/ucs/sys/sys.c @@ -814,7 +814,7 @@ ucs_status_t ucs_sysv_alloc(size_t *size, size_t max_size, void **address_p, } } - ucs_memtrack_allocated(ptr, alloc_size UCS_MEMTRACK_VAL); + ucs_memtrack_allocated(ptr, alloc_size, alloc_name); *address_p = ptr; *size = alloc_size; return UCS_OK; @@ -835,7 +835,7 @@ ucs_status_t ucs_sysv_free(void *address) } ucs_status_t ucs_mmap_alloc(size_t *size, void **address_p, - int flags UCS_MEMTRACK_ARG) + int flags, const char* alloc_name) { size_t alloc_length; void *addr; @@ -843,7 +843,7 @@ ucs_status_t ucs_mmap_alloc(size_t *size, void **address_p, alloc_length = ucs_align_up_pow2(*size, ucs_get_page_size()); addr = ucs_mmap(*address_p, alloc_length, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | flags, -1, 0 UCS_MEMTRACK_VAL); + MAP_PRIVATE | MAP_ANON | flags, -1, 0, alloc_name); if (addr == MAP_FAILED) { return UCS_ERR_NO_MEMORY; } diff --git a/src/ucs/sys/sys.h b/src/ucs/sys/sys.h index e723ebabc17..3d9cb7719e3 100644 --- a/src/ucs/sys/sys.h +++ b/src/ucs/sys/sys.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include @@ -271,7 +271,7 @@ ucs_status_t ucs_sysv_free(void *address); * @param flags Flags to pass to the mmap() system call */ ucs_status_t ucs_mmap_alloc(size_t *size, void **address_p, - int flags UCS_MEMTRACK_ARG); + int flags, const char* alloc_name); /** * Release memory allocated via mmap API. diff --git a/src/ucs/time/timer_wheel.c b/src/ucs/time/timer_wheel.c index 9924949d3a7..bc336795ad8 100644 --- a/src/ucs/time/timer_wheel.c +++ b/src/ucs/time/timer_wheel.c @@ -12,7 +12,7 @@ #include #include -#include +#include #include diff --git a/src/ucs/time/timerq.c b/src/ucs/time/timerq.c index a72528bbc4a..5e9795bbfdb 100644 --- a/src/ucs/time/timerq.c +++ b/src/ucs/time/timerq.c @@ -11,7 +11,7 @@ #include "timerq.h" #include -#include +#include #include #include diff --git a/src/ucs/type/class.c b/src/ucs/type/class.c index 46ba925e312..4db549800b7 100644 --- a/src/ucs/type/class.c +++ b/src/ucs/type/class.c @@ -11,7 +11,7 @@ #include "class.h" #include -#include +#include #include diff --git a/src/uct/base/uct_component.c b/src/uct/base/uct_component.c index 549b18fbde0..f578cd7eec8 100644 --- a/src/uct/base/uct_component.c +++ b/src/uct/base/uct_component.c @@ -11,7 +11,7 @@ #include "uct_component.h" #include -#include +#include #include #include #include diff --git a/src/uct/base/uct_md.c b/src/uct/base/uct_md.c index b1dd3a0b726..3dd763d6946 100644 --- a/src/uct/base/uct_md.c +++ b/src/uct/base/uct_md.c @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/uct/base/uct_mem.c b/src/uct/base/uct_mem.c index 743998bf825..1ee633eb175 100644 --- a/src/uct/base/uct_mem.c +++ b/src/uct/base/uct_mem.c @@ -165,8 +165,8 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, break; } - ret = ucs_posix_memalign(&address, huge_page_size, alloc_length - UCS_MEMTRACK_VAL); + ret = ucs_posix_memalign(&address, huge_page_size, alloc_length, + alloc_name); if (ret != 0) { ucs_trace("failed to allocate %zu bytes using THP: %m", alloc_length); } else { @@ -192,7 +192,7 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, alloc_length = min_length; ret = ucs_posix_memalign(&address, UCS_SYS_CACHE_LINE_SIZE, - alloc_length UCS_MEMTRACK_VAL); + alloc_length, alloc_name); if (ret == 0) { goto allocated_without_md; } @@ -206,8 +206,7 @@ ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, address = addr; status = ucs_mmap_alloc(&alloc_length, &address, - uct_mem_get_mmap_flags(flags) - UCS_MEMTRACK_VAL); + uct_mem_get_mmap_flags(flags), alloc_name); if (status== UCS_OK) { goto allocated_without_md; } diff --git a/src/uct/cuda/cuda_copy/cuda_copy_ep.c b/src/uct/cuda/cuda_copy/cuda_copy_ep.c index d30f608d9bf..bcf75188cbb 100644 --- a/src/uct/cuda/cuda_copy/cuda_copy_ep.c +++ b/src/uct/cuda/cuda_copy/cuda_copy_ep.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/uct/cuda/cuda_copy/cuda_copy_md.c b/src/uct/cuda/cuda_copy/cuda_copy_md.c index 0758a05b5c6..1ed25ab37fe 100644 --- a/src/uct/cuda/cuda_copy/cuda_copy_md.c +++ b/src/uct/cuda/cuda_copy/cuda_copy_md.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/uct/cuda/cuda_ipc/cuda_ipc_cache.c b/src/uct/cuda/cuda_ipc/cuda_ipc_cache.c index accbb046439..78f09fea4b8 100644 --- a/src/uct/cuda/cuda_ipc/cuda_ipc_cache.c +++ b/src/uct/cuda/cuda_ipc/cuda_ipc_cache.c @@ -10,7 +10,7 @@ #include "cuda_ipc_cache.h" #include -#include +#include #include #include #include diff --git a/src/uct/cuda/cuda_ipc/cuda_ipc_ep.c b/src/uct/cuda/cuda_ipc/cuda_ipc_ep.c index a28e592af3f..7a63b32c6cb 100644 --- a/src/uct/cuda/cuda_ipc/cuda_ipc_ep.c +++ b/src/uct/cuda/cuda_ipc/cuda_ipc_ep.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/uct/cuda/cuda_ipc/cuda_ipc_md.c b/src/uct/cuda/cuda_ipc/cuda_ipc_md.c index 908de0419d0..708a7785d56 100644 --- a/src/uct/cuda/cuda_ipc/cuda_ipc_md.c +++ b/src/uct/cuda/cuda_ipc/cuda_ipc_md.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/uct/cuda/gdr_copy/gdr_copy_ep.c b/src/uct/cuda/gdr_copy/gdr_copy_ep.c index 1e0f965cf48..acf2bc22587 100644 --- a/src/uct/cuda/gdr_copy/gdr_copy_ep.c +++ b/src/uct/cuda/gdr_copy/gdr_copy_ep.c @@ -13,7 +13,7 @@ #include "gdr_copy_iface.h" #include -#include +#include #include #include #include diff --git a/src/uct/cuda/gdr_copy/gdr_copy_md.c b/src/uct/cuda/gdr_copy/gdr_copy_md.c index b6979780d02..6bba1034e95 100644 --- a/src/uct/cuda/gdr_copy/gdr_copy_md.c +++ b/src/uct/cuda/gdr_copy/gdr_copy_md.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/uct/ib/base/ib_device.c b/src/uct/ib/base/ib_device.c index 94578a7b49b..6f7537c1a6a 100644 --- a/src/uct/ib/base/ib_device.c +++ b/src/uct/ib/base/ib_device.c @@ -12,7 +12,7 @@ #include "ib_md.h" #include -#include +#include #include #include #include @@ -464,8 +464,8 @@ uct_ib_device_init_nb_close_ctx(int fd, uct_ib_device_nb_close_ctx **ctx_p) ctx->buff_size = ucs_get_page_size() * 2; ctx->cmd_fd = fd; - status = ucs_mmap_alloc(&ctx->buff_size, &ctx->buff, 0 - UCS_MEMTRACK_NAME("ibv cleanup buff")); + status = ucs_mmap_alloc(&ctx->buff_size, &ctx->buff, 0, + "ibv cleanup buff"); if (status != UCS_OK) { ucs_error("cleanup buffer allocation failed"); goto err_alloc; diff --git a/src/uct/ib/rc/base/rc_ep.c b/src/uct/ib/rc/base/rc_ep.c index cb87b5e8aa8..f0deb4d9f16 100644 --- a/src/uct/ib/rc/base/rc_ep.c +++ b/src/uct/ib/rc/base/rc_ep.c @@ -13,7 +13,7 @@ #include "rc_iface.h" #include -#include +#include #include #include #include diff --git a/src/uct/ib/rc/base/rc_iface.c b/src/uct/ib/rc/base/rc_iface.c index 8f4f22b264e..3a1a9089b67 100644 --- a/src/uct/ib/rc/base/rc_iface.c +++ b/src/uct/ib/rc/base/rc_iface.c @@ -12,7 +12,7 @@ #include "rc_ep.h" #include -#include +#include #include #include diff --git a/src/uct/ib/ud/accel/ud_mlx5.c b/src/uct/ib/ud/accel/ud_mlx5.c index 3492e5830a0..31019ee4b3a 100644 --- a/src/uct/ib/ud/accel/ud_mlx5.c +++ b/src/uct/ib/ud/accel/ud_mlx5.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include /* For htonl */ diff --git a/src/uct/ib/ud/base/ud_ep.c b/src/uct/ib/ud/base/ud_ep.c index d26ac8a43ec..4f7c5778dd7 100644 --- a/src/uct/ib/ud/base/ud_ep.c +++ b/src/uct/ib/ud/base/ud_ep.c @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include diff --git a/src/uct/ib/ud/base/ud_iface.c b/src/uct/ib/ud/base/ud_iface.c index 9a734220478..98311e3e8d1 100644 --- a/src/uct/ib/ud/base/ud_iface.c +++ b/src/uct/ib/ud/base/ud_iface.c @@ -13,7 +13,7 @@ #include "ud_inl.h" #include -#include +#include #include #include #include diff --git a/src/uct/ib/ud/verbs/ud_verbs.c b/src/uct/ib/ud/verbs/ud_verbs.c index 0e957771fd1..aa73803329d 100644 --- a/src/uct/ib/ud/verbs/ud_verbs.c +++ b/src/uct/ib/ud/verbs/ud_verbs.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include /* For htonl */ diff --git a/src/uct/rocm/copy/rocm_copy_ep.c b/src/uct/rocm/copy/rocm_copy_ep.c index 04095ad2515..375cbb2428f 100644 --- a/src/uct/rocm/copy/rocm_copy_ep.c +++ b/src/uct/rocm/copy/rocm_copy_ep.c @@ -12,7 +12,7 @@ #include #include -#include +#include #include #include diff --git a/src/uct/rocm/copy/rocm_copy_md.c b/src/uct/rocm/copy/rocm_copy_md.c index 2ef768d63d2..7b1e52f69a4 100644 --- a/src/uct/rocm/copy/rocm_copy_md.c +++ b/src/uct/rocm/copy/rocm_copy_md.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/uct/rocm/gdr/rocm_gdr_ep.c b/src/uct/rocm/gdr/rocm_gdr_ep.c index 0ed879dd355..3e48f5bef81 100644 --- a/src/uct/rocm/gdr/rocm_gdr_ep.c +++ b/src/uct/rocm/gdr/rocm_gdr_ep.c @@ -11,7 +11,7 @@ #include "rocm_gdr_iface.h" #include -#include +#include #include #include diff --git a/src/uct/rocm/gdr/rocm_gdr_md.c b/src/uct/rocm/gdr/rocm_gdr_md.c index d2e03a7682f..53d67ffde2c 100644 --- a/src/uct/rocm/gdr/rocm_gdr_md.c +++ b/src/uct/rocm/gdr/rocm_gdr_md.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 6575225b9d7..ca5da55a8c8 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -12,7 +12,7 @@ #include "rocm_ipc_cache.h" #include -#include +#include #include #include #include diff --git a/src/uct/sm/mm/base/mm_iface.h b/src/uct/sm/mm/base/mm_iface.h index f686127fad0..df9f3ed143a 100644 --- a/src/uct/sm/mm/base/mm_iface.h +++ b/src/uct/sm/mm/base/mm_iface.h @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/uct/sm/mm/base/mm_md.h b/src/uct/sm/mm/base/mm_md.h index a74f57364a2..4ad97ddf6e2 100644 --- a/src/uct/sm/mm/base/mm_md.h +++ b/src/uct/sm/mm/base/mm_md.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include diff --git a/src/uct/sm/mm/posix/mm_posix.c b/src/uct/sm/mm/posix/mm_posix.c index 76357ad5191..2dbe8ed686f 100644 --- a/src/uct/sm/mm/posix/mm_posix.c +++ b/src/uct/sm/mm/posix/mm_posix.c @@ -11,7 +11,7 @@ #include #include -#include +#include #include #include #include @@ -288,7 +288,7 @@ uct_posix_mmap(void **address_p, size_t *length_p, int flags, int fd, #endif result = ucs_mmap(*address_p, aligned_length, UCT_POSIX_MMAP_PROT, - MAP_SHARED | flags, fd, 0 UCS_MEMTRACK_VAL); + MAP_SHARED | flags, fd, 0, alloc_name); if (result == MAP_FAILED) { ucs_log(err_level, "shared memory mmap(addr=%p, length=%zu, flags=%s%s, fd=%d) failed: %m", diff --git a/src/uct/sm/mm/sysv/mm_sysv.c b/src/uct/sm/mm/sysv/mm_sysv.c index cd84ab47c29..bf4361bbe8a 100644 --- a/src/uct/sm/mm/sysv/mm_sysv.c +++ b/src/uct/sm/mm/sysv/mm_sysv.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include diff --git a/src/uct/sm/mm/xpmem/mm_xpmem.c b/src/uct/sm/mm/xpmem/mm_xpmem.c index 6c59ac29db4..e56064df439 100644 --- a/src/uct/sm/mm/xpmem/mm_xpmem.c +++ b/src/uct/sm/mm/xpmem/mm_xpmem.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/uct/sm/scopy/cma/cma_md.h b/src/uct/sm/scopy/cma/cma_md.h index 9ab24494b3c..b5b715ed7c4 100644 --- a/src/uct/sm/scopy/cma/cma_md.h +++ b/src/uct/sm/scopy/cma/cma_md.h @@ -9,7 +9,7 @@ #define UCT_CMA_MD_H_ #include -#include +#include #include #include diff --git a/src/uct/sm/scopy/knem/knem_md.h b/src/uct/sm/scopy/knem/knem_md.h index 051078a59d3..021f4089b8b 100644 --- a/src/uct/sm/scopy/knem/knem_md.h +++ b/src/uct/sm/scopy/knem/knem_md.h @@ -9,7 +9,7 @@ #define UCT_KNEM_MD_H_ #include -#include +#include #include #include #include diff --git a/test/apps/test_memtrack_limit.c b/test/apps/test_memtrack_limit.c index 16f8c61e007..83bc20f6d33 100644 --- a/test/apps/test_memtrack_limit.c +++ b/test/apps/test_memtrack_limit.c @@ -9,7 +9,7 @@ #endif #include -#include +#include #include int main(int argc, char **argv) diff --git a/test/gtest/ucs/test_memtrack.cc b/test/gtest/ucs/test_memtrack.cc index 7ef5ccd34ef..5c6955968ab 100644 --- a/test/gtest/ucs/test_memtrack.cc +++ b/test/gtest/ucs/test_memtrack.cc @@ -7,7 +7,7 @@ #include extern "C" { -#include +#include #include } @@ -19,8 +19,6 @@ extern "C" { #include -#ifdef ENABLE_MEMTRACK - class test_memtrack : public ucs::test { protected: static const size_t ALLOC_SIZE = 10000; @@ -205,5 +203,3 @@ UCS_TEST_F(test_memtrack, custom) { test_total(1, ALLOC_SIZE); } - -#endif diff --git a/test/gtest/uct/ib/test_ib_md.cc b/test/gtest/uct/ib/test_ib_md.cc index 6fbbb9db78a..3b6aa18f95d 100644 --- a/test/gtest/uct/ib/test_ib_md.cc +++ b/test/gtest/uct/ib/test_ib_md.cc @@ -48,8 +48,7 @@ void test_ib_md::ib_md_umr_check(void *rkey_buffer, buffer = NULL; alloc_size = size; - status = ucs_mmap_alloc(&alloc_size, &buffer, 0 - UCS_MEMTRACK_NAME("test_umr")); + status = ucs_mmap_alloc(&alloc_size, &buffer, 0, "test_umr"); ASSERT_UCS_OK(status); uct_mem_h memh; diff --git a/test/gtest/uct/test_p2p_err.cc b/test/gtest/uct/test_p2p_err.cc index fb8bc593353..958f8aba2d2 100644 --- a/test/gtest/uct/test_p2p_err.cc +++ b/test/gtest/uct/test_p2p_err.cc @@ -119,8 +119,8 @@ class uct_p2p_err_test : public uct_p2p_test { static void* get_unused_address(size_t length) { void *address = NULL; - ucs_status_t status = ucs_mmap_alloc(&length, &address, 0 - UCS_MEMTRACK_NAME("test_dummy")); + ucs_status_t status = ucs_mmap_alloc(&length, &address, 0, + "test_dummy"); ASSERT_UCS_OK(status, << "length = " << length); status = ucs_mmap_free(address, length); ASSERT_UCS_OK(status);