From dfacabf3c567d399df3d36acf32d27e233848ee6 Mon Sep 17 00:00:00 2001 From: Todd Kordenbrock Date: Tue, 19 Oct 2021 11:25:36 -0500 Subject: [PATCH] btl-portals4: - move recv init inside the interface init loop - fix uninitialized variable compiler warnings - fix unused variable compiler warnings - fix undefined variable failures in debug builds mtl-portals4: - move verbose output to the end of open() to avoid complicated message generation - there is no function to add-fetch uint32_t, so change event_count type to opal_atomic_int32_t coll-portals4: - move declaration of loop index variables into the loop - fix hidden loop index compiler warnings - use REQUEST_COMPLETE to mark ompi_request_t complete - remove incorrect req_complete initialization osc-portals4: - replace nonstatndard __FUNCTION__ references with __FILE__ - use REQUEST_COMPLETE to mark ompi_request_t complete - remove incorrect req_complete initialization - add atomic compatible bytes_committed member to ompi_osc_portals4_request_t - the request status _ucount member shouldn't be incremented directly with opal_atomic_add_fetch_32(); instead increment bytes_committed and assigned the result to _ucount - remove initialization of nonexistent module members - osc_set_info and osc_get_info Signed-off-by: Todd Kordenbrock (cherry picked from commit 2fc4020f3c0fcd8c43fc1f5c6f65df76741f6266) --- .../coll/portals4/coll_portals4_component.c | 7 +- ompi/mca/coll/portals4/coll_portals4_gather.c | 28 ++--- .../mca/coll/portals4/coll_portals4_request.c | 2 +- .../mca/coll/portals4/coll_portals4_request.h | 1 - .../mca/mtl/portals4/mtl_portals4_component.c | 55 ++++----- ompi/mca/mtl/portals4/mtl_portals4_request.h | 2 +- ompi/mca/mtl/portals4/mtl_portals4_send.c | 2 +- ompi/mca/osc/portals4/osc_portals4_comm.c | 114 +++++++++--------- .../mca/osc/portals4/osc_portals4_component.c | 5 +- ompi/mca/osc/portals4/osc_portals4_request.c | 2 +- ompi/mca/osc/portals4/osc_portals4_request.h | 3 +- opal/mca/btl/portals4/btl_portals4.c | 27 +++-- 12 files changed, 120 insertions(+), 128 deletions(-) diff --git a/ompi/mca/coll/portals4/coll_portals4_component.c b/ompi/mca/coll/portals4/coll_portals4_component.c index dcb6f284ed9..1fb89ea203a 100644 --- a/ompi/mca/coll/portals4/coll_portals4_component.c +++ b/ompi/mca/coll/portals4/coll_portals4_component.c @@ -655,7 +655,11 @@ portals4_module_enable(mca_coll_base_module_t *module, return OMPI_SUCCESS; } - +#if OPAL_ENABLE_DEBUG +/* These string maps are only used for debugging output. + * They will be compiled-out when OPAL is configured + * without --enable-debug. + */ static char *failtype[] = { "PTL_NI_OK", "PTL_NI_PERM_VIOLATION", @@ -689,6 +693,7 @@ static char *evname[] = { "PTL_EVENT_SEARCH", "PTL_EVENT_LINK" }; +#endif /* Target EQ */ static int diff --git a/ompi/mca/coll/portals4/coll_portals4_gather.c b/ompi/mca/coll/portals4/coll_portals4_gather.c index 274e9d4ee89..54b7acb0e84 100644 --- a/ompi/mca/coll/portals4/coll_portals4_gather.c +++ b/ompi/mca/coll/portals4/coll_portals4_gather.c @@ -76,7 +76,7 @@ static ompi_coll_portals4_tree_t* ompi_coll_portals4_build_in_order_bmtree( struct ompi_communicator_t* comm, int root ) { - int childs = 0, rank, vrank, vparent, size, mask = 1, remote, i; + int childs = 0, rank, vrank, vparent, size, mask = 1, remote; ompi_coll_portals4_tree_t *bmtree; /* @@ -97,7 +97,7 @@ ompi_coll_portals4_build_in_order_bmtree( struct ompi_communicator_t* comm, bmtree->tree_bmtree = 1; bmtree->tree_root = MPI_UNDEFINED; bmtree->tree_nextsize = MPI_UNDEFINED; - for(i=0;itree_next[i] = -1; } @@ -520,8 +520,6 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc int vrank=-1; - int32_t i=0; - ompi_coll_portals4_tree_t* bmtree; int32_t expected_ops =0; @@ -606,7 +604,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc "%s:%d: packed_size=%lu, fragment_size=%lu", __FILE__, __LINE__, request->u.gather.packed_size, mca_coll_portals4_component.ni_limits.max_msg_size)); - for (int i =0; i < bmtree->tree_nextsize; i++) { + for (int i = 0; i < bmtree->tree_nextsize; i++) { int child_vrank = VRANK(bmtree->tree_next[i], request->u.gather.root_rank, request->u.gather.size); int sub_tree_size = get_tree_numdescendants_of(comm, child_vrank) + 1; ptl_size_t local_number_of_fragment = ((sub_tree_size * request->u.gather.packed_size) + mca_coll_portals4_component.ni_limits.max_msg_size -1) / mca_coll_portals4_component.ni_limits.max_msg_size; @@ -688,7 +686,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc /************************************/ /* put Recv-ACK to each child */ /************************************/ - for (i=0;itree_nextsize;i++) { + for (int i = 0; i < bmtree->tree_nextsize; i++) { int32_t child=bmtree->tree_next[i]; ret = PtlTriggeredPut(request->u.gather.sync_mdh, 0, @@ -730,7 +728,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc /**********************************/ /* put RTR to each child */ /**********************************/ - for (i=0;itree_nextsize;i++) { + for (int i = 0; i < bmtree->tree_nextsize; i++) { int32_t child=bmtree->tree_next[i]; ret = PtlTriggeredPut(request->u.gather.sync_mdh, 0, @@ -750,7 +748,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc /**********************************/ /* put RTR to each child */ /**********************************/ - for (i=0;itree_nextsize;i++) { + for (int i = 0; i < bmtree->tree_nextsize; i++) { int32_t child=bmtree->tree_next[i]; ret = PtlPut(request->u.gather.sync_mdh, 0, @@ -818,8 +816,6 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct int8_t i_am_root; - int32_t i=0; - int32_t expected_ops =0; int32_t expected_acks=0; @@ -975,7 +971,7 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct /* root puts Recv-ACK to all other ranks */ /*****************************************/ if (i_am_root) { - for (i=0;iu.gather.size;i++) { + for (int i = 0; i < request->u.gather.size; i++) { if (i == request->u.gather.root_rank) { continue; } ret = PtlTriggeredPut(request->u.gather.sync_mdh, 0, @@ -1019,7 +1015,7 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct /* root puts RTR to all other ranks */ /************************************/ if (i_am_root) { - for (i=0;iu.gather.size;i++) { + for (int i = 0; i < request->u.gather.size; i++) { if (i == request->u.gather.root_rank) { continue; } ret = PtlTriggeredPut(request->u.gather.sync_mdh, 0, @@ -1041,7 +1037,7 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct /* root puts RTR to all other ranks */ /************************************/ if (i_am_root) { - for (i=0;iu.gather.size;i++) { + for (int i = 0; i < request->u.gather.size; i++) { if (i == request->u.gather.root_rank) { continue; } ret = PtlPut(request->u.gather.sync_mdh, 0, @@ -1093,7 +1089,6 @@ ompi_coll_portals4_gather_intra_binomial_bottom(struct ompi_communicator_t *comm ompi_coll_portals4_request_t *request) { int ret, line; - int i; OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output, "coll:portals4:gather_intra_binomial_bottom enter rank %d", request->u.gather.my_rank)); @@ -1109,7 +1104,7 @@ ompi_coll_portals4_gather_intra_binomial_bottom(struct ompi_communicator_t *comm struct iovec iov; size_t max_data; - for (i=0;iu.gather.size;i++) { + for (int i = 0; i < request->u.gather.size; i++) { uint64_t offset = request->u.gather.unpack_dst_extent * request->u.gather.unpack_dst_count * ((request->u.gather.my_rank + i) % request->u.gather.size); opal_output_verbose(30, ompi_coll_base_framework.framework_output, @@ -1161,7 +1156,6 @@ ompi_coll_portals4_gather_intra_linear_bottom(struct ompi_communicator_t *comm, ompi_coll_portals4_request_t *request) { int ret, line; - int i; OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output, "coll:portals4:gather_intra_linear_bottom enter rank %d", request->u.gather.my_rank)); @@ -1177,7 +1171,7 @@ ompi_coll_portals4_gather_intra_linear_bottom(struct ompi_communicator_t *comm, struct iovec iov; size_t max_data; - for (i=0;iu.gather.size;i++) { + for (int i = 0; i < request->u.gather.size; i++) { ompi_coll_portals4_create_recv_converter (&request->u.gather.recv_converter, request->u.gather.unpack_dst_buf + (request->u.gather.unpack_dst_extent*request->u.gather.unpack_dst_count*i), ompi_comm_peer_lookup(comm, request->u.gather.my_rank), diff --git a/ompi/mca/coll/portals4/coll_portals4_request.c b/ompi/mca/coll/portals4/coll_portals4_request.c index 001594f5d5c..9bebe56825c 100644 --- a/ompi/mca/coll/portals4/coll_portals4_request.c +++ b/ompi/mca/coll/portals4/coll_portals4_request.c @@ -27,7 +27,7 @@ request_free(struct ompi_request_t **ompi_req) ompi_coll_portals4_request_t *request = (ompi_coll_portals4_request_t*) *ompi_req; - if (true != request->super.req_complete) { + if (!REQUEST_COMPLETE(&request->super)) { return MPI_ERR_REQUEST; } diff --git a/ompi/mca/coll/portals4/coll_portals4_request.h b/ompi/mca/coll/portals4/coll_portals4_request.h index f78c2dee631..7c845698a79 100644 --- a/ompi/mca/coll/portals4/coll_portals4_request.h +++ b/ompi/mca/coll/portals4/coll_portals4_request.h @@ -176,7 +176,6 @@ OBJ_CLASS_DECLARATION(ompi_coll_portals4_request_t); req = (ompi_coll_portals4_request_t*) item; \ OMPI_REQUEST_INIT(&req->super, false); \ req->super.req_mpi_object.comm = comm; \ - req->super.req_complete = false; \ req->super.req_state = OMPI_REQUEST_ACTIVE; \ } while (0) diff --git a/ompi/mca/mtl/portals4/mtl_portals4_component.c b/ompi/mca/mtl/portals4/mtl_portals4_component.c index 57777ff5f19..94cd0468ad7 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_component.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_component.c @@ -227,36 +227,6 @@ ompi_mtl_portals4_component_open(void) sizeof(ompi_mtl_portals4_request_t) - sizeof(struct mca_mtl_request_t); - opal_output_verbose(1, ompi_mtl_base_framework.framework_output, - "Flow control: " -#if OMPI_MTL_PORTALS4_FLOW_CONTROL - "yes" -#else - "no" -#endif - ); - opal_output_verbose(1, ompi_mtl_base_framework.framework_output, - "Max message size: %lu", (unsigned long) - ompi_mtl_portals4.max_msg_size_mtl); - opal_output_verbose(1, ompi_mtl_base_framework.framework_output, - "Short limit: %d", (int) - ompi_mtl_portals4.short_limit); - opal_output_verbose(1, ompi_mtl_base_framework.framework_output, - "Eager limit: %d", (int) - ompi_mtl_portals4.eager_limit); - opal_output_verbose(1, ompi_mtl_base_framework.framework_output, - "Short receive blocks: %d", - ompi_mtl_portals4.recv_short_num); - opal_output_verbose(1, ompi_mtl_base_framework.framework_output, - "Send queue size: %d", ompi_mtl_portals4.send_queue_size); - opal_output_verbose(1, ompi_mtl_base_framework.framework_output, - "Recv queue size: %d", ompi_mtl_portals4.recv_queue_size); - opal_output_verbose(1, ompi_mtl_base_framework.framework_output, - "Long protocol: %s", - (ompi_mtl_portals4.protocol == eager) ? "Eager" : - (ompi_mtl_portals4.protocol == rndv) ? "Rendezvous" : - "Other"); - OBJ_CONSTRUCT(&ompi_mtl_portals4.fl_message, opal_free_list_t); opal_free_list_init(&ompi_mtl_portals4.fl_message, sizeof(ompi_mtl_portals4_message_t) + @@ -289,6 +259,31 @@ ompi_mtl_portals4_component_open(void) ompi_mtl_portals4.use_flowctl=0; #endif + opal_output_verbose(1, ompi_mtl_base_framework.framework_output, + "Flow control: %s", + ompi_mtl_portals4.use_flowctl ? "yes" : "no"); + opal_output_verbose(1, ompi_mtl_base_framework.framework_output, + "Max message size: %lu", (unsigned long) + ompi_mtl_portals4.max_msg_size_mtl); + opal_output_verbose(1, ompi_mtl_base_framework.framework_output, + "Short limit: %d", (int) + ompi_mtl_portals4.short_limit); + opal_output_verbose(1, ompi_mtl_base_framework.framework_output, + "Eager limit: %d", (int) + ompi_mtl_portals4.eager_limit); + opal_output_verbose(1, ompi_mtl_base_framework.framework_output, + "Short receive blocks: %d", + ompi_mtl_portals4.recv_short_num); + opal_output_verbose(1, ompi_mtl_base_framework.framework_output, + "Send queue size: %d", ompi_mtl_portals4.send_queue_size); + opal_output_verbose(1, ompi_mtl_base_framework.framework_output, + "Recv queue size: %d", ompi_mtl_portals4.recv_queue_size); + opal_output_verbose(1, ompi_mtl_base_framework.framework_output, + "Long protocol: %s", + (ompi_mtl_portals4.protocol == eager) ? "Eager" : + (ompi_mtl_portals4.protocol == rndv) ? "Rendezvous" : + "Other"); + return OMPI_SUCCESS; } diff --git a/ompi/mca/mtl/portals4/mtl_portals4_request.h b/ompi/mca/mtl/portals4/mtl_portals4_request.h index c7e3c31e47a..a16f15195fd 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_request.h +++ b/ompi/mca/mtl/portals4/mtl_portals4_request.h @@ -55,7 +55,7 @@ struct ompi_mtl_portals4_isend_request_t { #endif ptl_size_t length; int32_t pending_get; - uint32_t event_count; + int32_t event_count; }; typedef struct ompi_mtl_portals4_isend_request_t ompi_mtl_portals4_isend_request_t; diff --git a/ompi/mca/mtl/portals4/mtl_portals4_send.c b/ompi/mca/mtl/portals4/mtl_portals4_send.c index 27291eed559..c70b00706f5 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_send.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_send.c @@ -40,7 +40,7 @@ ompi_mtl_portals4_callback(ptl_event_t *ev, ompi_mtl_portals4_base_request_t* ptl_base_request, bool *complete) { - int retval = OMPI_SUCCESS, ret, val, add = 1; + int retval = OMPI_SUCCESS, ret = 0, val = 0, add = 1; ompi_mtl_portals4_isend_request_t* ptl_request = (ompi_mtl_portals4_isend_request_t*) ptl_base_request; diff --git a/ompi/mca/osc/portals4/osc_portals4_comm.c b/ompi/mca/osc/portals4/osc_portals4_comm.c index b125f2aee50..e41a1f68276 100644 --- a/ompi/mca/osc/portals4/osc_portals4_comm.c +++ b/ompi/mca/osc/portals4/osc_portals4_comm.c @@ -183,7 +183,7 @@ number_of_fragments(ptl_size_t length, ptl_size_t maxlength) { ptl_size_t nb_frag = length == 0 ? 1 : (length - 1) / maxlength + 1; OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d : %ld fragment(s)", __FUNCTION__, __LINE__, nb_frag)); + "%s,%d : %ld fragment(s)", __FILE__, __LINE__, nb_frag)); return nb_frag; } @@ -225,7 +225,7 @@ segmentedPut(int64_t *opcount, opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlPut failed with return value %d", - __FUNCTION__, __LINE__, ret); + __FILE__, __LINE__, ret); return ret; } put_length -= frag_length; @@ -269,7 +269,7 @@ segmentedGet(int64_t *opcount, opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlGet failed with return value %d", - __FUNCTION__, __LINE__, ret); + __FILE__, __LINE__, ret); return ret; } get_length -= frag_length; @@ -318,7 +318,7 @@ segmentedAtomic(int64_t *opcount, opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlAtomic failed with return value %d", - __FUNCTION__, __LINE__, ret); + __FILE__, __LINE__, ret); return ret; } length -= frag_length; @@ -370,7 +370,7 @@ segmentedFetchAtomic(int64_t *opcount, opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlFetchAtomic failed with return value %d", - __FUNCTION__, __LINE__, ret); + __FILE__, __LINE__, ret); return ret; } length -= frag_length; @@ -422,7 +422,7 @@ segmentedSwap(int64_t *opcount, opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlSwap failed with return value %d", - __FUNCTION__, __LINE__, ret); + __FILE__, __LINE__, ret); return ret; } length -= frag_length; @@ -551,7 +551,7 @@ get_to_iovec(ompi_osc_portals4_module_t *module, OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", - __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); + __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); ret = PtlGet(module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, @@ -563,7 +563,7 @@ get_to_iovec(ompi_osc_portals4_module_t *module, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d PtlGet() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } @@ -636,7 +636,7 @@ atomic_get_to_iovec(ompi_osc_portals4_module_t *module, OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", - __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); + __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); ret = segmentedGet(&module->opcount, module->origin_iovec_md_h, (ptl_size_t) origin_lb, @@ -720,7 +720,7 @@ put_from_iovec(ompi_osc_portals4_module_t *module, OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", - __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); + __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); ret = PtlPut(module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, @@ -734,7 +734,7 @@ put_from_iovec(ompi_osc_portals4_module_t *module, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d PtlPut() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } @@ -807,7 +807,7 @@ atomic_put_from_iovec(ompi_osc_portals4_module_t *module, OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", - __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); + __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); ret = segmentedPut(&module->opcount, module->origin_iovec_md_h, (ptl_size_t) origin_lb, @@ -1487,7 +1487,7 @@ atomic_to_noncontig(ompi_osc_portals4_module_t *module, (unsigned long) target_iovec[target_iov_index].iov_len)); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Atomic", __FUNCTION__, __LINE__)); + "%s,%d Atomic", __FILE__, __LINE__)); ret = PtlAtomic(md_h, (ptl_size_t)origin_iovec[origin_iov_index].iov_base, atomic_len, @@ -1843,7 +1843,7 @@ swap_from_noncontig(ompi_osc_portals4_module_t *module, if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlSwap failed with return value %d", - __FUNCTION__, __LINE__, ret); + __FILE__, __LINE__, ret); opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } @@ -1994,7 +1994,7 @@ fetch_atomic_from_noncontig(ompi_osc_portals4_module_t *module, if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlFetchAtomic failed with return value %d", - __FUNCTION__, __LINE__, ret); + __FILE__, __LINE__, ret); opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } @@ -2065,7 +2065,7 @@ ompi_osc_portals4_rput(const void *origin_addr, OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d put_to_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { @@ -2084,7 +2084,7 @@ ompi_osc_portals4_rput(const void *origin_addr, OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d put_from_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -2105,7 +2105,7 @@ ompi_osc_portals4_rput(const void *origin_addr, OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d RPut(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", - __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); + __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); ret = segmentedPut(&module->opcount, module->req_md_h, (ptl_size_t) origin_addr + origin_lb, @@ -2176,7 +2176,7 @@ ompi_osc_portals4_rget(void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d get_from_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { @@ -2194,7 +2194,7 @@ ompi_osc_portals4_rget(void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d get_to_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -2212,7 +2212,7 @@ ompi_osc_portals4_rget(void *origin_addr, request->ops_expected += number_of_fragments(length, mca_osc_portals4_component.ptl_max_msg_size); OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, - "%s,%d RGet", __FUNCTION__, __LINE__)); + "%s,%d RGet", __FILE__, __LINE__)); ret = segmentedGet(&module->opcount, module->req_md_h, (ptl_size_t) origin_addr + origin_lb, @@ -2287,7 +2287,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_put_to_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -2308,7 +2308,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_to_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } @@ -2329,7 +2329,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_put_from_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -2349,7 +2349,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_from_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } @@ -2376,7 +2376,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Put", __FUNCTION__, __LINE__)); + "%s,%d Put", __FILE__, __LINE__)); ret = segmentedPut(&module->opcount, module->req_md_h, md_offset + origin_lb, @@ -2414,7 +2414,7 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Atomic", __FUNCTION__, __LINE__)); + "%s,%d Atomic", __FILE__, __LINE__)); ret = PtlAtomic(module->req_md_h, md_offset + sent + origin_lb, msg_length, @@ -2505,7 +2505,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d swap_from_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } @@ -2527,7 +2527,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_get_from_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } @@ -2554,7 +2554,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d fetch_atomic_from_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } @@ -2581,7 +2581,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d swap_to_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } @@ -2602,7 +2602,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_get_to_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } @@ -2627,7 +2627,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d fetch_atomic_to_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } @@ -2709,7 +2709,7 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, request->ops_expected += number_of_fragments(length, module->fetch_atomic_max); OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, - "%s,%d MPI_Get_accumulate", __FUNCTION__, __LINE__)); + "%s,%d MPI_Get_accumulate", __FILE__, __LINE__)); ret = segmentedGet(&module->opcount, module->req_md_h, (ptl_size_t) md_offset + result_lb, @@ -2835,7 +2835,7 @@ ompi_osc_portals4_put(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d put_to_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { @@ -2853,7 +2853,7 @@ ompi_osc_portals4_put(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d put_from_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -2870,7 +2870,7 @@ ompi_osc_portals4_put(const void *origin_addr, OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", - __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); + __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); ret = segmentedPut(&module->opcount, module->md_h, (ptl_size_t) origin_addr + origin_lb, @@ -2934,7 +2934,7 @@ ompi_osc_portals4_get(void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d get_from_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { @@ -2952,7 +2952,7 @@ ompi_osc_portals4_get(void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d get_to_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -2968,7 +2968,7 @@ ompi_osc_portals4_get(void *origin_addr, length = size * origin_count; OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, - "%s,%d Get", __FUNCTION__, __LINE__)); + "%s,%d Get", __FILE__, __LINE__)); ret = segmentedGet(&module->opcount, module->md_h, (ptl_size_t) origin_addr + origin_lb, @@ -3035,7 +3035,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_put_to_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -3055,7 +3055,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_to_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } @@ -3075,7 +3075,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_put_from_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -3094,7 +3094,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_from_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } @@ -3117,7 +3117,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr, if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Put", __FUNCTION__, __LINE__)); + "%s,%d Put", __FILE__, __LINE__)); ret = segmentedPut(&module->opcount, module->md_h, md_offset + origin_lb, @@ -3152,7 +3152,7 @@ ompi_osc_portals4_accumulate(const void *origin_addr, (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Atomic", __FUNCTION__, __LINE__)); + "%s,%d Atomic", __FILE__, __LINE__)); ret = PtlAtomic(module->md_h, md_offset + sent + origin_lb, msg_length, @@ -3236,7 +3236,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d swap_from_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else if (MPI_NO_OP == op) { @@ -3257,7 +3257,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_get_from_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -3283,7 +3283,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d fetch_atomic_from_noncontig() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } @@ -3309,7 +3309,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d swap_to_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else if (MPI_NO_OP == op) { @@ -3329,7 +3329,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_get_to_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } else { @@ -3353,7 +3353,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d fetch_atomic_to_iovec() failed: ret = %d", - __FUNCTION__, __LINE__, ret)); + __FILE__, __LINE__, ret)); return ret; } } @@ -3423,7 +3423,7 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, md_offset = (ptl_size_t) result_addr; OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, - "%s,%d MPI_Get_accumulate", __FUNCTION__, __LINE__)); + "%s,%d MPI_Get_accumulate", __FILE__, __LINE__)); ret = segmentedGet(&module->opcount, module->md_h, (ptl_size_t) md_offset + result_lb, @@ -3544,7 +3544,7 @@ ompi_osc_portals4_compare_and_swap(const void *origin_addr, (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, - "%s,%d Swap", __FUNCTION__, __LINE__)); + "%s,%d Swap", __FILE__, __LINE__)); ret = PtlSwap(module->md_h, result_md_offset, module->md_h, @@ -3615,7 +3615,7 @@ ompi_osc_portals4_fetch_and_op(const void *origin_addr, (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Swap", __FUNCTION__, __LINE__)); + "%s,%d Swap", __FILE__, __LINE__)); ret = PtlSwap(module->md_h, result_md_offset, module->md_h, @@ -3637,7 +3637,7 @@ ompi_osc_portals4_fetch_and_op(const void *origin_addr, (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Get", __FUNCTION__, __LINE__)); + "%s,%d Get", __FILE__, __LINE__)); ret = PtlGet(module->md_h, md_offset, length, @@ -3661,7 +3661,7 @@ ompi_osc_portals4_fetch_and_op(const void *origin_addr, origin_md_offset = (ptl_size_t) origin_addr; OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d FetchAtomic", __FUNCTION__, __LINE__)); + "%s,%d FetchAtomic", __FILE__, __LINE__)); ret = PtlFetchAtomic(module->md_h, result_md_offset, module->md_h, diff --git a/ompi/mca/osc/portals4/osc_portals4_component.c b/ompi/mca/osc/portals4/osc_portals4_component.c index 8a4781e3af6..f0df81e2758 100644 --- a/ompi/mca/osc/portals4/osc_portals4_component.c +++ b/ompi/mca/osc/portals4/osc_portals4_component.c @@ -98,9 +98,6 @@ ompi_osc_portals4_module_t ompi_osc_portals4_module_template = { ompi_osc_portals4_flush_all, ompi_osc_portals4_flush_local, ompi_osc_portals4_flush_local_all, - - ompi_osc_portals4_set_info, - ompi_osc_portals4_get_info } }; @@ -230,7 +227,7 @@ progress_callback(void) } req = (ompi_osc_portals4_request_t*) ev.user_ptr; - opal_atomic_add_fetch_size_t(&req->super.req_status._ucount, ev.mlength); + req->super.req_status._ucount = opal_atomic_add_fetch_32(&req->bytes_committed, ev.mlength); ops = opal_atomic_add_fetch_32(&req->ops_committed, 1); if (ops == req->ops_expected) { ompi_request_complete(&req->super, true); diff --git a/ompi/mca/osc/portals4/osc_portals4_request.c b/ompi/mca/osc/portals4/osc_portals4_request.c index 19461ec0580..cf5388af6b0 100644 --- a/ompi/mca/osc/portals4/osc_portals4_request.c +++ b/ompi/mca/osc/portals4/osc_portals4_request.c @@ -29,7 +29,7 @@ request_free(struct ompi_request_t **ompi_req) ompi_osc_portals4_request_t *request = (ompi_osc_portals4_request_t*) *ompi_req; - if (true != request->super.req_complete) { + if (!REQUEST_COMPLETE(&request->super)) { return MPI_ERR_REQUEST; } diff --git a/ompi/mca/osc/portals4/osc_portals4_request.h b/ompi/mca/osc/portals4/osc_portals4_request.h index ae1be6f44d2..6d0c4b1f2d5 100644 --- a/ompi/mca/osc/portals4/osc_portals4_request.h +++ b/ompi/mca/osc/portals4/osc_portals4_request.h @@ -19,6 +19,7 @@ struct ompi_osc_portals4_request_t { ompi_request_t super; int32_t ops_expected; volatile int32_t ops_committed; + volatile int32_t bytes_committed; }; typedef struct ompi_osc_portals4_request_t ompi_osc_portals4_request_t; @@ -31,11 +32,11 @@ OBJ_CLASS_DECLARATION(ompi_osc_portals4_request_t); req = (ompi_osc_portals4_request_t*) item; \ OMPI_REQUEST_INIT(&req->super, false); \ req->super.req_mpi_object.win = win; \ - req->super.req_complete = false; \ req->super.req_state = OMPI_REQUEST_ACTIVE; \ req->super.req_status.MPI_ERROR = MPI_SUCCESS; \ req->ops_expected = 0; \ req->ops_committed = 0; \ + req->bytes_committed = 0; \ } while (0) #define OMPI_OSC_PORTALS4_REQUEST_RETURN(req) \ diff --git a/opal/mca/btl/portals4/btl_portals4.c b/opal/mca/btl/portals4/btl_portals4.c index 80e28ef47bc..7addb795e1d 100644 --- a/opal/mca/btl/portals4/btl_portals4.c +++ b/opal/mca/btl/portals4/btl_portals4.c @@ -188,14 +188,14 @@ btl_portals4_init_interface(void) goto error; } OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlMEAppend (overflow list) OK for NI %d", interface)); - } - ret = mca_btl_portals4_recv_enable(portals4_btl); - if (PTL_OK != ret) { - opal_output_verbose(1, opal_btl_base_framework.framework_output, - "%s:%d: Initialization of recv buffer failed: %d", - __FILE__, __LINE__, ret); - goto error; + ret = mca_btl_portals4_recv_enable(portals4_btl); + if (PTL_OK != ret) { + opal_output_verbose(1, opal_btl_base_framework.framework_output, + "%s:%d: Initialization of recv buffer failed: %d", __FILE__, __LINE__, + ret); + goto error; + } } return OPAL_SUCCESS; @@ -558,15 +558,16 @@ mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base, size_t* size, uint32_t flags) { - struct mca_btl_portals4_module_t* portals4_btl = (struct mca_btl_portals4_module_t*) btl_base; - mca_btl_portals4_frag_t* frag; + mca_btl_portals4_frag_t* frag = NULL; size_t max_data = *size; struct iovec iov; uint32_t iov_count = 1; int ret; OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, - "mca_btl_portals4_prepare_src NI=%d reserve=%ld size=%ld max_data=%ld\n", portals4_btl->interface_num, reserve, *size, max_data)); + "mca_btl_portals4_prepare_src NI=%d reserve=%ld size=%ld max_data=%ld\n", + ((struct mca_btl_portals4_module_t *) btl_base)->interface_num, + reserve, *size, max_data)); if (0 != reserve || 0 != opal_convertor_need_buffers(convertor)) { OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "mca_btl_portals4_prepare_src NEED BUFFERS or RESERVE\n")); @@ -670,11 +671,11 @@ mca_btl_portals4_deregister_mem(mca_btl_base_module_t *btl_base, mca_btl_base_registration_handle_t *handle) { int ret; - struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t*) btl_base; OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, - "mca_btl_portals4_deregister_mem NI=%d handle=%p key=%ld me_h=%d\n", - portals4_btl->interface_num, (void *)handle, handle->key, handle->me_h)); + "mca_btl_portals4_deregister_mem NI=%d handle=%p key=%ld me_h=%d\n", + ((struct mca_btl_portals4_module_t *) btl_base)->interface_num, + (void *) handle, handle->key, handle->me_h)); if (!PtlHandleIsEqual(handle->me_h, PTL_INVALID_HANDLE)) { ret = PtlMEUnlink(handle->me_h);