coll/libnbc: fix race condition with multi threaded apps

protect the mca_coll_libnbc_component.active_requests list with the new mca_coll_libnbc_component.lock mutex. Thanks Jie Hu for the report Signed-off-by: Gilles Gouaillardet <gilles@rist.or.jp> (back-ported from commit 2c94a3a)
open-mpi · Nov 21, 2016 · 53ce62f · 53ce62f
1 parent 800063e
commit 53ce62f
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 0 deletions.
diff --git a/ompi/mca/coll/libnbc/coll_libnbc.h b/ompi/mca/coll/libnbc/coll_libnbc.h
@@ -13,6 +13,8 @@
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -71,6 +73,7 @@ struct ompi_coll_libnbc_component_t {
     opal_list_t active_requests;
     int32_t active_comms;
     opal_atomic_lock_t progress_lock;
+    opal_mutex_t lock;
 };
 typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t;
 

diff --git a/ompi/mca/coll/libnbc/coll_libnbc_component.c b/ompi/mca/coll/libnbc/coll_libnbc_component.c
@@ -89,6 +89,7 @@ libnbc_open(void)
 
     OBJ_CONSTRUCT(&mca_coll_libnbc_component.requests, ompi_free_list_t);
     OBJ_CONSTRUCT(&mca_coll_libnbc_component.active_requests, opal_list_t);
+    OBJ_CONSTRUCT(&mca_coll_libnbc_component.lock, opal_mutex_t);
     ret = ompi_free_list_init(&mca_coll_libnbc_component.requests,
                               sizeof(ompi_coll_libnbc_request_t),
                               OBJ_CLASS(ompi_coll_libnbc_request_t),
@@ -116,6 +117,7 @@ libnbc_close(void)
 
     OBJ_DESTRUCT(&mca_coll_libnbc_component.requests);
     OBJ_DESTRUCT(&mca_coll_libnbc_component.active_requests);
+    OBJ_DESTRUCT(&mca_coll_libnbc_component.lock);
 
     return OMPI_SUCCESS;
 }
@@ -242,19 +244,25 @@ ompi_coll_libnbc_progress(void)
 
     if (opal_atomic_trylock(&mca_coll_libnbc_component.progress_lock)) return 0;
 
+    OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock);
     OPAL_LIST_FOREACH_SAFE(request, next, &mca_coll_libnbc_component.active_requests,
                            ompi_coll_libnbc_request_t) {
+        OPAL_THREAD_UNLOCK(&mca_coll_libnbc_component.lock);
         if (NBC_OK == NBC_Progress(request)) {
             /* done, remove and complete */
+            OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock);
             opal_list_remove_item(&mca_coll_libnbc_component.active_requests,
                                   &request->super.super.super);
+            OPAL_THREAD_UNLOCK(&mca_coll_libnbc_component.lock);
 
             request->super.req_status.MPI_ERROR = OMPI_SUCCESS;
             OPAL_THREAD_LOCK(&ompi_request_lock);
             ompi_request_complete(&request->super, true);
             OPAL_THREAD_UNLOCK(&ompi_request_lock);
         }
+        OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock);
     }
+    OPAL_THREAD_UNLOCK(&mca_coll_libnbc_component.lock);
 
     opal_atomic_unlock(&mca_coll_libnbc_component.progress_lock);
 

diff --git a/ompi/mca/coll/libnbc/nbc.c b/ompi/mca/coll/libnbc/nbc.c
@@ -659,7 +659,9 @@ int NBC_Start(NBC_Handle *handle, NBC_Schedule *schedule) {
   res = NBC_Start_round(handle);
   if((NBC_OK != res)) { printf("Error in NBC_Start_round() (%i)\n", res); return res; }
 
+  OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock);
   opal_list_append(&mca_coll_libnbc_component.active_requests, &(handle->super.super.super));
+  OPAL_THREAD_UNLOCK(&mca_coll_libnbc_component.lock);
 
   return NBC_OK;
 }