Merge pull request #789 from sourceryinstitute/vehre/issue-781-clear-…

…communicator-4-mpich

Fix test failures with mpich 4+.

CMakeLists.txt

@@ -644,6 +644,11 @@ define_property(TARGET
   FULL_DOCS "Property to mark executable targets run as tests that they require 2^n images."
   )
 
+#-------------------------------------------------------------------------------------
+# Add global openmpi property, because using a variable an setting in parent scope did
+# not work as expected, i.e., not at all, on Linux Fedora 39.
+#-------------------------------------------------------------------------------------
+define_property(GLOBAL PROPERTY openmpi BRIEF_DOCS "True when mpi is openMPI.")
 
 #-------------------------------
 # Recurse into the src directory
@@ -715,6 +720,7 @@ function(add_caf_test name num_caf_img test_target)
     endif()
   endif()
   # Add a host file for OMPI
+  get_property(openmpi GLOBAL PROPERTY openmpi)
   if ( openmpi )
     set(test_parameters --hostfile ${CMAKE_BINARY_DIR}/hostfile)
   endif()

src/runtime-libraries/mpi/CMakeLists.txt

@@ -147,13 +147,17 @@ execute_process(COMMAND ${MPIEXEC_EXECUTABLE} --version
   OUTPUT_VARIABLE mpi_version_out)
 if (mpi_version_out MATCHES "[Oo]pen[ -][Mm][Pp][Ii]")
   message( STATUS "OpenMPI detected")
-  set ( openmpi true PARENT_SCOPE)
+  set_property(GLOBAL PROPERTY openmpi true)
   # Write out a host file because OMPI's mpiexec is dumb
   file(WRITE ${CMAKE_BINARY_DIR}/hostfile "${HOST_NAME} slots=${N_CPU}\n")
   message( STATUS "hostfile written to: ${CMAKE_BINARY_DIR}/hostfile")
   if(NOT DEFINED ENV{TRAVIS})
     message( STATUS "Open-MPI back end detected, passing --allow-run-as-root to allow tests to pass when run with sudo or as root." )
   endif()
+elseif (mpi_version_out MATCHES "HYDRA")
+  message(STATUS "MPICH detected")
+  target_compile_definitions(caf_mpi PRIVATE MPI_CLEAR_COMM_BEFORE_FREE)
+  target_compile_definitions(caf_mpi_static PRIVATE MPI_CLEAR_COMM_BEFORE_FREE)
 endif ()
 
 if("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")

src/runtime-libraries/mpi/mpi_caf.c

@@ -1104,6 +1104,24 @@ finalize_internal(int status_code)
     ierr = MPI_Finalize(); chk_err(ierr);
   }
 #else
+#ifdef MPI_CLEAR_COMM_BEFORE_FREE
+  {
+    int probe_flag;
+    MPI_Status status;
+    do {
+      ierr = MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, CAF_COMM_WORLD, &probe_flag,
+                        &status); /* error is not of interest. */
+      if (probe_flag) {
+        int cnt;
+        MPI_Get_count(&status, MPI_BYTE, &cnt);
+        void * buf = alloca(cnt);
+        ierr = MPI_Recv(buf, cnt, MPI_BYTE, status.MPI_SOURCE, status.MPI_TAG,
+                        CAF_COMM_WORLD, &status); chk_err(ierr);
+      }
+    } while (probe_flag);
+  }
+#endif
+  dprint("freeing caf's communicator.\n");
   ierr = MPI_Comm_free(&CAF_COMM_WORLD); chk_err(ierr);
 
   CAF_Win_unlock_all(*stat_tok);
@@ -1112,6 +1130,7 @@ finalize_internal(int status_code)
   /* Only call Finalize if CAF runtime Initialized MPI. */
   if (caf_owns_mpi)
   {
+    dprint("Finalizing MPI.\n");
     ierr = MPI_Finalize(); chk_err(ierr);
   }
 #endif