protect more MPI communication with a CollectiveMutex

- also replace a MPI_Probe with MPI_ANY_TAG

diff --git a/include/deal.II/base/mpi.h b/include/deal.II/base/mpi.h
index 318db3730de887262f5389df79f8a8849031b2f3..3eddaf43ea2f3b92cf8769682fd2fc0ec1134ec5 100644 (file)
--- a/include/deal.II/base/mpi.h
+++ b/include/deal.II/base/mpi.h
@@ -1504,6 +1504,10 @@ namespace Utilities
         Utilities::MPI::compute_point_to_point_communication_pattern(comm,
                                                                      send_to);
 
+      // Protect the following communication:
+      static CollectiveMutex      mutex;
+      CollectiveMutex::ScopedLock lock(mutex, comm);
+
       // Sending buffers
       std::vector<std::vector<char>> buffers_to_send(send_to.size());
       std::vector<MPI_Request>       buffer_send_requests(send_to.size());

diff --git a/include/deal.II/fe/fe_tools_extrapolate.templates.h b/include/deal.II/fe/fe_tools_extrapolate.templates.h
index 82321e6492385f441403029e7b241de036d61e8a..2d69b02f43e633cb228e185fb94be1e6f206a88c 100644 (file)
--- a/include/deal.II/fe/fe_tools_extrapolate.templates.h
+++ b/include/deal.II/fe/fe_tools_extrapolate.templates.h
@@ -1148,6 +1148,10 @@ namespace FETools
       std::vector<MPI_Request>                 requests(cells_to_send.size());
       std::vector<unsigned int>                destinations;
 
+      // Protect the communcation below:
+      static Utilities::MPI::CollectiveMutex      mutex;
+      Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex, communicator);
+
       // send data
       unsigned int idx = 0;
       for (typename std::vector<CellData>::const_iterator it =

diff --git a/include/deal.II/grid/grid_tools.h b/include/deal.II/grid/grid_tools.h
index bbb4808f52e412e6e9e915a4844cabfcb0271b19..9726c0b736f2ecf2d87e1631a35722a18913fa1c 100644 (file)
--- a/include/deal.II/grid/grid_tools.h
+++ b/include/deal.II/grid/grid_tools.h
@@ -3957,6 +3957,10 @@ namespace GridTools
             }
         }
 
+    // Protect the following communcation:
+    static Utilities::MPI::CollectiveMutex      mutex;
+    Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex,
+                                                     tria->get_communicator());
 
     // 2. send our messages
     std::set<dealii::types::subdomain_id> ghost_owners   = tria->ghost_owners();

diff --git a/source/distributed/tria.cc b/source/distributed/tria.cc
index 528e8234df1d333a9f5ed3e687355d7ea0016be6..4e0d27039aa210ffca38428e06d0aa06288f9609 100644 (file)
--- a/source/distributed/tria.cc
+++ b/source/distributed/tria.cc
@@ -4224,7 +4224,13 @@ namespace parallel
             needs_to_get_cells);
         }
 
-      // sending
+      // Send information
+
+      // We need to protect this communication below using a mutex:
+      static Utilities::MPI::CollectiveMutex      mutex;
+      Utilities::MPI::CollectiveMutex::ScopedLock lock(
+        mutex, this->get_communicator());
+
       std::vector<std::vector<char>> sendbuffers(needs_to_get_cells.size());
       std::vector<std::vector<char>>::iterator buffer = sendbuffers.begin();
       std::vector<MPI_Request>  requests(needs_to_get_cells.size());

diff --git a/source/dofs/dof_handler_policy.cc b/source/dofs/dof_handler_policy.cc
index d53a01886035dd8e55313b6420841c39053dbe56..20b20a36eb95f7d56b460996b8126c796018a654 100644 (file)
--- a/source/dofs/dof_handler_policy.cc
+++ b/source/dofs/dof_handler_policy.cc
@@ -4428,6 +4428,13 @@ namespace internal
           Assert(level_ghost_owners.size() == neighbor_cell_list.size(),
                  ExcInternalError());
 
+
+          // Before sending & receiving, make sure we protect this section with
+          // a mutex:
+          static Utilities::MPI::CollectiveMutex      mutex;
+          Utilities::MPI::CollectiveMutex::ScopedLock lock(
+            mutex, tria.get_communicator());
+
           //* send our requests:
           std::vector<MPI_Request> requests(level_ghost_owners.size());
           {

diff --git a/source/lac/sparsity_tools.cc b/source/lac/sparsity_tools.cc
index 5f2865eac5e8994df246396064b734952950248f..a3a7584e4e20cc68056c597b7e9af32411b369c9 100644 (file)
--- a/source/lac/sparsity_tools.cc
+++ b/source/lac/sparsity_tools.cc
@@ -1176,6 +1176,10 @@ namespace SparsityTools
 
 
     // send data
+
+    static Utilities::MPI::CollectiveMutex      mutex;
+    Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex, mpi_comm);
+
     {
       unsigned int idx = 0;
       for (const auto &sparsity_line : send_data)
@@ -1199,9 +1203,8 @@ namespace SparsityTools
         {
           MPI_Status status;
           int        len;
-          int ierr = MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &status);
+          int        ierr = MPI_Probe(MPI_ANY_SOURCE, 124, mpi_comm, &status);
           AssertThrowMPI(ierr);
-          Assert(status.MPI_TAG == 124, ExcInternalError());
 
           ierr = MPI_Get_count(&status, DEAL_II_DOF_INDEX_MPI_TYPE, &len);
           AssertThrowMPI(ierr);

diff --git a/source/multigrid/mg_transfer_internal.cc b/source/multigrid/mg_transfer_internal.cc
index c62814f6f511a1ee5de824080e6cb2dafc22be3a..a11aa5b66cbec27f040384084b25a5925b6970a8 100644 (file)
--- a/source/multigrid/mg_transfer_internal.cc
+++ b/source/multigrid/mg_transfer_internal.cc
@@ -290,6 +290,12 @@ namespace internal
                                                        level_dof_indices[i]);
             }
 
+
+          // Protect the send/recv logic with a mutex:
+          static Utilities::MPI::CollectiveMutex      mutex;
+          Utilities::MPI::CollectiveMutex::ScopedLock lock(
+            mutex, tria->get_communicator());
+
           // * send
           std::vector<MPI_Request> requests;
           {