From: Timo Heister <timo.heister@gmail.com>
Date: Mon, 4 Nov 2019 19:23:56 +0000 (-0500)
Subject: protect more MPI communication with a CollectiveMutex
X-Git-Tag: v9.2.0-rc1~910^2~1
X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=34fbddf7264e6d7031912452afa7f0ef2750ce75;p=dealii.git

protect more MPI communication with a CollectiveMutex

- also replace a MPI_Probe with MPI_ANY_TAG
---

diff --git a/include/deal.II/base/mpi.h b/include/deal.II/base/mpi.h
index 318db3730d..3eddaf43ea 100644
--- a/include/deal.II/base/mpi.h
+++ b/include/deal.II/base/mpi.h
@@ -1504,6 +1504,10 @@ namespace Utilities
         Utilities::MPI::compute_point_to_point_communication_pattern(comm,
                                                                      send_to);
 
+      // Protect the following communication:
+      static CollectiveMutex      mutex;
+      CollectiveMutex::ScopedLock lock(mutex, comm);
+
       // Sending buffers
       std::vector<std::vector<char>> buffers_to_send(send_to.size());
       std::vector<MPI_Request>       buffer_send_requests(send_to.size());
diff --git a/include/deal.II/fe/fe_tools_extrapolate.templates.h b/include/deal.II/fe/fe_tools_extrapolate.templates.h
index 82321e6492..2d69b02f43 100644
--- a/include/deal.II/fe/fe_tools_extrapolate.templates.h
+++ b/include/deal.II/fe/fe_tools_extrapolate.templates.h
@@ -1148,6 +1148,10 @@ namespace FETools
       std::vector<MPI_Request>                 requests(cells_to_send.size());
       std::vector<unsigned int>                destinations;
 
+      // Protect the communcation below:
+      static Utilities::MPI::CollectiveMutex      mutex;
+      Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex, communicator);
+
       // send data
       unsigned int idx = 0;
       for (typename std::vector<CellData>::const_iterator it =
diff --git a/include/deal.II/grid/grid_tools.h b/include/deal.II/grid/grid_tools.h
index bbb4808f52..9726c0b736 100644
--- a/include/deal.II/grid/grid_tools.h
+++ b/include/deal.II/grid/grid_tools.h
@@ -3957,6 +3957,10 @@ namespace GridTools
             }
         }
 
+    // Protect the following communcation:
+    static Utilities::MPI::CollectiveMutex      mutex;
+    Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex,
+                                                     tria->get_communicator());
 
     // 2. send our messages
     std::set<dealii::types::subdomain_id> ghost_owners   = tria->ghost_owners();
diff --git a/source/distributed/tria.cc b/source/distributed/tria.cc
index 528e8234df..4e0d27039a 100644
--- a/source/distributed/tria.cc
+++ b/source/distributed/tria.cc
@@ -4224,7 +4224,13 @@ namespace parallel
             needs_to_get_cells);
         }
 
-      // sending
+      // Send information
+
+      // We need to protect this communication below using a mutex:
+      static Utilities::MPI::CollectiveMutex      mutex;
+      Utilities::MPI::CollectiveMutex::ScopedLock lock(
+        mutex, this->get_communicator());
+
       std::vector<std::vector<char>> sendbuffers(needs_to_get_cells.size());
       std::vector<std::vector<char>>::iterator buffer = sendbuffers.begin();
       std::vector<MPI_Request>  requests(needs_to_get_cells.size());
diff --git a/source/dofs/dof_handler_policy.cc b/source/dofs/dof_handler_policy.cc
index d53a018860..20b20a36eb 100644
--- a/source/dofs/dof_handler_policy.cc
+++ b/source/dofs/dof_handler_policy.cc
@@ -4428,6 +4428,13 @@ namespace internal
           Assert(level_ghost_owners.size() == neighbor_cell_list.size(),
                  ExcInternalError());
 
+
+          // Before sending & receiving, make sure we protect this section with
+          // a mutex:
+          static Utilities::MPI::CollectiveMutex      mutex;
+          Utilities::MPI::CollectiveMutex::ScopedLock lock(
+            mutex, tria.get_communicator());
+
           //* send our requests:
           std::vector<MPI_Request> requests(level_ghost_owners.size());
           {
diff --git a/source/lac/sparsity_tools.cc b/source/lac/sparsity_tools.cc
index 5f2865eac5..a3a7584e4e 100644
--- a/source/lac/sparsity_tools.cc
+++ b/source/lac/sparsity_tools.cc
@@ -1176,6 +1176,10 @@ namespace SparsityTools
 
 
     // send data
+
+    static Utilities::MPI::CollectiveMutex      mutex;
+    Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex, mpi_comm);
+
     {
       unsigned int idx = 0;
       for (const auto &sparsity_line : send_data)
@@ -1199,9 +1203,8 @@ namespace SparsityTools
         {
           MPI_Status status;
           int        len;
-          int ierr = MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &status);
+          int        ierr = MPI_Probe(MPI_ANY_SOURCE, 124, mpi_comm, &status);
           AssertThrowMPI(ierr);
-          Assert(status.MPI_TAG == 124, ExcInternalError());
 
           ierr = MPI_Get_count(&status, DEAL_II_DOF_INDEX_MPI_TYPE, &len);
           AssertThrowMPI(ierr);
diff --git a/source/multigrid/mg_transfer_internal.cc b/source/multigrid/mg_transfer_internal.cc
index c62814f6f5..a11aa5b66c 100644
--- a/source/multigrid/mg_transfer_internal.cc
+++ b/source/multigrid/mg_transfer_internal.cc
@@ -290,6 +290,12 @@ namespace internal
                                                        level_dof_indices[i]);
             }
 
+
+          // Protect the send/recv logic with a mutex:
+          static Utilities::MPI::CollectiveMutex      mutex;
+          Utilities::MPI::CollectiveMutex::ScopedLock lock(
+            mutex, tria->get_communicator());
+
           // * send
           std::vector<MPI_Request> requests;
           {