From 34fbddf7264e6d7031912452afa7f0ef2750ce75 Mon Sep 17 00:00:00 2001 From: Timo Heister Date: Mon, 4 Nov 2019 14:23:56 -0500 Subject: [PATCH] protect more MPI communication with a CollectiveMutex - also replace a MPI_Probe with MPI_ANY_TAG --- include/deal.II/base/mpi.h | 4 ++++ include/deal.II/fe/fe_tools_extrapolate.templates.h | 4 ++++ include/deal.II/grid/grid_tools.h | 4 ++++ source/distributed/tria.cc | 8 +++++++- source/dofs/dof_handler_policy.cc | 7 +++++++ source/lac/sparsity_tools.cc | 7 +++++-- source/multigrid/mg_transfer_internal.cc | 6 ++++++ 7 files changed, 37 insertions(+), 3 deletions(-) diff --git a/include/deal.II/base/mpi.h b/include/deal.II/base/mpi.h index 318db3730d..3eddaf43ea 100644 --- a/include/deal.II/base/mpi.h +++ b/include/deal.II/base/mpi.h @@ -1504,6 +1504,10 @@ namespace Utilities Utilities::MPI::compute_point_to_point_communication_pattern(comm, send_to); + // Protect the following communication: + static CollectiveMutex mutex; + CollectiveMutex::ScopedLock lock(mutex, comm); + // Sending buffers std::vector> buffers_to_send(send_to.size()); std::vector buffer_send_requests(send_to.size()); diff --git a/include/deal.II/fe/fe_tools_extrapolate.templates.h b/include/deal.II/fe/fe_tools_extrapolate.templates.h index 82321e6492..2d69b02f43 100644 --- a/include/deal.II/fe/fe_tools_extrapolate.templates.h +++ b/include/deal.II/fe/fe_tools_extrapolate.templates.h @@ -1148,6 +1148,10 @@ namespace FETools std::vector requests(cells_to_send.size()); std::vector destinations; + // Protect the communcation below: + static Utilities::MPI::CollectiveMutex mutex; + Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex, communicator); + // send data unsigned int idx = 0; for (typename std::vector::const_iterator it = diff --git a/include/deal.II/grid/grid_tools.h b/include/deal.II/grid/grid_tools.h index bbb4808f52..9726c0b736 100644 --- a/include/deal.II/grid/grid_tools.h +++ b/include/deal.II/grid/grid_tools.h @@ -3957,6 +3957,10 @@ namespace GridTools } } + // Protect the following communcation: + static Utilities::MPI::CollectiveMutex mutex; + Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex, + tria->get_communicator()); // 2. send our messages std::set ghost_owners = tria->ghost_owners(); diff --git a/source/distributed/tria.cc b/source/distributed/tria.cc index 528e8234df..4e0d27039a 100644 --- a/source/distributed/tria.cc +++ b/source/distributed/tria.cc @@ -4224,7 +4224,13 @@ namespace parallel needs_to_get_cells); } - // sending + // Send information + + // We need to protect this communication below using a mutex: + static Utilities::MPI::CollectiveMutex mutex; + Utilities::MPI::CollectiveMutex::ScopedLock lock( + mutex, this->get_communicator()); + std::vector> sendbuffers(needs_to_get_cells.size()); std::vector>::iterator buffer = sendbuffers.begin(); std::vector requests(needs_to_get_cells.size()); diff --git a/source/dofs/dof_handler_policy.cc b/source/dofs/dof_handler_policy.cc index d53a018860..20b20a36eb 100644 --- a/source/dofs/dof_handler_policy.cc +++ b/source/dofs/dof_handler_policy.cc @@ -4428,6 +4428,13 @@ namespace internal Assert(level_ghost_owners.size() == neighbor_cell_list.size(), ExcInternalError()); + + // Before sending & receiving, make sure we protect this section with + // a mutex: + static Utilities::MPI::CollectiveMutex mutex; + Utilities::MPI::CollectiveMutex::ScopedLock lock( + mutex, tria.get_communicator()); + //* send our requests: std::vector requests(level_ghost_owners.size()); { diff --git a/source/lac/sparsity_tools.cc b/source/lac/sparsity_tools.cc index 5f2865eac5..a3a7584e4e 100644 --- a/source/lac/sparsity_tools.cc +++ b/source/lac/sparsity_tools.cc @@ -1176,6 +1176,10 @@ namespace SparsityTools // send data + + static Utilities::MPI::CollectiveMutex mutex; + Utilities::MPI::CollectiveMutex::ScopedLock lock(mutex, mpi_comm); + { unsigned int idx = 0; for (const auto &sparsity_line : send_data) @@ -1199,9 +1203,8 @@ namespace SparsityTools { MPI_Status status; int len; - int ierr = MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &status); + int ierr = MPI_Probe(MPI_ANY_SOURCE, 124, mpi_comm, &status); AssertThrowMPI(ierr); - Assert(status.MPI_TAG == 124, ExcInternalError()); ierr = MPI_Get_count(&status, DEAL_II_DOF_INDEX_MPI_TYPE, &len); AssertThrowMPI(ierr); diff --git a/source/multigrid/mg_transfer_internal.cc b/source/multigrid/mg_transfer_internal.cc index c62814f6f5..a11aa5b66c 100644 --- a/source/multigrid/mg_transfer_internal.cc +++ b/source/multigrid/mg_transfer_internal.cc @@ -290,6 +290,12 @@ namespace internal level_dof_indices[i]); } + + // Protect the send/recv logic with a mutex: + static Utilities::MPI::CollectiveMutex mutex; + Utilities::MPI::CollectiveMutex::ScopedLock lock( + mutex, tria->get_communicator()); + // * send std::vector requests; { -- 2.39.5