From: Wolfgang Bangerth <bangerth@colostate.edu>
Date: Wed, 12 Jan 2022 14:35:51 +0000 (-0700)
Subject: Keep track of which answers we have already received.
X-Git-Tag: v9.4.0-rc1~593^2~4
X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0a366661e33aebd139d0a391aa3ab3bfcb3e1134;p=dealii.git

Keep track of which answers we have already received.
---

diff --git a/include/deal.II/base/mpi_consensus_algorithms.h b/include/deal.II/base/mpi_consensus_algorithms.h
index b70d43abc7..dcb0a46ff2 100644
--- a/include/deal.II/base/mpi_consensus_algorithms.h
+++ b/include/deal.II/base/mpi_consensus_algorithms.h
@@ -330,6 +330,11 @@ namespace Utilities
          */
         std::vector<std::unique_ptr<MPI_Request>> request_requests;
 
+        /**
+         * The ranks of processes from which we are still expecting answers.
+         */
+        std::set<unsigned int> outstanding_answers;
+
         // request for barrier
         MPI_Request barrier_request;
 #endif
diff --git a/include/deal.II/base/mpi_consensus_algorithms.templates.h b/include/deal.II/base/mpi_consensus_algorithms.templates.h
index c6d21aa189..6385752d2a 100644
--- a/include/deal.II/base/mpi_consensus_algorithms.templates.h
+++ b/include/deal.II/base/mpi_consensus_algorithms.templates.h
@@ -197,6 +197,7 @@ namespace Utilities
         // 2) allocate memory
         send_requests.resize(n_targets);
         send_buffers.resize(n_targets);
+        outstanding_answers.clear();
 
         {
           // 4) send and receive
@@ -218,6 +219,8 @@ namespace Utilities
                                     this->comm,
                                     &send_requests[index]);
               AssertThrowMPI(ierr);
+
+              outstanding_answers.insert(rank);
             }
         }
 #endif
@@ -236,7 +239,8 @@ namespace Utilities
         // immediately with a return code that indicates whether
         // it has found a message from a given process with a given
         // tag
-        for (const unsigned int target : targets)
+        std::set<unsigned int> received_answers;
+        for (const unsigned int target : outstanding_answers)
           {
             const int tag_deliver = Utilities::MPI::internal::Tags::
               consensus_algorithm_nbx_process_deliver;
@@ -251,13 +255,43 @@ namespace Utilities
 
             // If there is no pending message from that process,
             // then we are clearly not done receiving everything
-            // yet -- so return false:
+            // yet -- so return false.
+            //
+            // Before doing so, remove the ranks for which we have
+            // (in previous loop iterations) found answers from the
+            // list of outstanding answers.
+            //
+            // Note that it is possible that we have indeed received
+            // more messages, just from ranks that we would consider
+            // *after* the current one. We could keep checking for
+            // those as well, but we don't need to: We will eventually
+            // get to those once the message from the rank we are
+            // currently processing has arrived, at which point we
+            // will run the current loop beyong the current 'target'
+            // rank.
             if (request_is_pending == 0)
-              return false;
+              {
+                for (const unsigned int r : received_answers)
+                  outstanding_answers.erase(r);
+                return false;
+              }
+            else
+              {
+                // We can record that we have received an answer
+                // from this process. We would like to just remove this
+                // rank from 'outstanding_answers' then, but that would
+                // break the loop we are currently in, so just keep
+                // track of that and remove these ranks after we are
+                // done with the loop (either by completing the loop
+                // or via the 'if' above).
+                received_answers.insert(target);
+              }
           }
 
         // If we have made it here, then we have received an answer
         // from everyone and can return true:
+        Assert(received_answers == outstanding_answers, ExcInternalError());
+        outstanding_answers.clear();
         return true;
 
 #else