From e5edf08459cbd9ea24310dd8a5b7f4e71ebdc729 Mon Sep 17 00:00:00 2001
From: benjamin <benjamin.brands@fau.de>
Date: Sun, 28 Jan 2018 11:38:09 +0100
Subject: [PATCH] move implemetation of MPI::some_to_some

---
 include/deal.II/base/mpi.h           | 79 +++++++++++++++++++++++++++
 include/deal.II/base/mpi.templates.h | 81 ----------------------------
 tests/base/mpi_some_to_some_01.cc    |  2 +-
 tests/base/mpi_some_to_some_02.cc    |  2 +-
 4 files changed, 81 insertions(+), 83 deletions(-)

diff --git a/include/deal.II/base/mpi.h b/include/deal.II/base/mpi.h
index 50a5842d27..8abb223e19 100644
--- a/include/deal.II/base/mpi.h
+++ b/include/deal.II/base/mpi.h
@@ -551,6 +551,85 @@ namespace Utilities
                            mpi_communicator, ArrayView<T>(minima, N));
     }
 
+    template<typename T>
+    std::map<unsigned int, T>
+    some_to_some(const MPI_Comm &comm,
+                 const std::map<unsigned int, T> &objects_to_send)
+    {
+#ifndef DEAL_II_WITH_MPI
+      (void)comm;
+      Assert(objects_to_send.size() == 0, ExcMessage("Cannot send to more than one processor."));
+      Assert(objects_to_send.find(0) != objects_to_send.end() || objects_to_send.size() == 0,
+             ExcMessage("Can only send to myself or to nobody."));
+      return objects_to_send;
+#else
+
+      std::vector<unsigned int> send_to(objects_to_send.size());
+      {
+        unsigned int i=0;
+        for (const auto &m: objects_to_send)
+          send_to[i++] = m.first;
+      }
+      AssertDimension(send_to.size(), objects_to_send.size());
+
+      const auto receive_from =
+        Utilities::MPI::compute_point_to_point_communication_pattern(comm, send_to);
+
+      // Sending buffers
+      std::vector<std::vector<char> > buffers_to_send(send_to.size());
+      std::vector<MPI_Request> buffer_send_requests(send_to.size());
+      {
+        unsigned int i = 0;
+        for (const auto &rank_obj : objects_to_send)
+          {
+            const auto &rank = rank_obj.first;
+            buffers_to_send[i] = Utilities::pack(rank_obj.second);
+            const int ierr = MPI_Isend(buffers_to_send[i].data(),
+                                       buffers_to_send[i].size(), MPI_CHAR,
+                                       rank, 21, comm, &buffer_send_requests[i]);
+            AssertThrowMPI(ierr);
+            ++i;
+          }
+      }
+
+      // Receiving buffers
+      std::map<unsigned int, T> received_objects;
+      {
+        std::vector<char> buffer;
+        // We do this on a first come/first served basis
+        for (unsigned int i = 0; i<receive_from.size(); ++i)
+          {
+            // Probe what's going on. Take data from the first available sender
+            MPI_Status status;
+            int ierr = MPI_Probe(MPI_ANY_SOURCE, 21, comm, &status);
+            AssertThrowMPI(ierr);
+
+            // Length of the message
+            int len;
+            ierr = MPI_Get_count(&status, MPI_CHAR, &len);
+            AssertThrowMPI(ierr);
+            buffer.resize(len);
+
+            // Source rank
+            const unsigned int rank = status.MPI_SOURCE;
+
+            // Actually receive the message
+            ierr = MPI_Recv(buffer.data(), len, MPI_CHAR,
+                            rank, 21, comm, MPI_STATUS_IGNORE);
+            AssertThrowMPI(ierr);
+            Assert(received_objects.find(rank) == received_objects.end(),
+                   ExcInternalError("I should not receive again from this rank"));
+            received_objects[rank] = Utilities::unpack<T>(buffer);
+          }
+      }
+
+      // Wait to have sent all objects.
+      MPI_Waitall(send_to.size(), buffer_send_requests.data(),MPI_STATUSES_IGNORE);
+
+      return received_objects;
+#endif // deal.II with MPI
+    }
+
     template<typename T>
     std::vector<T> all_gather(const MPI_Comm       &comm,
                               const T &object)
diff --git a/include/deal.II/base/mpi.templates.h b/include/deal.II/base/mpi.templates.h
index f0cf9c208a..5c8cc75a96 100644
--- a/include/deal.II/base/mpi.templates.h
+++ b/include/deal.II/base/mpi.templates.h
@@ -190,87 +190,6 @@ namespace Utilities
 
 
 
-    template<typename T>
-    std::map<unsigned int, T>
-    some_to_some(const MPI_Comm &comm,
-                 const std::map<unsigned int, T> &objects_to_send)
-    {
-#ifndef DEAL_II_WITH_MPI
-      (void)comm;
-      Assert(objects_to_send.size() == 0, ExcMessage("Cannot send to more than one processor."));
-      Assert(objects_to_send.find(0) != objects_to_send.end() || objects_to_send.size() == 0,
-             ExcMessage("Can only send to myself or to nobody."));
-      return objects_to_send;
-#else
-
-      std::vector<unsigned int> send_to(objects_to_send.size());
-      {
-        unsigned int i=0;
-        for (const auto &m: objects_to_send)
-          send_to[i++] = m.first;
-      }
-      AssertDimension(send_to.size(), objects_to_send.size());
-
-      const auto receive_from =
-        Utilities::MPI::compute_point_to_point_communication_pattern(comm, send_to);
-
-      // Sending buffers
-      std::vector<std::vector<char> > buffers_to_send(send_to.size());
-      std::vector<MPI_Request> buffer_send_requests(send_to.size());
-      {
-        unsigned int i = 0;
-        for (const auto &rank_obj : objects_to_send)
-          {
-            const auto &rank = rank_obj.first;
-            buffers_to_send[i] = Utilities::pack(rank_obj.second);
-            const int ierr = MPI_Isend(buffers_to_send[i].data(),
-                                       buffers_to_send[i].size(), MPI_CHAR,
-                                       rank, 21, comm, &buffer_send_requests[i]);
-            AssertThrowMPI(ierr);
-            ++i;
-          }
-      }
-
-      // Receiving buffers
-      std::map<unsigned int, T> received_objects;
-      {
-        std::vector<char> buffer;
-        // We do this on a first come/first served basis
-        for (unsigned int i = 0; i<receive_from.size(); ++i)
-          {
-            // Probe what's going on. Take data from the first available sender
-            MPI_Status status;
-            int ierr = MPI_Probe(MPI_ANY_SOURCE, 21, comm, &status);
-            AssertThrowMPI(ierr);
-
-            // Length of the message
-            int len;
-            ierr = MPI_Get_count(&status, MPI_CHAR, &len);
-            AssertThrowMPI(ierr);
-            buffer.resize(len);
-
-            // Source rank
-            const unsigned int rank = status.MPI_SOURCE;
-
-            // Actually receive the message
-            ierr = MPI_Recv(buffer.data(), len, MPI_CHAR,
-                            rank, 21, comm, MPI_STATUS_IGNORE);
-            AssertThrowMPI(ierr);
-            Assert(received_objects.find(rank) == received_objects.end(),
-                   ExcInternalError("I should not receive again from this rank"));
-            received_objects[rank] = Utilities::unpack<T>(buffer);
-          }
-      }
-
-      // Wait to have sent all objects.
-      MPI_Waitall(send_to.size(), buffer_send_requests.data(),MPI_STATUSES_IGNORE);
-
-      return received_objects;
-#endif // deal.II with MPI
-    }
-
-
-
     template <typename T>
     T sum (const T &t,
            const MPI_Comm &mpi_communicator)
diff --git a/tests/base/mpi_some_to_some_01.cc b/tests/base/mpi_some_to_some_01.cc
index f9319414ae..e5bca2c7b6 100644
--- a/tests/base/mpi_some_to_some_01.cc
+++ b/tests/base/mpi_some_to_some_01.cc
@@ -19,7 +19,7 @@
 #include "../tests.h"
 
 #include <deal.II/base/point.h>
-#include <deal.II/base/mpi.templates.h>
+#include <deal.II/base/mpi.h>
 
 #include <vector>
 
diff --git a/tests/base/mpi_some_to_some_02.cc b/tests/base/mpi_some_to_some_02.cc
index 8a442b9400..383fb15f30 100644
--- a/tests/base/mpi_some_to_some_02.cc
+++ b/tests/base/mpi_some_to_some_02.cc
@@ -19,8 +19,8 @@
 #include "../tests.h"
 
 #include <deal.II/base/point.h>
-#include <deal.II/base/mpi.templates.h>
 #include <deal.II/base/mpi.h>
+
 #include <vector>
 #include <tuple>
 
-- 
2.39.5