From 32c87607c635913a719b640cf5ad81eb070c9d51 Mon Sep 17 00:00:00 2001
From: David Wells <wellsd2@rpi.edu>
Date: Tue, 24 May 2016 21:07:56 -0400
Subject: [PATCH] Reduce code duplication in all_reduce.

There were previously four versions of this function that all did
approximately the same thing.

In addition, fix a test that failed as a result of this change.
---
 include/deal.II/base/mpi.h               | 109 ++++++++---------------
 tests/mpi/collective_02_dealii_vector.cc |   2 +-
 2 files changed, 40 insertions(+), 71 deletions(-)

diff --git a/include/deal.II/base/mpi.h b/include/deal.II/base/mpi.h
index 183d141db2..95d4711398 100644
--- a/include/deal.II/base/mpi.h
+++ b/include/deal.II/base/mpi.h
@@ -489,28 +489,52 @@ namespace Utilities
 
       template <typename T>
       inline
-      T all_reduce (const MPI_Op &mpi_op,
-                    const T &t,
-                    const MPI_Comm &mpi_communicator)
+      void all_reduce (const MPI_Op      &mpi_op,
+                       const T *const    values,
+                       const MPI_Comm    &mpi_communicator,
+                       T                 *output,
+                       const std::size_t  size)
       {
 #ifdef DEAL_II_WITH_MPI
         if (job_supports_mpi())
           {
-            T output;
-            MPI_Allreduce (const_cast<void *>(static_cast<const void *>(&t)),
-                           &output, 1, internal::mpi_type_id(&t), mpi_op,
+            MPI_Allreduce (values != output
+                           ?
+                           // TODO This const_cast is only needed for older
+                           // (e.g., openMPI 1.6, released in 2012)
+                           // implementations of MPI-2. It is not needed as of
+                           // MPI-3 and we should remove it at some point in
+                           // the future.
+                           const_cast<void *>(static_cast<const void *> (values))
+                           :
+                           MPI_IN_PLACE,
+                           static_cast<void *>(output),
+                           static_cast<int>(size),
+                           internal::mpi_type_id(values),
+                           mpi_op,
                            mpi_communicator);
-            return output;
           }
         else
 #endif
           {
             (void)mpi_op;
             (void)mpi_communicator;
-            return t;
+            for (std::size_t i=0; i<size; ++i)
+              output[i] = values[i];
           }
       }
 
+      template <typename T>
+      inline
+      T all_reduce (const MPI_Op &mpi_op,
+                    const T &t,
+                    const MPI_Comm &mpi_communicator)
+      {
+        T output;
+        all_reduce(mpi_op, &t, mpi_communicator, &output, 1);
+        return output;
+      }
+
       template <typename T, unsigned int N>
       inline
       void all_reduce (const MPI_Op &mpi_op,
@@ -518,25 +542,7 @@ namespace Utilities
                        const MPI_Comm &mpi_communicator,
                        T (&output)[N])
       {
-#ifdef DEAL_II_WITH_MPI
-        if (job_supports_mpi())
-          {
-            MPI_Allreduce ((&values[0] != &output[0]
-                            ?
-                            const_cast<void *>(static_cast<const void *>(&values[0]))
-                            :
-                            MPI_IN_PLACE),
-                           &output[0], N, internal::mpi_type_id(values), mpi_op,
-                           mpi_communicator);
-          }
-        else
-#endif
-          {
-            (void)mpi_op;
-            (void)mpi_communicator;
-            for (unsigned int i=0; i<N; ++i)
-              output[i] = values[i];
-          }
+        all_reduce(mpi_op, values, mpi_communicator, output, N);
       }
 
       template <typename T>
@@ -546,25 +552,9 @@ namespace Utilities
                        const MPI_Comm       &mpi_communicator,
                        std::vector<T>       &output)
       {
-#ifdef DEAL_II_WITH_MPI
-        if (job_supports_mpi())
-          {
-            output.resize (values.size());
-            MPI_Allreduce ((&values[0] != &output[0]
-                            ?
-                            const_cast<void *>(static_cast<const void *>(&values[0]))
-                            :
-                            MPI_IN_PLACE),
-                           &output[0], values.size(), internal::mpi_type_id((T *)0), mpi_op,
-                           mpi_communicator);
-          }
-        else
-#endif
-          {
-            (void)mpi_op;
-            (void)mpi_communicator;
-            output = values;
-          }
+        Assert(values.size() == output.size(),
+               ExcDimensionMismatch(values.size(), output.size()));
+        all_reduce(mpi_op, &values[0], mpi_communicator, &output[0], values.size());
       }
 
       template <typename T>
@@ -574,31 +564,10 @@ namespace Utilities
                        const MPI_Comm  &mpi_communicator,
                        Vector<T>  &output)
       {
-#ifdef DEAL_II_WITH_MPI
-        if (job_supports_mpi())
-          {
-            if (values.begin() != output.begin())
-              output.reinit (values.size());
-
-            MPI_Allreduce ((values.begin() != output.begin()
-                            ?
-                            const_cast<void *>(static_cast<const void *>(values.begin()))
-                            :
-                            MPI_IN_PLACE),
-                           output.begin(), values.size(), internal::mpi_type_id((T *)0), mpi_op,
-                           mpi_communicator);
-          }
-        else
-#endif
-          {
-            (void)mpi_op;
-            (void)mpi_communicator;
-            output = values;
-          }
+        Assert(values.size() == output.size(),
+               ExcDimensionMismatch(values.size(), output.size()));
+        all_reduce(mpi_op, values.begin(), mpi_communicator, output.begin(), values.size());
       }
-
-
-
     }
 
 
diff --git a/tests/mpi/collective_02_dealii_vector.cc b/tests/mpi/collective_02_dealii_vector.cc
index 8ee9c5a463..1c88c6995f 100644
--- a/tests/mpi/collective_02_dealii_vector.cc
+++ b/tests/mpi/collective_02_dealii_vector.cc
@@ -47,7 +47,7 @@ void test()
     Vector<double> values(2);
     values[0] = 1.5;
     values[1] = 2.5;
-    Vector<double> sums;
+    Vector<double> sums(2);
     Utilities::MPI::sum (values,
                          MPI_COMM_WORLD,
                          sums);
-- 
2.39.5