From e310308e12cfbcb43a5b561956340f129f72018a Mon Sep 17 00:00:00 2001 From: David Wells Date: Mon, 30 May 2016 16:11:05 -0400 Subject: [PATCH] Move templates in mpi.h to mpi.templates.h. --- cmake/config/template-arguments.in | 8 + include/deal.II/base/mpi.h | 255 +----------------------- include/deal.II/base/mpi.templates.h | 263 +++++++++++++++++++++++++ source/base/CMakeLists.txt | 1 + source/base/mpi.cc | 6 +- source/base/mpi.inst.in | 60 ++++++ tests/mpi/renumber_cuthill_mckee.cc | 1 + tests/mpi/renumber_cuthill_mckee_02.cc | 1 + tests/tests.h | 1 + 9 files changed, 346 insertions(+), 250 deletions(-) create mode 100644 include/deal.II/base/mpi.templates.h create mode 100644 source/base/mpi.inst.in diff --git a/cmake/config/template-arguments.in b/cmake/config/template-arguments.in index b3e4595a62..adf41b2ab8 100644 --- a/cmake/config/template-arguments.in +++ b/cmake/config/template-arguments.in @@ -2,6 +2,14 @@ BOOL := { true; false } REAL_SCALARS := { double; float } COMPLEX_SCALARS := { std::complex; std::complex } +MPI_SCALARS := { int; + long int; + unsigned int; + unsigned long int; + unsigned long long int; + float; + double; + long double } DERIVATIVE_TENSORS := { double; Tensor<1,deal_II_dimension>; diff --git a/include/deal.II/base/mpi.h b/include/deal.II/base/mpi.h index 943cb75f1f..735c4b0731 100644 --- a/include/deal.II/base/mpi.h +++ b/include/deal.II/base/mpi.h @@ -434,282 +434,43 @@ namespace Utilities */ bool job_supports_mpi (); +#ifndef DOXYGEN + // declaration for an internal function that lives in mpi.templates.h namespace internal { -#ifdef DEAL_II_WITH_MPI - /** - * Return the corresponding MPI data type id for the argument given. - */ - inline MPI_Datatype mpi_type_id (const int *) - { - return MPI_INT; - } - - - inline MPI_Datatype mpi_type_id (const long int *) - { - return MPI_LONG; - } - - - inline MPI_Datatype mpi_type_id (const unsigned int *) - { - return MPI_UNSIGNED; - } - - - inline MPI_Datatype mpi_type_id (const unsigned long int *) - { - return MPI_UNSIGNED_LONG; - } - - - inline MPI_Datatype mpi_type_id (const unsigned long long int *) - { - return MPI_UNSIGNED_LONG_LONG; - } - - - inline MPI_Datatype mpi_type_id (const float *) - { - return MPI_FLOAT; - } - - - inline MPI_Datatype mpi_type_id (const double *) - { - return MPI_DOUBLE; - } - - - inline MPI_Datatype mpi_type_id (const long double *) - { - return MPI_LONG_DOUBLE; - } -#endif - template - inline void all_reduce (const MPI_Op &mpi_op, const T *const values, const MPI_Comm &mpi_communicator, T *output, - const std::size_t size) - { -#ifdef DEAL_II_WITH_MPI - if (job_supports_mpi()) - { - MPI_Allreduce (values != output - ? - // TODO This const_cast is only needed for older - // (e.g., openMPI 1.6, released in 2012) - // implementations of MPI-2. It is not needed as of - // MPI-3 and we should remove it at some point in - // the future. - const_cast(static_cast (values)) - : - MPI_IN_PLACE, - static_cast(output), - static_cast(size), - internal::mpi_type_id(values), - mpi_op, - mpi_communicator); - } - else -#endif - { - (void)mpi_op; - (void)mpi_communicator; - for (std::size_t i=0; i - inline - T all_reduce (const MPI_Op &mpi_op, - const T &t, - const MPI_Comm &mpi_communicator) - { - T output; - all_reduce(mpi_op, &t, mpi_communicator, &output, 1); - return output; - } - - template - inline - void all_reduce (const MPI_Op &mpi_op, - const T (&values)[N], - const MPI_Comm &mpi_communicator, - T (&output)[N]) - { - all_reduce(mpi_op, values, mpi_communicator, output, N); - } - - template - inline - void all_reduce (const MPI_Op &mpi_op, - const std::vector &values, - const MPI_Comm &mpi_communicator, - std::vector &output) - { - Assert(values.size() == output.size(), - ExcDimensionMismatch(values.size(), output.size())); - all_reduce(mpi_op, &values[0], mpi_communicator, &output[0], values.size()); - } - - template - inline - void all_reduce (const MPI_Op &mpi_op, - const Vector &values, - const MPI_Comm &mpi_communicator, - Vector &output) - { - Assert(values.size() == output.size(), - ExcDimensionMismatch(values.size(), output.size())); - all_reduce(mpi_op, values.begin(), mpi_communicator, output.begin(), values.size()); - } + const std::size_t size); } - - template - inline - T sum (const T &t, - const MPI_Comm &mpi_communicator) - { - return internal::all_reduce(MPI_SUM, t, mpi_communicator); - } - - + // Since these depend on N they must live in the header file template - inline void sum (const T (&values)[N], const MPI_Comm &mpi_communicator, T (&sums)[N]) { - internal::all_reduce(MPI_SUM, values, mpi_communicator, sums); - } - - - template - inline - void sum (const std::vector &values, - const MPI_Comm &mpi_communicator, - std::vector &sums) - { - internal::all_reduce(MPI_SUM, values, mpi_communicator, sums); + internal::all_reduce(MPI_SUM, values, mpi_communicator, sums, N); } - template - inline - void sum (const Vector &values, - const MPI_Comm &mpi_communicator, - Vector &sums) - { - internal::all_reduce(MPI_SUM, values, mpi_communicator, sums); - } - - - template - inline - Tensor - sum (const Tensor &local, - const MPI_Comm &mpi_communicator) - { - const unsigned int n_entries = Tensor::n_independent_components; - Number entries[ Tensor::n_independent_components ]; - - for (unsigned int i=0; i< n_entries; ++i) - entries[i] = local[ local.unrolled_to_component_indices(i) ]; - - Number global_entries[ Tensor::n_independent_components ]; - dealii::Utilities::MPI::sum( entries, mpi_communicator, global_entries ); - - Tensor global; - for (unsigned int i=0; i< n_entries; ++i) - global[ global.unrolled_to_component_indices(i) ] = global_entries[i]; - - return global; - } - - template - inline - SymmetricTensor - sum (const SymmetricTensor &local, - const MPI_Comm &mpi_communicator) - { - const unsigned int n_entries = SymmetricTensor::n_independent_components; - Number entries[ SymmetricTensor::n_independent_components ]; - - for (unsigned int i=0; i< n_entries; ++i) - entries[i] = local[ local.unrolled_to_component_indices(i) ]; - - Number global_entries[ SymmetricTensor::n_independent_components ]; - dealii::Utilities::MPI::sum( entries, mpi_communicator, global_entries ); - - SymmetricTensor global; - for (unsigned int i=0; i< n_entries; ++i) - global[ global.unrolled_to_component_indices(i) ] = global_entries[i]; - - return global; - } - - template - inline - T max (const T &t, - const MPI_Comm &mpi_communicator) - { - return internal::all_reduce(MPI_MAX, t, mpi_communicator); - } - - template - inline void max (const T (&values)[N], const MPI_Comm &mpi_communicator, T (&maxima)[N]) { - internal::all_reduce(MPI_MAX, values, mpi_communicator, maxima); - } - - - template - inline - void max (const std::vector &values, - const MPI_Comm &mpi_communicator, - std::vector &maxima) - { - internal::all_reduce(MPI_MAX, values, mpi_communicator, maxima); - } - - - template - inline - T min (const T &t, - const MPI_Comm &mpi_communicator) - { - return internal::all_reduce(MPI_MIN, t, mpi_communicator); + internal::all_reduce(MPI_MAX, values, mpi_communicator, maxima, N); } - template - inline void min (const T (&values)[N], const MPI_Comm &mpi_communicator, T (&minima)[N]) { - internal::all_reduce(MPI_MIN, values, mpi_communicator, minima); - } - - - template - inline - void min (const std::vector &values, - const MPI_Comm &mpi_communicator, - std::vector &minima) - { - internal::all_reduce(MPI_MIN, values, mpi_communicator, minima); + internal::all_reduce(MPI_MIN, values, mpi_communicator, minima, N); } +#endif inline diff --git a/include/deal.II/base/mpi.templates.h b/include/deal.II/base/mpi.templates.h new file mode 100644 index 0000000000..f4108f2142 --- /dev/null +++ b/include/deal.II/base/mpi.templates.h @@ -0,0 +1,263 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2011 - 2016 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE at +// the top level of the deal.II distribution. +// +// --------------------------------------------------------------------- + +#ifndef dealii__mpi_templates_h +#define dealii__mpi_templates_h + +#include +#include +#include +#include +#include +#include + +#include + +DEAL_II_NAMESPACE_OPEN + +namespace Utilities +{ + namespace MPI + { + namespace internal + { +#ifdef DEAL_II_WITH_MPI + /** + * Return the corresponding MPI data type id for the argument given. + */ + MPI_Datatype mpi_type_id (const int *) + { + return MPI_INT; + } + + + MPI_Datatype mpi_type_id (const long int *) + { + return MPI_LONG; + } + + + MPI_Datatype mpi_type_id (const unsigned int *) + { + return MPI_UNSIGNED; + } + + + MPI_Datatype mpi_type_id (const unsigned long int *) + { + return MPI_UNSIGNED_LONG; + } + + + MPI_Datatype mpi_type_id (const unsigned long long int *) + { + return MPI_UNSIGNED_LONG_LONG; + } + + + MPI_Datatype mpi_type_id (const float *) + { + return MPI_FLOAT; + } + + + MPI_Datatype mpi_type_id (const double *) + { + return MPI_DOUBLE; + } + + + MPI_Datatype mpi_type_id (const long double *) + { + return MPI_LONG_DOUBLE; + } +#endif + + template + void all_reduce (const MPI_Op &mpi_op, + const T *const values, + const MPI_Comm &mpi_communicator, + T *output, + const std::size_t size) + { +#ifdef DEAL_II_WITH_MPI + if (job_supports_mpi()) + { + MPI_Allreduce (values != output + ? + // TODO This const_cast is only needed for older + // (e.g., openMPI 1.6, released in 2012) + // implementations of MPI-2. It is not needed as of + // MPI-3 and we should remove it at some point in + // the future. + const_cast(static_cast(values)) + : + MPI_IN_PLACE, + static_cast(output), + static_cast(size), + internal::mpi_type_id(values), + mpi_op, + mpi_communicator); + } + else +#endif + { + (void)mpi_op; + (void)mpi_communicator; + for (std::size_t i=0; i + T all_reduce (const MPI_Op &mpi_op, + const T &t, + const MPI_Comm &mpi_communicator) + { + T output; + all_reduce(mpi_op, &t, mpi_communicator, &output, 1); + return output; + } + + template + void all_reduce (const MPI_Op &mpi_op, + const std::vector &values, + const MPI_Comm &mpi_communicator, + std::vector &output) + { + Assert(values.size() == output.size(), + ExcDimensionMismatch(values.size(), output.size())); + all_reduce(mpi_op, &values[0], mpi_communicator, &output[0], values.size()); + } + + template + void all_reduce (const MPI_Op &mpi_op, + const Vector &values, + const MPI_Comm &mpi_communicator, + Vector &output) + { + Assert(values.size() == output.size(), + ExcDimensionMismatch(values.size(), output.size())); + all_reduce(mpi_op, values.begin(), mpi_communicator, output.begin(), values.size()); + } + } + + + template + T sum (const T &t, + const MPI_Comm &mpi_communicator) + { + return internal::all_reduce(MPI_SUM, t, mpi_communicator); + } + + + template + void sum (const std::vector &values, + const MPI_Comm &mpi_communicator, + std::vector &sums) + { + internal::all_reduce(MPI_SUM, values, mpi_communicator, sums); + } + + template + void sum (const Vector &values, + const MPI_Comm &mpi_communicator, + Vector &sums) + { + internal::all_reduce(MPI_SUM, values, mpi_communicator, sums); + } + + + template + Tensor + sum (const Tensor &local, + const MPI_Comm &mpi_communicator) + { + const unsigned int n_entries = Tensor::n_independent_components; + Number entries[ Tensor::n_independent_components ]; + + for (unsigned int i=0; i< n_entries; ++i) + entries[i] = local[ local.unrolled_to_component_indices(i) ]; + + Number global_entries[ Tensor::n_independent_components ]; + dealii::Utilities::MPI::sum( entries, mpi_communicator, global_entries ); + + Tensor global; + for (unsigned int i=0; i< n_entries; ++i) + global[ global.unrolled_to_component_indices(i) ] = global_entries[i]; + + return global; + } + + template + SymmetricTensor + sum (const SymmetricTensor &local, + const MPI_Comm &mpi_communicator) + { + const unsigned int n_entries = SymmetricTensor::n_independent_components; + Number entries[ SymmetricTensor::n_independent_components ]; + + for (unsigned int i=0; i< n_entries; ++i) + entries[i] = local[ local.unrolled_to_component_indices(i) ]; + + Number global_entries[ SymmetricTensor::n_independent_components ]; + dealii::Utilities::MPI::sum( entries, mpi_communicator, global_entries ); + + SymmetricTensor global; + for (unsigned int i=0; i< n_entries; ++i) + global[ global.unrolled_to_component_indices(i) ] = global_entries[i]; + + return global; + } + + template + T max (const T &t, + const MPI_Comm &mpi_communicator) + { + return internal::all_reduce(MPI_MAX, t, mpi_communicator); + } + + + template + void max (const std::vector &values, + const MPI_Comm &mpi_communicator, + std::vector &maxima) + { + internal::all_reduce(MPI_MAX, values, mpi_communicator, maxima); + } + + + template + T min (const T &t, + const MPI_Comm &mpi_communicator) + { + return internal::all_reduce(MPI_MIN, t, mpi_communicator); + } + + + template + void min (const std::vector &values, + const MPI_Comm &mpi_communicator, + std::vector &minima) + { + internal::all_reduce(MPI_MIN, values, mpi_communicator, minima); + } + } // end of namespace MPI +} // end of namespace Utilities + + +DEAL_II_NAMESPACE_CLOSE + +#endif diff --git a/source/base/CMakeLists.txt b/source/base/CMakeLists.txt index 587b1dada9..9df7b72fb6 100644 --- a/source/base/CMakeLists.txt +++ b/source/base/CMakeLists.txt @@ -75,6 +75,7 @@ SET(_inst data_out_base.inst.in function.inst.in function_time.inst.in + mpi.inst.in polynomials_rannacher_turek.inst.in tensor_function.inst.in time_stepping.inst.in diff --git a/source/base/mpi.cc b/source/base/mpi.cc index 858a621f4f..4c86df4589 100644 --- a/source/base/mpi.cc +++ b/source/base/mpi.cc @@ -15,6 +15,7 @@ #include +#include #include #include #include @@ -506,10 +507,9 @@ namespace Utilities } #endif } - - +#include "mpi.inst" + } } // end of namespace MPI - } // end of namespace Utilities DEAL_II_NAMESPACE_CLOSE diff --git a/source/base/mpi.inst.in b/source/base/mpi.inst.in new file mode 100644 index 0000000000..531f86beb1 --- /dev/null +++ b/source/base/mpi.inst.in @@ -0,0 +1,60 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2016 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE at +// the top level of the deal.II distribution. +// +// --------------------------------------------------------------------- + + +for (S : MPI_SCALARS) +{ + template + void sum (const Vector &, const MPI_Comm &, Vector &); + + template + S sum (const S &, const MPI_Comm &); + + template + void sum (const std::vector &, const MPI_Comm &, std::vector &); + + template + S max (const S &, const MPI_Comm &); + + template + void max (const std::vector &, const MPI_Comm &, std::vector &); + + template + S min (const S &, const MPI_Comm &); + + template + void min (const std::vector &, const MPI_Comm &, std::vector &); +} + + +for (S : REAL_SCALARS; rank: RANKS; dim : SPACE_DIMENSIONS) +{ + template + Tensor + sum (const Tensor &, const MPI_Comm &); +} + + +// Recall that SymmetricTensor only makes sense for rank 2 and 4 +for (S : REAL_SCALARS; dim : SPACE_DIMENSIONS) +{ + template + SymmetricTensor<2,dim,S> + sum<2,dim,S> (const SymmetricTensor<2,dim,S> &, const MPI_Comm &); + + template + SymmetricTensor<4,dim,S> + sum<4,dim,S> (const SymmetricTensor<4,dim,S> &, const MPI_Comm &); +} diff --git a/tests/mpi/renumber_cuthill_mckee.cc b/tests/mpi/renumber_cuthill_mckee.cc index c72a61c3ee..9fab96ae2b 100644 --- a/tests/mpi/renumber_cuthill_mckee.cc +++ b/tests/mpi/renumber_cuthill_mckee.cc @@ -22,6 +22,7 @@ #include "../tests.h" #include #include +#include #include #include #include diff --git a/tests/mpi/renumber_cuthill_mckee_02.cc b/tests/mpi/renumber_cuthill_mckee_02.cc index d0ecc31957..99de134f94 100644 --- a/tests/mpi/renumber_cuthill_mckee_02.cc +++ b/tests/mpi/renumber_cuthill_mckee_02.cc @@ -22,6 +22,7 @@ #include "../tests.h" #include #include +#include #include #include #include diff --git a/tests/tests.h b/tests/tests.h index 7a1dd16a80..d8ae920990 100644 --- a/tests/tests.h +++ b/tests/tests.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include -- 2.39.5