From 07cb8b131063bb5cf15df3d8401512cc7736694b Mon Sep 17 00:00:00 2001 From: Peter Munch Date: Mon, 23 Mar 2020 10:47:56 +0100 Subject: [PATCH] Remove template argument from LinearAlgebra::SharedMPI::Partitioner --- include/deal.II/lac/la_sm_partitioner.h | 60 +++++++++++++------- include/deal.II/lac/la_sm_vector.h | 6 +- include/deal.II/lac/la_sm_vector.templates.h | 17 +++--- include/deal.II/matrix_free/matrix_free.h | 7 +-- 4 files changed, 54 insertions(+), 36 deletions(-) diff --git a/include/deal.II/lac/la_sm_partitioner.h b/include/deal.II/lac/la_sm_partitioner.h index 2b60c5e4fd..7973662c90 100644 --- a/include/deal.II/lac/la_sm_partitioner.h +++ b/include/deal.II/lac/la_sm_partitioner.h @@ -30,7 +30,6 @@ namespace LinearAlgebra { namespace SharedMPI { - template class Partitioner : public LinearAlgebra::CommunicationPatternBase { public: @@ -228,18 +227,21 @@ namespace LinearAlgebra send_sm_req.data(), MPI_STATUSES_IGNORE); } - - buffer.resize(send_remote_ptr.back()); } + template void update_ghost_values_start( - Number * data_this, - std::vector &data_others, - const unsigned int communication_channel = 0) const + Number * data_this, + std::vector & data_others, + dealii::AlignedVector &buffer, + const unsigned int communication_channel = 0) const { (void)data_others; + if (send_remote_ptr.back() != buffer.size()) + buffer.resize(send_remote_ptr.back()); + int dummy; for (unsigned int i = 0; i < recv_sm_ranks.size(); i++) MPI_Irecv(&dummy, @@ -285,10 +287,14 @@ namespace LinearAlgebra } } + template void - update_ghost_values_finish(Number * data_this, - std::vector &data_others) const + update_ghost_values_finish(Number * data_this, + std::vector & data_others, + dealii::AlignedVector &buffer) const { + (void)buffer; + for (unsigned int c = 0; c < recv_sm_ranks.size(); c++) { int i; @@ -318,21 +324,28 @@ namespace LinearAlgebra MPI_STATUSES_IGNORE); } + template void - update_ghost_values(Number * data_this, - std::vector &data_others) const + update_ghost_values(Number * data_this, + std::vector & data_others, + dealii::AlignedVector &buffer) const { - update_ghost_values_start(data_this, data_others); - update_ghost_values_finish(data_this, data_others); + update_ghost_values_start(data_this, data_others, buffer); + update_ghost_values_finish(data_this, data_others, buffer); } + template void - compress_start(Number * data_this, - std::vector &data_others, - const unsigned int communication_channel = 0) const + compress_start(Number * data_this, + std::vector & data_others, + dealii::AlignedVector &buffer, + const unsigned int communication_channel = 0) const { (void)data_others; + if (send_remote_ptr.back() != buffer.size()) + buffer.resize(send_remote_ptr.back()); + int dummy; for (unsigned int i = 0; i < recv_sm_ranks.size(); i++) MPI_Isend(&dummy, @@ -371,9 +384,11 @@ namespace LinearAlgebra send_remote_req.data() + i); } + template void - compress_finish(Number * data_this, - std::vector &data_others) const + compress_finish(Number * data_this, + std::vector & data_others, + dealii::AlignedVector &buffer) const { for (unsigned int c = 0; c < send_sm_ranks.size(); c++) { @@ -419,11 +434,14 @@ namespace LinearAlgebra MPI_STATUSES_IGNORE); } + template void - compress(Number *data_this, std::vector &data_others) const + compress(Number * data_this, + std::vector & data_others, + dealii::AlignedVector &buffer) const { - compress_start(data_this, data_others); - compress_finish(data_this, data_others); + compress_start(data_this, data_others, buffer); + compress_finish(data_this, data_others, buffer); } private: @@ -432,8 +450,6 @@ namespace LinearAlgebra unsigned int n_local_elements; - mutable AlignedVector buffer; - std::vector recv_remote_ranks; std::vector recv_remote_ptr = {0}; mutable std::vector recv_remote_req; diff --git a/include/deal.II/lac/la_sm_vector.h b/include/deal.II/lac/la_sm_vector.h index 4e89827b01..423e3403e1 100644 --- a/include/deal.II/lac/la_sm_vector.h +++ b/include/deal.II/lac/la_sm_vector.h @@ -159,7 +159,7 @@ namespace LinearAlgebra void reinit( const std::shared_ptr &partitioner, - const std::shared_ptr> &partitioner_sm); + const std::shared_ptr &partitioner_sm); void swap(Vector &v); @@ -477,7 +477,7 @@ namespace LinearAlgebra std::shared_ptr partitioner; - std::shared_ptr> partitioner_sm; + std::shared_ptr partitioner_sm; size_type allocated_size; @@ -491,7 +491,7 @@ namespace LinearAlgebra thread_loop_partitioner; // needed? - mutable MemorySpaceData import_data; + mutable dealii::AlignedVector import_data; mutable bool vector_is_ghosted; diff --git a/include/deal.II/lac/la_sm_vector.templates.h b/include/deal.II/lac/la_sm_vector.templates.h index 41841b283a..796cfff466 100644 --- a/include/deal.II/lac/la_sm_vector.templates.h +++ b/include/deal.II/lac/la_sm_vector.templates.h @@ -206,8 +206,7 @@ namespace LinearAlgebra // is only used as temporary storage for compress() and // update_ghost_values, and we might have vectors where we never // call these methods and hence do not need to have the storage. - import_data.values.reset(); - import_data.values_dev.reset(); + import_data.clear(); // TODO thread_loop_partitioner = v.thread_loop_partitioner; } @@ -246,7 +245,7 @@ namespace LinearAlgebra void Vector::reinit( const std::shared_ptr &partitioner, - const std::shared_ptr> & partitioner_sm) + const std::shared_ptr & partitioner_sm) { clear_mpi_requests(); this->partitioner = partitioner; @@ -265,8 +264,7 @@ namespace LinearAlgebra // is only used as temporary storage for compress() and // update_ghost_values, and we might have vectors where we never // call these methods and hence do not need to have the storage. - import_data.values.reset(); - import_data.values_dev.reset(); + import_data.clear(); vector_is_ghosted = false; } @@ -523,6 +521,7 @@ namespace LinearAlgebra ExcMessage("Cannot call compress() on a ghosted vector")); partitioner_sm->compress_start(data.values.get(), data.others, + import_data, communication_channel); } @@ -536,7 +535,9 @@ namespace LinearAlgebra Assert(::dealii::VectorOperation::values::add == operation, ExcNotImplemented()); vector_is_ghosted = false; - partitioner_sm->compress_finish(data.values.get(), data.others); + partitioner_sm->compress_finish(data.values.get(), + data.others, + import_data); } @@ -548,6 +549,7 @@ namespace LinearAlgebra { partitioner_sm->update_ghost_values_start(data.values.get(), data.others, + import_data, communication_channel); } @@ -558,7 +560,8 @@ namespace LinearAlgebra Vector::update_ghost_values_finish() const { partitioner_sm->update_ghost_values_finish(data.values.get(), - data.others); + data.others, + import_data); vector_is_ghosted = true; } diff --git a/include/deal.II/matrix_free/matrix_free.h b/include/deal.II/matrix_free/matrix_free.h index b4509d9c67..922bb61b1d 100644 --- a/include/deal.II/matrix_free/matrix_free.h +++ b/include/deal.II/matrix_free/matrix_free.h @@ -2095,9 +2095,8 @@ private: // TODO: move it to DoFInfo mutable std::map< unsigned int, - std::map< - const MPI_Comm, - std::shared_ptr>>> + std::map>> partitioner_sm; /** @@ -2240,7 +2239,7 @@ MatrixFree::initialize_dof_vector( if (partitioner_sm[comp][comm_sm] == nullptr) partitioner_sm[comp][comm_sm] = - std::make_shared>( + std::make_shared( part->get_communicator(), comm_sm, part->locally_owned_range(), -- 2.39.5