From ef0923382e07a2141b242fb8656a7192c7b70fe0 Mon Sep 17 00:00:00 2001 From: Peter Munch Date: Sat, 21 Mar 2020 19:32:27 +0100 Subject: [PATCH] Implement initialization --- include/deal.II/lac/la_sm_partitioner.h | 17 +++++++++--- include/deal.II/lac/la_sm_vector.h | 19 +++++-------- include/deal.II/lac/la_sm_vector.templates.h | 28 +++++++++++++------- include/deal.II/matrix_free/matrix_free.h | 13 +++++---- tests/sm/vector_sm_01.cc | 4 +-- 5 files changed, 48 insertions(+), 33 deletions(-) diff --git a/include/deal.II/lac/la_sm_partitioner.h b/include/deal.II/lac/la_sm_partitioner.h index 02cc66481c..aac7cc57e7 100644 --- a/include/deal.II/lac/la_sm_partitioner.h +++ b/include/deal.II/lac/la_sm_partitioner.h @@ -18,6 +18,7 @@ #include + #include #include @@ -33,7 +34,10 @@ namespace LinearAlgebra class Partitioner : public LinearAlgebra::CommunicationPatternBase { public: - Partitioner(const MPI_Comm &comm, const MPI_Comm &comm_sm, const IndexSet &is_locally_owned, const IndexSet &is_locally_ghost) + Partitioner(const MPI_Comm &comm, + const MPI_Comm &comm_sm, + const IndexSet &is_locally_owned, + const IndexSet &is_locally_ghost) : comm(comm) , comm_sm(comm_sm) { @@ -46,6 +50,12 @@ namespace LinearAlgebra return comm; } + const MPI_Comm & + get_sm_mpi_communicator() const + { + return comm_sm; + } + void reinit(const IndexSet &is_locally_owned, const IndexSet &is_locally_ghost, @@ -58,8 +68,7 @@ namespace LinearAlgebra } void - reinit(const IndexSet &is_locally_owned, - const IndexSet &is_locally_ghost) + reinit(const IndexSet &is_locally_owned, const IndexSet &is_locally_ghost) { this->n_local_elements = is_locally_owned.n_elements(); @@ -226,7 +235,7 @@ namespace LinearAlgebra private: const MPI_Comm &comm; const MPI_Comm &comm_sm; - + unsigned int n_local_elements; AlignedVector buffer; diff --git a/include/deal.II/lac/la_sm_vector.h b/include/deal.II/lac/la_sm_vector.h index 5a60b1dfab..02c8508a29 100644 --- a/include/deal.II/lac/la_sm_vector.h +++ b/include/deal.II/lac/la_sm_vector.h @@ -494,7 +494,7 @@ namespace LinearAlgebra clear_mpi_requests(); void - resize_val(const size_type new_allocated_size); + resize_val(const size_type new_allocated_size, const MPI_Comm &comm_sm); template friend class Vector; @@ -514,17 +514,15 @@ namespace LinearAlgebra struct Policy { static inline typename Vector::iterator - begin(MemorySpaceData &) + begin(MemorySpaceData &data) { - Assert(false, ExcNotImplemented()); - return nullptr; + return data.values.get(); } static inline typename Vector::const_iterator - begin(const MemorySpaceData &) + begin(const MemorySpaceData &data) { - Assert(false, ExcNotImplemented()); - return nullptr; + return data.values.get(); } static inline Number * @@ -561,8 +559,7 @@ namespace LinearAlgebra inline typename Vector::size_type Vector::local_size() const { - Assert(false, ExcNotImplemented()); - return 0; + return partitioner->local_size(); } @@ -638,7 +635,6 @@ namespace LinearAlgebra inline typename Vector::iterator Vector::begin() { - Assert(false, ExcNotImplemented()); return internal::Policy::begin(data); } @@ -648,7 +644,6 @@ namespace LinearAlgebra inline typename Vector::const_iterator Vector::begin() const { - Assert(false, ExcNotImplemented()); return internal::Policy::begin(data); } @@ -658,7 +653,6 @@ namespace LinearAlgebra inline typename Vector::iterator Vector::end() { - Assert(false, ExcNotImplemented()); return internal::Policy::begin(data) + partitioner->local_size(); } @@ -669,7 +663,6 @@ namespace LinearAlgebra inline typename Vector::const_iterator Vector::end() const { - Assert(false, ExcNotImplemented()); return internal::Policy::begin(data) + partitioner->local_size(); } diff --git a/include/deal.II/lac/la_sm_vector.templates.h b/include/deal.II/lac/la_sm_vector.templates.h index a55fd3d6af..62826b57a5 100644 --- a/include/deal.II/lac/la_sm_vector.templates.h +++ b/include/deal.II/lac/la_sm_vector.templates.h @@ -48,7 +48,8 @@ namespace LinearAlgebra static void resize_val(const types::global_dof_index new_alloc_size, types::global_dof_index & allocated_size, - MemorySpaceData & data) + MemorySpaceData & data, + const MPI_Comm & comm_shared) { Assert(allocated_size == 0, ExcNotImplemented()); Assert(data.values == nullptr, ExcNotImplemented()); @@ -57,7 +58,6 @@ namespace LinearAlgebra // TODO std::vector data_others; - MPI_Comm comm_shared; data.values_win = new MPI_Win; Number *data_this = (Number *)malloc(0); @@ -105,11 +105,15 @@ namespace LinearAlgebra template void - Vector::resize_val(const size_type new_alloc_size) + Vector::resize_val(const size_type new_alloc_size, + const MPI_Comm &comm_sm) { internal::la_parallel_vector_templates_functions< Number, - MemorySpaceType>::resize_val(new_alloc_size, allocated_size, data); + MemorySpaceType>::resize_val(new_alloc_size, + allocated_size, + data, + comm_sm); } @@ -181,7 +185,7 @@ namespace LinearAlgebra // set vector size and allocate memory const size_type new_allocated_size = this->partitioner->local_size() + this->partitioner->n_ghost_indices(); - resize_val(new_allocated_size); + resize_val(new_allocated_size, partitioner_sm->get_sm_mpi_communicator()); // initialize to zero this->operator=(Number()); @@ -274,7 +278,6 @@ namespace LinearAlgebra template inline Vector::~Vector() { - Assert(false, ExcNotImplemented()); try { clear_mpi_requests(); @@ -363,7 +366,10 @@ namespace LinearAlgebra void Vector::zero_out_ghosts() const { - Assert(false, ExcNotImplemented()); + if (data.values != nullptr) + std::fill_n(data.values.get() + partitioner->local_size(), + partitioner->n_ghost_indices(), + Number()); } @@ -439,8 +445,12 @@ namespace LinearAlgebra Vector & Vector::operator=(const Number s) { - Assert(false, ExcNotImplemented()); - (void)s; + const size_type this_size = local_size(); + if (this_size > 0) + std::fill_n(data.values.get(), this_size, s); + + if (s == Number()) + zero_out_ghosts(); return *this; } diff --git a/include/deal.II/matrix_free/matrix_free.h b/include/deal.II/matrix_free/matrix_free.h index d8841bad93..4f1f67e5d4 100644 --- a/include/deal.II/matrix_free/matrix_free.h +++ b/include/deal.II/matrix_free/matrix_free.h @@ -1482,7 +1482,7 @@ public: template void initialize_dof_vector(LinearAlgebra::SharedMPI::Vector &vec, - const MPI_Comm & comm_sm, + const MPI_Comm & comm_sm, const unsigned int dof_handler_index = 0) const; /** @@ -2231,17 +2231,20 @@ template inline void MatrixFree::initialize_dof_vector( LinearAlgebra::SharedMPI::Vector &vec, - const MPI_Comm & comm_sm, + const MPI_Comm & comm_sm, const unsigned int comp) const { AssertIndexRange(comp, n_components()); - const auto & part = dof_info[comp].vector_partitioner; - + const auto &part = dof_info[comp].vector_partitioner; + if (partitioner_sm[comp][comm_sm] == nullptr) partitioner_sm[comp][comm_sm] = std::make_shared>( - part->get_communicator(), comm_sm, part->locally_owned_range(), part->ghost_indices()); + part->get_communicator(), + comm_sm, + part->locally_owned_range(), + part->ghost_indices()); vec.reinit(part, partitioner_sm[comp][comm_sm]); } diff --git a/tests/sm/vector_sm_01.cc b/tests/sm/vector_sm_01.cc index 1506d1064c..ec3622a90a 100644 --- a/tests/sm/vector_sm_01.cc +++ b/tests/sm/vector_sm_01.cc @@ -61,9 +61,9 @@ test(const int n_refinements, const int degree, const int group_size) matrix_free.reinit(mapping, dof_handler, constraint, quad, additional_data); MPI_Comm comm = MPI_COMM_WORLD; - + const unsigned int rank = Utilities::MPI::this_mpi_process(comm); - + MPI_Comm comm_sm; MPI_Comm_split(comm, rank / group_size, rank, &comm_sm); -- 2.39.5