From 85f3028b3965f383d61c9325a4cee052b16e3f51 Mon Sep 17 00:00:00 2001 From: Daniel Arndt Date: Fri, 17 Feb 2023 14:10:54 -0500 Subject: [PATCH] Fix Tpetra detection and unit tests --- cmake/configure/configure_20_trilinos.cmake | 7 +- include/deal.II/base/index_set.h | 2 +- include/deal.II/lac/read_write_vector.h | 15 +-- .../deal.II/lac/read_write_vector.templates.h | 17 ++-- .../trilinos_tpetra_communication_pattern.h | 10 +- include/deal.II/lac/trilinos_tpetra_vector.h | 11 ++- .../lac/trilinos_tpetra_vector.templates.h | 95 +++++++++++-------- include/deal.II/lac/vector_element_access.h | 22 +++-- source/base/index_set.cc | 14 +-- .../trilinos_tpetra_communication_pattern.cc | 24 ++--- 10 files changed, 124 insertions(+), 93 deletions(-) diff --git a/cmake/configure/configure_20_trilinos.cmake b/cmake/configure/configure_20_trilinos.cmake index 593e026471..3655c55c92 100644 --- a/cmake/configure/configure_20_trilinos.cmake +++ b/cmake/configure/configure_20_trilinos.cmake @@ -233,6 +233,7 @@ macro(feature_trilinos_find_external var) list(APPEND CMAKE_REQUIRED_INCLUDES ${MPI_CXX_INCLUDE_PATH}) list(APPEND CMAKE_REQUIRED_LIBRARIES ${Trilinos_LIBRARIES} ${MPI_LIBRARIES}) + list(APPEND CMAKE_REQUIRED_FLAGS ${TRILINOS_CXX_FLAGS}) # For the case of Trilinos being compiled with openmp support the # following Tpetra test needs -fopenmp to succeed. Make sure that we @@ -249,11 +250,10 @@ macro(feature_trilinos_find_external var) " #include #include - int - main() + int main() { using LO = int; - using GO = ${_global_index_type}; + using GO = std::make_signed_t<${_global_index_type}>; using map_type = Tpetra::Map; Teuchos::RCP dummy_map = Teuchos::rcp(new map_type()); Tpetra::Vector dummy_vector(dummy_map); @@ -283,6 +283,7 @@ macro(feature_trilinos_find_external var) list(APPEND CMAKE_REQUIRED_INCLUDES ${MPI_CXX_INCLUDE_PATH}) list(APPEND CMAKE_REQUIRED_LIBRARIES ${Trilinos_LIBRARIES} ${MPI_LIBRARIES}) + list(APPEND CMAKE_REQUIRED_FLAGS ${TRILINOS_CXX_FLAGS}) CHECK_CXX_SOURCE_COMPILES( " diff --git a/include/deal.II/base/index_set.h b/include/deal.II/base/index_set.h index b17c18282c..722a3010ad 100644 --- a/include/deal.II/base/index_set.h +++ b/include/deal.II/base/index_set.h @@ -496,7 +496,7 @@ public: const bool overlapping = false) const; # ifdef DEAL_II_TRILINOS_WITH_TPETRA - Tpetra::Map + Tpetra::Map> make_tpetra_map(const MPI_Comm &communicator = MPI_COMM_WORLD, const bool overlapping = false) const; # endif diff --git a/include/deal.II/lac/read_write_vector.h b/include/deal.II/lac/read_write_vector.h index 14b6f445ab..6236f67e0c 100644 --- a/include/deal.II/lac/read_write_vector.h +++ b/include/deal.II/lac/read_write_vector.h @@ -657,13 +657,14 @@ namespace LinearAlgebra * used directly. */ void - import( - const Tpetra::Vector &tpetra_vector, - const IndexSet & locally_owned_elements, - VectorOperation::values operation, - const MPI_Comm & mpi_comm, - const std::shared_ptr - &communication_pattern); + import(const Tpetra:: + Vector> + & tpetra_vector, + const IndexSet & locally_owned_elements, + VectorOperation::values operation, + const MPI_Comm & mpi_comm, + const std::shared_ptr + &communication_pattern); # endif /** diff --git a/include/deal.II/lac/read_write_vector.templates.h b/include/deal.II/lac/read_write_vector.templates.h index c97afd5e3d..a85caa9a18 100644 --- a/include/deal.II/lac/read_write_vector.templates.h +++ b/include/deal.II/lac/read_write_vector.templates.h @@ -577,10 +577,11 @@ namespace LinearAlgebra template void ReadWriteVector::import( - const Tpetra::Vector &vector, - const IndexSet & source_elements, - VectorOperation::values operation, - const MPI_Comm & mpi_comm, + const Tpetra:: + Vector> &vector, + const IndexSet & source_elements, + VectorOperation::values operation, + const MPI_Comm & mpi_comm, const std::shared_ptr &communication_pattern) { @@ -620,11 +621,11 @@ namespace LinearAlgebra "LinearAlgebra::TpetraWrappers::CommunicationPattern.")); } - Tpetra::Export tpetra_export( - tpetra_comm_pattern->get_tpetra_export()); + Tpetra::Export> + tpetra_export(tpetra_comm_pattern->get_tpetra_export()); - Tpetra::Vector target_vector( - tpetra_export.getSourceMap()); + Tpetra::Vector> + target_vector(tpetra_export.getSourceMap()); target_vector.doImport(vector, tpetra_export, Tpetra::REPLACE); const auto *new_values = target_vector.getData().get(); diff --git a/include/deal.II/lac/trilinos_tpetra_communication_pattern.h b/include/deal.II/lac/trilinos_tpetra_communication_pattern.h index a746580583..862dadc24d 100644 --- a/include/deal.II/lac/trilinos_tpetra_communication_pattern.h +++ b/include/deal.II/lac/trilinos_tpetra_communication_pattern.h @@ -68,13 +68,13 @@ namespace LinearAlgebra /** * Return the underlying Tpetra::Import object. */ - const Tpetra::Import & + const Tpetra::Import> & get_tpetra_import() const; /** * Return the underlying Tpetra::Export object. */ - const Tpetra::Export & + const Tpetra::Export> & get_tpetra_export() const; private: @@ -86,13 +86,15 @@ namespace LinearAlgebra /** * Shared pointer to the Tpetra::Import object used. */ - std::unique_ptr> + std::unique_ptr< + Tpetra::Import>> tpetra_import; /** * Shared pointer to the Tpetra::Export object used. */ - std::unique_ptr> + std::unique_ptr< + Tpetra::Export>> tpetra_export; }; } // end of namespace TpetraWrappers diff --git a/include/deal.II/lac/trilinos_tpetra_vector.h b/include/deal.II/lac/trilinos_tpetra_vector.h index 26c1f1675a..6e3bdab2f7 100644 --- a/include/deal.II/lac/trilinos_tpetra_vector.h +++ b/include/deal.II/lac/trilinos_tpetra_vector.h @@ -337,14 +337,15 @@ namespace LinearAlgebra * Return a const reference to the underlying Trilinos * Tpetra::Vector class. */ - const Tpetra::Vector & - trilinos_vector() const; + const Tpetra:: + Vector> & + trilinos_vector() const; /** * Return a (modifiable) reference to the underlying Trilinos * Tpetra::Vector class. */ - Tpetra::Vector & + Tpetra::Vector> & trilinos_vector(); /** @@ -398,7 +399,9 @@ namespace LinearAlgebra /** * Pointer to the actual Tpetra vector object. */ - std::unique_ptr> + std::unique_ptr< + Tpetra:: + Vector>> vector; /** diff --git a/include/deal.II/lac/trilinos_tpetra_vector.templates.h b/include/deal.II/lac/trilinos_tpetra_vector.templates.h index 154e45ddf0..54db805e38 100644 --- a/include/deal.II/lac/trilinos_tpetra_vector.templates.h +++ b/include/deal.II/lac/trilinos_tpetra_vector.templates.h @@ -47,9 +47,12 @@ namespace LinearAlgebra template Vector::Vector() : Subscriptor() - , vector(new Tpetra::Vector( - Teuchos::RCP>( - new Tpetra::Map( + , vector(new Tpetra::Vector>( + Teuchos::RCP< + Tpetra::Map>>( + new Tpetra::Map>( 0, 0, Utilities::Trilinos::tpetra_comm_self())))) @@ -60,7 +63,9 @@ namespace LinearAlgebra template Vector::Vector(const Vector &V) : Subscriptor() - , vector(new Tpetra::Vector( + , vector(new Tpetra::Vector>( V.trilinos_vector(), Teuchos::Copy)) {} @@ -71,9 +76,12 @@ namespace LinearAlgebra Vector::Vector(const IndexSet ¶llel_partitioner, const MPI_Comm &communicator) : Subscriptor() - , vector(new Tpetra::Vector( - Teuchos::rcp(new Tpetra::Map( - parallel_partitioner.make_tpetra_map(communicator, false))))) + , vector(new Tpetra::Vector>( + Teuchos::rcp( + new Tpetra::Map>( + parallel_partitioner.make_tpetra_map(communicator, false))))) {} @@ -84,12 +92,15 @@ namespace LinearAlgebra const MPI_Comm &communicator, const bool omit_zeroing_entries) { - Tpetra::Map input_map = + Tpetra::Map> input_map = parallel_partitioner.make_tpetra_map(communicator, false); if (vector->getMap()->isSameAs(input_map) == false) vector = std::make_unique< - Tpetra::Vector>(Teuchos::rcp( - new Tpetra::Map(input_map))); + Tpetra:: + Vector>>( + Teuchos::rcp( + new Tpetra::Map>( + input_map))); else if (omit_zeroing_entries == false) { vector->putScalar(0.); @@ -131,8 +142,8 @@ namespace LinearAlgebra { if (size() == V.size()) { - Tpetra::Import data_exchange( - vector->getMap(), V.trilinos_vector().getMap()); + Tpetra::Import> + data_exchange(vector->getMap(), V.trilinos_vector().getMap()); vector->doImport(V.trilinos_vector(), data_exchange, @@ -140,7 +151,9 @@ namespace LinearAlgebra } else vector = std::make_unique< - Tpetra::Vector>( + Tpetra::Vector>>( V.trilinos_vector()); } @@ -201,22 +214,25 @@ namespace LinearAlgebra "LinearAlgebra::TpetraWrappers::CommunicationPattern.")); } - Tpetra::Export tpetra_export( - tpetra_comm_pattern->get_tpetra_export()); - Tpetra::Vector source_vector( - tpetra_export.getSourceMap()); - - source_vector.template sync(); - auto x_2d = source_vector.template getLocalView(); - auto x_1d = Kokkos::subview(x_2d, Kokkos::ALL(), 0); - source_vector.template modify(); - const size_t localLength = source_vector.getLocalLength(); - auto values_it = V.begin(); - for (size_t k = 0; k < localLength; ++k) - x_1d(k) = *values_it++; - source_vector.template sync< - typename Tpetra::Vector:: - device_type::memory_space>(); + Tpetra::Export> + tpetra_export(tpetra_comm_pattern->get_tpetra_export()); + Tpetra::Vector> + source_vector(tpetra_export.getSourceMap()); + + { + source_vector.template sync(); + auto x_2d = source_vector.template getLocalView(); + auto x_1d = Kokkos::subview(x_2d, Kokkos::ALL(), 0); + source_vector.template modify(); + const size_t localLength = source_vector.getLocalLength(); + auto values_it = V.begin(); + for (size_t k = 0; k < localLength; ++k) + x_1d(k) = *values_it++; + source_vector.template sync< + typename Tpetra:: + Vector>:: + device_type::memory_space>(); + } if (operation == VectorOperation::insert) vector->doExport(source_vector, tpetra_export, Tpetra::REPLACE); else if (operation == VectorOperation::add) @@ -274,10 +290,11 @@ namespace LinearAlgebra // TODO: Tpetra doesn't have a combine mode that also updates local // elements, maybe there is a better workaround. - Tpetra::Vector dummy( - vector->getMap(), false); - Tpetra::Import data_exchange( - down_V.trilinos_vector().getMap(), dummy.getMap()); + Tpetra:: + Vector> + dummy(vector->getMap(), false); + Tpetra::Import> + data_exchange(down_V.trilinos_vector().getMap(), dummy.getMap()); dummy.doImport(down_V.trilinos_vector(), data_exchange, @@ -338,8 +355,9 @@ namespace LinearAlgebra vector_1d(k) += a; } vector->template sync< - typename Tpetra::Vector:: - device_type::memory_space>(); + typename Tpetra:: + Vector>:: + device_type::memory_space>(); } @@ -603,8 +621,9 @@ namespace LinearAlgebra template - const Tpetra::Vector & - Vector::trilinos_vector() const + const Tpetra:: + Vector> & + Vector::trilinos_vector() const { return *vector; } @@ -612,7 +631,7 @@ namespace LinearAlgebra template - Tpetra::Vector & + Tpetra::Vector> & Vector::trilinos_vector() { return *vector; diff --git a/include/deal.II/lac/vector_element_access.h b/include/deal.II/lac/vector_element_access.h index 5893aba997..2d2307769a 100644 --- a/include/deal.II/lac/vector_element_access.h +++ b/include/deal.II/lac/vector_element_access.h @@ -157,8 +157,8 @@ namespace internal LinearAlgebra::TpetraWrappers::Vector &V) { // Extract local indices in the vector. - Tpetra::Vector vector = - V.trilinos_vector(); + Tpetra::Vector> + vector = V.trilinos_vector(); TrilinosWrappers::types::int_type trilinos_i = vector.getMap()->getLocalElement( static_cast(i)); @@ -170,8 +170,9 @@ namespace internal vector.template modify(); vector_1d(trilinos_i) += value; vector.template sync< - typename Tpetra::Vector:: - device_type::memory_space>(); + typename Tpetra:: + Vector>:: + device_type::memory_space>(); } @@ -184,8 +185,8 @@ namespace internal LinearAlgebra::TpetraWrappers::Vector &V) { // Extract local indices in the vector. - Tpetra::Vector vector = - V.trilinos_vector(); + Tpetra::Vector> + vector = V.trilinos_vector(); TrilinosWrappers::types::int_type trilinos_i = vector.getMap()->getLocalElement( static_cast(i)); @@ -197,8 +198,9 @@ namespace internal vector.template modify(); vector_1d(trilinos_i) = value; vector.template sync< - typename Tpetra::Vector:: - device_type::memory_space>(); + typename Tpetra:: + Vector>:: + device_type::memory_space>(); } @@ -210,8 +212,8 @@ namespace internal const types::global_dof_index i) { // Extract local indices in the vector. - Tpetra::Vector vector = - V.trilinos_vector(); + Tpetra::Vector> + vector = V.trilinos_vector(); TrilinosWrappers::types::int_type trilinos_i = vector.getMap()->getLocalElement( static_cast(i)); diff --git a/source/base/index_set.cc b/source/base/index_set.cc index bc68b5fefc..81e38f538f 100644 --- a/source/base/index_set.cc +++ b/source/base/index_set.cc @@ -716,7 +716,7 @@ IndexSet::fill_index_vector(std::vector &indices) const #ifdef DEAL_II_WITH_TRILINOS # ifdef DEAL_II_TRILINOS_WITH_TPETRA -Tpetra::Map +Tpetra::Map> IndexSet::make_tpetra_map(const MPI_Comm &communicator, const bool overlapping) const { @@ -752,7 +752,7 @@ IndexSet::make_tpetra_map(const MPI_Comm &communicator, const bool linear = overlapping ? false : is_ascending_and_one_to_one(communicator); if (linear) - return Tpetra::Map( + return Tpetra::Map>( size(), n_elements(), 0, @@ -764,11 +764,13 @@ IndexSet::make_tpetra_map(const MPI_Comm &communicator, ); else { - const std::vector indices = get_index_vector(); - std::vector int_indices(indices.size()); + const std::vector indices = get_index_vector(); + std::vector> int_indices( + indices.size()); std::copy(indices.begin(), indices.end(), int_indices.begin()); - const Teuchos::ArrayView arr_view(int_indices); - return Tpetra::Map( + const Teuchos::ArrayView> + arr_view(int_indices); + return Tpetra::Map>( size(), arr_view, 0, diff --git a/source/lac/trilinos_tpetra_communication_pattern.cc b/source/lac/trilinos_tpetra_communication_pattern.cc index afddac8413..34c08d6332 100644 --- a/source/lac/trilinos_tpetra_communication_pattern.cc +++ b/source/lac/trilinos_tpetra_communication_pattern.cc @@ -51,22 +51,22 @@ namespace LinearAlgebra { comm = std::make_shared(communicator); - auto vector_space_vector_map = - Teuchos::rcp(new Tpetra::Map( + auto vector_space_vector_map = Teuchos::rcp( + new Tpetra::Map>( vector_space_vector_index_set.make_tpetra_map(*comm, false))); - auto read_write_vector_map = - Teuchos::rcp(new Tpetra::Map( + auto read_write_vector_map = Teuchos::rcp( + new Tpetra::Map>( read_write_vector_index_set.make_tpetra_map(*comm, true))); // Target map is read_write_vector_map // Source map is vector_space_vector_map. This map must have uniquely // owned GID. - tpetra_import = - std::make_unique>( - read_write_vector_map, vector_space_vector_map); - tpetra_export = - std::make_unique>( - read_write_vector_map, vector_space_vector_map); + tpetra_import = std::make_unique< + Tpetra::Import>>( + read_write_vector_map, vector_space_vector_map); + tpetra_export = std::make_unique< + Tpetra::Export>>( + read_write_vector_map, vector_space_vector_map); } @@ -79,7 +79,7 @@ namespace LinearAlgebra - const Tpetra::Import & + const Tpetra::Import> & CommunicationPattern::get_tpetra_import() const { return *tpetra_import; @@ -87,7 +87,7 @@ namespace LinearAlgebra - const Tpetra::Export & + const Tpetra::Export> & CommunicationPattern::get_tpetra_export() const { return *tpetra_export; -- 2.39.5