From: Sebastian Kinnewig Date: Fri, 17 Nov 2023 17:56:29 +0000 (+0100) Subject: Add parallel support to TpetraWrappers::Vector. X-Git-Tag: relicensing~148^2~1 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f7d9a353ea5cf70f5576f21b95120d6b08ee26f5;p=dealii.git Add parallel support to TpetraWrappers::Vector. --- diff --git a/include/deal.II/lac/affine_constraints.templates.h b/include/deal.II/lac/affine_constraints.templates.h index fe265b8594..b99edfb2fe 100644 --- a/include/deal.II/lac/affine_constraints.templates.h +++ b/include/deal.II/lac/affine_constraints.templates.h @@ -2650,6 +2650,35 @@ namespace internal } #endif + + +#ifdef DEAL_II_TRILINOS_WITH_TPETRA + template + inline void + import_vector_with_ghost_elements( + const LinearAlgebra::TpetraWrappers::Vector &vec, + const IndexSet &locally_owned_elements, + const IndexSet &needed_elements, + LinearAlgebra::TpetraWrappers::Vector &output, + const std::bool_constant /*is_block_vector*/) + { + Assert(!vec.has_ghost_elements(), ExcGhostsPresent()); + IndexSet parallel_partitioner = locally_owned_elements; + parallel_partitioner.add_indices(needed_elements); + + const MPI_Comm mpi_comm = + Trilinos::teuchos_comm_to_mpi_comm(vec.trilinos_rcp()->getMap()->getComm()); + + output.reinit(locally_owned_elements, + needed_elements, + mpi_comm); + + output = vec; + } +#endif // DEAL_II_TRILINOS_WITH_TPETRA + + + #ifdef DEAL_II_WITH_PETSC inline void import_vector_with_ghost_elements( diff --git a/include/deal.II/lac/read_write_vector.templates.h b/include/deal.II/lac/read_write_vector.templates.h index 710c5c7f85..652990854b 100644 --- a/include/deal.II/lac/read_write_vector.templates.h +++ b/include/deal.II/lac/read_write_vector.templates.h @@ -1036,12 +1036,12 @@ namespace LinearAlgebra const MPI_Comm mpi_comm) { source_stored_elements = source_index_set; - TpetraWrappers::CommunicationPattern epetra_comm_pattern( + TpetraWrappers::CommunicationPattern tpetra_comm_pattern( source_stored_elements, stored_elements, mpi_comm); comm_pattern = std::make_shared( source_stored_elements, stored_elements, mpi_comm); - return epetra_comm_pattern; + return tpetra_comm_pattern; } # endif diff --git a/include/deal.II/lac/trilinos_tpetra_vector.h b/include/deal.II/lac/trilinos_tpetra_vector.h index 50a037292f..856bb7456e 100644 --- a/include/deal.II/lac/trilinos_tpetra_vector.h +++ b/include/deal.II/lac/trilinos_tpetra_vector.h @@ -260,6 +260,10 @@ namespace LinearAlgebra using MapType = Tpetra::Map; using VectorType = Tpetra::Vector; + using ExportType = + Tpetra::Export; + using ImportType = + Tpetra::Import; /** * @name 1: Basic Object-handling @@ -420,6 +424,32 @@ namespace LinearAlgebra reference operator()(const size_type index); + /** + * Provide read-only access to an element. + * + * When using a vector distributed with MPI, this operation only makes + * sense for elements that are actually present on the calling processor. + * Otherwise, an exception is thrown. + */ + Number + operator()(const size_type index) const; + + /** + * Provide access to a given element, both read and write. + * + * Exactly the same as operator(). + */ + reference + operator[](const size_type index); + + /** + * Provide read-only access to an element. + * + * Exactly the same as operator(). + */ + Number + operator[](const size_type index) const; + /** @} */ @@ -611,8 +641,7 @@ namespace LinearAlgebra /** @{ */ /** - * This function always returns false and is present only for backward - * compatibility. + * Return whether the vector has ghost elements or not. */ bool has_ghost_elements() const; @@ -631,6 +660,14 @@ namespace LinearAlgebra size_type locally_owned_size() const; + /** + * Return the state of the vector, i.e., whether compress() needs to be + * called after an operation requiring data exchange. A call to compress() + * is also needed when the method set() or add() has been called. + */ + bool + is_compressed() const; + /** * Return the underlying MPI communicator. */ @@ -664,6 +701,11 @@ namespace LinearAlgebra * necessary after writing into a vector element-by-element and before * anything else can be done on it. * + * @param operation The compress mode (Add or Insert) + * in case the vector has not been written to since the last time this + * function was called. The argument is ignored if the vector has been + * added or written to since the last time compress() was called. + * * See * @ref GlossCompress "Compressing distributed objects" * for more information. @@ -739,6 +781,47 @@ namespace LinearAlgebra */ DeclException0(ExcVectorTypeNotCompatible); + /* + * Access to a an element that is not (locally-)owned. + * + * @ingroup Exceptions + */ + DeclException4( + ExcAccessToNonLocalElement, + size_type, + size_type, + size_type, + size_type, + << "You are trying to access element " << arg1 + << " of a distributed vector, but this element is not stored " + << "on the current processor. Note: There are " << arg2 + << " elements stored " + << "on the current processor from within the range [" << arg3 << ',' + << arg4 << "] but Trilinos vectors need not store contiguous " + << "ranges on each processor, and not every element in " + << "this range may in fact be stored locally." + << "\n\n" + << "A common source for this kind of problem is that you " + << "are passing a 'fully distributed' vector into a function " + << "that needs read access to vector elements that correspond " + << "to degrees of freedom on ghost cells (or at least to " + << "'locally active' degrees of freedom that are not also " + << "'locally owned'). You need to pass a vector that has these " + << "elements as ghost entries."); + + /** + * Missing index set. + * + * @ingroup Exceptions + */ + DeclExceptionMsg(ExcMissingIndexSet, + "To compress a vector, a locally_relevant_dofs " + "index set, and a locally_owned_dofs index set " + "must be provided. These index sets must be " + "provided either when the vector is initialized " + "or when compress is called. See the documentation " + "of compress() for more information."); + /** * Exception thrown by an error in Trilinos. * @@ -759,15 +842,38 @@ namespace LinearAlgebra create_tpetra_comm_pattern(const IndexSet &source_index_set, const MPI_Comm mpi_comm); + /** + * Store whether the vector has ghost elements or not. + * + * If the vector has no ghost elements, it can only access and modify + * entries included in the locally owned index set. + * And if the vector has ghost elements it can access and modify + * entries included in the locally relevant index set. + */ + bool has_ghost; + /** * Teuchos::RCP to the actual Tpetra vector object. */ Teuchos::RCP vector; + /** + * Pointer to the Tpetra::Map storing the owned elemenets + * per rank. This map has a one-to-one mapping. + */ + Teuchos::RCP locally_owned_map; + + /** + * Pointer to the Tpetra::Map storing the relevant elements + * per rank. There are entries in this map, that belong to multiple + * ranks. + */ + Teuchos::RCP locally_relevant_map; + /** * IndexSet of the elements of the last imported vector. */ - ::dealii::IndexSet source_stored_elements; + dealii::IndexSet source_stored_elements; /** * CommunicationPattern for the communication between the @@ -787,7 +893,16 @@ namespace LinearAlgebra inline bool Vector::has_ghost_elements() const { - return false; + return has_ghost; + } + + + + template + inline bool + Vector::is_compressed() const + { + return !has_ghost; } @@ -826,10 +941,27 @@ namespace LinearAlgebra for (size_type i = 0; i < n_elements; ++i) { - const size_type row = indices[i]; - const TrilinosWrappers::types::int_type local_row = + const size_type row = indices[i]; + TrilinosWrappers::types::int_type local_row = vector->getMap()->getLocalElement(row); +# if DEAL_II_TRILINOS_VERSION_GTE(14, 0, 0) + Assert( + local_row != Teuchos::OrdinalTraits::invalid(), + ExcAccessToNonLocalElement(row, + vector->getMap()->getLocalNumElements(), + vector->getMap()->getMinLocalIndex(), + vector->getMap()->getMaxLocalIndex())); +# else + Assert( + local_row != Teuchos::OrdinalTraits::invalid(), + ExcAccessToNonLocalElement(row, + vector->getMap()->getNodeNumElements(), + vector->getMap()->getMinLocalIndex(), + vector->getMap()->getMaxLocalIndex())); + +# endif + if (local_row != Teuchos::OrdinalTraits::invalid()) vector_1d(local_row) += values[i]; } @@ -863,10 +995,26 @@ namespace LinearAlgebra for (size_type i = 0; i < n_elements; ++i) { - const size_type row = indices[i]; - const TrilinosWrappers::types::int_type local_row = + const size_type row = indices[i]; + TrilinosWrappers::types::int_type local_row = vector->getMap()->getLocalElement(row); +# if DEAL_II_TRILINOS_VERSION_GTE(14, 0, 0) + Assert( + local_row != Teuchos::OrdinalTraits::invalid(), + ExcAccessToNonLocalElement(row, + vector->getMap()->getLocalNumElements(), + vector->getMap()->getMinLocalIndex(), + vector->getMap()->getMaxLocalIndex())); +# else + Assert( + local_row != Teuchos::OrdinalTraits::invalid(), + ExcAccessToNonLocalElement(row, + vector->getMap()->getNodeNumElements(), + vector->getMap()->getMinLocalIndex(), + vector->getMap()->getMaxLocalIndex())); +# endif + if (local_row != Teuchos::OrdinalTraits::invalid()) vector_1d(local_row) = values[i]; } @@ -887,6 +1035,20 @@ namespace LinearAlgebra return internal::VectorReference(*this, index); } + template + inline internal::VectorReference + Vector::operator[](const size_type index) + { + return operator()(index); + } + + template + inline Number + Vector::operator[](const size_type index) const + { + return operator()(index); + } + # ifndef DOXYGEN // VectorReference diff --git a/include/deal.II/lac/trilinos_tpetra_vector.templates.h b/include/deal.II/lac/trilinos_tpetra_vector.templates.h index ad5b04d76e..2a7f11edb6 100644 --- a/include/deal.II/lac/trilinos_tpetra_vector.templates.h +++ b/include/deal.II/lac/trilinos_tpetra_vector.templates.h @@ -47,6 +47,7 @@ namespace LinearAlgebra template Vector::Vector() : Subscriptor() + , has_ghost(false) , vector(Utilities::Trilinos::internal::make_rcp( Utilities::Trilinos::internal::make_rcp( 0, @@ -59,9 +60,12 @@ namespace LinearAlgebra template Vector::Vector(const Vector &V) : Subscriptor() + , has_ghost(V.has_ghost) , vector(Utilities::Trilinos::internal::make_rcp( V.trilinos_vector(), Teuchos::Copy)) + , locally_owned_map(V.locally_owned_map) + , locally_relevant_map(V.locally_relevant_map) {} @@ -69,8 +73,20 @@ namespace LinearAlgebra template Vector::Vector(const Teuchos::RCP V) : Subscriptor() + , has_ghost(false) , vector(V) - {} + { + if (vector->getMap()->isOneToOne()) + { + locally_owned_map = + Teuchos::rcp_const_cast(vector->getMap()); + } + else + { + locally_relevant_map = + Teuchos::rcp_const_cast(vector->getMap()); + } + } @@ -78,9 +94,13 @@ namespace LinearAlgebra Vector::Vector(const IndexSet ¶llel_partitioner, const MPI_Comm communicator) : Subscriptor() - , vector(Utilities::Trilinos::internal::make_rcp( - parallel_partitioner.make_tpetra_map_rcp(communicator, false))) - {} + , has_ghost(false) + , locally_owned_map( + parallel_partitioner.make_tpetra_map_rcp(communicator, false)) + { + vector = Utilities::Trilinos::internal::make_rcp( + locally_owned_map); + } @@ -90,12 +110,23 @@ namespace LinearAlgebra const MPI_Comm communicator, const bool omit_zeroing_entries) { - Teuchos::RCP input_map = + // release memory before reallocation + vector.reset(); + locally_owned_map.reset(); + locally_relevant_map.reset(); + source_stored_elements.clear(); + + // Here we assume, that the index set is non-overlapping. + // If the index set is overlapping the function + // make_tpetra_map_rcp() will throw an assert. + has_ghost = false; + locally_owned_map = parallel_partitioner.make_tpetra_map_rcp(communicator, false); - if (vector->getMap()->isSameAs(*input_map) == false) - vector = Utilities::Trilinos::internal::make_rcp(input_map); - else if (omit_zeroing_entries == false) + if (!vector->getMap()->isSameAs(*locally_owned_map)) + vector = Utilities::Trilinos::internal::make_rcp( + locally_owned_map); + if (!omit_zeroing_entries) { vector->putScalar(0.); } @@ -109,13 +140,23 @@ namespace LinearAlgebra const IndexSet &ghost_entries, const MPI_Comm communicator) { + // release memory before reallocation + vector.reset(); + locally_owned_map.reset(); + locally_relevant_map.reset(); + + locally_owned_map = + locally_owned_entries.make_tpetra_map_rcp(communicator, false); + IndexSet parallel_partitioner = locally_owned_entries; parallel_partitioner.add_indices(ghost_entries); - - Teuchos::RCP input_map = + locally_relevant_map = parallel_partitioner.make_tpetra_map_rcp(communicator, true); - vector = Utilities::Trilinos::internal::make_rcp(input_map); + vector = Utilities::Trilinos::internal::make_rcp( + locally_relevant_map); + + has_ghost = false; } @@ -139,11 +180,9 @@ namespace LinearAlgebra ArrayView &elements) const { AssertDimension(indices.size(), elements.size()); - const auto &vector = trilinos_vector(); - const auto &map = vector.getMap(); # if DEAL_II_TRILINOS_VERSION_GTE(13, 2, 0) - auto vector_2d = vector.template getLocalView( + auto vector_2d = vector->template getLocalView( Tpetra::Access::ReadOnly); # else /* @@ -156,16 +195,35 @@ namespace LinearAlgebra * Let us choose to simply ignore this problem for such an old * Trilinos version. */ - auto vector_2d = vector.template getLocalView(); + auto vector_2d = vector->template getLocalView(); # endif auto vector_1d = Kokkos::subview(vector_2d, Kokkos::ALL(), 0); for (unsigned int i = 0; i < indices.size(); ++i) { AssertIndexRange(indices[i], size()); - const auto trilinos_i = map->getLocalElement( - static_cast(indices[i])); - elements[i] = vector_1d(trilinos_i); + const size_type row = indices[i]; + TrilinosWrappers::types::int_type local_row = + vector->getMap()->getLocalElement(row); + + + Assert( + local_row != Teuchos::OrdinalTraits::invalid(), +# if DEAL_II_TRILINOS_VERSION_GTE(14, 0, 0) + ExcAccessToNonLocalElement(row, + vector->getMap()->getLocalNumElements(), + vector->getMap()->getMinLocalIndex(), + vector->getMap()->getMaxLocalIndex())); +# else + ExcAccessToNonLocalElement(row, + vector->getMap()->getNodeNumElements(), + vector->getMap()->getMinLocalIndex(), + vector->getMap()->getMaxLocalIndex())); + +# endif + + if (local_row != Teuchos::OrdinalTraits::invalid()) + elements[i] = vector_1d(local_row); } } @@ -179,22 +237,45 @@ namespace LinearAlgebra // - First case: both vectors have the same layout. // - Second case: both vectors have the same size but different layout. // - Third case: the vectors have different size. - if (vector->getMap()->isSameAs(*(V.trilinos_vector().getMap()))) - *vector = V.trilinos_vector(); + if (vector->getMap()->isSameAs(*V.vector->getMap())) + { + *vector = *V.vector; + } + else if (size() == V.size()) + { + // We expect the origin vector to have a one-to-one map, otherwise + // we can not call Import + Assert(V.vector->getMap()->isOneToOne(), + ExcMessage( + "You are trying to map one vector distributed " + "between processors, where some elements belong " + "to multiple processors, onto another distribution " + "pattern, where some elements belong to multiple " + "processors. It is unclear how to deal with elements " + "in the vector belonging to multiple processors. " + "Therefore, compress() must be called on this " + "vector first.")); + + Teuchos::RCP importer = + Tpetra::createImport(V.vector->getMap(), vector->getMap()); + + // Since we are distributing the vector from a one-to-one map + // we can always use the VectorOperation::insert / Tpetra::INSERT + // here. + vector->doImport(*V.vector, *importer, Tpetra::INSERT); + } else { - if (size() == V.size()) - { - Tpetra::Import data_exchange( - vector->getMap(), V.trilinos_vector().getMap()); - - vector->doImport(V.trilinos_vector(), - data_exchange, - Tpetra::REPLACE); - } - else - vector = Utilities::Trilinos::internal::make_rcp( - V.trilinos_vector()); + vector.reset(); + vector = Utilities::Trilinos::internal::make_rcp( + V.vector->getMap()); + Tpetra::deep_copy(*vector, *V.vector); + + has_ghost = V.has_ghost; + locally_owned_map = V.locally_owned_map; + locally_relevant_map = V.locally_relevant_map; + source_stored_elements = V.source_stored_elements; + tpetra_comm_pattern = V.tpetra_comm_pattern; } return *this; @@ -253,8 +334,8 @@ namespace LinearAlgebra "LinearAlgebra::TpetraWrappers::CommunicationPattern.")); } - Teuchos::RCP> - tpetra_export = tpetra_comm_pattern->get_tpetra_export_rcp(); + Teuchos::RCP tpetra_export = + tpetra_comm_pattern->get_tpetra_export_rcp(); VectorType source_vector(tpetra_export->getSourceMap()); @@ -280,12 +361,15 @@ namespace LinearAlgebra device_type::memory_space>(); # endif } + Tpetra::CombineMode tpetra_operation = Tpetra::ZERO; if (operation == VectorOperation::insert) - vector->doExport(source_vector, *tpetra_export, Tpetra::REPLACE); + tpetra_operation = Tpetra::INSERT; else if (operation == VectorOperation::add) - vector->doExport(source_vector, *tpetra_export, Tpetra::ADD); + tpetra_operation = Tpetra::ADD; else - AssertThrow(false, ExcNotImplemented()); + Assert(false, ExcNotImplemented()); + + vector->doExport(source_vector, *tpetra_export, tpetra_operation); } @@ -301,6 +385,7 @@ namespace LinearAlgebra } + template void Vector::import_elements(const ReadWriteVector &V, @@ -358,9 +443,8 @@ namespace LinearAlgebra // elements, maybe there is a better workaround. Tpetra::Vector dummy( vector->getMap(), false); - Tpetra::Import data_exchange( - V.trilinos_vector().getMap(), dummy.getMap()); - + ImportType data_exchange(V.trilinos_vector().getMap(), + dummy.getMap()); dummy.doImport(V.trilinos_vector(), data_exchange, Tpetra::INSERT); vector->update(1.0, dummy, 1.0); @@ -396,6 +480,45 @@ namespace LinearAlgebra + template + Number + Vector::operator()(const size_type index) const + { + // Get the local index + const TrilinosWrappers::types::int_type local_index = + vector->getMap()->getLocalElement( + static_cast(index)); + + Number value = 0.0; + + // If the element is not present on the current processor, we can't + // continue. This is the main difference to the el() function. + if (local_index == -1) + { +# if DEAL_II_TRILINOS_VERSION_GTE(14, 0, 0) + Assert( + false, + ExcAccessToNonLocalElement(index, + vector->getMap()->getLocalNumElements(), + vector->getMap()->getMinLocalIndex(), + vector->getMap()->getMaxLocalIndex())); +# else + Assert( + false, + ExcAccessToNonLocalElement(index, + vector->getMap()->getNodeNumElements(), + vector->getMap()->getMinLocalIndex(), + vector->getMap()->getMaxLocalIndex())); +# endif + } + else + value = vector->getData()[local_index]; + + return value; + } + + + template void Vector::add(const Number a) @@ -643,8 +766,43 @@ namespace LinearAlgebra template void - Vector::compress(const VectorOperation::values /*operation*/) - {} + Vector::compress(const VectorOperation::values operation) + { + Assert(has_ghost == false, + ExcMessage("Calling compress() is only useful if a vector " + "has been written into, but this is a vector with ghost " + "elements and consequently is read-only. It does " + "not make sense to call compress() for such " + "vectors.")); + + if (!vector->getMap()->isOneToOne()) + { + Tpetra::CombineMode tpetra_operation = Tpetra::ZERO; + if (operation == VectorOperation::insert) + tpetra_operation = Tpetra::INSERT; + else if (operation == VectorOperation::add) + tpetra_operation = Tpetra::ADD; + else + Assert(false, ExcNotImplemented()); + + if (locally_relevant_map.is_null()) + locally_relevant_map = + Teuchos::rcp_const_cast(vector->getMap()); + + Assert(!locally_relevant_map.is_null(), ExcMissingIndexSet()); + Assert(!locally_owned_map.is_null(), ExcMissingIndexSet()); + + VectorType dummy = VectorType(locally_owned_map.getConst()); + + Teuchos::RCP exporter = + Tpetra::createExport(locally_relevant_map.getConst(), + locally_owned_map.getConst()); + dummy.doExport(*vector, *exporter, tpetra_operation); + + *vector = dummy; + } + } + template @@ -714,13 +872,27 @@ namespace LinearAlgebra auto vector_1d = Kokkos::subview(vector_2d, Kokkos::ALL(), 0); const size_t local_length = vector->getLocalLength(); - if (across) - for (unsigned int i = 0; i < local_length; ++i) - out << vector_1d(i) << ' '; + if (size() != local_length) + { + out << "size:" << size() << " locally_owned_size:" << local_length + << " :" << std::endl; + for (size_type i = 0; i < local_length; ++i) + { + const TrilinosWrappers::types::int_type global_row = + vector->getMap()->getGlobalElement(i); + out << "[" << global_row << "]: " << vector_1d(i) << std::endl; + } + } else - for (unsigned int i = 0; i < local_length; ++i) - out << vector_1d(i) << std::endl; - out << std::endl; + { + if (across) + for (unsigned int i = 0; i < local_length; ++i) + out << vector_1d(i) << ' '; + else + for (unsigned int i = 0; i < local_length; ++i) + out << vector_1d(i) << std::endl; + out << std::endl; + } // restore the representation // of the vector @@ -754,10 +926,10 @@ namespace LinearAlgebra const MPI_Comm mpi_comm) { source_stored_elements = source_index_set; - tpetra_comm_pattern = Utilities::Trilinos::internal::make_rcp< - TpetraWrappers::CommunicationPattern>(locally_owned_elements(), - source_index_set, - mpi_comm); + + tpetra_comm_pattern = + Teuchos::rcp(new TpetraWrappers::CommunicationPattern( + locally_owned_elements(), source_index_set, mpi_comm)); } } // namespace TpetraWrappers } // namespace LinearAlgebra diff --git a/include/deal.II/numerics/data_out_dof_data.templates.h b/include/deal.II/numerics/data_out_dof_data.templates.h index 32ce2c2212..a811f44557 100644 --- a/include/deal.II/numerics/data_out_dof_data.templates.h +++ b/include/deal.II/numerics/data_out_dof_data.templates.h @@ -829,6 +829,21 @@ namespace internal } #endif +#ifdef DEAL_II_TRILINOS_WITH_TPETRA + template + void + copy_locally_owned_data_from( + const LinearAlgebra::TpetraWrappers::Vector &src, + LinearAlgebra::distributed::Vector &dst) + { + // ReadWriteVector does not work for ghosted + // TrilinosWrappers::MPI::Vector objects. Fall back to copy the + // entries manually. + for (const auto i : dst.locally_owned_elements()) + dst[i] = src[i]; + } +#endif + /** * Create a ghosted-copy of a block dof vector. */ diff --git a/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=off.mpirun=2.output b/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=off.mpirun=2.output index 638319c2f1..ba921e5ad6 100644 --- a/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=off.mpirun=2.output +++ b/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=off.mpirun=2.output @@ -1,13 +1,17 @@ DEAL:0::Tpetra first import add: -2.000e+00 1.000e+02 +size:4 locally_owned_size:2 : +[0]: 2.000e+00 +[1]: 1.000e+02 IndexSet: {[0,2]} [0]: 1.000e+00 [1]: 1.000e+02 [2]: 1.000e+01 DEAL:0::Tpetra second import add: -4.000e+00 2.000e+02 +size:4 locally_owned_size:2 : +[0]: 4.000e+00 +[1]: 2.000e+02 IndexSet: {[0,2]} [0]: 2.000e+00 @@ -33,13 +37,17 @@ IndexSet: {[0,2]} [2]: 1.000e+02 DEAL:1::Tpetra first import add: -1.000e+01 1.010e+02 +size:4 locally_owned_size:2 : +[2]: 1.000e+01 +[3]: 1.010e+02 IndexSet: {0, 3} [0]: 1.000e+00 [3]: 1.010e+02 DEAL:1::Tpetra second import add: -3.000e+01 2.010e+02 +size:4 locally_owned_size:2 : +[2]: 3.000e+01 +[3]: 2.010e+02 IndexSet: {0, 3} [0]: 2.000e+00 diff --git a/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=off.mpirun=4.output b/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=off.mpirun=4.output index f70ed13b09..f643d65be1 100644 --- a/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=off.mpirun=4.output +++ b/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=off.mpirun=4.output @@ -1,13 +1,17 @@ DEAL:0::Tpetra first import add: -4.000e+00 1.000e+02 +size:8 locally_owned_size:2 : +[0]: 4.000e+00 +[1]: 1.000e+02 IndexSet: {[0,2]} [0]: 1.000e+00 [1]: 1.000e+02 [2]: 1.000e+01 DEAL:0::Tpetra second import add: -8.000e+00 2.000e+02 +size:8 locally_owned_size:2 : +[0]: 8.000e+00 +[1]: 2.000e+02 IndexSet: {[0,2]} [0]: 2.000e+00 @@ -33,14 +37,18 @@ IndexSet: {[0,2]} [2]: 1.000e+02 DEAL:1::Tpetra first import add: -1.000e+01 1.010e+02 +size:8 locally_owned_size:2 : +[2]: 1.000e+01 +[3]: 1.010e+02 IndexSet: {0, [3,4]} [0]: 1.000e+00 [3]: 1.010e+02 [4]: 1.100e+01 DEAL:1::Tpetra second import add: -3.000e+01 2.010e+02 +size:8 locally_owned_size:2 : +[2]: 3.000e+01 +[3]: 2.010e+02 IndexSet: {0, [3,4]} [0]: 2.000e+00 @@ -67,14 +75,18 @@ IndexSet: {0, [3,4]} DEAL:2::Tpetra first import add: -1.100e+01 1.020e+02 +size:8 locally_owned_size:2 : +[4]: 1.100e+01 +[5]: 1.020e+02 IndexSet: {0, [5,6]} [0]: 1.000e+00 [5]: 1.020e+02 [6]: 1.200e+01 DEAL:2::Tpetra second import add: -3.200e+01 2.020e+02 +size:8 locally_owned_size:2 : +[4]: 3.200e+01 +[5]: 2.020e+02 IndexSet: {0, [5,6]} [0]: 2.000e+00 @@ -101,13 +113,17 @@ IndexSet: {0, [5,6]} DEAL:3::Tpetra first import add: -1.200e+01 1.030e+02 +size:8 locally_owned_size:2 : +[6]: 1.200e+01 +[7]: 1.030e+02 IndexSet: {0, 7} [0]: 1.000e+00 [7]: 1.030e+02 DEAL:3::Tpetra second import add: -3.400e+01 2.030e+02 +size:8 locally_owned_size:2 : +[6]: 3.400e+01 +[7]: 2.030e+02 IndexSet: {0, 7} [0]: 2.000e+00 diff --git a/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=on.mpirun=2.output b/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=on.mpirun=2.output index 638319c2f1..ba921e5ad6 100644 --- a/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=on.mpirun=2.output +++ b/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=on.mpirun=2.output @@ -1,13 +1,17 @@ DEAL:0::Tpetra first import add: -2.000e+00 1.000e+02 +size:4 locally_owned_size:2 : +[0]: 2.000e+00 +[1]: 1.000e+02 IndexSet: {[0,2]} [0]: 1.000e+00 [1]: 1.000e+02 [2]: 1.000e+01 DEAL:0::Tpetra second import add: -4.000e+00 2.000e+02 +size:4 locally_owned_size:2 : +[0]: 4.000e+00 +[1]: 2.000e+02 IndexSet: {[0,2]} [0]: 2.000e+00 @@ -33,13 +37,17 @@ IndexSet: {[0,2]} [2]: 1.000e+02 DEAL:1::Tpetra first import add: -1.000e+01 1.010e+02 +size:4 locally_owned_size:2 : +[2]: 1.000e+01 +[3]: 1.010e+02 IndexSet: {0, 3} [0]: 1.000e+00 [3]: 1.010e+02 DEAL:1::Tpetra second import add: -3.000e+01 2.010e+02 +size:4 locally_owned_size:2 : +[2]: 3.000e+01 +[3]: 2.010e+02 IndexSet: {0, 3} [0]: 2.000e+00 diff --git a/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=on.mpirun=4.output b/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=on.mpirun=4.output index f70ed13b09..f643d65be1 100644 --- a/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=on.mpirun=4.output +++ b/tests/trilinos/tpetra_vector_03.with_trilinos_with_tpetra=on.with_complex_values=on.mpirun=4.output @@ -1,13 +1,17 @@ DEAL:0::Tpetra first import add: -4.000e+00 1.000e+02 +size:8 locally_owned_size:2 : +[0]: 4.000e+00 +[1]: 1.000e+02 IndexSet: {[0,2]} [0]: 1.000e+00 [1]: 1.000e+02 [2]: 1.000e+01 DEAL:0::Tpetra second import add: -8.000e+00 2.000e+02 +size:8 locally_owned_size:2 : +[0]: 8.000e+00 +[1]: 2.000e+02 IndexSet: {[0,2]} [0]: 2.000e+00 @@ -33,14 +37,18 @@ IndexSet: {[0,2]} [2]: 1.000e+02 DEAL:1::Tpetra first import add: -1.000e+01 1.010e+02 +size:8 locally_owned_size:2 : +[2]: 1.000e+01 +[3]: 1.010e+02 IndexSet: {0, [3,4]} [0]: 1.000e+00 [3]: 1.010e+02 [4]: 1.100e+01 DEAL:1::Tpetra second import add: -3.000e+01 2.010e+02 +size:8 locally_owned_size:2 : +[2]: 3.000e+01 +[3]: 2.010e+02 IndexSet: {0, [3,4]} [0]: 2.000e+00 @@ -67,14 +75,18 @@ IndexSet: {0, [3,4]} DEAL:2::Tpetra first import add: -1.100e+01 1.020e+02 +size:8 locally_owned_size:2 : +[4]: 1.100e+01 +[5]: 1.020e+02 IndexSet: {0, [5,6]} [0]: 1.000e+00 [5]: 1.020e+02 [6]: 1.200e+01 DEAL:2::Tpetra second import add: -3.200e+01 2.020e+02 +size:8 locally_owned_size:2 : +[4]: 3.200e+01 +[5]: 2.020e+02 IndexSet: {0, [5,6]} [0]: 2.000e+00 @@ -101,13 +113,17 @@ IndexSet: {0, [5,6]} DEAL:3::Tpetra first import add: -1.200e+01 1.030e+02 +size:8 locally_owned_size:2 : +[6]: 1.200e+01 +[7]: 1.030e+02 IndexSet: {0, 7} [0]: 1.000e+00 [7]: 1.030e+02 DEAL:3::Tpetra second import add: -3.400e+01 2.030e+02 +size:8 locally_owned_size:2 : +[6]: 3.400e+01 +[7]: 2.030e+02 IndexSet: {0, 7} [0]: 2.000e+00