From cd9568be790ce27003158d6acdcecc851ea17085 Mon Sep 17 00:00:00 2001 From: Wolfgang Bangerth Date: Fri, 7 Aug 2009 20:34:09 +0000 Subject: [PATCH] Use a separate communicator for Trilinos objects by forking the existing communicator into a unique and separate one. git-svn-id: https://svn.dealii.org/trunk@19212 0785d39b-7218-0410-832d-ea1e28bc413d --- .../lac/include/lac/trilinos_sparse_matrix.h | 15 +- deal.II/lac/source/trilinos_sparse_matrix.cc | 131 +++++++++--------- 2 files changed, 79 insertions(+), 67 deletions(-) diff --git a/deal.II/lac/include/lac/trilinos_sparse_matrix.h b/deal.II/lac/include/lac/trilinos_sparse_matrix.h index a288fa6fb0..a09d7b98f0 100755 --- a/deal.II/lac/include/lac/trilinos_sparse_matrix.h +++ b/deal.II/lac/include/lac/trilinos_sparse_matrix.h @@ -20,11 +20,13 @@ #include #include -#include #include #include #include +#include + + #ifdef DEAL_II_USE_TRILINOS # define TrilinosScalar double @@ -1906,6 +1908,17 @@ namespace TrilinosWrappers private: + + /** + * A pointer to the communicator used + * for all operations in this object. + * + * Note that we create a new + * communicator (with a unique MPI ID) + * for each object if we are running in + * parallel. + */ + boost::scoped_ptr communicator; /** * Epetra Trilinos diff --git a/deal.II/lac/source/trilinos_sparse_matrix.cc b/deal.II/lac/source/trilinos_sparse_matrix.cc index 86d04fa04e..34df2630fc 100755 --- a/deal.II/lac/source/trilinos_sparse_matrix.cc +++ b/deal.II/lac/source/trilinos_sparse_matrix.cc @@ -13,6 +13,7 @@ #include +#include #include #include #include @@ -90,12 +91,10 @@ namespace TrilinosWrappers // interface. SparseMatrix::SparseMatrix () : -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - row_map (0, 0, Epetra_MpiComm(MPI_COMM_SELF)), -#else - row_map (0, 0, Epetra_SerialComm()), -#endif - col_map (row_map), + communicator (Utilities::Trilinos::duplicate_communicator + (Utilities::Trilinos::comm_self())), + row_map (0, 0, *communicator), + col_map (0, 0, *communicator), last_action (Zero), compressed (true), matrix (std::auto_ptr @@ -109,7 +108,8 @@ namespace TrilinosWrappers SparseMatrix::SparseMatrix (const Epetra_Map &InputMap, const unsigned int n_max_entries_per_row) : - row_map (InputMap), + communicator (Utilities::Trilinos::duplicate_communicator (InputMap.Comm())), + row_map (Utilities::Trilinos::duplicate_map (InputMap, *communicator)), col_map (row_map), last_action (Zero), compressed (true), @@ -123,7 +123,8 @@ namespace TrilinosWrappers SparseMatrix::SparseMatrix (const Epetra_Map &InputMap, const std::vector &n_entries_per_row) : - row_map (InputMap), + communicator (Utilities::Trilinos::duplicate_communicator (InputMap.Comm())), + row_map (Utilities::Trilinos::duplicate_map (InputMap, *communicator)), col_map (row_map), last_action (Zero), compressed (true), @@ -139,8 +140,9 @@ namespace TrilinosWrappers const Epetra_Map &InputColMap, const unsigned int n_max_entries_per_row) : - row_map (InputRowMap), - col_map (InputColMap), + communicator (Utilities::Trilinos::duplicate_communicator (InputRowMap.Comm())), + row_map (Utilities::Trilinos::duplicate_map (InputRowMap, *communicator)), + col_map (Utilities::Trilinos::duplicate_map (InputColMap, *communicator)), last_action (Zero), compressed (true), matrix (std::auto_ptr @@ -154,8 +156,9 @@ namespace TrilinosWrappers const Epetra_Map &InputColMap, const std::vector &n_entries_per_row) : - row_map (InputRowMap), - col_map (InputColMap), + communicator (Utilities::Trilinos::duplicate_communicator (InputRowMap.Comm())), + row_map (Utilities::Trilinos::duplicate_map (InputRowMap, *communicator)), + col_map (Utilities::Trilinos::duplicate_map (InputColMap, *communicator)), last_action (Zero), compressed (true), matrix (std::auto_ptr @@ -170,13 +173,10 @@ namespace TrilinosWrappers const unsigned int n, const unsigned int n_max_entries_per_row) : -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - row_map (m, 0, Epetra_MpiComm(MPI_COMM_SELF)), - col_map (n, 0, Epetra_MpiComm(MPI_COMM_SELF)), -#else - row_map (m, 0, Epetra_SerialComm()), - col_map (n, 0, Epetra_SerialComm()), -#endif + communicator (Utilities::Trilinos::duplicate_communicator + (Utilities::Trilinos::comm_self())), + row_map (m, 0, *communicator), + col_map (n, 0, *communicator), last_action (Zero), compressed (true), // on one processor only, we know how the @@ -199,13 +199,10 @@ namespace TrilinosWrappers const unsigned int n, const std::vector &n_entries_per_row) : -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - row_map (m, 0, Epetra_MpiComm(MPI_COMM_SELF)), - col_map (n, 0, Epetra_MpiComm(MPI_COMM_SELF)), -#else - row_map (m, 0, Epetra_SerialComm()), - col_map (n, 0, Epetra_SerialComm()), -#endif + communicator (Utilities::Trilinos::duplicate_communicator + (Utilities::Trilinos::comm_self())), + row_map (m, 0, *communicator), + col_map (n, 0, *communicator), last_action (Zero), compressed (true), matrix (std::auto_ptr @@ -219,8 +216,14 @@ namespace TrilinosWrappers SparseMatrix::SparseMatrix (const SparsityPattern &InputSP) : Subscriptor(), - row_map (InputSP.range_partitioner()), - col_map (InputSP.domain_partitioner()), + communicator (Utilities::Trilinos::duplicate_communicator + (InputSP.range_partitioner().Comm())), + row_map (Utilities::Trilinos:: + duplicate_map (InputSP.range_partitioner(), + *communicator)), + col_map (Utilities::Trilinos:: + duplicate_map (InputSP.domain_partitioner(), + *communicator)), last_action (Zero), compressed (true), matrix (std::auto_ptr @@ -238,8 +241,14 @@ namespace TrilinosWrappers SparseMatrix::SparseMatrix (const SparseMatrix &InputMatrix) : Subscriptor(), - row_map (InputMatrix.row_map), - col_map (InputMatrix.col_map), + communicator (Utilities::Trilinos::duplicate_communicator + (InputMatrix.row_map.Comm())), + row_map (Utilities::Trilinos:: + duplicate_map (InputMatrix.row_map, + *communicator)), + col_map (Utilities::Trilinos:: + duplicate_map (InputMatrix.col_map, + *communicator)), last_action (Zero), compressed (true), matrix (std::auto_ptr @@ -256,8 +265,8 @@ namespace TrilinosWrappers SparseMatrix & SparseMatrix::copy_from (const SparseMatrix &m) { - row_map = m.row_map; - col_map = m.col_map; + row_map = Utilities::Trilinos::duplicate_map (m.row_map, *communicator); + col_map = Utilities::Trilinos::duplicate_map (m.col_map, *communicator); // check whether we need to update the // partitioner or can just copy the data: // in case we have the same distribution, @@ -277,18 +286,12 @@ namespace TrilinosWrappers void SparseMatrix::reinit (const SparsityType &sparsity_pattern) { -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - Epetra_MpiComm trilinos_communicator (MPI_COMM_SELF); -#else - Epetra_SerialComm trilinos_communicator; -#endif - const Epetra_Map rows (sparsity_pattern.n_rows(), 0, - trilinos_communicator); + Utilities::Trilinos::comm_self()); const Epetra_Map columns (sparsity_pattern.n_cols(), 0, - trilinos_communicator); + Utilities::Trilinos::comm_self()); reinit (rows, columns, sparsity_pattern); } @@ -325,8 +328,8 @@ namespace TrilinosWrappers sparsity_pattern.n_cols())); } - row_map = input_row_map; - col_map = input_col_map; + row_map = Utilities::Trilinos::duplicate_map (input_row_map, *communicator); + col_map = Utilities::Trilinos::duplicate_map (input_col_map, *communicator); std::vector n_entries_per_row(n_rows); @@ -447,8 +450,8 @@ namespace TrilinosWrappers sparsity_pattern.n_cols())); } - row_map = input_row_map; - col_map = input_col_map; + row_map = Utilities::Trilinos::duplicate_map (input_row_map, *communicator); + col_map = Utilities::Trilinos::duplicate_map (input_col_map, *communicator); std::vector n_entries_per_row(n_rows); @@ -511,8 +514,10 @@ namespace TrilinosWrappers // sparsity pattern. matrix.reset(); - row_map = sparsity_pattern.range_partitioner(); - col_map = sparsity_pattern.domain_partitioner(); + row_map = Utilities::Trilinos::duplicate_map (sparsity_pattern.range_partitioner(), + *communicator); + col_map = Utilities::Trilinos::duplicate_map (sparsity_pattern.domain_partitioner(), + *communicator); AssertThrow (sparsity_pattern.trilinos_sparsity_pattern().Filled() == true, ExcMessage("The Trilinos sparsity pattern has not been compressed")); @@ -530,8 +535,11 @@ namespace TrilinosWrappers { matrix.reset(); - row_map = sparse_matrix.row_map; - col_map = sparse_matrix.col_map; + row_map = Utilities::Trilinos::duplicate_map (sparse_matrix.range_partitioner(), + *communicator); + col_map = Utilities::Trilinos::duplicate_map (sparse_matrix.domain_partitioner(), + *communicator); + matrix = std::auto_ptr (new Epetra_FECrsMatrix(Copy, sparse_matrix.trilinos_sparsity_pattern(), false)); @@ -547,18 +555,12 @@ namespace TrilinosWrappers const double drop_tolerance, const bool copy_values) { -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - Epetra_MpiComm trilinos_communicator (MPI_COMM_SELF); -#else - Epetra_SerialComm trilinos_communicator; -#endif - const Epetra_Map rows (dealii_sparse_matrix.m(), 0, - trilinos_communicator); + Utilities::Trilinos::comm_self()); const Epetra_Map columns (dealii_sparse_matrix.n(), 0, - trilinos_communicator); + Utilities::Trilinos::comm_self()); reinit (rows, columns, dealii_sparse_matrix, drop_tolerance, copy_values); } @@ -605,8 +607,8 @@ namespace TrilinosWrappers ExcDimensionMismatch (input_col_map.NumGlobalElements(), dealii_sparse_matrix.n())); - row_map = input_row_map; - col_map = input_col_map; + row_map = Utilities::Trilinos::duplicate_map (input_row_map, *communicator); + col_map = Utilities::Trilinos::duplicate_map (input_col_map, *communicator); std::vector n_entries_per_row(n_rows); @@ -656,8 +658,8 @@ namespace TrilinosWrappers Assert (input_matrix.Filled()==true, ExcMessage("Input CrsMatrix has not called FillComplete()!")); - row_map = input_matrix.RangeMap(); - col_map = input_matrix.DomainMap(); + row_map = Utilities::Trilinos::duplicate_map (input_matrix.RangeMap(), *communicator); + col_map = Utilities::Trilinos::duplicate_map (input_matrix.DomainMap(), *communicator); const Epetra_CrsGraph *graph = &input_matrix.Graph(); @@ -687,12 +689,9 @@ namespace TrilinosWrappers // the pointer and generate an // empty matrix. matrix.reset(); - -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - row_map = Epetra_Map (0, 0, Epetra_MpiComm(MPI_COMM_SELF)); -#else - row_map = Epetra_Map (0, 0, Epetra_SerialComm()); -#endif + + row_map = Epetra_Map (0, 0, + Utilities::Trilinos::comm_self()); col_map = row_map; -- 2.39.5