From: kronbichler Date: Thu, 4 Sep 2008 18:49:25 +0000 (+0000) Subject: Updated a few things to make parallel run possible. Some more things will need to... X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=846f44321f51c5f9c4fbad9d448f4dcb91823028;p=dealii-svn.git Updated a few things to make parallel run possible. Some more things will need to be changed, though. git-svn-id: https://svn.dealii.org/trunk@16746 0785d39b-7218-0410-832d-ea1e28bc413d --- diff --git a/deal.II/examples/step-32/step-32.cc b/deal.II/examples/step-32/step-32.cc index 661ce22747..c3ffe8fc17 100644 --- a/deal.II/examples/step-32/step-32.cc +++ b/deal.II/examples/step-32/step-32.cc @@ -261,9 +261,6 @@ namespace LinearSolvers const SmartPointer > m_inverse; const PreconditionerA &a_preconditioner; - - mutable TrilinosWrappers::Vector tmp; - }; @@ -276,8 +273,7 @@ namespace LinearSolvers : stokes_matrix (&S), m_inverse (&Mpinv), - a_preconditioner (Apreconditioner), - tmp (stokes_matrix->block(1,1).row_map) + a_preconditioner (Apreconditioner) {} template @@ -286,6 +282,7 @@ namespace LinearSolvers const TrilinosWrappers::BlockVector &src) const { a_preconditioner.vmult (dst.block(0), src.block(0)); + TrilinosWrappers::Vector tmp (src.block(1), true); stokes_matrix->block(1,0).residual(tmp, dst.block(0), src.block(1)); tmp *= -1; m_inverse->vmult (dst.block(1), tmp); @@ -630,6 +627,8 @@ void BoussinesqFlowProblem::setup_dofs () { stokes_matrix.clear (); + BlockSparsityPattern stokes_sparsity_pattern (2,2); + if (trilinos_communicator.MyPID() == 0) { BlockCompressedSetSparsityPattern csp (2,2); @@ -653,13 +652,10 @@ void BoussinesqFlowProblem::setup_dofs () DoFTools::make_sparsity_pattern (stokes_dof_handler, coupling, csp); stokes_constraints.condense (csp); - BlockSparsityPattern stokes_sparsity_pattern; stokes_sparsity_pattern.copy_from (csp); - - stokes_matrix.reinit (stokes_partitioner, stokes_sparsity_pattern); } - stokes_matrix.collect_sizes(); + stokes_matrix.reinit (stokes_partitioner, stokes_sparsity_pattern); } { @@ -667,6 +663,8 @@ void BoussinesqFlowProblem::setup_dofs () Mp_preconditioner.reset (); stokes_preconditioner_matrix.clear (); + BlockSparsityPattern stokes_preconditioner_sparsity_pattern (2,2); + if (trilinos_communicator.MyPID() == 0) { BlockCompressedSetSparsityPattern csp (2,2); @@ -689,15 +687,14 @@ void BoussinesqFlowProblem::setup_dofs () DoFTools::make_sparsity_pattern (stokes_dof_handler, coupling, csp); stokes_constraints.condense (csp); - BlockSparsityPattern stokes_preconditioner_sparsity_pattern; stokes_preconditioner_sparsity_pattern.copy_from (csp); - - stokes_preconditioner_matrix.reinit (stokes_partitioner, - stokes_preconditioner_sparsity_pattern); } - stokes_preconditioner_matrix.collect_sizes(); + stokes_preconditioner_matrix.reinit (stokes_partitioner, + stokes_preconditioner_sparsity_pattern); + } + std::cout << " bla " << std::flush; temperature_partitioner = Epetra_Map (n_T, 0, trilinos_communicator); { @@ -705,25 +702,23 @@ void BoussinesqFlowProblem::setup_dofs () temperature_stiffness_matrix.clear (); temperature_matrix.clear (); + SparsityPattern temperature_sparsity_pattern; + if (trilinos_communicator.MyPID() == 0) { CompressedSetSparsityPattern csp (n_T, n_T); DoFTools::make_sparsity_pattern (temperature_dof_handler, csp); temperature_constraints.condense (csp); - SparsityPattern temperature_sparsity_pattern; temperature_sparsity_pattern.copy_from (csp); - - temperature_matrix.reinit (temperature_partitioner, - temperature_sparsity_pattern); - temperature_mass_matrix.reinit (temperature_partitioner, - temperature_sparsity_pattern); - temperature_stiffness_matrix.reinit (temperature_partitioner, - temperature_sparsity_pattern); } - temperature_matrix.compress(); - temperature_mass_matrix.compress(); - temperature_stiffness_matrix.compress(); + + temperature_matrix.reinit (temperature_partitioner, + temperature_sparsity_pattern); + temperature_mass_matrix.reinit (temperature_partitioner, + temperature_sparsity_pattern); + temperature_stiffness_matrix.reinit (temperature_partitioner, + temperature_sparsity_pattern); } stokes_solution.reinit (stokes_partitioner); @@ -789,8 +784,8 @@ BoussinesqFlowProblem::assemble_stokes_preconditioner () cell->get_dof_indices (local_dof_indices); stokes_constraints.distribute_local_to_global (local_matrix, - local_dof_indices, - stokes_preconditioner_matrix); + local_dof_indices, + stokes_preconditioner_matrix); } stokes_preconditioner_matrix.compress(); @@ -1529,7 +1524,7 @@ void BoussinesqFlowProblem::run () old_old_temperature_solution = old_temperature_solution; old_temperature_solution = temperature_solution; } - while (time <= 100); + while (time <= 1); } @@ -1544,6 +1539,7 @@ int main (int argc, char *argv[]) (void)argv; #endif + sleep (20); try { deallog.depth_console (0); diff --git a/deal.II/lac/include/lac/trilinos_sparse_matrix.h b/deal.II/lac/include/lac/trilinos_sparse_matrix.h index 14ba1f4784..5cda675d0e 100755 --- a/deal.II/lac/include/lac/trilinos_sparse_matrix.h +++ b/deal.II/lac/include/lac/trilinos_sparse_matrix.h @@ -1093,9 +1093,10 @@ namespace TrilinosWrappers int, int, << "You tried to access element (" << arg1 << "/" << arg2 << ")" - << " of a sparse matrix, but it appears to not " + << " of a sparse matrix, but it appears to not" << " exist in the Trilinos sparsity pattern."); + private: /** * Epetra Trilinos mapping of the * matrix rows that @@ -1115,18 +1116,6 @@ namespace TrilinosWrappers */ Epetra_Map col_map; - /** - * A sparse matrix object in - * Trilinos to be used for - * finite element based problems - * which allows for assembling into - * non-local elements. - * The actual type, a sparse - * matrix, is set in the constructor. - */ - std::auto_ptr matrix; - - protected: /** * Trilinos doesn't allow to mix additions * to matrix entries and overwriting @@ -1148,6 +1137,19 @@ namespace TrilinosWrappers * or an insertion. */ Epetra_CombineMode last_action; + + public: + /** + * A sparse matrix object in + * Trilinos to be used for + * finite element based problems + * which allows for assembling into + * non-local elements. + * The actual type, a sparse + * matrix, is set in the constructor. + */ + std::auto_ptr matrix; + }; diff --git a/deal.II/lac/include/lac/trilinos_vector.h b/deal.II/lac/include/lac/trilinos_vector.h index f50e788425..4e3fe40250 100755 --- a/deal.II/lac/include/lac/trilinos_vector.h +++ b/deal.II/lac/include/lac/trilinos_vector.h @@ -83,7 +83,7 @@ namespace TrilinosWrappers * as to only allow the actual vector * class to create it. */ - VectorReference (Vector &vector, + VectorReference (Vector &vector, const unsigned int index); public: @@ -862,6 +862,7 @@ namespace TrilinosWrappers << " are stored locally and can be accessed."); + private: /** * An Epetra map used to map vector data * accross multiple processes. This is @@ -871,16 +872,7 @@ namespace TrilinosWrappers */ Epetra_Map map; - /** - * An Epetra distibuted vector type. - * Requires an existing Epetra_Map for - * storing data. - */ - std::auto_ptr vector; - - - private: - /** + /** * Trilinos doesn't allow to mix additions * to matrix entries and overwriting * them (to make synchronisation of @@ -902,6 +894,17 @@ namespace TrilinosWrappers */ Epetra_CombineMode last_action; + public: + /** + * An Epetra distibuted vector type. + * Requires an existing Epetra_Map for + * storing data. + * TODO: Should become private at some + * point. + */ + std::auto_ptr vector; + + /** * Make the reference class a friend. */ diff --git a/deal.II/lac/source/trilinos_block_sparse_matrix.cc b/deal.II/lac/source/trilinos_block_sparse_matrix.cc index 90731a9cc4..1b8d171490 100644 --- a/deal.II/lac/source/trilinos_block_sparse_matrix.cc +++ b/deal.II/lac/source/trilinos_block_sparse_matrix.cc @@ -95,12 +95,15 @@ namespace TrilinosWrappers { const unsigned int n_block_rows = input_maps.size(); - Assert (n_block_rows == block_sparsity_pattern.n_block_rows(), - ExcDimensionMismatch (n_block_rows, - block_sparsity_pattern.n_block_rows())); - Assert (n_block_rows == block_sparsity_pattern.n_block_cols(), - ExcDimensionMismatch (n_block_rows, - block_sparsity_pattern.n_block_cols())); + if (input_maps[0].Comm().MyPID()==0) + { + Assert (n_block_rows == block_sparsity_pattern.n_block_rows(), + ExcDimensionMismatch (n_block_rows, + block_sparsity_pattern.n_block_rows())); + Assert (n_block_rows == block_sparsity_pattern.n_block_cols(), + ExcDimensionMismatch (n_block_rows, + block_sparsity_pattern.n_block_cols())); + } // Call the other basic reinit function ... reinit (n_block_rows, n_block_rows); @@ -113,6 +116,8 @@ namespace TrilinosWrappers this->block(r,c).reinit(input_maps[r],input_maps[c], block_sparsity_pattern.block(r,c)); } + + collect_sizes(); } @@ -144,6 +149,8 @@ namespace TrilinosWrappers dealii_block_sparse_matrix.block(r,c), drop_tolerance); } + + collect_sizes(); } diff --git a/deal.II/lac/source/trilinos_precondition_amg.cc b/deal.II/lac/source/trilinos_precondition_amg.cc index 95c98345fd..d9c44149a1 100755 --- a/deal.II/lac/source/trilinos_precondition_amg.cc +++ b/deal.II/lac/source/trilinos_precondition_amg.cc @@ -175,11 +175,11 @@ namespace TrilinosWrappers void PreconditionAMG::vmult (dealii::Vector &dst, const dealii::Vector &src) const { - Assert (Map->SameAs(Matrix->row_map), + Assert (Map->SameAs(Matrix->matrix->RowMap()), ExcMessage("The sparse matrix given to the preconditioner uses " "a map that is not compatible. Check ML preconditioner " "setup.")); - Assert (Map->SameAs(Matrix->col_map), + Assert (Map->SameAs(Matrix->matrix->RowMap()), ExcMessage("The sparse matrix given to the preconditioner uses " "a map that is not compatible. Check ML preconditioner " "setup.")); diff --git a/deal.II/lac/source/trilinos_sparse_matrix.cc b/deal.II/lac/source/trilinos_sparse_matrix.cc index 1bea1a6616..14075ee65e 100755 --- a/deal.II/lac/source/trilinos_sparse_matrix.cc +++ b/deal.II/lac/source/trilinos_sparse_matrix.cc @@ -85,9 +85,9 @@ namespace TrilinosWrappers row_map (0, 0, Epetra_SerialComm()), #endif col_map (row_map), + last_action (Insert), matrix (std::auto_ptr - (new Epetra_FECrsMatrix(Copy, row_map, 0))), - last_action (Insert) + (new Epetra_FECrsMatrix(Copy, row_map, 0))) {} SparseMatrix::SparseMatrix (const Epetra_Map &InputMap, @@ -95,10 +95,10 @@ namespace TrilinosWrappers : row_map (InputMap), col_map (row_map), + last_action (Insert), matrix (std::auto_ptr (new Epetra_FECrsMatrix(Copy, row_map, - int(n_max_entries_per_row), false))), - last_action (Insert) + int(n_max_entries_per_row), false))) {} SparseMatrix::SparseMatrix (const Epetra_Map &InputMap, @@ -106,11 +106,11 @@ namespace TrilinosWrappers : row_map (InputMap), col_map (row_map), + last_action (Insert), matrix (std::auto_ptr (new Epetra_FECrsMatrix(Copy, row_map, (int*)const_cast(&(n_entries_per_row[0])), - true))), - last_action (Insert) + true))) {} SparseMatrix::SparseMatrix (const Epetra_Map &InputRowMap, @@ -119,10 +119,10 @@ namespace TrilinosWrappers : row_map (InputRowMap), col_map (InputColMap), + last_action (Insert), matrix (std::auto_ptr (new Epetra_FECrsMatrix(Copy, row_map, col_map, - int(n_max_entries_per_row), false))), - last_action (Insert) + int(n_max_entries_per_row), false))) {} SparseMatrix::SparseMatrix (const Epetra_Map &InputRowMap, @@ -131,11 +131,11 @@ namespace TrilinosWrappers : row_map (InputRowMap), col_map (InputColMap), + last_action (Insert), matrix (std::auto_ptr (new Epetra_FECrsMatrix(Copy, row_map, col_map, (int*)const_cast(&(n_entries_per_row[0])), - true))), - last_action (Insert) + true))) {} @@ -160,44 +160,48 @@ namespace TrilinosWrappers { unsigned int n_rows = sparsity_pattern.n_rows(); - Assert (matrix->NumGlobalRows() == (int)sparsity_pattern.n_rows(), - ExcDimensionMismatch (matrix->NumGlobalRows(), - sparsity_pattern.n_rows())); - - // Trilinos seems to have a bug for - // rectangular matrices at this point, - // so do not check for consistent - // column numbers here. - // - // - // this bug is filed in the Sandia - // bugzilla under #4123 and should be - // fixed for version 9.0 -// Assert (matrix->NumGlobalCols() == (int)sparsity_pattern.n_cols(), -// ExcDimensionMismatch (matrix->NumGlobalCols(), -// sparsity_pattern.n_cols())); - - std::vector n_entries_per_row(n_rows); - - for (unsigned int row=0; row values; - std::vector row_indices; - - for (unsigned int row=0; rowInsertGlobalValues(row, row_length, - &values[0], &row_indices[0]); + Assert (matrix->NumGlobalRows() == (int)sparsity_pattern.n_rows(), + ExcDimensionMismatch (matrix->NumGlobalRows(), + sparsity_pattern.n_rows())); + + // Trilinos seems to have a bug for + // rectangular matrices at this point, + // so do not check for consistent + // column numbers here. + // + // + // this bug is filed in the Sandia + // bugzilla under #4123 and should be + // fixed for version 9.0 +// Assert (matrix->NumGlobalCols() == (int)sparsity_pattern.n_cols(), +// ExcDimensionMismatch (matrix->NumGlobalCols(), +// sparsity_pattern.n_cols())); + + std::vector n_entries_per_row(n_rows); + + for (unsigned int row=0; row values; + std::vector row_indices; + + for (unsigned int row=0; rowInsertGlobalValues(row, row_length, + &values[0], &row_indices[0]); + } } + compress(); // In the end, the matrix needs to // be compressed in order to be // really ready. However, that is @@ -229,12 +233,15 @@ namespace TrilinosWrappers unsigned int n_rows = sparsity_pattern.n_rows(); + if (row_map.Comm().MyPID() == 0) + { Assert (input_row_map.NumGlobalElements() == (int)sparsity_pattern.n_rows(), ExcDimensionMismatch (input_row_map.NumGlobalElements(), sparsity_pattern.n_rows())); Assert (input_col_map.NumGlobalElements() == (int)sparsity_pattern.n_cols(), ExcDimensionMismatch (input_col_map.NumGlobalElements(), sparsity_pattern.n_cols())); + } row_map = input_row_map; col_map = input_col_map; @@ -326,7 +333,7 @@ namespace TrilinosWrappers const int n_row_entries = index; matrix->InsertGlobalValues(row, n_row_entries, - &values[0], &row_indices[0]); + &values[0], &row_indices[0]); } } @@ -730,9 +737,9 @@ namespace TrilinosWrappers { Assert (&src != &dst, ExcSourceEqualsDestination()); - Assert (col_map.SameAs(src.map), + Assert (col_map.SameAs(src.vector->Map()), ExcMessage ("Column map of matrix does not fit with vector map!")); - Assert (row_map.SameAs(dst.map), + Assert (row_map.SameAs(dst.vector->Map()), ExcMessage ("Row map of matrix does not fit with vector map!")); if (!matrix->Filled()) @@ -750,9 +757,9 @@ namespace TrilinosWrappers { Assert (&src != &dst, ExcSourceEqualsDestination()); - Assert (row_map.SameAs(src.map), + Assert (row_map.SameAs(src.vector->Map()), ExcMessage ("Row map of matrix does not fit with vector map!")); - Assert (col_map.SameAs(dst.map), + Assert (col_map.SameAs(dst.vector->Map()), ExcMessage ("Column map of matrix does not fit with vector map!")); if (!matrix->Filled()) @@ -795,7 +802,7 @@ namespace TrilinosWrappers TrilinosScalar SparseMatrix::matrix_norm_square (const Vector &v) const { - Vector tmp(v.map); + Vector tmp(v,true); vmult (tmp, v); return tmp*v; } @@ -804,9 +811,9 @@ namespace TrilinosWrappers TrilinosScalar SparseMatrix::matrix_scalar_product (const Vector &u, - const Vector &v) const + const Vector &v) const { - Vector tmp(v.map); + Vector tmp(v,true); vmult (tmp, v); return u*tmp; } diff --git a/deal.II/lac/source/trilinos_vector.cc b/deal.II/lac/source/trilinos_vector.cc index bd99c0eeaa..5405f9c9f6 100755 --- a/deal.II/lac/source/trilinos_vector.cc +++ b/deal.II/lac/source/trilinos_vector.cc @@ -51,26 +51,26 @@ namespace TrilinosWrappers #else map (0,0,Epetra_SerialComm()), #endif + last_action (Insert), vector(std::auto_ptr - (new Epetra_FEVector(map))), - last_action (Insert) + (new Epetra_FEVector(map))) {} Vector::Vector (unsigned int GlobalSize, Epetra_Comm &Comm) : map (GlobalSize, 0, Comm), + last_action (Insert), vector (std::auto_ptr - (new Epetra_FEVector(map))), - last_action (Insert) + (new Epetra_FEVector(map))) {} Vector::Vector (const Epetra_Map &InputMap) : map (InputMap), + last_action (Insert), vector (std::auto_ptr - (new Epetra_FEVector(map))), - last_action (Insert) + (new Epetra_FEVector(map))) {} @@ -78,9 +78,9 @@ namespace TrilinosWrappers const bool fast) : map (v.map), + last_action (Insert), vector(std::auto_ptr - (new Epetra_FEVector(v.map,!fast))), - last_action (Insert) + (new Epetra_FEVector(v.map,!fast))) {}