From: Martin Kronbichler Date: Mon, 20 Jan 2014 12:03:12 +0000 (+0000) Subject: Simplify exchange of data in Trilinos sparse matrix reinit: Can use the nonlocal_grap... X-Git-Tag: v8.2.0-rc1~1000 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ec41a659ae3102ff6967976a715b32721ca4610b;p=dealii.git Simplify exchange of data in Trilinos sparse matrix reinit: Can use the nonlocal_graph directly. Apparently, I would need another entry point into distribute_sparsity_pattern to fix the deal.II transfer. git-svn-id: https://svn.dealii.org/trunk@32264 0785d39b-7218-0410-832d-ea1e28bc413d --- diff --git a/deal.II/source/lac/sparsity_tools.cc b/deal.II/source/lac/sparsity_tools.cc index afa8e0fa3d..d5f08ee540 100644 --- a/deal.II/source/lac/sparsity_tools.cc +++ b/deal.II/source/lac/sparsity_tools.cc @@ -601,7 +601,7 @@ namespace SparsityTools Assert (status.MPI_TAG==124, ExcInternalError()); MPI_Get_count(&status, MPI_BYTE, &len); - Assert( len%sizeof(unsigned int)==0, ExcInternalError()); + Assert( len%sizeof(size_type)==0, ExcInternalError()); recv_buf.resize(len/sizeof(size_type)); diff --git a/deal.II/source/lac/trilinos_sparse_matrix.cc b/deal.II/source/lac/trilinos_sparse_matrix.cc index 88e828f356..0443687b93 100644 --- a/deal.II/source/lac/trilinos_sparse_matrix.cc +++ b/deal.II/source/lac/trilinos_sparse_matrix.cc @@ -593,38 +593,8 @@ namespace TrilinosWrappers AssertDimension (sparsity_pattern.n_rows(), static_cast(n_global_elements(input_row_map))); - - // exchange data if requested with deal.II's own function -#ifdef DEAL_II_WITH_MPI - const Epetra_MpiComm* communicator = - dynamic_cast(&input_row_map.Comm()); - if (exchange_data && communicator != 0) - { - std::vector - owned_per_proc(communicator->NumProc(), -1); - size_type my_elements = input_row_map.NumMyElements(); - MPI_Allgather(&my_elements, 1, - Utilities::MPI::internal::mpi_type_id(&my_elements), - &owned_per_proc[0], 1, - Utilities::MPI::internal::mpi_type_id(&my_elements), - communicator->Comm()); - - AssertDimension(std::accumulate(owned_per_proc.begin(), - owned_per_proc.end(), size_type()), sparsity_pattern.n_rows()); - - SparsityTools::distribute_sparsity_pattern - (const_cast(sparsity_pattern), - owned_per_proc, communicator->Comm(), sparsity_pattern.row_index_set()); - } -#endif - - if (input_row_map.Comm().MyPID() == 0) - { - AssertDimension (sparsity_pattern.n_rows(), - static_cast(n_global_elements(input_row_map))); - AssertDimension (sparsity_pattern.n_cols(), - static_cast(n_global_elements(input_col_map))); - } + AssertDimension (sparsity_pattern.n_cols(), + static_cast(n_global_elements(input_col_map))); column_space_map.reset (new Epetra_Map (input_col_map)); @@ -690,7 +660,8 @@ namespace TrilinosWrappers if (input_row_map.Comm().NumProc() > 1) { graph.reset (new Epetra_CrsGraph (Copy, input_row_map, - &n_entries_per_row[0], true)); + &n_entries_per_row[0], + exchange_data ? false : true)); if (have_ghost_rows == true) nonlocal_graph.reset (new Epetra_CrsGraph (Copy, off_processor_map, &n_entries_per_ghost_row[0], @@ -724,15 +695,7 @@ namespace TrilinosWrappers } } - graph->FillComplete(input_col_map, input_row_map); - graph->OptimizeStorage(); - - AssertDimension (sparsity_pattern.n_cols(),static_cast( - n_global_cols(*graph))); - - matrix.reset (new Epetra_FECrsMatrix(Copy, *graph, false)); - - // and now the same operations for the nonlocal graph and matrix + // finalize nonlocal graph and create nonlocal matrix if (nonlocal_graph.get() != 0) { if (nonlocal_graph->IndicesAreGlobal() == false && @@ -748,8 +711,24 @@ namespace TrilinosWrappers nonlocal_graph->FillComplete(input_col_map, input_row_map); nonlocal_graph->OptimizeStorage(); + // insert data from nonlocal graph into the final sparsity pattern + if (exchange_data) + { + Epetra_Export exporter(nonlocal_graph->RowMap(), input_row_map); + int ierr = graph->Export(*nonlocal_graph, exporter, Add); + Assert (ierr==0, ExcTrilinosError(ierr)); + } + nonlocal_matrix.reset (new Epetra_CrsMatrix(Copy, *nonlocal_graph)); } + + graph->FillComplete(input_col_map, input_row_map); + graph->OptimizeStorage(); + + AssertDimension (sparsity_pattern.n_cols(),static_cast( + n_global_cols(*graph))); + + matrix.reset (new Epetra_FECrsMatrix(Copy, *graph, false)); last_action = Zero; // In the end, the matrix needs to be compressed in order to be really