From: Peter Munch Date: Fri, 20 Sep 2019 06:43:52 +0000 (+0200) Subject: Compute number of locally owned active cells per process on-demand. X-Git-Tag: v9.2.0-rc1~1064^2 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5eb3561664a032f9f9723d96fad06711e8f955fc;p=dealii.git Compute number of locally owned active cells per process on-demand. --- diff --git a/doc/news/changes/incompatibilities/20190920PeterMunch b/doc/news/changes/incompatibilities/20190920PeterMunch new file mode 100644 index 0000000000..d830fdfbbc --- /dev/null +++ b/doc/news/changes/incompatibilities/20190920PeterMunch @@ -0,0 +1,5 @@ +Changed: The parallel::TriangulationBase class does not store the number of active cells +of all MPI processes any more. Instead, the information is computed on-demand when calling +the function parallel::TriangulationBase::compute_n_locally_owned_active_cells_per_processor. +
+(Peter Munch, 2019/09/20) diff --git a/include/deal.II/distributed/tria_base.h b/include/deal.II/distributed/tria_base.h index 53749beb99..fafcd5d1c2 100644 --- a/include/deal.II/distributed/tria_base.h +++ b/include/deal.II/distributed/tria_base.h @@ -1,6 +1,6 @@ // --------------------------------------------------------------------- // -// Copyright (C) 2008 - 2018 by the deal.II authors +// Copyright (C) 2008 - 2019 by the deal.II authors // // This file is part of the deal.II library. // @@ -79,10 +79,11 @@ namespace parallel * that contribute to this triangulation. The element of this vector * indexed by locally_owned_subdomain() equals the result of * n_locally_owned_active_cells(). + * + * @note This function involves global communication! */ - const std::vector & - n_locally_owned_active_cells_per_processor() const; - + std::vector + compute_n_locally_owned_active_cells_per_processor() const; /** * Return the number of active cells in the triangulation that are locally @@ -195,10 +196,9 @@ namespace parallel struct NumberCache { /** - * This vector stores the number of locally owned active cells per MPI - * rank. + * Number of locally owned active cells of this MPI rank. */ - std::vector n_locally_owned_active_cells; + unsigned int n_locally_owned_active_cells; /** * The total number of active cells (sum of @p * n_locally_owned_active_cells). diff --git a/source/distributed/tria_base.cc b/source/distributed/tria_base.cc index 44252fa12f..b3c6d4b5c7 100644 --- a/source/distributed/tria_base.cc +++ b/source/distributed/tria_base.cc @@ -56,7 +56,6 @@ namespace parallel ExcMessage("You compiled deal.II without MPI support, for " "which parallel::TriangulationBase is not available.")); #endif - number_cache.n_locally_owned_active_cells.resize(n_subdomains); } @@ -98,8 +97,6 @@ namespace parallel this->dealii::Triangulation::memory_consumption() + MemoryConsumption::memory_consumption(mpi_communicator) + MemoryConsumption::memory_consumption(my_subdomain) + - MemoryConsumption::memory_consumption( - number_cache.n_locally_owned_active_cells) + MemoryConsumption::memory_consumption( number_cache.n_global_active_cells) + MemoryConsumption::memory_consumption(number_cache.n_global_levels); @@ -117,7 +114,7 @@ namespace parallel template TriangulationBase::NumberCache::NumberCache() - : n_global_active_cells(0) + : n_locally_owned_active_cells(0) , n_global_levels(0) {} @@ -125,7 +122,7 @@ namespace parallel unsigned int TriangulationBase::n_locally_owned_active_cells() const { - return number_cache.n_locally_owned_active_cells[my_subdomain]; + return number_cache.n_locally_owned_active_cells; } template @@ -143,11 +140,32 @@ namespace parallel } template - const std::vector & - TriangulationBase::n_locally_owned_active_cells_per_processor() - const + std::vector + TriangulationBase:: + compute_n_locally_owned_active_cells_per_processor() const { - return number_cache.n_locally_owned_active_cells; + ; +#ifdef DEAL_II_WITH_MPI + std::vector n_locally_owned_active_cells_per_processor( + Utilities::MPI::n_mpi_processes(this->mpi_communicator), 0); + + if (this->n_levels() > 0) + { + const int ierr = + MPI_Allgather(&number_cache.n_locally_owned_active_cells, + 1, + MPI_UNSIGNED, + n_locally_owned_active_cells_per_processor.data(), + 1, + MPI_UNSIGNED, + this->mpi_communicator); + AssertThrowMPI(ierr); + } + + return n_locally_owned_active_cells_per_processor; +#else + return {number_cache.n_locally_owned_active_cells}; +#endif } template @@ -162,16 +180,9 @@ namespace parallel void TriangulationBase::update_number_cache() { - Assert(number_cache.n_locally_owned_active_cells.size() == - Utilities::MPI::n_mpi_processes(this->mpi_communicator), - ExcInternalError()); - - std::fill(number_cache.n_locally_owned_active_cells.begin(), - number_cache.n_locally_owned_active_cells.end(), - 0); - number_cache.ghost_owners.clear(); number_cache.level_ghost_owners.clear(); + number_cache.n_locally_owned_active_cells = 0; if (this->n_levels() == 0) { @@ -206,25 +217,11 @@ namespace parallel cell != this->end(); ++cell) if (cell->subdomain_id() == my_subdomain) - ++number_cache.n_locally_owned_active_cells[my_subdomain]; - - unsigned int send_value = - number_cache.n_locally_owned_active_cells[my_subdomain]; - const int ierr = - MPI_Allgather(&send_value, - 1, - MPI_UNSIGNED, - number_cache.n_locally_owned_active_cells.data(), - 1, - MPI_UNSIGNED, - this->mpi_communicator); - AssertThrowMPI(ierr); + ++number_cache.n_locally_owned_active_cells; number_cache.n_global_active_cells = - std::accumulate(number_cache.n_locally_owned_active_cells.begin(), - number_cache.n_locally_owned_active_cells.end(), - /* ensure sum is computed with correct data type:*/ - static_cast(0)); + Utilities::MPI::sum(number_cache.n_locally_owned_active_cells, + this->mpi_communicator); number_cache.n_global_levels = Utilities::MPI::max(this->n_levels(), this->mpi_communicator); } diff --git a/tests/mpi/cell_weights_01.cc b/tests/mpi/cell_weights_01.cc index 32fa0bac5e..932e9a2858 100644 --- a/tests/mpi/cell_weights_01.cc +++ b/tests/mpi/cell_weights_01.cc @@ -53,10 +53,12 @@ test() // (roughly) equal between all processors tr.repartition(); + const auto n_locally_owned_active_cells_per_processor = + tr.compute_n_locally_owned_active_cells_per_processor(); if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) for (unsigned int p = 0; p < numproc; ++p) deallog << "processor " << p << ": " - << tr.n_locally_owned_active_cells_per_processor()[p] + << n_locally_owned_active_cells_per_processor[p] << " locally owned active cells" << std::endl; } diff --git a/tests/mpi/cell_weights_01_back_and_forth_01.cc b/tests/mpi/cell_weights_01_back_and_forth_01.cc index 9ebbc79689..2686080140 100644 --- a/tests/mpi/cell_weights_01_back_and_forth_01.cc +++ b/tests/mpi/cell_weights_01_back_and_forth_01.cc @@ -87,11 +87,12 @@ test() std::placeholders::_2)); tr.repartition(); - + const auto n_locally_owned_active_cells_per_processor = + tr.compute_n_locally_owned_active_cells_per_processor(); if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) for (unsigned int p = 0; p < numproc; ++p) deallog << "processor " << p << ": " - << tr.n_locally_owned_active_cells_per_processor()[p] + << n_locally_owned_active_cells_per_processor[p] << " locally owned active cells" << std::endl; } diff --git a/tests/mpi/cell_weights_01_back_and_forth_02.cc b/tests/mpi/cell_weights_01_back_and_forth_02.cc index 4d28e2a0b2..edbffd902f 100644 --- a/tests/mpi/cell_weights_01_back_and_forth_02.cc +++ b/tests/mpi/cell_weights_01_back_and_forth_02.cc @@ -72,11 +72,12 @@ test() tr.signals.cell_weight.disconnect_all_slots(); tr.repartition(); - + const auto n_locally_owned_active_cells_per_processor = + tr.compute_n_locally_owned_active_cells_per_processor(); if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) for (unsigned int p = 0; p < numproc; ++p) deallog << "processor " << p << ": " - << tr.n_locally_owned_active_cells_per_processor()[p] + << n_locally_owned_active_cells_per_processor[p] << " locally owned active cells" << std::endl; } diff --git a/tests/mpi/cell_weights_02.cc b/tests/mpi/cell_weights_02.cc index 20fd4cd4a3..862b0644fc 100644 --- a/tests/mpi/cell_weights_02.cc +++ b/tests/mpi/cell_weights_02.cc @@ -59,10 +59,12 @@ test() std::bind(&cell_weight, std::placeholders::_1, std::placeholders::_2)); tr.refine_global(1); + const auto n_locally_owned_active_cells_per_processor = + tr.compute_n_locally_owned_active_cells_per_processor(); if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) for (unsigned int p = 0; p < numproc; ++p) deallog << "processor " << p << ": " - << tr.n_locally_owned_active_cells_per_processor()[p] + << n_locally_owned_active_cells_per_processor[p] << " locally owned active cells" << std::endl; } diff --git a/tests/mpi/cell_weights_03.cc b/tests/mpi/cell_weights_03.cc index de63dea8e3..5a44ca85eb 100644 --- a/tests/mpi/cell_weights_03.cc +++ b/tests/mpi/cell_weights_03.cc @@ -67,10 +67,12 @@ test() tr.repartition(); + const auto n_locally_owned_active_cells_per_processor = + tr.compute_n_locally_owned_active_cells_per_processor(); if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) for (unsigned int p = 0; p < numproc; ++p) deallog << "processor " << p << ": " - << tr.n_locally_owned_active_cells_per_processor()[p] + << n_locally_owned_active_cells_per_processor[p] << " locally owned active cells" << std::endl; // let each processor sum up its weights diff --git a/tests/mpi/cell_weights_04.cc b/tests/mpi/cell_weights_04.cc index 367eb75caf..fab4b09040 100644 --- a/tests/mpi/cell_weights_04.cc +++ b/tests/mpi/cell_weights_04.cc @@ -61,10 +61,12 @@ test() tr.refine_global(1); + const auto n_locally_owned_active_cells_per_processor = + tr.compute_n_locally_owned_active_cells_per_processor(); if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) for (unsigned int p = 0; p < numproc; ++p) deallog << "processor " << p << ": " - << tr.n_locally_owned_active_cells_per_processor()[p] + << n_locally_owned_active_cells_per_processor[p] << " locally owned active cells" << std::endl; // let each processor sum up its weights diff --git a/tests/mpi/cell_weights_05.cc b/tests/mpi/cell_weights_05.cc index df14695434..b0c2bee4d9 100644 --- a/tests/mpi/cell_weights_05.cc +++ b/tests/mpi/cell_weights_05.cc @@ -80,10 +80,12 @@ test() tr.repartition(); + const auto n_locally_owned_active_cells_per_processor = + tr.compute_n_locally_owned_active_cells_per_processor(); if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) for (unsigned int p = 0; p < numproc; ++p) deallog << "processor " << p << ": " - << tr.n_locally_owned_active_cells_per_processor()[p] + << n_locally_owned_active_cells_per_processor[p] << " locally owned active cells" << std::endl; // let each processor sum up its weights diff --git a/tests/mpi/cell_weights_06.cc b/tests/mpi/cell_weights_06.cc index 3d1e6b7f4a..b57d4f3f10 100644 --- a/tests/mpi/cell_weights_06.cc +++ b/tests/mpi/cell_weights_06.cc @@ -79,10 +79,12 @@ test() std::bind(&cell_weight, std::placeholders::_1, std::placeholders::_2)); tr.repartition(); + const auto n_locally_owned_active_cells_per_processor = + tr.compute_n_locally_owned_active_cells_per_processor(); if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) for (unsigned int p = 0; p < numproc; ++p) deallog << "processor " << p << ": " - << tr.n_locally_owned_active_cells_per_processor()[p] + << n_locally_owned_active_cells_per_processor[p] << " locally owned active cells" << std::endl; // let each processor sum up its weights diff --git a/tests/mpi/hp_step-40.cc b/tests/mpi/hp_step-40.cc index ba44608f2c..44918b2fb0 100644 --- a/tests/mpi/hp_step-40.cc +++ b/tests/mpi/hp_step-40.cc @@ -329,11 +329,12 @@ namespace Step40 pcout << " Number of active cells: " << triangulation.n_global_active_cells() << std::endl << " "; + const auto n_locally_owned_active_cells_per_processor = + triangulation.compute_n_locally_owned_active_cells_per_processor(); for (unsigned int i = 0; i < Utilities::MPI::n_mpi_processes(mpi_communicator); ++i) - pcout << triangulation.n_locally_owned_active_cells_per_processor()[i] - << '+'; + pcout << n_locally_owned_active_cells_per_processor[i] << '+'; pcout << std::endl; pcout << " Number of degrees of freedom: " << dof_handler.n_dofs() diff --git a/tests/mpi/hp_step-40_variable_01.cc b/tests/mpi/hp_step-40_variable_01.cc index 73e117f835..90427a37a7 100644 --- a/tests/mpi/hp_step-40_variable_01.cc +++ b/tests/mpi/hp_step-40_variable_01.cc @@ -332,11 +332,12 @@ namespace Step40 pcout << " Number of active cells: " << triangulation.n_global_active_cells() << std::endl << " "; + const auto n_locally_owned_active_cells_per_processor = + triangulation.compute_n_locally_owned_active_cells_per_processor(); for (unsigned int i = 0; i < Utilities::MPI::n_mpi_processes(mpi_communicator); ++i) - pcout << triangulation.n_locally_owned_active_cells_per_processor()[i] - << '+'; + pcout << n_locally_owned_active_cells_per_processor[i] << '+'; pcout << std::endl; pcout << " Number of degrees of freedom: " << dof_handler.n_dofs() diff --git a/tests/mpi/step-40.cc b/tests/mpi/step-40.cc index 6253d4b7a5..092bb521b8 100644 --- a/tests/mpi/step-40.cc +++ b/tests/mpi/step-40.cc @@ -314,11 +314,12 @@ namespace Step40 pcout << " Number of active cells: " << triangulation.n_global_active_cells() << std::endl << " "; + const auto n_locally_owned_active_cells_per_processor = + triangulation.compute_n_locally_owned_active_cells_per_processor(); for (unsigned int i = 0; i < Utilities::MPI::n_mpi_processes(mpi_communicator); ++i) - pcout << triangulation.n_locally_owned_active_cells_per_processor()[i] - << '+'; + pcout << n_locally_owned_active_cells_per_processor[i] << '+'; pcout << std::endl; pcout << " Number of degrees of freedom: " << dof_handler.n_dofs() diff --git a/tests/mpi/step-40_cuthill_mckee.cc b/tests/mpi/step-40_cuthill_mckee.cc index 5225ccaaa3..65a0289b10 100644 --- a/tests/mpi/step-40_cuthill_mckee.cc +++ b/tests/mpi/step-40_cuthill_mckee.cc @@ -378,11 +378,12 @@ namespace Step40 pcout << " Number of active cells: " << triangulation.n_global_active_cells() << std::endl << " "; + const auto n_locally_owned_active_cells_per_processor = + triangulation.compute_n_locally_owned_active_cells_per_processor(); for (unsigned int i = 0; i < Utilities::MPI::n_mpi_processes(mpi_communicator); ++i) - pcout << triangulation.n_locally_owned_active_cells_per_processor()[i] - << '+'; + pcout << n_locally_owned_active_cells_per_processor[i] << '+'; pcout << std::endl; pcout << " Number of degrees of freedom: " << dof_handler.n_dofs() diff --git a/tests/mpi/step-40_cuthill_mckee_MPI-subset.cc b/tests/mpi/step-40_cuthill_mckee_MPI-subset.cc index b658255126..7516e42d80 100644 --- a/tests/mpi/step-40_cuthill_mckee_MPI-subset.cc +++ b/tests/mpi/step-40_cuthill_mckee_MPI-subset.cc @@ -379,11 +379,12 @@ namespace Step40 pcout << " Number of active cells: " << triangulation.n_global_active_cells() << std::endl << " "; + const auto n_locally_owned_active_cells_per_processor = + triangulation.compute_n_locally_owned_active_cells_per_processor(); for (unsigned int i = 0; i < Utilities::MPI::n_mpi_processes(mpi_communicator); ++i) - pcout << triangulation.n_locally_owned_active_cells_per_processor()[i] - << '+'; + pcout << n_locally_owned_active_cells_per_processor[i] << '+'; pcout << std::endl; pcout << " Number of degrees of freedom: " << dof_handler.n_dofs() diff --git a/tests/mpi/step-40_direct_solver.cc b/tests/mpi/step-40_direct_solver.cc index 6b80fca5c7..1a43bb5cee 100644 --- a/tests/mpi/step-40_direct_solver.cc +++ b/tests/mpi/step-40_direct_solver.cc @@ -292,11 +292,12 @@ namespace Step40 pcout << " Number of active cells: " << triangulation.n_global_active_cells() << std::endl << " "; + const auto n_locally_owned_active_cells_per_processor = + triangulation.compute_n_locally_owned_active_cells_per_processor(); for (unsigned int i = 0; i < Utilities::MPI::n_mpi_processes(mpi_communicator); ++i) - pcout << triangulation.n_locally_owned_active_cells_per_processor()[i] - << '+'; + pcout << n_locally_owned_active_cells_per_processor[i] << '+'; pcout << std::endl; pcout << " Number of degrees of freedom: " << dof_handler.n_dofs()