From 4f3cef4df741e5687e8efcc23b0a520eac8ed3c7 Mon Sep 17 00:00:00 2001 From: Timo Heister Date: Mon, 18 Jan 2016 22:27:23 -0500 Subject: [PATCH] setup missing parallel level neighbors Level ghosts were not set up if neighbor is finer and we have no cells on that finest level. This causes a situation where Triangulation::level_ghost_owners is not symmetric causing a deadlock (and maybe also wrong information down the line). This causes a few test output changes... --- source/distributed/tria.cc | 27 +++- source/dofs/dof_handler_policy.cc | 123 ++++++++++-------- .../constrained_dofs_01.mpirun=3.output | 6 +- ...sfer_04.with_trilinos=true.mpirun=2.output | 8 +- ...a.with_trilinos=true.mpirun=2.debug.output | 12 +- 5 files changed, 104 insertions(+), 72 deletions(-) diff --git a/source/distributed/tria.cc b/source/distributed/tria.cc index 160f40ea62..52295521d1 100644 --- a/source/distributed/tria.cc +++ b/source/distributed/tria.cc @@ -3400,26 +3400,41 @@ namespace parallel // unless the neighbor is active and coarser. It can end up on a // different processor. Luckily, the level_subdomain_id can be // figured out without communication, because the cell is active - // (and so level_subdomain_id=subdomain_id): + // (and so level_subdomain_id=subdomain_id). Finally, also consider + // the opposite case: if we are the coarser neighbor for another + // processor, also mark them. for (typename Triangulation::cell_iterator cell = this->begin(); cell!=this->end(); ++cell) { - if (cell->level_subdomain_id()!=this->locally_owned_subdomain()) + if (cell->level_subdomain_id() == numbers::artificial_subdomain_id) continue; + bool cell_level_mine = cell->level_subdomain_id() == this->locally_owned_subdomain(); + for (unsigned int f=0; f::faces_per_cell; ++f) { - if (cell->face(f)->at_boundary()) + if (cell->face(f)->at_boundary() || cell->neighbor(f)->level() >= cell->level()) continue; - if (cell->neighbor(f)->level() < cell->level() - && - cell->neighbor(f)->level_subdomain_id()!=this->locally_owned_subdomain()) + + bool neighbor_level_mine = cell->neighbor(f)->level_subdomain_id() == this->locally_owned_subdomain(); + + if (cell_level_mine && !neighbor_level_mine) { + // set the neighbor level_id up Assert(cell->neighbor(f)->active(), ExcInternalError()); Assert(cell->neighbor(f)->subdomain_id() != numbers::artificial_subdomain_id, ExcInternalError()); Assert(cell->neighbor(f)->level_subdomain_id() == numbers::artificial_subdomain_id || cell->neighbor(f)->level_subdomain_id() == cell->neighbor(f)->subdomain_id(), ExcInternalError()); cell->neighbor(f)->set_level_subdomain_id(cell->neighbor(f)->subdomain_id()); } + else if (!cell_level_mine && neighbor_level_mine) + { + // set the current cell up because it is a neighbor for us + Assert(cell->active(), ExcInternalError()); + Assert(cell->subdomain_id() != numbers::artificial_subdomain_id, ExcInternalError()); + Assert(cell->level_subdomain_id() == numbers::artificial_subdomain_id + || cell->level_subdomain_id() == cell->subdomain_id(), ExcInternalError()); + cell->set_level_subdomain_id(cell->subdomain_id()); + } } } diff --git a/source/dofs/dof_handler_policy.cc b/source/dofs/dof_handler_policy.cc index c84fe75afd..29c74979e5 100644 --- a/source/dofs/dof_handler_policy.cc +++ b/source/dofs/dof_handler_policy.cc @@ -1533,6 +1533,26 @@ namespace internal } + // Finally, if we are neighboring a coarser cell, add them to + // the destination list + if (dealii_cell->active()) + { + for (unsigned int f=0; f::faces_per_cell; ++f) + { + if (dealii_cell->at_boundary(f)) + continue; + typename DoFHandler::level_cell_iterator neighbor = dealii_cell->neighbor(f); + if (neighbor->level()>=level) + continue; + + dealii::types::subdomain_id dest = neighbor->level_subdomain_id(); + Assert(dest != dealii::numbers::artificial_subdomain_id, ExcInternalError()); + if (dest != tria.locally_owned_subdomain()) + send_to.insert(dest); + + } + } + // send if we have something to send if (send_to.size() > 0) @@ -1981,24 +2001,23 @@ namespace internal cellmap_t; cellmap_t needs_to_get_cells; - if (level < tr->n_levels()) - for (typename DoFHandler::level_cell_iterator - cell = dof_handler.begin(0); - cell != dof_handler.end(0); - ++cell) - { - typename dealii::internal::p4est::types::quadrant p4est_coarse_cell; - internal::p4est::init_coarse_quadrant(p4est_coarse_cell); - - fill_mg_dofindices_recursively - (*tr, - coarse_cell_to_p4est_tree_permutation[cell->index()], - cell, - p4est_coarse_cell, - vertices_with_ghost_neighbors, - needs_to_get_cells, - level); - } + for (typename DoFHandler::level_cell_iterator + cell = dof_handler.begin(0); + cell != dof_handler.end(0); + ++cell) + { + typename dealii::internal::p4est::types::quadrant p4est_coarse_cell; + internal::p4est::init_coarse_quadrant(p4est_coarse_cell); + + fill_mg_dofindices_recursively + (*tr, + coarse_cell_to_p4est_tree_permutation[cell->index()], + cell, + p4est_coarse_cell, + vertices_with_ghost_neighbors, + needs_to_get_cells, + level); + } //sending @@ -2032,44 +2051,42 @@ namespace internal } - // mark all own cells, that miss some - // dof_data and collect the neighbors - // that are going to send stuff to us + // mark all own cells, that miss some dof_data and collect the + // neighbors that are going to send stuff to us std::set senders; - if (level < tr->n_levels()) - { - std::vector local_dof_indices; - typename DoFHandler::level_cell_iterator - cell, endc = dof_handler.end(level); + { + std::vector local_dof_indices; + typename DoFHandler::level_cell_iterator + cell, endc = dof_handler.end(level); - for (cell = dof_handler.begin(level); cell != endc; ++cell) - { - if (cell->level_subdomain_id()==dealii::numbers::artificial_subdomain_id) - { - //artificial - } - else if (cell->level_subdomain_id()==dof_handler.get_triangulation().locally_owned_subdomain()) - { - //own - local_dof_indices.resize (cell->get_fe().dofs_per_cell); - cell->get_mg_dof_indices (local_dof_indices); - if (local_dof_indices.end() != - std::find (local_dof_indices.begin(), - local_dof_indices.end(), - DoFHandler::invalid_dof_index)) - cell->set_user_flag(); - else - cell->clear_user_flag(); - } - else - { - //ghost - if (cell->user_flag_set()) - senders.insert(cell->level_subdomain_id()); - } - } + for (cell = dof_handler.begin(level); cell != endc; ++cell) + { + if (cell->level_subdomain_id()==dealii::numbers::artificial_subdomain_id) + { + //artificial + } + else if (cell->level_subdomain_id()==dof_handler.get_triangulation().locally_owned_subdomain()) + { + //own + local_dof_indices.resize (cell->get_fe().dofs_per_cell); + cell->get_mg_dof_indices (local_dof_indices); + if (local_dof_indices.end() != + std::find (local_dof_indices.begin(), + local_dof_indices.end(), + DoFHandler::invalid_dof_index)) + cell->set_user_flag(); + else + cell->clear_user_flag(); + } + else + { + //ghost + if (cell->user_flag_set()) + senders.insert(cell->level_subdomain_id()); + } + } - } + } //* 5. receive ghostcelldata diff --git a/tests/multigrid/constrained_dofs_01.mpirun=3.output b/tests/multigrid/constrained_dofs_01.mpirun=3.output index e3d6d45af0..e45b5c97b3 100644 --- a/tests/multigrid/constrained_dofs_01.mpirun=3.output +++ b/tests/multigrid/constrained_dofs_01.mpirun=3.output @@ -26,11 +26,11 @@ DEAL:0::{[0,17], 21} DEAL:0::ok ok DEAL:0::Level 3: DEAL:0::get_refinement_edge_indices: -DEAL:0::{} +DEAL:0::{0, 2, [6,7]} DEAL:0::get_boundary_indices: -DEAL:0::{} +DEAL:0::{[0,1]} DEAL:0::relevant: -DEAL:0::{} +DEAL:0::{[0,3], [6,7]} DEAL:0::ok ok DEAL:1::FE_Q<2>(1) diff --git a/tests/multigrid/transfer_04.with_trilinos=true.mpirun=2.output b/tests/multigrid/transfer_04.with_trilinos=true.mpirun=2.output index dc1f852173..eab98a5f76 100644 --- a/tests/multigrid/transfer_04.with_trilinos=true.mpirun=2.output +++ b/tests/multigrid/transfer_04.with_trilinos=true.mpirun=2.output @@ -82,7 +82,7 @@ DEAL:1::cell=0_2:31 level_subdomain_id=1 DEAL:1::cell=0_2:32 level_subdomain_id=1 DEAL:1::cell=0_2:33 level_subdomain_id=1 DEAL:1::cell=0_3:010 level_subdomain_id=4294967294 -DEAL:1::cell=0_3:011 level_subdomain_id=4294967294 +DEAL:1::cell=0_3:011 level_subdomain_id=0 DEAL:1::cell=0_3:012 level_subdomain_id=4294967294 DEAL:1::cell=0_3:013 level_subdomain_id=0 DEAL:1::cell=0_3:030 level_subdomain_id=0 @@ -106,8 +106,8 @@ DEAL:1::cell=0_3:211 level_subdomain_id=1 DEAL:1::cell=0_3:212 level_subdomain_id=1 DEAL:1::cell=0_3:213 level_subdomain_id=1 DEAL:1::cell=0_4:0320 level_subdomain_id=4294967294 -DEAL:1::cell=0_4:0321 level_subdomain_id=4294967294 -DEAL:1::cell=0_4:0322 level_subdomain_id=4294967294 -DEAL:1::cell=0_4:0323 level_subdomain_id=4294967294 +DEAL:1::cell=0_4:0321 level_subdomain_id=0 +DEAL:1::cell=0_4:0322 level_subdomain_id=0 +DEAL:1::cell=0_4:0323 level_subdomain_id=0 DEAL:1::ok diff --git a/tests/multigrid/transfer_04a.with_trilinos=true.mpirun=2.debug.output b/tests/multigrid/transfer_04a.with_trilinos=true.mpirun=2.debug.output index a60ed7643f..d7e70ff539 100644 --- a/tests/multigrid/transfer_04a.with_trilinos=true.mpirun=2.debug.output +++ b/tests/multigrid/transfer_04a.with_trilinos=true.mpirun=2.debug.output @@ -81,13 +81,13 @@ DEAL:1::cell=0_3:001 level_subdomain_id=4294967294 DEAL:1::cell=0_3:002 level_subdomain_id=4294967294 DEAL:1::cell=0_3:003 level_subdomain_id=4294967294 DEAL:1::cell=0_3:010 level_subdomain_id=4294967294 -DEAL:1::cell=0_3:011 level_subdomain_id=4294967294 -DEAL:1::cell=0_3:012 level_subdomain_id=4294967294 -DEAL:1::cell=0_3:013 level_subdomain_id=4294967294 +DEAL:1::cell=0_3:011 level_subdomain_id=0 +DEAL:1::cell=0_3:012 level_subdomain_id=0 +DEAL:1::cell=0_3:013 level_subdomain_id=0 DEAL:1::cell=0_3:020 level_subdomain_id=4294967294 -DEAL:1::cell=0_3:021 level_subdomain_id=4294967294 -DEAL:1::cell=0_3:022 level_subdomain_id=4294967294 -DEAL:1::cell=0_3:023 level_subdomain_id=4294967294 +DEAL:1::cell=0_3:021 level_subdomain_id=0 +DEAL:1::cell=0_3:022 level_subdomain_id=0 +DEAL:1::cell=0_3:023 level_subdomain_id=0 copy_indices[1] 22 5 copy_indices[1] 23 7 copy_indices[1] 24 8 -- 2.39.5