setup missing parallel level neighbors

Level ghosts were not set up if neighbor is finer and we have no cells
on that finest level. This causes a situation where
Triangulation::level_ghost_owners is not symmetric causing a deadlock
(and maybe also wrong information down the line). This causes a few test
output changes...

diff --git a/source/distributed/tria.cc b/source/distributed/tria.cc
index 160f40ea625863aca62284cec8a9668325dd98cb..52295521d1fc4b8bf3886b9c033fd825b4301326 100644 (file)
--- a/source/distributed/tria.cc
+++ b/source/distributed/tria.cc
@@ -3400,26 +3400,41 @@ namespace parallel
           // unless the neighbor is active and coarser. It can end up on a
           // different processor. Luckily, the level_subdomain_id can be
           // figured out without communication, because the cell is active
-          // (and so level_subdomain_id=subdomain_id):
+          // (and so level_subdomain_id=subdomain_id). Finally, also consider
+          // the opposite case: if we are the coarser neighbor for another
+          // processor, also mark them.
           for (typename Triangulation<dim,spacedim>::cell_iterator cell = this->begin(); cell!=this->end(); ++cell)
             {
-              if (cell->level_subdomain_id()!=this->locally_owned_subdomain())
+              if (cell->level_subdomain_id() == numbers::artificial_subdomain_id)
                 continue;
 
+              bool cell_level_mine = cell->level_subdomain_id() == this->locally_owned_subdomain();
+
               for (unsigned int f=0; f<GeometryInfo<dim>::faces_per_cell; ++f)
                 {
-                  if (cell->face(f)->at_boundary())
+                  if (cell->face(f)->at_boundary() || cell->neighbor(f)->level() >= cell->level())
                     continue;
-                  if (cell->neighbor(f)->level() < cell->level()
-                      &&
-                      cell->neighbor(f)->level_subdomain_id()!=this->locally_owned_subdomain())
+
+                  bool neighbor_level_mine = cell->neighbor(f)->level_subdomain_id() == this->locally_owned_subdomain();
+
+                  if (cell_level_mine && !neighbor_level_mine)
                     {
+                      // set the neighbor level_id up
                       Assert(cell->neighbor(f)->active(), ExcInternalError());
                       Assert(cell->neighbor(f)->subdomain_id() != numbers::artificial_subdomain_id, ExcInternalError());
                       Assert(cell->neighbor(f)->level_subdomain_id() == numbers::artificial_subdomain_id
                              || cell->neighbor(f)->level_subdomain_id() == cell->neighbor(f)->subdomain_id(), ExcInternalError());
                       cell->neighbor(f)->set_level_subdomain_id(cell->neighbor(f)->subdomain_id());
                     }
+                  else if (!cell_level_mine && neighbor_level_mine)
+                    {
+                      // set the current cell up because it is a neighbor for us
+                      Assert(cell->active(), ExcInternalError());
+                      Assert(cell->subdomain_id() != numbers::artificial_subdomain_id, ExcInternalError());
+                      Assert(cell->level_subdomain_id() == numbers::artificial_subdomain_id
+                             || cell->level_subdomain_id() == cell->subdomain_id(), ExcInternalError());
+                      cell->set_level_subdomain_id(cell->subdomain_id());
+                    }
                 }
 
             }

diff --git a/source/dofs/dof_handler_policy.cc b/source/dofs/dof_handler_policy.cc
index c84fe75afddbbf07eb691e74988c3ed3ba10167b..29c74979e577e723e4ca5db4adab1cffedca91d1 100644 (file)
--- a/source/dofs/dof_handler_policy.cc
+++ b/source/dofs/dof_handler_policy.cc
@@ -1533,6 +1533,26 @@ namespace internal
 
                 }
 
+              // Finally, if we are neighboring a coarser cell, add them to
+              // the destination list
+              if (dealii_cell->active())
+                {
+                  for (unsigned int f=0; f<GeometryInfo<dim>::faces_per_cell; ++f)
+                    {
+                      if (dealii_cell->at_boundary(f))
+                        continue;
+                      typename DoFHandler<dim,spacedim>::level_cell_iterator neighbor = dealii_cell->neighbor(f);
+                      if (neighbor->level()>=level)
+                        continue;
+
+                      dealii::types::subdomain_id dest = neighbor->level_subdomain_id();
+                      Assert(dest != dealii::numbers::artificial_subdomain_id, ExcInternalError());
+                      if (dest != tria.locally_owned_subdomain())
+                        send_to.insert(dest);
+
+                    }
+                }
+
 
               // send if we have something to send
               if (send_to.size() > 0)
@@ -1981,24 +2001,23 @@ namespace internal
           cellmap_t;
           cellmap_t needs_to_get_cells;
 
-          if (level < tr->n_levels())
-            for (typename DoFHandler<dim,spacedim>::level_cell_iterator
-                 cell = dof_handler.begin(0);
-                 cell != dof_handler.end(0);
-                 ++cell)
-              {
-                typename dealii::internal::p4est::types<dim>::quadrant p4est_coarse_cell;
-                internal::p4est::init_coarse_quadrant<dim>(p4est_coarse_cell);
-
-                fill_mg_dofindices_recursively<dim,spacedim>
-                (*tr,
-                 coarse_cell_to_p4est_tree_permutation[cell->index()],
-                 cell,
-                 p4est_coarse_cell,
-                 vertices_with_ghost_neighbors,
-                 needs_to_get_cells,
-                 level);
-              }
+          for (typename DoFHandler<dim,spacedim>::level_cell_iterator
+               cell = dof_handler.begin(0);
+               cell != dof_handler.end(0);
+               ++cell)
+            {
+              typename dealii::internal::p4est::types<dim>::quadrant p4est_coarse_cell;
+              internal::p4est::init_coarse_quadrant<dim>(p4est_coarse_cell);
+
+              fill_mg_dofindices_recursively<dim,spacedim>
+              (*tr,
+               coarse_cell_to_p4est_tree_permutation[cell->index()],
+               cell,
+               p4est_coarse_cell,
+               vertices_with_ghost_neighbors,
+               needs_to_get_cells,
+               level);
+            }
 
 
           //sending
@@ -2032,44 +2051,42 @@ namespace internal
             }
 
 
-          // mark all own cells, that miss some
-          // dof_data and collect the neighbors
-          // that are going to send stuff to us
+          // mark all own cells, that miss some dof_data and collect the
+          // neighbors that are going to send stuff to us
           std::set<dealii::types::subdomain_id> senders;
-          if (level < tr->n_levels())
-            {
-              std::vector<dealii::types::global_dof_index> local_dof_indices;
-              typename DoFHandler<dim,spacedim>::level_cell_iterator
-              cell, endc = dof_handler.end(level);
+          {
+            std::vector<dealii::types::global_dof_index> local_dof_indices;
+            typename DoFHandler<dim,spacedim>::level_cell_iterator
+            cell, endc = dof_handler.end(level);
 
-              for (cell = dof_handler.begin(level); cell != endc; ++cell)
-                {
-                  if (cell->level_subdomain_id()==dealii::numbers::artificial_subdomain_id)
-                    {
-                      //artificial
-                    }
-                  else if (cell->level_subdomain_id()==dof_handler.get_triangulation().locally_owned_subdomain())
-                    {
-                      //own
-                      local_dof_indices.resize (cell->get_fe().dofs_per_cell);
-                      cell->get_mg_dof_indices (local_dof_indices);
-                      if (local_dof_indices.end() !=
-                          std::find (local_dof_indices.begin(),
-                                     local_dof_indices.end(),
-                                     DoFHandler<dim,spacedim>::invalid_dof_index))
-                        cell->set_user_flag();
-                      else
-                        cell->clear_user_flag();
-                    }
-                  else
-                    {
-                      //ghost
-                      if (cell->user_flag_set())
-                        senders.insert(cell->level_subdomain_id());
-                    }
-                }
+            for (cell = dof_handler.begin(level); cell != endc; ++cell)
+              {
+                if (cell->level_subdomain_id()==dealii::numbers::artificial_subdomain_id)
+                  {
+                    //artificial
+                  }
+                else if (cell->level_subdomain_id()==dof_handler.get_triangulation().locally_owned_subdomain())
+                  {
+                    //own
+                    local_dof_indices.resize (cell->get_fe().dofs_per_cell);
+                    cell->get_mg_dof_indices (local_dof_indices);
+                    if (local_dof_indices.end() !=
+                        std::find (local_dof_indices.begin(),
+                                   local_dof_indices.end(),
+                                   DoFHandler<dim,spacedim>::invalid_dof_index))
+                      cell->set_user_flag();
+                    else
+                      cell->clear_user_flag();
+                  }
+                else
+                  {
+                    //ghost
+                    if (cell->user_flag_set())
+                      senders.insert(cell->level_subdomain_id());
+                  }
+              }
 
-            }
+          }
 
 
           //* 5. receive ghostcelldata

diff --git a/tests/multigrid/constrained_dofs_01.mpirun=3.output b/tests/multigrid/constrained_dofs_01.mpirun=3.output
index e3d6d45af0f64c5971a71facd55965ac36d5a640..e45b5c97b37f951d330e434971b5559eff2daac1 100644 (file)
--- a/tests/multigrid/constrained_dofs_01.mpirun=3.output
+++ b/tests/multigrid/constrained_dofs_01.mpirun=3.output
@@ -26,11 +26,11 @@ DEAL:0::{[0,17], 21}
 DEAL:0::ok ok 
 DEAL:0::Level 3:
 DEAL:0::get_refinement_edge_indices:
-DEAL:0::{}
+DEAL:0::{0, 2, [6,7]}
 DEAL:0::get_boundary_indices:
-DEAL:0::{}
+DEAL:0::{[0,1]}
 DEAL:0::relevant:
-DEAL:0::{}
+DEAL:0::{[0,3], [6,7]}
 DEAL:0::ok ok 
 
 DEAL:1::FE_Q<2>(1)

diff --git a/tests/multigrid/transfer_04.with_trilinos=true.mpirun=2.output b/tests/multigrid/transfer_04.with_trilinos=true.mpirun=2.output
index dc1f852173fd2d9415302f08dccf3fd74880d021..eab98a5f76b3eeb412a1c848a0a66fc5fee2f025 100644 (file)
--- a/tests/multigrid/transfer_04.with_trilinos=true.mpirun=2.output
+++ b/tests/multigrid/transfer_04.with_trilinos=true.mpirun=2.output
@@ -82,7 +82,7 @@ DEAL:1::cell=0_2:31 level_subdomain_id=1
 DEAL:1::cell=0_2:32 level_subdomain_id=1
 DEAL:1::cell=0_2:33 level_subdomain_id=1
 DEAL:1::cell=0_3:010 level_subdomain_id=4294967294
-DEAL:1::cell=0_3:011 level_subdomain_id=4294967294
+DEAL:1::cell=0_3:011 level_subdomain_id=0
 DEAL:1::cell=0_3:012 level_subdomain_id=4294967294
 DEAL:1::cell=0_3:013 level_subdomain_id=0
 DEAL:1::cell=0_3:030 level_subdomain_id=0
@@ -106,8 +106,8 @@ DEAL:1::cell=0_3:211 level_subdomain_id=1
 DEAL:1::cell=0_3:212 level_subdomain_id=1
 DEAL:1::cell=0_3:213 level_subdomain_id=1
 DEAL:1::cell=0_4:0320 level_subdomain_id=4294967294
-DEAL:1::cell=0_4:0321 level_subdomain_id=4294967294
-DEAL:1::cell=0_4:0322 level_subdomain_id=4294967294
-DEAL:1::cell=0_4:0323 level_subdomain_id=4294967294
+DEAL:1::cell=0_4:0321 level_subdomain_id=0
+DEAL:1::cell=0_4:0322 level_subdomain_id=0
+DEAL:1::cell=0_4:0323 level_subdomain_id=0
 DEAL:1::ok
 

diff --git a/tests/multigrid/transfer_04a.with_trilinos=true.mpirun=2.debug.output b/tests/multigrid/transfer_04a.with_trilinos=true.mpirun=2.debug.output
index a60ed7643f0521e2eeab0e4950ec7a0c65eef20e..d7e70ff5398bfa0f5259849071334054d9c78ef9 100644 (file)
--- a/tests/multigrid/transfer_04a.with_trilinos=true.mpirun=2.debug.output
+++ b/tests/multigrid/transfer_04a.with_trilinos=true.mpirun=2.debug.output
@@ -81,13 +81,13 @@ DEAL:1::cell=0_3:001 level_subdomain_id=4294967294
 DEAL:1::cell=0_3:002 level_subdomain_id=4294967294
 DEAL:1::cell=0_3:003 level_subdomain_id=4294967294
 DEAL:1::cell=0_3:010 level_subdomain_id=4294967294
-DEAL:1::cell=0_3:011 level_subdomain_id=4294967294
-DEAL:1::cell=0_3:012 level_subdomain_id=4294967294
-DEAL:1::cell=0_3:013 level_subdomain_id=4294967294
+DEAL:1::cell=0_3:011 level_subdomain_id=0
+DEAL:1::cell=0_3:012 level_subdomain_id=0
+DEAL:1::cell=0_3:013 level_subdomain_id=0
 DEAL:1::cell=0_3:020 level_subdomain_id=4294967294
-DEAL:1::cell=0_3:021 level_subdomain_id=4294967294
-DEAL:1::cell=0_3:022 level_subdomain_id=4294967294
-DEAL:1::cell=0_3:023 level_subdomain_id=4294967294
+DEAL:1::cell=0_3:021 level_subdomain_id=0
+DEAL:1::cell=0_3:022 level_subdomain_id=0
+DEAL:1::cell=0_3:023 level_subdomain_id=0
 copy_indices[1]        22      5
 copy_indices[1]        23      7
 copy_indices[1]        24      8