From 4e31e67294d0288c01275dc7bed290454fe66d0e Mon Sep 17 00:00:00 2001
From: David Wells <drwells@email.unc.edu>
Date: Tue, 3 May 2022 16:12:24 -0400
Subject: [PATCH] Fix a DoFRenumbering issue with p::s::T.

This can happen if some processors don't have any degrees of freedom (e.g., we
run in parallel with only one cell in the triangulation). This crashed in a
spectacular way since this test would ultimately call MPI_Allgather with
different types at the same time.

The correct workaround here is to verify that all processors are doing the same
type of renumbering.
---
 source/dofs/dof_handler_policy.cc             | 12 +++++++--
 tests/sharedtria/dof_04.cc                    | 17 ++++++++----
 .../dof_04.with_metis=true.mpirun=3.output    | 27 ++++++++++++-------
 3 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/source/dofs/dof_handler_policy.cc b/source/dofs/dof_handler_policy.cc
index f86e09bb06..58d71ca312 100644
--- a/source/dofs/dof_handler_policy.cc
+++ b/source/dofs/dof_handler_policy.cc
@@ -3456,11 +3456,19 @@ namespace internal
 
         std::vector<types::global_dof_index> global_gathered_numbers(
           this->dof_handler->n_dofs(), 0);
-        // as we call DoFRenumbering::subdomain_wise (*dof_handler) from
+        // as we call DoFRenumbering::subdomain_wise(*dof_handler) from
         // distribute_dofs(), we need to support sequential-like input.
         // Distributed-like input from, for example, component_wise renumbering
         // is also supported.
-        if (new_numbers.size() == this->dof_handler->n_dofs())
+        const bool uses_sequential_numbering =
+          new_numbers.size() == this->dof_handler->n_dofs();
+        bool all_use_sequential_numbering = false;
+        Utilities::MPI::internal::all_reduce<bool>(
+          MPI_LAND,
+          ArrayView<const bool>(&uses_sequential_numbering, 1),
+          tr->get_communicator(),
+          ArrayView<bool>(&all_use_sequential_numbering, 1));
+        if (all_use_sequential_numbering)
           {
             global_gathered_numbers = new_numbers;
           }
diff --git a/tests/sharedtria/dof_04.cc b/tests/sharedtria/dof_04.cc
index aa08376fa9..42c6427d6a 100644
--- a/tests/sharedtria/dof_04.cc
+++ b/tests/sharedtria/dof_04.cc
@@ -41,7 +41,7 @@
 
 template <int dim>
 void
-test()
+test(const unsigned int n_global_refinements)
 {
   parallel::shared::Triangulation<dim> triangulation(
     MPI_COMM_WORLD,
@@ -54,7 +54,7 @@ test()
   DoFHandler<dim> dof_handler(triangulation);
 
   GridGenerator::hyper_cube(triangulation);
-  triangulation.refine_global(2);
+  triangulation.refine_global(n_global_refinements);
 
   const unsigned int n_refinements[] = {0, 4, 3, 2};
   for (unsigned int i = 0; i < n_refinements[dim]; ++i)
@@ -158,11 +158,18 @@ main(int argc, char *argv[])
 
   MPILogInitAll all;
 
-  deallog.push("2d");
-  test<2>();
+  deallog.push("2d, no global refinements");
+  test<2>(0);
+  deallog.pop();
+
+  // To preserve output with an older version of this test reseed here:
+  Testing::srand(1);
+
+  deallog.push("2d, global refinements");
+  test<2>(2);
   deallog.pop();
 
   deallog.push("3d");
-  test<3>();
+  test<3>(2);
   deallog.pop();
 }
diff --git a/tests/sharedtria/dof_04.with_metis=true.mpirun=3.output b/tests/sharedtria/dof_04.with_metis=true.mpirun=3.output
index ae56d843df..4a96e0233b 100644
--- a/tests/sharedtria/dof_04.with_metis=true.mpirun=3.output
+++ b/tests/sharedtria/dof_04.with_metis=true.mpirun=3.output
@@ -1,20 +1,29 @@
 
-DEAL:0:2d::n_dofs: 818
-DEAL:0:2d::n_dofs: 1754
-DEAL:0:2d::n_dofs: 3056
+DEAL:0:2d, no global refinements::n_dofs: 114
+DEAL:0:2d, no global refinements::n_dofs: 272
+DEAL:0:2d, no global refinements::n_dofs: 652
+DEAL:0:2d, global refinements::n_dofs: 818
+DEAL:0:2d, global refinements::n_dofs: 1754
+DEAL:0:2d, global refinements::n_dofs: 3056
 DEAL:0:3d::n_dofs: 13282
 DEAL:0:3d::n_dofs: 41826
 
-DEAL:1:2d::n_dofs: 818
-DEAL:1:2d::n_dofs: 1754
-DEAL:1:2d::n_dofs: 3056
+DEAL:1:2d, no global refinements::n_dofs: 114
+DEAL:1:2d, no global refinements::n_dofs: 272
+DEAL:1:2d, no global refinements::n_dofs: 652
+DEAL:1:2d, global refinements::n_dofs: 818
+DEAL:1:2d, global refinements::n_dofs: 1754
+DEAL:1:2d, global refinements::n_dofs: 3056
 DEAL:1:3d::n_dofs: 13282
 DEAL:1:3d::n_dofs: 41826
 
 
-DEAL:2:2d::n_dofs: 818
-DEAL:2:2d::n_dofs: 1754
-DEAL:2:2d::n_dofs: 3056
+DEAL:2:2d, no global refinements::n_dofs: 114
+DEAL:2:2d, no global refinements::n_dofs: 272
+DEAL:2:2d, no global refinements::n_dofs: 652
+DEAL:2:2d, global refinements::n_dofs: 818
+DEAL:2:2d, global refinements::n_dofs: 1754
+DEAL:2:2d, global refinements::n_dofs: 3056
 DEAL:2:3d::n_dofs: 13282
 DEAL:2:3d::n_dofs: 41826
 
-- 
2.39.5