From 6614e3282e5f7b0a194c4976f14c44b60df08863 Mon Sep 17 00:00:00 2001
From: Daniel Arndt <arndtd@ornl.gov>
Date: Mon, 24 Jul 2023 08:55:06 -0400
Subject: [PATCH] Require PETSc to be configured with MPI support

---
 cmake/configure/configure_20_petsc.cmake      | 11 ++--
 .../incompatibilities/20230724DanielArndt     |  3 ++
 include/deal.II/base/config.h.in              |  2 +-
 include/deal.II/base/mpi_stub.h               |  2 +-
 source/lac/petsc_communication_pattern.cc     | 50 ++-----------------
 source/lac/petsc_matrix_base.cc               |  2 -
 source/lac/petsc_parallel_vector.cc           |  4 --
 source/lac/petsc_vector_base.cc               |  2 -
 ...true.with_petsc_with_complex=false.output} |  0
 ...raits_is_serial_03.with_petsc=true.output} |  0
 ...itioner.with_petsc_with_hypre=true.output} |  0
 ..._hypre=true.with_64bit_indices=off.output} |  0
 ...omeramg.with_petsc_with_hypre=true.output} |  0
 ...mmetric.with_petsc_with_hypre=true.output} |  0
 ...ces=off.with_petsc_with_hypre=true.output} |  0
 15 files changed, 13 insertions(+), 63 deletions(-)
 create mode 100644 doc/news/changes/incompatibilities/20230724DanielArndt
 rename tests/lac/{utilities_01.with_mpi=true.with_petsc=true.with_petsc_with_complex=false.output => utilities_01.with_petsc=true.with_petsc_with_complex=false.output} (100%)
 rename tests/lac/{vector_type_traits_is_serial_03.with_petsc=true.with_mpi=true.output => vector_type_traits_is_serial_03.with_petsc=true.output} (100%)
 rename tests/petsc/{different_matrix_preconditioner.with_mpi=true.with_petsc_with_hypre=true.output => different_matrix_preconditioner.with_petsc_with_hypre=true.output} (100%)
 rename tests/petsc/{reinit_preconditioner_02.with_petsc_with_hypre=true.with_64bit_indices=off.with_mpi=true.output => reinit_preconditioner_02.with_petsc_with_hypre=true.with_64bit_indices=off.output} (100%)
 rename tests/petsc/{solver_03_precondition_boomeramg.with_mpi=true.with_petsc_with_hypre=true.output => solver_03_precondition_boomeramg.with_petsc_with_hypre=true.output} (100%)
 rename tests/petsc/{solver_03_precondition_boomeramg_symmetric.with_mpi=true.with_petsc_with_hypre=true.output => solver_03_precondition_boomeramg_symmetric.with_petsc_with_hypre=true.output} (100%)
 rename tests/petsc/{solver_03_precondition_parasails.with_64bit_indices=off.with_petsc_with_hypre=true.with_mpi=true.output => solver_03_precondition_parasails.with_64bit_indices=off.with_petsc_with_hypre=true.output} (100%)

diff --git a/cmake/configure/configure_20_petsc.cmake b/cmake/configure/configure_20_petsc.cmake
index 850a2f6af9..9998b71779 100644
--- a/cmake/configure/configure_20_petsc.cmake
+++ b/cmake/configure/configure_20_petsc.cmake
@@ -17,7 +17,7 @@
 # Configuration for the petsc library:
 #
 
-set(FEATURE_PETSC_AFTER MPI)
+set(FEATURE_PETSC_DEPENDS MPI)
 
 
 macro(feature_petsc_find_external var)
@@ -51,17 +51,14 @@ macro(feature_petsc_find_external var)
     # _NOT_ enabled.
     # So we check for this:
     #
-    if( (PETSC_WITH_MPIUNI AND DEAL_II_WITH_MPI)
-         OR
-         (NOT PETSC_WITH_MPIUNI AND NOT DEAL_II_WITH_MPI))
+    if(PETSC_WITH_MPIUNI)
       message(STATUS "Could not find a sufficient PETSc installation: "
-        "PETSc has to be configured with the same MPI configuration as deal.II."
+        "PETSc has to be configured with MPI support."
         )
       set(PETSC_ADDITIONAL_ERROR_STRING
         ${PETSC_ADDITIONAL_ERROR_STRING}
         "Could not find a sufficient PETSc installation:\n"
-        "PETSc has to be configured with the same MPI configuration as deal.II, but found:\n"
-        "  DEAL_II_WITH_MPI = ${DEAL_II_WITH_MPI}\n"
+        "PETSc has to be configured with the MPI support, but found:\n"
         "  PETSC_WITH_MPI   = (NOT ${PETSC_WITH_MPIUNI})\n"
         )
       set(${var} FALSE)
diff --git a/doc/news/changes/incompatibilities/20230724DanielArndt b/doc/news/changes/incompatibilities/20230724DanielArndt
new file mode 100644
index 0000000000..5e7a01c426
--- /dev/null
+++ b/doc/news/changes/incompatibilities/20230724DanielArndt
@@ -0,0 +1,3 @@
+Removed: Support for PETSc without MPI has been removed.
+<br>
+(Daniel Arndt, 2023/07/24)
diff --git a/include/deal.II/base/config.h.in b/include/deal.II/base/config.h.in
index ae74303461..9203616264 100644
--- a/include/deal.II/base/config.h.in
+++ b/include/deal.II/base/config.h.in
@@ -572,7 +572,7 @@ _Pragma("GCC diagnostic pop")
  * Some systems require including mpi.h before stdio.h which happens in
  * types.h
  */
-#if defined(DEAL_II_WITH_MPI) || defined(DEAL_II_WITH_PETSC)
+#if defined(DEAL_II_WITH_MPI)
 DEAL_II_DISABLE_EXTRA_DIAGNOSTICS
 #  include <mpi.h>
 DEAL_II_ENABLE_EXTRA_DIAGNOSTICS
diff --git a/include/deal.II/base/mpi_stub.h b/include/deal.II/base/mpi_stub.h
index aab8574451..695fe40f30 100644
--- a/include/deal.II/base/mpi_stub.h
+++ b/include/deal.II/base/mpi_stub.h
@@ -22,7 +22,7 @@
 // types and global constants for the no-MPI case. This way we can still use,
 // e.g., MPI_Comm in the API.
 
-#if defined(DEAL_II_WITH_MPI) || defined(DEAL_II_WITH_PETSC)
+#if defined(DEAL_II_WITH_MPI)
 #  include <mpi.h>
 #else
 // without MPI, we would still like to use
diff --git a/source/lac/petsc_communication_pattern.cc b/source/lac/petsc_communication_pattern.cc
index c2eb6c87b6..61124d2233 100644
--- a/source/lac/petsc_communication_pattern.cc
+++ b/source/lac/petsc_communication_pattern.cc
@@ -277,23 +277,13 @@ namespace PETScWrappers
     const ArrayView<const Number> &src,
     const ArrayView<Number> &      dst) const
   {
-#  ifdef DEAL_II_WITH_MPI
     auto datatype = Utilities::MPI::mpi_type_id_for_type<Number>;
 
-#    if DEAL_II_PETSC_VERSION_LT(3, 15, 0)
+#  if DEAL_II_PETSC_VERSION_LT(3, 15, 0)
     AssertPETSc(PetscSFBcastBegin(sf, datatype, src.data(), dst.data()));
-#    else
+#  else
     AssertPETSc(
       PetscSFBcastBegin(sf, datatype, src.data(), dst.data(), MPI_REPLACE));
-#    endif
-
-#  else
-
-    (void)src;
-    (void)dst;
-    Assert(false,
-           ExcMessage("This program is running without MPI. There should "
-                      "not be anything to import or export!"));
 #  endif
   }
 
@@ -305,23 +295,13 @@ namespace PETScWrappers
     const ArrayView<const Number> &src,
     const ArrayView<Number> &      dst) const
   {
-#  ifdef DEAL_II_WITH_MPI
     auto datatype = Utilities::MPI::mpi_type_id_for_type<Number>;
 
-#    if DEAL_II_PETSC_VERSION_LT(3, 15, 0)
+#  if DEAL_II_PETSC_VERSION_LT(3, 15, 0)
     AssertPETSc(PetscSFBcastEnd(sf, datatype, src.data(), dst.data()));
-#    else
+#  else
     AssertPETSc(
       PetscSFBcastEnd(sf, datatype, src.data(), dst.data(), MPI_REPLACE));
-#    endif
-
-#  else
-
-    (void)src;
-    (void)dst;
-    Assert(false,
-           ExcMessage("This program is running without MPI. There should "
-                      "not be anything to import or export!"));
 #  endif
   }
 
@@ -346,22 +326,11 @@ namespace PETScWrappers
     const ArrayView<const Number> &src,
     const ArrayView<Number> &      dst) const
   {
-#  ifdef DEAL_II_WITH_MPI
     MPI_Op mpiop    = (op == VectorOperation::insert) ? MPI_REPLACE : MPI_SUM;
     auto   datatype = Utilities::MPI::mpi_type_id_for_type<Number>;
 
     AssertPETSc(
       PetscSFReduceBegin(sf, datatype, src.data(), dst.data(), mpiop));
-
-#  else
-
-    (void)op;
-    (void)src;
-    (void)dst;
-    Assert(false,
-           ExcMessage("This program is running without MPI. There should "
-                      "not be anything to import or export!"));
-#  endif
   }
 
 
@@ -373,21 +342,10 @@ namespace PETScWrappers
     const ArrayView<const Number> &src,
     const ArrayView<Number> &      dst) const
   {
-#  ifdef DEAL_II_WITH_MPI
     MPI_Op mpiop    = (op == VectorOperation::insert) ? MPI_REPLACE : MPI_SUM;
     auto   datatype = Utilities::MPI::mpi_type_id_for_type<Number>;
 
     AssertPETSc(PetscSFReduceEnd(sf, datatype, src.data(), dst.data(), mpiop));
-
-#  else
-
-    (void)op;
-    (void)src;
-    (void)dst;
-    Assert(false,
-           ExcMessage("This program is running without MPI. There should "
-                      "not be anything to import or export!"));
-#  endif
   }
 
 
diff --git a/source/lac/petsc_matrix_base.cc b/source/lac/petsc_matrix_base.cc
index 2949542d66..4246a864e6 100644
--- a/source/lac/petsc_matrix_base.cc
+++ b/source/lac/petsc_matrix_base.cc
@@ -247,7 +247,6 @@ namespace PETScWrappers
   {
     {
 #  ifdef DEBUG
-#    ifdef DEAL_II_WITH_MPI
       // Check that all processors agree that last_action is the same (or none!)
 
       int my_int_last_action = last_action;
@@ -265,7 +264,6 @@ namespace PETScWrappers
                     (VectorOperation::add | VectorOperation::insert),
                   ExcMessage("Error: not all processors agree on the last "
                              "VectorOperation before this compress() call."));
-#    endif
 #  endif
     }
 
diff --git a/source/lac/petsc_parallel_vector.cc b/source/lac/petsc_parallel_vector.cc
index 91a40a087e..2db8381ddd 100644
--- a/source/lac/petsc_parallel_vector.cc
+++ b/source/lac/petsc_parallel_vector.cc
@@ -349,15 +349,11 @@ namespace PETScWrappers
     Vector::all_zero() const
     {
       unsigned int has_nonzero = VectorBase::all_zero() ? 0 : 1;
-#  ifdef DEAL_II_WITH_MPI
       // in parallel, check that the vector
       // is zero on _all_ processors.
       unsigned int num_nonzero =
         Utilities::MPI::sum(has_nonzero, this->get_mpi_communicator());
       return num_nonzero == 0;
-#  else
-      return has_nonzero == 0;
-#  endif
     }
 
 
diff --git a/source/lac/petsc_vector_base.cc b/source/lac/petsc_vector_base.cc
index c8e0946464..5cfc76f878 100644
--- a/source/lac/petsc_vector_base.cc
+++ b/source/lac/petsc_vector_base.cc
@@ -542,7 +542,6 @@ namespace PETScWrappers
   {
     {
 #  ifdef DEBUG
-#    ifdef DEAL_II_WITH_MPI
       // Check that all processors agree that last_action is the same (or none!)
 
       int my_int_last_action = last_action;
@@ -560,7 +559,6 @@ namespace PETScWrappers
                     (VectorOperation::add | VectorOperation::insert),
                   ExcMessage("Error: not all processors agree on the last "
                              "VectorOperation before this compress() call."));
-#    endif
 #  endif
     }
 
diff --git a/tests/lac/utilities_01.with_mpi=true.with_petsc=true.with_petsc_with_complex=false.output b/tests/lac/utilities_01.with_petsc=true.with_petsc_with_complex=false.output
similarity index 100%
rename from tests/lac/utilities_01.with_mpi=true.with_petsc=true.with_petsc_with_complex=false.output
rename to tests/lac/utilities_01.with_petsc=true.with_petsc_with_complex=false.output
diff --git a/tests/lac/vector_type_traits_is_serial_03.with_petsc=true.with_mpi=true.output b/tests/lac/vector_type_traits_is_serial_03.with_petsc=true.output
similarity index 100%
rename from tests/lac/vector_type_traits_is_serial_03.with_petsc=true.with_mpi=true.output
rename to tests/lac/vector_type_traits_is_serial_03.with_petsc=true.output
diff --git a/tests/petsc/different_matrix_preconditioner.with_mpi=true.with_petsc_with_hypre=true.output b/tests/petsc/different_matrix_preconditioner.with_petsc_with_hypre=true.output
similarity index 100%
rename from tests/petsc/different_matrix_preconditioner.with_mpi=true.with_petsc_with_hypre=true.output
rename to tests/petsc/different_matrix_preconditioner.with_petsc_with_hypre=true.output
diff --git a/tests/petsc/reinit_preconditioner_02.with_petsc_with_hypre=true.with_64bit_indices=off.with_mpi=true.output b/tests/petsc/reinit_preconditioner_02.with_petsc_with_hypre=true.with_64bit_indices=off.output
similarity index 100%
rename from tests/petsc/reinit_preconditioner_02.with_petsc_with_hypre=true.with_64bit_indices=off.with_mpi=true.output
rename to tests/petsc/reinit_preconditioner_02.with_petsc_with_hypre=true.with_64bit_indices=off.output
diff --git a/tests/petsc/solver_03_precondition_boomeramg.with_mpi=true.with_petsc_with_hypre=true.output b/tests/petsc/solver_03_precondition_boomeramg.with_petsc_with_hypre=true.output
similarity index 100%
rename from tests/petsc/solver_03_precondition_boomeramg.with_mpi=true.with_petsc_with_hypre=true.output
rename to tests/petsc/solver_03_precondition_boomeramg.with_petsc_with_hypre=true.output
diff --git a/tests/petsc/solver_03_precondition_boomeramg_symmetric.with_mpi=true.with_petsc_with_hypre=true.output b/tests/petsc/solver_03_precondition_boomeramg_symmetric.with_petsc_with_hypre=true.output
similarity index 100%
rename from tests/petsc/solver_03_precondition_boomeramg_symmetric.with_mpi=true.with_petsc_with_hypre=true.output
rename to tests/petsc/solver_03_precondition_boomeramg_symmetric.with_petsc_with_hypre=true.output
diff --git a/tests/petsc/solver_03_precondition_parasails.with_64bit_indices=off.with_petsc_with_hypre=true.with_mpi=true.output b/tests/petsc/solver_03_precondition_parasails.with_64bit_indices=off.with_petsc_with_hypre=true.output
similarity index 100%
rename from tests/petsc/solver_03_precondition_parasails.with_64bit_indices=off.with_petsc_with_hypre=true.with_mpi=true.output
rename to tests/petsc/solver_03_precondition_parasails.with_64bit_indices=off.with_petsc_with_hypre=true.output
-- 
2.39.5