From 208418a5eb7a83f210f4b7cfb39575784cb5d130 Mon Sep 17 00:00:00 2001
From: Daniel Arndt <arndtd@ornl.gov>
Date: Fri, 25 Aug 2023 15:24:51 -0400
Subject: [PATCH] FEValues: Allow checking for cell similarities in
 multi-threaded mode

---
 include/deal.II/fe/fe_values_base.h           | 11 ++++++++++
 .../matrix_free/cuda_matrix_free.templates.h  |  1 +
 source/fe/fe_values_base.cc                   | 21 ++++++++++++-------
 ...iming_matrix_free_kokkos.release.run_only} |  0
 4 files changed, 26 insertions(+), 7 deletions(-)
 rename tests/performance/{timing_matrix_free_kokkos.threads=1.mpirun=max.exclusive.release.run_only => timing_matrix_free_kokkos.release.run_only} (100%)

diff --git a/include/deal.II/fe/fe_values_base.h b/include/deal.II/fe/fe_values_base.h
index 3947ac1ff1..1b275f1ee6 100644
--- a/include/deal.II/fe/fe_values_base.h
+++ b/include/deal.II/fe/fe_values_base.h
@@ -222,6 +222,12 @@ public:
    */
   virtual ~FEValuesBase() override;
 
+  /**
+   * Explicitly allow to check for cell similarity. By default, this is only
+   * enabled when the number of threads is 1.
+   */
+  void
+  allow_check_for_cell_similarity(bool allow);
 
   /// @name Access to shape function values
   ///
@@ -1741,6 +1747,11 @@ private:
    */
   dealii::internal::FEValuesViews::Cache<dim, spacedim> fe_values_views_cache;
 
+  /**
+   * Whether checking for cell similarity is allowed.
+   */
+  bool check_for_cell_similarity_allowed;
+
   // Make the view classes friends of this class, since they access internal
   // data.
   template <int, int>
diff --git a/include/deal.II/matrix_free/cuda_matrix_free.templates.h b/include/deal.II/matrix_free/cuda_matrix_free.templates.h
index 68a47ae8da..c72b6f090b 100644
--- a/include/deal.II/matrix_free/cuda_matrix_free.templates.h
+++ b/include/deal.II/matrix_free/cuda_matrix_free.templates.h
@@ -118,6 +118,7 @@ namespace CUDAWrappers
     {
       local_dof_indices.resize(data->dofs_per_cell);
       lexicographic_dof_indices.resize(dofs_per_cell);
+      fe_values.allow_check_for_cell_similarity(true);
     }
 
 
diff --git a/source/fe/fe_values_base.cc b/source/fe/fe_values_base.cc
index c25863ca85..cb932cdb1b 100644
--- a/source/fe/fe_values_base.cc
+++ b/source/fe/fe_values_base.cc
@@ -227,6 +227,7 @@ FEValuesBase<dim, spacedim>::FEValuesBase(
   , fe(&fe, typeid(*this).name())
   , cell_similarity(CellSimilarity::Similarity::none)
   , fe_values_views_cache(*this)
+  , check_for_cell_similarity_allowed(MultithreadInfo::n_threads() == 1)
 {
   Assert(n_q_points > 0,
          ExcMessage("There is nothing useful you can do with an FEValues "
@@ -246,6 +247,15 @@ FEValuesBase<dim, spacedim>::~FEValuesBase()
 
 
 
+template <int dim, int spacedim>
+void
+FEValuesBase<dim, spacedim>::allow_check_for_cell_similarity(bool allow)
+{
+  check_for_cell_similarity_allowed = allow;
+}
+
+
+
 namespace internal
 {
   // put shape function part of get_function_xxx methods into separate
@@ -1413,15 +1423,12 @@ FEValuesBase<dim, spacedim>::check_cell_similarity(
   // initialized to the first cell the thread sees. As this number might
   // different between different runs (after all, the tasks are scheduled
   // dynamically onto threads), this slight deviation leads to difference in
-  // roundoff errors that propagate through the program. Therefore, we need to
-  // disable CellSimilarity in case there is more than one thread in the
-  // problem. This will likely not affect many MPI test cases as there
+  // roundoff errors that propagate through the program. Therefore, we
+  // disable CellSimilarity by default in case there is more than one thread in
+  // the problem. This will likely not affect many MPI test cases as there
   // multithreading is disabled on default, but in many other situations
   // because we rarely explicitly set the number of threads.
-  //
-  // TODO: Is it reasonable to introduce a flag "unsafe" in the constructor of
-  // FEValues to re-enable this feature?
-  if (MultithreadInfo::n_threads() > 1)
+  if (!check_for_cell_similarity_allowed)
     {
       cell_similarity = CellSimilarity::none;
       return;
diff --git a/tests/performance/timing_matrix_free_kokkos.threads=1.mpirun=max.exclusive.release.run_only b/tests/performance/timing_matrix_free_kokkos.release.run_only
similarity index 100%
rename from tests/performance/timing_matrix_free_kokkos.threads=1.mpirun=max.exclusive.release.run_only
rename to tests/performance/timing_matrix_free_kokkos.release.run_only
-- 
2.39.5