From: Martin Kronbichler <martin.kronbichler@uni-a.de>
Date: Tue, 16 May 2023 12:04:24 +0000 (+0200)
Subject: Avoid setting face vectorization boundary in serial
X-Git-Tag: v9.5.0-rc1~215^2
X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=68b45bfb89df739433a5ae8b097c19e39b21e36c;p=dealii.git

Avoid setting face vectorization boundary in serial
---

diff --git a/include/deal.II/matrix_free/matrix_free.templates.h b/include/deal.II/matrix_free/matrix_free.templates.h
index 7162655951..4f66144506 100644
--- a/include/deal.II/matrix_free/matrix_free.templates.h
+++ b/include/deal.II/matrix_free/matrix_free.templates.h
@@ -1951,14 +1951,23 @@ MatrixFree<dim, Number, VectorizedArrayType>::initialize_indices(
 
       std::vector<bool> hard_vectorization_boundary(
         task_info.face_partition_data.size(), false);
-      if (task_info.scheme == internal::MatrixFreeFunctions::TaskInfo::none &&
-          task_info.partition_row_index[2] <
-            task_info.face_partition_data.size())
-        hard_vectorization_boundary[task_info.partition_row_index[2]] = true;
+      if (task_info.scheme == internal::MatrixFreeFunctions::TaskInfo::none)
+        {
+          // In case we do an MPI data exchange, we must make sure to first
+          // complete all face integrals with results in the ghost range
+          if (task_info.partition_row_index[2] <
+              task_info.face_partition_data.size())
+            hard_vectorization_boundary[task_info.partition_row_index[2]] =
+              true;
+        }
       else
-        std::fill(hard_vectorization_boundary.begin(),
-                  hard_vectorization_boundary.end(),
-                  true);
+        {
+          // If we do threading, we need to strictly adhere to each partition,
+          // as we can't schedule work on conflicting face batches
+          std::fill(hard_vectorization_boundary.begin(),
+                    hard_vectorization_boundary.end(),
+                    true);
+        }
 
       internal::MatrixFreeFunctions::collect_faces_vectorization(
         face_setup.inner_faces,