]> https://gitweb.dealii.org/ - dealii.git/commitdiff
include ghosted cell in MF: support for active level operator. Testing.
authorMichał Wichrowski <mtwichrowski@gmail.com>
Thu, 6 Feb 2025 21:41:09 +0000 (22:41 +0100)
committerMichał Wichrowski <mtwichrowski@gmail.com>
Thu, 6 Feb 2025 21:41:09 +0000 (22:41 +0100)
doc/news/changes/minor/20250130Wichrowski [new file with mode: 0644]
include/deal.II/matrix_free/matrix_free.templates.h
tests/matrix_free/matrix_vector_26.cc [new file with mode: 0644]
tests/matrix_free/matrix_vector_26.with_trilinos=true.with_p4est=true.mpirun=2.output [new file with mode: 0644]

diff --git a/doc/news/changes/minor/20250130Wichrowski b/doc/news/changes/minor/20250130Wichrowski
new file mode 100644 (file)
index 0000000..760266f
--- /dev/null
@@ -0,0 +1,2 @@
+New: Matrix-Free now can build datastructures for ghosted cells. The functionality can be accessed  by AdditionalData::store_ghost_cells.
+(Michał Wichrowski, 2025/01/30)
\ No newline at end of file
index f55e4e8bf013ac72d55bb71779d38d19a24aa150..d943f50a01960832a662e6cc39926aabfc944760 100644 (file)
@@ -925,17 +925,24 @@ namespace internal
     // steps through all children and adds the active cells recursively
     template <typename InIterator>
     void
-    resolve_cell(const InIterator                                   &cell,
-                 std::vector<std::pair<unsigned int, unsigned int>> &cell_its)
+    resolve_cell(
+      const InIterator                                   &cell,
+      std::vector<std::pair<unsigned int, unsigned int>> &cell_its,
+      std::vector<std::pair<unsigned int, unsigned int>> &ghost_cell_its)
     {
       if (cell->has_children())
         for (unsigned int child = 0; child < cell->n_children(); ++child)
-          resolve_cell(cell->child(child), cell_its);
+          resolve_cell(cell->child(child), cell_its, ghost_cell_its);
       else if (cell->is_locally_owned())
         {
           Assert(cell->is_active(), ExcInternalError());
           cell_its.emplace_back(cell->level(), cell->index());
         }
+      else if (cell->is_ghost())
+        {
+          Assert(cell->is_active(), ExcInternalError());
+          ghost_cell_its.emplace_back(cell->level(), cell->index());
+        }
     }
   } // namespace MatrixFreeFunctions
 } // namespace internal
@@ -969,11 +976,10 @@ MatrixFree<dim, Number, VectorizedArrayType>::initialize_dof_handlers(
       // when setting up neighboring relations between cells for thread
       // parallelization
       for (const auto &cell : tria.cell_iterators_on_level(0))
-        internal::MatrixFreeFunctions::resolve_cell(cell, cell_level_index);
+        internal::MatrixFreeFunctions::resolve_cell(cell,
+                                                    cell_level_index,
+                                                    ghosted_cell_index);
 
-      Assert(additional_data.store_ghost_cells == false,
-             ExcMessage(
-               "Storing ghost cells is only supported for level operator."));
       Assert(task_info.n_procs > 1 ||
                cell_level_index.size() == tria.n_active_cells(),
              ExcInternalError());
@@ -997,13 +1003,11 @@ MatrixFree<dim, Number, VectorizedArrayType>::initialize_dof_handlers(
   // cell_level_index_end_local to the size of cell_level_index.
   cell_level_index_end_local = cell_level_index.size();
 
-  Assert(ghosted_cell_index.size() == 0 || additional_data.store_ghost_cells,
-         ExcInternalError());
-
   // If ghost cells are stored, add them to the end of the cell_level_index
-  cell_level_index.insert(cell_level_index.end(),
-                          ghosted_cell_index.begin(),
-                          ghosted_cell_index.end());
+  if (additional_data.store_ghost_cells == true)
+    cell_level_index.insert(cell_level_index.end(),
+                            ghosted_cell_index.begin(),
+                            ghosted_cell_index.end());
 }
 
 
diff --git a/tests/matrix_free/matrix_vector_26.cc b/tests/matrix_free/matrix_vector_26.cc
new file mode 100644 (file)
index 0000000..5841d5a
--- /dev/null
@@ -0,0 +1,271 @@
+// ------------------------------------------------------------------------
+//
+// SPDX-License-Identifier: LGPL-2.1-or-later
+// Copyright (C) 2013 - 2023 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// Part of the source code is dual licensed under Apache-2.0 WITH
+// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
+// governing the source code and code contributions can be found in
+// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
+//
+// ------------------------------------------------------------------------
+
+
+
+// same as matrix_vector_10.cc but with matrix free stores data for ghosted
+// cells too
+
+#include <deal.II/base/function.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/distributed/tria.h>
+
+#include <deal.II/dofs/dof_handler.h>
+#include <deal.II/dofs/dof_tools.h>
+
+#include <deal.II/fe/fe_q.h>
+#include <deal.II/fe/fe_values.h>
+
+#include <deal.II/grid/grid_generator.h>
+#include <deal.II/grid/manifold_lib.h>
+
+#include <deal.II/lac/affine_constraints.h>
+#include <deal.II/lac/trilinos_sparse_matrix.h>
+#include <deal.II/lac/trilinos_sparsity_pattern.h>
+
+#include <deal.II/numerics/vector_tools.h>
+
+#include <iostream>
+
+#include "../tests.h"
+
+#include "matrix_vector_mf.h"
+
+
+
+template <int dim, int fe_degree>
+void
+test()
+{
+  using number = double;
+
+  parallel::distributed::Triangulation<dim> tria(MPI_COMM_WORLD);
+  GridGenerator::hyper_cube(tria);
+  tria.refine_global(1);
+  typename Triangulation<dim>::active_cell_iterator cell = tria.begin_active(),
+                                                    endc = tria.end();
+  cell                                                   = tria.begin_active();
+  for (; cell != endc; ++cell)
+    if (cell->is_locally_owned())
+      if (cell->center().norm() < 0.2)
+        cell->set_refine_flag();
+  tria.execute_coarsening_and_refinement();
+  if (dim < 3 && fe_degree < 2)
+    tria.refine_global(2);
+  else
+    tria.refine_global(1);
+  if (tria.begin(tria.n_levels() - 1)->is_locally_owned())
+    tria.begin(tria.n_levels() - 1)->set_refine_flag();
+  if (tria.last()->is_locally_owned())
+    tria.last()->set_refine_flag();
+  tria.execute_coarsening_and_refinement();
+  cell = tria.begin_active();
+  for (unsigned int i = 0; i < 10 - 3 * dim; ++i)
+    {
+      cell                 = tria.begin_active();
+      unsigned int counter = 0;
+      for (; cell != endc; ++cell, ++counter)
+        if (cell->is_locally_owned())
+          if (counter % (7 - i) == 0)
+            cell->set_refine_flag();
+      tria.execute_coarsening_and_refinement();
+    }
+
+  FE_Q<dim>       fe(fe_degree);
+  DoFHandler<dim> dof(tria);
+  dof.distribute_dofs(fe);
+
+  IndexSet owned_set = dof.locally_owned_dofs();
+  IndexSet relevant_set;
+  DoFTools::extract_locally_relevant_dofs(dof, relevant_set);
+
+  AffineConstraints<double> constraints(relevant_set);
+  DoFTools::make_hanging_node_constraints(dof, constraints);
+  VectorTools::interpolate_boundary_values(dof,
+                                           0,
+                                           Functions::ZeroFunction<dim>(),
+                                           constraints);
+  constraints.close();
+
+  deallog << "Testing " << dof.get_fe().get_name() << std::endl;
+  // std::cout << "Number of cells: " << tria.n_global_active_cells() <<
+  // std::endl; std::cout << "Number of degrees of freedom: " << dof.n_dofs() <<
+  // std::endl; std::cout << "Number of constraints: " <<
+  // constraints.n_constraints() << std::endl;
+
+  MatrixFree<dim, number> mf_data;
+  {
+    const QGauss<1>                                  quad(fe_degree + 1);
+    typename MatrixFree<dim, number>::AdditionalData data;
+    data.tasks_parallel_scheme = MatrixFree<dim, number>::AdditionalData::none;
+    data.tasks_block_size      = 7;
+    data.store_ghost_cells     = true;
+    mf_data.reinit(MappingQ1<dim>{}, dof, constraints, quad, data);
+  }
+
+  // Check if every ghosted cell is included in MatrixFree storage
+  {
+    using IndexLevel = std::pair<unsigned int, unsigned int>;
+    std::set<std::pair<unsigned int, unsigned int>> ghost_cells;
+
+
+
+    for (unsigned int batch = 0;
+         batch < mf_data.n_cell_batches() + mf_data.n_ghost_cell_batches();
+         ++batch)
+      {
+        for (unsigned int lane = 0;
+             lane < mf_data.n_active_entries_per_cell_batch(batch);
+             ++lane)
+          {
+            const typename Triangulation<dim>::active_cell_iterator cell =
+              mf_data.get_cell_iterator(batch, lane);
+            IndexLevel index_level(cell->index(), cell->level());
+            ghost_cells.insert(index_level);
+          }
+      }
+
+    for (const auto &cell : tria.active_cell_iterators())
+      if (cell->is_ghost())
+        {
+          IndexLevel index_level(cell->index(), cell->level());
+
+          AssertThrow(ghost_cells.find(index_level) != ghost_cells.end(),
+                      ExcMessage(
+                        "MatrixFree does not store all ghost cells. "
+                        "Index: " +
+                        std::to_string(index_level.first) +
+                        ", level: " + std::to_string(index_level.second)));
+        }
+  }
+
+  MatrixFreeTest<dim,
+                 fe_degree,
+                 number,
+                 LinearAlgebra::distributed::Vector<number>>
+                                             mf(mf_data);
+  LinearAlgebra::distributed::Vector<number> in, out, ref;
+  mf_data.initialize_dof_vector(in);
+  out.reinit(in);
+  ref.reinit(in);
+
+  for (unsigned int i = 0; i < in.locally_owned_size(); ++i)
+    {
+      const unsigned int glob_index = owned_set.nth_index_in_set(i);
+      if (constraints.is_constrained(glob_index))
+        continue;
+      in.local_element(i) = random_value<double>();
+    }
+
+  mf.vmult(out, in);
+
+
+  // assemble trilinos sparse matrix with
+  // (\nabla v, \nabla u) + (v, 10 * u) for
+  // reference
+  TrilinosWrappers::SparseMatrix sparse_matrix;
+  {
+    TrilinosWrappers::SparsityPattern csp(owned_set, MPI_COMM_WORLD);
+    DoFTools::make_sparsity_pattern(dof,
+                                    csp,
+                                    constraints,
+                                    true,
+                                    Utilities::MPI::this_mpi_process(
+                                      MPI_COMM_WORLD));
+    csp.compress();
+    sparse_matrix.reinit(csp);
+  }
+  {
+    QGauss<dim> quadrature_formula(fe_degree + 1);
+
+    FEValues<dim> fe_values(dof.get_fe(),
+                            quadrature_formula,
+                            update_values | update_gradients |
+                              update_JxW_values);
+
+    const unsigned int dofs_per_cell = dof.get_fe().dofs_per_cell;
+    const unsigned int n_q_points    = quadrature_formula.size();
+
+    FullMatrix<double> cell_matrix(dofs_per_cell, dofs_per_cell);
+    std::vector<types::global_dof_index> local_dof_indices(dofs_per_cell);
+
+    typename DoFHandler<dim>::active_cell_iterator cell = dof.begin_active(),
+                                                   endc = dof.end();
+    for (; cell != endc; ++cell)
+      if (cell->is_locally_owned())
+        {
+          cell_matrix = 0;
+          fe_values.reinit(cell);
+
+          for (unsigned int q_point = 0; q_point < n_q_points; ++q_point)
+            for (unsigned int i = 0; i < dofs_per_cell; ++i)
+              {
+                for (unsigned int j = 0; j < dofs_per_cell; ++j)
+                  cell_matrix(i, j) +=
+                    ((fe_values.shape_grad(i, q_point) *
+                        fe_values.shape_grad(j, q_point) +
+                      10. * fe_values.shape_value(i, q_point) *
+                        fe_values.shape_value(j, q_point)) *
+                     fe_values.JxW(q_point));
+              }
+
+          cell->get_dof_indices(local_dof_indices);
+          constraints.distribute_local_to_global(cell_matrix,
+                                                 local_dof_indices,
+                                                 sparse_matrix);
+        }
+  }
+  sparse_matrix.compress(VectorOperation::add);
+
+  sparse_matrix.vmult(ref, in);
+  out -= ref;
+  const double diff_norm = out.linfty_norm();
+
+  deallog << "Norm of difference: " << diff_norm << std::endl << std::endl;
+}
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      deallog.push("2d");
+      test<2, 1>();
+      test<2, 2>();
+      deallog.pop();
+
+      deallog.push("3d");
+      test<3, 1>();
+      test<3, 2>();
+      deallog.pop();
+    }
+  else
+    {
+      test<2, 1>();
+      test<2, 2>();
+      test<3, 1>();
+      test<3, 2>();
+    }
+}
diff --git a/tests/matrix_free/matrix_vector_26.with_trilinos=true.with_p4est=true.mpirun=2.output b/tests/matrix_free/matrix_vector_26.with_trilinos=true.with_p4est=true.mpirun=2.output
new file mode 100644 (file)
index 0000000..25e13d5
--- /dev/null
@@ -0,0 +1,13 @@
+
+DEAL:0:2d::Testing FE_Q<2>(1)
+DEAL:0:2d::Norm of difference: 0
+DEAL:0:2d::
+DEAL:0:2d::Testing FE_Q<2>(2)
+DEAL:0:2d::Norm of difference: 0
+DEAL:0:2d::
+DEAL:0:3d::Testing FE_Q<3>(1)
+DEAL:0:3d::Norm of difference: 0
+DEAL:0:3d::
+DEAL:0:3d::Testing FE_Q<3>(2)
+DEAL:0:3d::Norm of difference: 0
+DEAL:0:3d::

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.