From: Michał Wichrowski Date: Thu, 6 Feb 2025 21:41:09 +0000 (+0100) Subject: include ghosted cell in MF: support for active level operator. Testing. X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9b79e56d23ba1d4db91f67e4d7f9b5e87e0ea619;p=dealii.git include ghosted cell in MF: support for active level operator. Testing. --- diff --git a/doc/news/changes/minor/20250130Wichrowski b/doc/news/changes/minor/20250130Wichrowski new file mode 100644 index 0000000000..760266f3f9 --- /dev/null +++ b/doc/news/changes/minor/20250130Wichrowski @@ -0,0 +1,2 @@ +New: Matrix-Free now can build datastructures for ghosted cells. The functionality can be accessed by AdditionalData::store_ghost_cells. +(Michał Wichrowski, 2025/01/30) \ No newline at end of file diff --git a/include/deal.II/matrix_free/matrix_free.templates.h b/include/deal.II/matrix_free/matrix_free.templates.h index f55e4e8bf0..d943f50a01 100644 --- a/include/deal.II/matrix_free/matrix_free.templates.h +++ b/include/deal.II/matrix_free/matrix_free.templates.h @@ -925,17 +925,24 @@ namespace internal // steps through all children and adds the active cells recursively template void - resolve_cell(const InIterator &cell, - std::vector> &cell_its) + resolve_cell( + const InIterator &cell, + std::vector> &cell_its, + std::vector> &ghost_cell_its) { if (cell->has_children()) for (unsigned int child = 0; child < cell->n_children(); ++child) - resolve_cell(cell->child(child), cell_its); + resolve_cell(cell->child(child), cell_its, ghost_cell_its); else if (cell->is_locally_owned()) { Assert(cell->is_active(), ExcInternalError()); cell_its.emplace_back(cell->level(), cell->index()); } + else if (cell->is_ghost()) + { + Assert(cell->is_active(), ExcInternalError()); + ghost_cell_its.emplace_back(cell->level(), cell->index()); + } } } // namespace MatrixFreeFunctions } // namespace internal @@ -969,11 +976,10 @@ MatrixFree::initialize_dof_handlers( // when setting up neighboring relations between cells for thread // parallelization for (const auto &cell : tria.cell_iterators_on_level(0)) - internal::MatrixFreeFunctions::resolve_cell(cell, cell_level_index); + internal::MatrixFreeFunctions::resolve_cell(cell, + cell_level_index, + ghosted_cell_index); - Assert(additional_data.store_ghost_cells == false, - ExcMessage( - "Storing ghost cells is only supported for level operator.")); Assert(task_info.n_procs > 1 || cell_level_index.size() == tria.n_active_cells(), ExcInternalError()); @@ -997,13 +1003,11 @@ MatrixFree::initialize_dof_handlers( // cell_level_index_end_local to the size of cell_level_index. cell_level_index_end_local = cell_level_index.size(); - Assert(ghosted_cell_index.size() == 0 || additional_data.store_ghost_cells, - ExcInternalError()); - // If ghost cells are stored, add them to the end of the cell_level_index - cell_level_index.insert(cell_level_index.end(), - ghosted_cell_index.begin(), - ghosted_cell_index.end()); + if (additional_data.store_ghost_cells == true) + cell_level_index.insert(cell_level_index.end(), + ghosted_cell_index.begin(), + ghosted_cell_index.end()); } diff --git a/tests/matrix_free/matrix_vector_26.cc b/tests/matrix_free/matrix_vector_26.cc new file mode 100644 index 0000000000..5841d5a9b4 --- /dev/null +++ b/tests/matrix_free/matrix_vector_26.cc @@ -0,0 +1,271 @@ +// ------------------------------------------------------------------------ +// +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (C) 2013 - 2023 by the deal.II authors +// +// This file is part of the deal.II library. +// +// Part of the source code is dual licensed under Apache-2.0 WITH +// LLVM-exception OR LGPL-2.1-or-later. Detailed license information +// governing the source code and code contributions can be found in +// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II. +// +// ------------------------------------------------------------------------ + + + +// same as matrix_vector_10.cc but with matrix free stores data for ghosted +// cells too + +#include +#include + +#include + +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include + +#include + +#include + +#include "../tests.h" + +#include "matrix_vector_mf.h" + + + +template +void +test() +{ + using number = double; + + parallel::distributed::Triangulation tria(MPI_COMM_WORLD); + GridGenerator::hyper_cube(tria); + tria.refine_global(1); + typename Triangulation::active_cell_iterator cell = tria.begin_active(), + endc = tria.end(); + cell = tria.begin_active(); + for (; cell != endc; ++cell) + if (cell->is_locally_owned()) + if (cell->center().norm() < 0.2) + cell->set_refine_flag(); + tria.execute_coarsening_and_refinement(); + if (dim < 3 && fe_degree < 2) + tria.refine_global(2); + else + tria.refine_global(1); + if (tria.begin(tria.n_levels() - 1)->is_locally_owned()) + tria.begin(tria.n_levels() - 1)->set_refine_flag(); + if (tria.last()->is_locally_owned()) + tria.last()->set_refine_flag(); + tria.execute_coarsening_and_refinement(); + cell = tria.begin_active(); + for (unsigned int i = 0; i < 10 - 3 * dim; ++i) + { + cell = tria.begin_active(); + unsigned int counter = 0; + for (; cell != endc; ++cell, ++counter) + if (cell->is_locally_owned()) + if (counter % (7 - i) == 0) + cell->set_refine_flag(); + tria.execute_coarsening_and_refinement(); + } + + FE_Q fe(fe_degree); + DoFHandler dof(tria); + dof.distribute_dofs(fe); + + IndexSet owned_set = dof.locally_owned_dofs(); + IndexSet relevant_set; + DoFTools::extract_locally_relevant_dofs(dof, relevant_set); + + AffineConstraints constraints(relevant_set); + DoFTools::make_hanging_node_constraints(dof, constraints); + VectorTools::interpolate_boundary_values(dof, + 0, + Functions::ZeroFunction(), + constraints); + constraints.close(); + + deallog << "Testing " << dof.get_fe().get_name() << std::endl; + // std::cout << "Number of cells: " << tria.n_global_active_cells() << + // std::endl; std::cout << "Number of degrees of freedom: " << dof.n_dofs() << + // std::endl; std::cout << "Number of constraints: " << + // constraints.n_constraints() << std::endl; + + MatrixFree mf_data; + { + const QGauss<1> quad(fe_degree + 1); + typename MatrixFree::AdditionalData data; + data.tasks_parallel_scheme = MatrixFree::AdditionalData::none; + data.tasks_block_size = 7; + data.store_ghost_cells = true; + mf_data.reinit(MappingQ1{}, dof, constraints, quad, data); + } + + // Check if every ghosted cell is included in MatrixFree storage + { + using IndexLevel = std::pair; + std::set> ghost_cells; + + + + for (unsigned int batch = 0; + batch < mf_data.n_cell_batches() + mf_data.n_ghost_cell_batches(); + ++batch) + { + for (unsigned int lane = 0; + lane < mf_data.n_active_entries_per_cell_batch(batch); + ++lane) + { + const typename Triangulation::active_cell_iterator cell = + mf_data.get_cell_iterator(batch, lane); + IndexLevel index_level(cell->index(), cell->level()); + ghost_cells.insert(index_level); + } + } + + for (const auto &cell : tria.active_cell_iterators()) + if (cell->is_ghost()) + { + IndexLevel index_level(cell->index(), cell->level()); + + AssertThrow(ghost_cells.find(index_level) != ghost_cells.end(), + ExcMessage( + "MatrixFree does not store all ghost cells. " + "Index: " + + std::to_string(index_level.first) + + ", level: " + std::to_string(index_level.second))); + } + } + + MatrixFreeTest> + mf(mf_data); + LinearAlgebra::distributed::Vector in, out, ref; + mf_data.initialize_dof_vector(in); + out.reinit(in); + ref.reinit(in); + + for (unsigned int i = 0; i < in.locally_owned_size(); ++i) + { + const unsigned int glob_index = owned_set.nth_index_in_set(i); + if (constraints.is_constrained(glob_index)) + continue; + in.local_element(i) = random_value(); + } + + mf.vmult(out, in); + + + // assemble trilinos sparse matrix with + // (\nabla v, \nabla u) + (v, 10 * u) for + // reference + TrilinosWrappers::SparseMatrix sparse_matrix; + { + TrilinosWrappers::SparsityPattern csp(owned_set, MPI_COMM_WORLD); + DoFTools::make_sparsity_pattern(dof, + csp, + constraints, + true, + Utilities::MPI::this_mpi_process( + MPI_COMM_WORLD)); + csp.compress(); + sparse_matrix.reinit(csp); + } + { + QGauss quadrature_formula(fe_degree + 1); + + FEValues fe_values(dof.get_fe(), + quadrature_formula, + update_values | update_gradients | + update_JxW_values); + + const unsigned int dofs_per_cell = dof.get_fe().dofs_per_cell; + const unsigned int n_q_points = quadrature_formula.size(); + + FullMatrix cell_matrix(dofs_per_cell, dofs_per_cell); + std::vector local_dof_indices(dofs_per_cell); + + typename DoFHandler::active_cell_iterator cell = dof.begin_active(), + endc = dof.end(); + for (; cell != endc; ++cell) + if (cell->is_locally_owned()) + { + cell_matrix = 0; + fe_values.reinit(cell); + + for (unsigned int q_point = 0; q_point < n_q_points; ++q_point) + for (unsigned int i = 0; i < dofs_per_cell; ++i) + { + for (unsigned int j = 0; j < dofs_per_cell; ++j) + cell_matrix(i, j) += + ((fe_values.shape_grad(i, q_point) * + fe_values.shape_grad(j, q_point) + + 10. * fe_values.shape_value(i, q_point) * + fe_values.shape_value(j, q_point)) * + fe_values.JxW(q_point)); + } + + cell->get_dof_indices(local_dof_indices); + constraints.distribute_local_to_global(cell_matrix, + local_dof_indices, + sparse_matrix); + } + } + sparse_matrix.compress(VectorOperation::add); + + sparse_matrix.vmult(ref, in); + out -= ref; + const double diff_norm = out.linfty_norm(); + + deallog << "Norm of difference: " << diff_norm << std::endl << std::endl; +} + + +int +main(int argc, char **argv) +{ + Utilities::MPI::MPI_InitFinalize mpi_initialization( + argc, argv, testing_max_num_threads()); + + unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD); + deallog.push(Utilities::int_to_string(myid)); + + if (myid == 0) + { + initlog(); + deallog << std::setprecision(4); + + deallog.push("2d"); + test<2, 1>(); + test<2, 2>(); + deallog.pop(); + + deallog.push("3d"); + test<3, 1>(); + test<3, 2>(); + deallog.pop(); + } + else + { + test<2, 1>(); + test<2, 2>(); + test<3, 1>(); + test<3, 2>(); + } +} diff --git a/tests/matrix_free/matrix_vector_26.with_trilinos=true.with_p4est=true.mpirun=2.output b/tests/matrix_free/matrix_vector_26.with_trilinos=true.with_p4est=true.mpirun=2.output new file mode 100644 index 0000000000..25e13d5f8f --- /dev/null +++ b/tests/matrix_free/matrix_vector_26.with_trilinos=true.with_p4est=true.mpirun=2.output @@ -0,0 +1,13 @@ + +DEAL:0:2d::Testing FE_Q<2>(1) +DEAL:0:2d::Norm of difference: 0 +DEAL:0:2d:: +DEAL:0:2d::Testing FE_Q<2>(2) +DEAL:0:2d::Norm of difference: 0 +DEAL:0:2d:: +DEAL:0:3d::Testing FE_Q<3>(1) +DEAL:0:3d::Norm of difference: 0 +DEAL:0:3d:: +DEAL:0:3d::Testing FE_Q<3>(2) +DEAL:0:3d::Norm of difference: 0 +DEAL:0:3d::