From f2ec7c776840ba135f79d0d765d8d7591bde752f Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 29 Nov 2018 19:45:19 +0100 Subject: [PATCH] add two tests: one pass and another still broken --- tests/mpi/periodicity_06.cc | 222 +++++++++++++++++++++ tests/mpi/periodicity_06.mpirun=2.output | 225 +++++++++++++++++++++ tests/mpi/periodicity_07.cc | 229 ++++++++++++++++++++++ tests/mpi/periodicity_07.mpirun=13.output | 3 + 4 files changed, 679 insertions(+) create mode 100644 tests/mpi/periodicity_06.cc create mode 100644 tests/mpi/periodicity_06.mpirun=2.output create mode 100644 tests/mpi/periodicity_07.cc create mode 100644 tests/mpi/periodicity_07.mpirun=13.output diff --git a/tests/mpi/periodicity_06.cc b/tests/mpi/periodicity_06.cc new file mode 100644 index 0000000000..b5f2e0bb1f --- /dev/null +++ b/tests/mpi/periodicity_06.cc @@ -0,0 +1,222 @@ +/* --------------------------------------------------------------------- + * + * Copyright (C) 2018 by the deal.II authors + * + * This file is part of the deal.II library. + * + * The deal.II library is free software; you can use it, redistribute + * it, and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * The full text of the license can be found in the file LICENSE.md at + * the top level directory of deal.II. + * + * --------------------------------------------------------------------- + */ + +// a unit test from https://github.com/dealii/dealii/issues/7053 +// we used to build incompatible constraints: +// === Process 0 +// Constraints: +// ... +// 16 32: 1 +// ... +// 52 32: 1 +// ... +// Owned DoFs: +// {[0,93]} +// Ghost DoFs: +// {[0,143]} +// Coordinates: +// 52@-20 20 -10 +// 32@20 -20 -10 +// +// vs +// +// === Process 1 +// Constraints: +// ... +// 32 16: 1 +// ... +// 52 16: 1 +// ... +// Owned DoFs: +// {[94,143]} +// Ghost DoFs: +// {[0,88], [94,143]} +// Coordinates: +// 52@-20 20 -10 +// 16@-20 -20 -10 + +#include + +#include + +#include +#include +#include + +#include +#include + +#include + +#include "../tests.h" + +template +void +test(const unsigned numRefinementLevels = 2) +{ + MPI_Comm mpi_communicator = MPI_COMM_WORLD; + + const unsigned int n_mpi_processes = + Utilities::MPI::n_mpi_processes(mpi_communicator); + const unsigned int this_mpi_process = + Utilities::MPI::this_mpi_process(mpi_communicator); + + const double L = 20; + dealii::parallel::distributed::Triangulation triangulation( + mpi_communicator); + GridGenerator::hyper_cube(triangulation, -L, L, /*colorize*/ false); + + // mark faces + for (auto cell : triangulation.active_cell_iterators()) + for (unsigned int f = 0; f < GeometryInfo::faces_per_cell; ++f) + { + const Point &face_center = cell->face(f)->center(); + if (cell->face(f)->at_boundary()) + { + unsigned int counter = 1; + for (unsigned int d = 0; d < dim; ++d) + { + if (std::abs(face_center[d] - L) < 1.0e-5) + cell->face(f)->set_boundary_id(counter); + ++counter; + if (std::abs(face_center[d] + L) < 1.0e-5) + cell->face(f)->set_boundary_id(counter); + ++counter; + } + } + } + + std::vector< + GridTools::PeriodicFacePair::cell_iterator>> + periodicity_vector; + for (int d = 0; d < dim; ++d) + GridTools::collect_periodic_faces(triangulation, + /*b_id1*/ 2 * d + 1, + /*b_id2*/ 2 * d + 2, + /*direction*/ d, + periodicity_vector); + + triangulation.add_periodicity(periodicity_vector); + + // refine mesh + triangulation.refine_global(1); + + Point corner; + for (unsigned int d = 0; d < dim; ++d) + corner[d] = -L; + + MappingQ1 mapping; + for (unsigned int ilevel = 0; ilevel < numRefinementLevels; ilevel++) + { + // pick an corner cell and refine + for (auto cell : triangulation.active_cell_iterators()) + { + try + { + const Point p_cell = + mapping.transform_real_to_unit_cell(cell, corner); + const double dist = + GeometryInfo::distance_to_unit_cell(p_cell); + + if (dist < 1e-08) + cell->set_refine_flag(); + } + catch (typename MappingQ1::ExcTransformationFailed) + {} + } + triangulation.execute_coarsening_and_refinement(); + } + + if (this_mpi_process == 0) + std::cout << "number of elements: " << triangulation.n_global_active_cells() + << std::endl; + + // create dofHandler + FESystem FE(FE_Q(QGaussLobatto<1>(2)), 1); + DoFHandler dofHandler(triangulation); + dofHandler.distribute_dofs(FE); + + // write mesh for visualization + DataOut data_out; + data_out.attach_dof_handler(dofHandler); + Vector subdomain(triangulation.n_active_cells()); + for (unsigned int i = 0; i < subdomain.size(); ++i) + subdomain(i) = triangulation.locally_owned_subdomain(); + data_out.add_data_vector(subdomain, "subdomain"); + data_out.build_patches(); + data_out.write_vtu_in_parallel(std::string("mesh.vtu").c_str(), + mpi_communicator); + + IndexSet locally_relevant_dofs; + DoFTools::extract_locally_relevant_dofs(dofHandler, locally_relevant_dofs); + + std::map> supportPoints; + DoFTools::map_dofs_to_support_points(MappingQ1(), + dofHandler, + supportPoints); + + /// creating combined hanging node and periodic constraint matrix + AffineConstraints constraints; + constraints.clear(); + constraints.reinit(locally_relevant_dofs); + DoFTools::make_hanging_node_constraints(dofHandler, constraints); + std::vector< + GridTools::PeriodicFacePair::cell_iterator>> + periodicity_vectorDof; + for (int d = 0; d < dim; ++d) + GridTools::collect_periodic_faces(dofHandler, + /*b_id1*/ 2 * d + 1, + /*b_id2*/ 2 * d + 2, + /*direction*/ d, + periodicity_vectorDof); + + DoFTools::make_periodicity_constraints>(periodicity_vectorDof, + constraints); + constraints.close(); + + for (unsigned int i = 0; i < n_mpi_processes; ++i) + { + if (this_mpi_process == i) + { + std::cout << "=== Process " << i << std::endl + << "Constraints:" << std::endl; + constraints.print(std::cout); + std::cout << "Owned DoFs:" << std::endl; + dofHandler.locally_owned_dofs().print(std::cout); + std::cout << "Ghost DoFs:" << std::endl; + locally_relevant_dofs.print(std::cout); + std::cout << "Coordinates:" << std::endl; + // we have issues with constraints for u_52 = .... + const unsigned int i = 52; + if (locally_relevant_dofs.is_element(i) && + constraints.is_constrained(i)) + { + std::cout << i << "@" << supportPoints[i] << std::endl; + for (auto c : *constraints.get_constraint_entries(i)) + std::cout << c.first << "@" << supportPoints[c.first] + << std::endl; + } + } + MPI_Barrier(mpi_communicator); + } +} + +int +main(int argc, char *argv[]) +{ + Utilities::MPI::MPI_InitFinalize mpi_initialization(argc, argv); + test<3>(); +} \ No newline at end of file diff --git a/tests/mpi/periodicity_06.mpirun=2.output b/tests/mpi/periodicity_06.mpirun=2.output new file mode 100644 index 0000000000..36e69ad116 --- /dev/null +++ b/tests/mpi/periodicity_06.mpirun=2.output @@ -0,0 +1,225 @@ +number of elements: 71 +=== Process 0 +Constraints: + 31 8: 1 + 32 16: 1 + 33 11: 1 + 36 9: 1 + 37 12: 1 + 40 17: 1 + 41 19: 1 + 43 23: 1 + 50 30: 1 + 51 0: 1 + 52 16: 1 + 53 4: 1 + 54 1: 1 + 55 5: 1 + 59 17: 1 + 60 18: 1 + 61 21: 1 + 64 44: 1 + 65 46: 1 + 66 26: 1 + 67 28: 1 + 68 30: 1 + 69 16: 1 + 71 56: 1 + 72 38: 1 + 73 17: 1 + 74 30: 1 + 75 0: 0.5 + 75 30: 0.5 + 76 8: 0.5 + 76 30: 0.5 + 77 0: 0.25 + 77 2: 0.25 + 77 8: 0.25 + 77 30: 0.25 + 78 16: 0.5 + 78 30: 0.5 + 79 0: 0.25 + 79 4: 0.25 + 79 16: 0.25 + 79 30: 0.25 + 80 8: 0.25 + 80 11: 0.25 + 80 16: 0.25 + 80 30: 0.25 + 82 0: 0.5 + 82 2: 0.5 + 83 0: 0.5 + 83 4: 0.5 + 84 0: 0.25 + 84 2: 0.25 + 84 4: 0.25 + 84 6: 0.25 + 85 2: 0.5 + 85 8: 0.5 + 86 8: 0.5 + 86 11: 0.5 + 87 2: 0.25 + 87 6: 0.25 + 87 8: 0.25 + 87 11: 0.25 + 88 2: 0.5 + 88 6: 0.5 + 89 4: 0.5 + 89 16: 0.5 + 90 11: 0.5 + 90 16: 0.5 + 91 4: 0.25 + 91 6: 0.25 + 91 11: 0.25 + 91 16: 0.25 + 92 4: 0.5 + 92 6: 0.5 + 93 6: 0.5 + 93 11: 0.5 + 103 30: 1 + 104 0: 1 + 105 8: 1 + 106 2: 1 + 107 1: 1 + 108 3: 1 + 109 9: 1 + 110 10: 1 + 111 14: 1 + 114 94: 1 + 115 96: 1 + 117 100: 1 + 118 26: 1 + 119 27: 1 + 120 30: 1 + 121 8: 1 + 122 34: 1 + 123 9: 1 + 127 94: 1 + 128 95: 1 + 129 98: 1 + 130 44: 1 + 131 45: 1 + 132 48: 1 + 133 30: 1 + 134 0: 1 + 135 1: 1 + 137 124: 1 + 138 112: 1 + 139 94: 1 + 140 62: 1 + 141 44: 1 + 142 26: 1 + 143 30: 1 +Owned DoFs: +{[0,93]} +Ghost DoFs: +{[0,143]} +Coordinates: +52@-20 20 -10 +16@-20 -20 -10 +=== Process 1 +Constraints: + 31 8: 1 + 32 16: 1 + 33 11: 1 + 36 9: 1 + 37 12: 1 + 40 17: 1 + 41 19: 1 + 43 23: 1 + 50 30: 1 + 51 0: 1 + 52 16: 1 + 53 4: 1 + 54 1: 1 + 55 5: 1 + 59 17: 1 + 60 18: 1 + 61 21: 1 + 64 44: 1 + 65 46: 1 + 66 26: 1 + 67 28: 1 + 68 30: 1 + 69 16: 1 + 71 56: 1 + 72 38: 1 + 73 17: 1 + 74 30: 1 + 75 0: 0.5 + 75 30: 0.5 + 76 8: 0.5 + 76 30: 0.5 + 77 0: 0.25 + 77 2: 0.25 + 77 8: 0.25 + 77 30: 0.25 + 78 16: 0.5 + 78 30: 0.5 + 79 0: 0.25 + 79 4: 0.25 + 79 16: 0.25 + 79 30: 0.25 + 80 8: 0.25 + 80 11: 0.25 + 80 16: 0.25 + 80 30: 0.25 + 82 0: 0.5 + 82 2: 0.5 + 83 0: 0.5 + 83 4: 0.5 + 84 0: 0.25 + 84 2: 0.25 + 84 4: 0.25 + 84 6: 0.25 + 85 2: 0.5 + 85 8: 0.5 + 86 8: 0.5 + 86 11: 0.5 + 87 2: 0.25 + 87 6: 0.25 + 87 8: 0.25 + 87 11: 0.25 + 88 2: 0.5 + 88 6: 0.5 + 103 30: 1 + 104 0: 1 + 105 8: 1 + 106 2: 1 + 107 1: 1 + 108 3: 1 + 109 9: 1 + 110 10: 1 + 111 14: 1 + 114 94: 1 + 115 96: 1 + 117 100: 1 + 118 26: 1 + 119 27: 1 + 120 30: 1 + 121 8: 1 + 122 34: 1 + 123 9: 1 + 127 94: 1 + 128 95: 1 + 129 98: 1 + 130 44: 1 + 131 45: 1 + 132 48: 1 + 133 30: 1 + 134 0: 1 + 135 1: 1 + 137 124: 1 + 138 112: 1 + 139 94: 1 + 140 62: 1 + 141 44: 1 + 142 26: 1 + 143 30: 1 +Owned DoFs: +{[94,143]} +Ghost DoFs: +{[0,88], [94,143]} +Coordinates: +52@-20 20 -10 +16@-20 -20 -10 diff --git a/tests/mpi/periodicity_07.cc b/tests/mpi/periodicity_07.cc new file mode 100644 index 0000000000..dc0bf0a9b4 --- /dev/null +++ b/tests/mpi/periodicity_07.cc @@ -0,0 +1,229 @@ +/* --------------------------------------------------------------------- + * + * Copyright (C) 2018 by the deal.II authors + * + * This file is part of the deal.II library. + * + * The deal.II library is free software; you can use it, redistribute + * it, and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * The full text of the license can be found in the file LICENSE.md at + * the top level directory of deal.II. + * + * --------------------------------------------------------------------- + */ + +// a variation of periodicity_06 that used to trigger +// AffineConstraints::is_consistent_in_parallel() on 13 mpi tasks. + +#include + +#include + +#include + +#include +#include +#include + +#include +#include + +#include + +#include "../tests.h" + +template +void +test(const unsigned numRefinementLevels = 2) +{ + MPI_Comm mpi_communicator = MPI_COMM_WORLD; + + const unsigned int n_mpi_processes = + Utilities::MPI::n_mpi_processes(mpi_communicator); + const unsigned int this_mpi_process = + Utilities::MPI::this_mpi_process(mpi_communicator); + + ConditionalOStream pcout(std::cout, this_mpi_process == 0); + + const double L = 20; + dealii::parallel::distributed::Triangulation triangulation( + mpi_communicator); + GridGenerator::hyper_cube(triangulation, -L, L, /*colorize*/ false); + + // mark faces + for (auto cell : triangulation.active_cell_iterators()) + for (unsigned int f = 0; f < GeometryInfo::faces_per_cell; ++f) + { + const Point &face_center = cell->face(f)->center(); + if (cell->face(f)->at_boundary()) + { + unsigned int counter = 1; + for (unsigned int d = 0; d < dim; ++d) + { + if (std::abs(face_center[d] - L) < 1.0e-5) + cell->face(f)->set_boundary_id(counter); + ++counter; + if (std::abs(face_center[d] + L) < 1.0e-5) + cell->face(f)->set_boundary_id(counter); + ++counter; + } + } + } + + std::vector< + GridTools::PeriodicFacePair::cell_iterator>> + periodicity_vector; + for (int d = 0; d < dim; ++d) + GridTools::collect_periodic_faces(triangulation, + /*b_id1*/ 2 * d + 1, + /*b_id2*/ 2 * d + 2, + /*direction*/ d, + periodicity_vector); + + triangulation.add_periodicity(periodicity_vector); + + // refine mesh + triangulation.refine_global(1); + + Point corner; + for (unsigned int d = 0; d < dim; ++d) + corner[d] = -L; + + MappingQ1 mapping; + for (unsigned int ilevel = 0; ilevel < numRefinementLevels; ilevel++) + { + // pick an corner cell and refine + for (auto cell : triangulation.active_cell_iterators()) + { + try + { + const Point p_cell = + mapping.transform_real_to_unit_cell(cell, corner); + const double dist = + GeometryInfo::distance_to_unit_cell(p_cell); + + if (dist < 1e-08) + cell->set_refine_flag(); + } + catch (typename MappingQ1::ExcTransformationFailed) + {} + } + triangulation.execute_coarsening_and_refinement(); + } + + pcout << "number of elements: " << triangulation.n_global_active_cells() + << std::endl; + + // create dofHandler + FESystem FE(FE_Q(QGaussLobatto<1>(2)), 1); + DoFHandler dofHandler(triangulation); + dofHandler.distribute_dofs(FE); + + // write mesh for visualization + DataOut data_out; + data_out.attach_dof_handler(dofHandler); + Vector subdomain(triangulation.n_active_cells()); + for (unsigned int i = 0; i < subdomain.size(); ++i) + subdomain(i) = triangulation.locally_owned_subdomain(); + data_out.add_data_vector(subdomain, "subdomain"); + data_out.build_patches(); + data_out.write_vtu_in_parallel(std::string("mesh.vtu").c_str(), + mpi_communicator); + + IndexSet locally_relevant_dofs; + DoFTools::extract_locally_relevant_dofs(dofHandler, locally_relevant_dofs); + + IndexSet locally_active_dofs; + DoFTools::extract_locally_active_dofs(dofHandler, locally_active_dofs); + + const std::vector &locally_owned_dofs = + dofHandler.locally_owned_dofs_per_processor(); + + std::map> supportPoints; + DoFTools::map_dofs_to_support_points(MappingQ1(), + dofHandler, + supportPoints); + + /// creating combined hanging node and periodic constraint matrix + AffineConstraints constraints; + constraints.clear(); + constraints.reinit(locally_relevant_dofs); + DoFTools::make_hanging_node_constraints(dofHandler, constraints); + + const bool hanging_consistent = + constraints.is_consistent_in_parallel(locally_owned_dofs, + locally_active_dofs, + mpi_communicator); + + pcout << "Hanging nodes constraints are consistent in parallel: " + << hanging_consistent << std::endl; + + std::vector< + GridTools::PeriodicFacePair::cell_iterator>> + periodicity_vectorDof; + for (int d = 0; d < dim; ++d) + GridTools::collect_periodic_faces(dofHandler, + /*b_id1*/ 2 * d + 1, + /*b_id2*/ 2 * d + 2, + /*direction*/ d, + periodicity_vectorDof); + + DoFTools::make_periodicity_constraints>(periodicity_vectorDof, + constraints); + constraints.close(); + + const bool consistent = + constraints.is_consistent_in_parallel(locally_owned_dofs, + locally_active_dofs, + mpi_communicator, + /*verbose*/ true); + + pcout << "Total constraints are consistent in parallel: " << consistent + << std::endl; + + /* verbose output of is_consistent_in_parallel() gives: + + Proc 10 got line 370 from 11 wrong values! + Proc 10 got line 374 from 11 wrong values! + Proc 10 got line 378 from 11 wrong values! + 3 inconsistent lines discovered! + + */ + + const std::vector wrong_lines = {{370, 374, 378}}; + + for (unsigned int i = 0; i < n_mpi_processes; ++i) + { + if (this_mpi_process == i) + { + std::cout << "=== Process " << i << std::endl; + // constraints.print(std::cout); + // std::cout << "Owned DoFs:" << std::endl; + // dofHandler.locally_owned_dofs().print(std::cout); + // std::cout << "Ghost DoFs:" << std::endl; + // locally_relevant_dofs.print(std::cout); + + for (auto ind : wrong_lines) + if (locally_relevant_dofs.is_element(ind) && + constraints.is_constrained(ind)) + { + std::cout << "Constraints for " << ind << " @ " + << supportPoints[ind] << ":" << std::endl; + for (auto c : *constraints.get_constraint_entries(ind)) + std::cout << " " << c.first << " @ " + << supportPoints[c.first] << " : " << c.second + << std::endl; + } + } + MPI_Barrier(mpi_communicator); + } +} + +int +main(int argc, char *argv[]) +{ + Utilities::MPI::MPI_InitFinalize mpi_initialization(argc, argv); + test<3>(4); +} diff --git a/tests/mpi/periodicity_07.mpirun=13.output b/tests/mpi/periodicity_07.mpirun=13.output new file mode 100644 index 0000000000..ed2f98d514 --- /dev/null +++ b/tests/mpi/periodicity_07.mpirun=13.output @@ -0,0 +1,3 @@ +number of elements: 183 +Hanging nodes constraints are consistent in parallel: 1 +Total constraints are consistent in parallel: 1 -- 2.39.5