From 8939e049b1d8a9abeeb4df71b712cb2fb7518ff9 Mon Sep 17 00:00:00 2001 From: Marc Fehling Date: Mon, 10 Feb 2020 23:49:04 +0100 Subject: [PATCH] Explicitly call p4est's repartition functions instead of p::d::Tria::repartition(). This avoids confusion with signals. New tests for serialization. --- source/distributed/tria.cc | 43 ++- source/hp/dof_handler.cc | 13 +- tests/mpi/cell_data_transfer_04.cc | 128 +++++++ ...ransfer_04.with_p4est=true.mpirun=5.output | 195 +++++++++++ tests/mpi/hp_active_fe_indices_transfer_04.cc | 137 ++++++++ ...ransfer_04.with_p4est=true.mpirun=5.output | 323 ++++++++++++++++++ 6 files changed, 828 insertions(+), 11 deletions(-) create mode 100644 tests/mpi/cell_data_transfer_04.cc create mode 100644 tests/mpi/cell_data_transfer_04.with_p4est=true.mpirun=5.output create mode 100644 tests/mpi/hp_active_fe_indices_transfer_04.cc create mode 100644 tests/mpi/hp_active_fe_indices_transfer_04.with_p4est=true.mpirun=5.output diff --git a/source/distributed/tria.cc b/source/distributed/tria.cc index 2477b09b0c..7ca5c02cbd 100644 --- a/source/distributed/tria.cc +++ b/source/distributed/tria.cc @@ -2826,12 +2826,43 @@ namespace parallel &connectivity); if (numcpus != Utilities::MPI::n_mpi_processes(this->mpi_communicator)) - // We are changing the number of CPUs so we need to repartition. - // Note that p4est actually distributes the cells between the changed - // number of CPUs and so everything works without this call, but - // this command changes the distribution for some reason, so we - // will leave it in here. - repartition(); + { + // We are changing the number of CPUs so we need to repartition. + // Note that p4est actually distributes the cells between the changed + // number of CPUs and so everything works without this call, but + // this command changes the distribution for some reason, so we + // will leave it in here. + if (this->signals.cell_weight.num_slots() == 0) + { + // no cell weights given -- call p4est's 'partition' without a + // callback for cell weights + dealii::internal::p4est::functions::partition( + parallel_forest, + /* prepare coarsening */ 1, + /* weight_callback */ nullptr); + } + else + { + // get cell weights for a weighted repartitioning. + const std::vector cell_weights = get_cell_weights(); + + PartitionWeights partition_weights(cell_weights); + + // attach (temporarily) a pointer to the cell weights through + // p4est's user_pointer object + Assert(parallel_forest->user_pointer == this, ExcInternalError()); + parallel_forest->user_pointer = &partition_weights; + + dealii::internal::p4est::functions::partition( + parallel_forest, + /* prepare coarsening */ 1, + /* weight_callback */ + &PartitionWeights::cell_weight); + + // reset the user pointer to its previous state + parallel_forest->user_pointer = this; + } + } try { diff --git a/source/hp/dof_handler.cc b/source/hp/dof_handler.cc index be33ba35e2..4ed5607f29 100644 --- a/source/hp/dof_handler.cc +++ b/source/hp/dof_handler.cc @@ -2133,7 +2133,7 @@ namespace hp // Unpack active_fe_indices. active_fe_index_transfer->active_fe_indices.resize( - tria->n_active_cells(), numbers::invalid_unsigned_int); + get_triangulation().n_active_cells(), numbers::invalid_unsigned_int); active_fe_index_transfer->cell_data_transfer->unpack( active_fe_index_transfer->active_fe_indices); @@ -2220,10 +2220,13 @@ namespace hp "if deal.II was configured to use p4est, but cmake did not find a " "valid p4est library.")); #else - Assert(active_fe_index_transfer != nullptr, ExcInternalError()); + if (fe_collection.size() > 0) + { + Assert(active_fe_index_transfer != nullptr, ExcInternalError()); - // Free memory. - active_fe_index_transfer.reset(); + // Free memory. + active_fe_index_transfer.reset(); + } #endif } @@ -2273,7 +2276,7 @@ namespace hp // Unpack active_fe_indices. active_fe_index_transfer->active_fe_indices.resize( - tria->n_active_cells(), numbers::invalid_unsigned_int); + get_triangulation().n_active_cells(), numbers::invalid_unsigned_int); active_fe_index_transfer->cell_data_transfer->deserialize( active_fe_index_transfer->active_fe_indices); diff --git a/tests/mpi/cell_data_transfer_04.cc b/tests/mpi/cell_data_transfer_04.cc new file mode 100644 index 0000000000..fc941fe0c7 --- /dev/null +++ b/tests/mpi/cell_data_transfer_04.cc @@ -0,0 +1,128 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2020 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + + + +// p::d::CellDataTransfer serialization with a different number of cpus + + +#include +#include + +#include + +#include + +#include "../tests.h" + + +template +void +test() +{ + unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD); + MPI_Comm com_all = MPI_COMM_WORLD; + MPI_Comm com_small; + + // split the communicator in proc 0,1,2 and 3,4 + MPI_Comm_split(com_all, (myid < 3) ? 0 : 1, myid, &com_small); + + // write with small com + if (myid < 3) + { + deallog << "writing with " << Utilities::MPI::n_mpi_processes(com_small) + << std::endl; + + // ------ setup ------ + parallel::distributed::Triangulation tria(com_small); + GridGenerator::subdivided_hyper_cube(tria, 2); + tria.refine_global(1); + + // ----- gather ----- + // store parent id of all cells + std::vector cell_ids(tria.n_active_cells()); + for (auto &cell : tria.active_cell_iterators()) + if (cell->is_locally_owned()) + { + const std::string parent_cellid = cell->parent()->id().to_string(); + const unsigned int parent_coarse_cell_id = + (unsigned int)std::stoul(parent_cellid); + cell_ids[cell->active_cell_index()] = parent_coarse_cell_id; + + deallog << "cellid=" << cell->id() + << " parentid=" << cell_ids[cell->active_cell_index()] + << std::endl; + } + + // ----- transfer ----- + parallel::distributed:: + CellDataTransfer> + cell_data_transfer(tria); + + cell_data_transfer.prepare_for_coarsening_and_refinement(cell_ids); + tria.save("file"); + } + + // make sure no processor is hanging + MPI_Barrier(MPI_COMM_WORLD); + + { + deallog << "reading with " << Utilities::MPI::n_mpi_processes(com_all) + << std::endl; + + // ------ setup ------ + parallel::distributed::Triangulation tria(com_all); + GridGenerator::subdivided_hyper_cube(tria, 2); + // triangulation has to be initialized with correct coarse cells + + // ----- transfer ----- + tria.load("file"); + + parallel::distributed:: + CellDataTransfer> + cell_data_transfer(tria); + + std::vector cell_ids(tria.n_active_cells()); + cell_data_transfer.deserialize(cell_ids); + + // ------ verify ------ + // check if all children adopted the correct id + for (auto &cell : tria.active_cell_iterators()) + if (cell->is_locally_owned()) + deallog << "cellid=" << cell->id() + << " parentid=" << cell_ids[(cell->active_cell_index())] + << std::endl; + } + + // make sure no processor is hanging + MPI_Barrier(MPI_COMM_WORLD); + + deallog << "OK" << std::endl; +} + + +int +main(int argc, char *argv[]) +{ + Utilities::MPI::MPI_InitFinalize mpi_initialization(argc, argv, 1); + MPILogInitAll log; + + deallog.push("2d"); + test<2, 2>(); + deallog.pop(); + deallog.push("3d"); + test<3, 3>(); + deallog.pop(); +} diff --git a/tests/mpi/cell_data_transfer_04.with_p4est=true.mpirun=5.output b/tests/mpi/cell_data_transfer_04.with_p4est=true.mpirun=5.output new file mode 100644 index 0000000000..5c77f952ea --- /dev/null +++ b/tests/mpi/cell_data_transfer_04.with_p4est=true.mpirun=5.output @@ -0,0 +1,195 @@ + +DEAL:0:2d::writing with 3 +DEAL:0:2d::cellid=0_1:0 parentid=0 +DEAL:0:2d::cellid=0_1:1 parentid=0 +DEAL:0:2d::cellid=0_1:2 parentid=0 +DEAL:0:2d::cellid=0_1:3 parentid=0 +DEAL:0:2d::reading with 5 +DEAL:0:2d::cellid=0_1:0 parentid=0 +DEAL:0:2d::cellid=0_1:1 parentid=0 +DEAL:0:2d::cellid=0_1:2 parentid=0 +DEAL:0:2d::cellid=0_1:3 parentid=0 +DEAL:0:2d::OK +DEAL:0:3d::writing with 3 +DEAL:0:3d::cellid=0_1:0 parentid=0 +DEAL:0:3d::cellid=0_1:1 parentid=0 +DEAL:0:3d::cellid=0_1:2 parentid=0 +DEAL:0:3d::cellid=0_1:3 parentid=0 +DEAL:0:3d::cellid=0_1:4 parentid=0 +DEAL:0:3d::cellid=0_1:5 parentid=0 +DEAL:0:3d::cellid=0_1:6 parentid=0 +DEAL:0:3d::cellid=0_1:7 parentid=0 +DEAL:0:3d::cellid=1_1:0 parentid=1 +DEAL:0:3d::cellid=1_1:1 parentid=1 +DEAL:0:3d::cellid=1_1:2 parentid=1 +DEAL:0:3d::cellid=1_1:3 parentid=1 +DEAL:0:3d::cellid=1_1:4 parentid=1 +DEAL:0:3d::cellid=1_1:5 parentid=1 +DEAL:0:3d::cellid=1_1:6 parentid=1 +DEAL:0:3d::cellid=1_1:7 parentid=1 +DEAL:0:3d::cellid=2_1:0 parentid=2 +DEAL:0:3d::cellid=2_1:1 parentid=2 +DEAL:0:3d::cellid=2_1:2 parentid=2 +DEAL:0:3d::cellid=2_1:3 parentid=2 +DEAL:0:3d::cellid=2_1:4 parentid=2 +DEAL:0:3d::cellid=2_1:5 parentid=2 +DEAL:0:3d::cellid=2_1:6 parentid=2 +DEAL:0:3d::cellid=2_1:7 parentid=2 +DEAL:0:3d::reading with 5 +DEAL:0:3d::cellid=0_1:0 parentid=0 +DEAL:0:3d::cellid=0_1:1 parentid=0 +DEAL:0:3d::cellid=0_1:2 parentid=0 +DEAL:0:3d::cellid=0_1:3 parentid=0 +DEAL:0:3d::cellid=0_1:4 parentid=0 +DEAL:0:3d::cellid=0_1:5 parentid=0 +DEAL:0:3d::cellid=0_1:6 parentid=0 +DEAL:0:3d::cellid=0_1:7 parentid=0 +DEAL:0:3d::cellid=1_1:0 parentid=1 +DEAL:0:3d::cellid=1_1:1 parentid=1 +DEAL:0:3d::cellid=1_1:2 parentid=1 +DEAL:0:3d::cellid=1_1:3 parentid=1 +DEAL:0:3d::cellid=1_1:4 parentid=1 +DEAL:0:3d::cellid=1_1:5 parentid=1 +DEAL:0:3d::cellid=1_1:6 parentid=1 +DEAL:0:3d::cellid=1_1:7 parentid=1 +DEAL:0:3d::OK + +DEAL:1:2d::writing with 3 +DEAL:1:2d::cellid=1_1:0 parentid=1 +DEAL:1:2d::cellid=1_1:1 parentid=1 +DEAL:1:2d::cellid=1_1:2 parentid=1 +DEAL:1:2d::cellid=1_1:3 parentid=1 +DEAL:1:2d::cellid=2_1:0 parentid=2 +DEAL:1:2d::cellid=2_1:1 parentid=2 +DEAL:1:2d::cellid=2_1:2 parentid=2 +DEAL:1:2d::cellid=2_1:3 parentid=2 +DEAL:1:2d::reading with 5 +DEAL:1:2d::cellid=1_1:0 parentid=1 +DEAL:1:2d::cellid=1_1:1 parentid=1 +DEAL:1:2d::cellid=1_1:2 parentid=1 +DEAL:1:2d::cellid=1_1:3 parentid=1 +DEAL:1:2d::OK +DEAL:1:3d::writing with 3 +DEAL:1:3d::cellid=3_1:0 parentid=3 +DEAL:1:3d::cellid=3_1:1 parentid=3 +DEAL:1:3d::cellid=3_1:2 parentid=3 +DEAL:1:3d::cellid=3_1:3 parentid=3 +DEAL:1:3d::cellid=3_1:4 parentid=3 +DEAL:1:3d::cellid=3_1:5 parentid=3 +DEAL:1:3d::cellid=3_1:6 parentid=3 +DEAL:1:3d::cellid=3_1:7 parentid=3 +DEAL:1:3d::cellid=4_1:0 parentid=4 +DEAL:1:3d::cellid=4_1:1 parentid=4 +DEAL:1:3d::cellid=4_1:2 parentid=4 +DEAL:1:3d::cellid=4_1:3 parentid=4 +DEAL:1:3d::cellid=4_1:4 parentid=4 +DEAL:1:3d::cellid=4_1:5 parentid=4 +DEAL:1:3d::cellid=4_1:6 parentid=4 +DEAL:1:3d::cellid=4_1:7 parentid=4 +DEAL:1:3d::reading with 5 +DEAL:1:3d::cellid=2_1:0 parentid=2 +DEAL:1:3d::cellid=2_1:1 parentid=2 +DEAL:1:3d::cellid=2_1:2 parentid=2 +DEAL:1:3d::cellid=2_1:3 parentid=2 +DEAL:1:3d::cellid=2_1:4 parentid=2 +DEAL:1:3d::cellid=2_1:5 parentid=2 +DEAL:1:3d::cellid=2_1:6 parentid=2 +DEAL:1:3d::cellid=2_1:7 parentid=2 +DEAL:1:3d::OK + + +DEAL:2:2d::writing with 3 +DEAL:2:2d::cellid=3_1:0 parentid=3 +DEAL:2:2d::cellid=3_1:1 parentid=3 +DEAL:2:2d::cellid=3_1:2 parentid=3 +DEAL:2:2d::cellid=3_1:3 parentid=3 +DEAL:2:2d::reading with 5 +DEAL:2:2d::OK +DEAL:2:3d::writing with 3 +DEAL:2:3d::cellid=5_1:0 parentid=5 +DEAL:2:3d::cellid=5_1:1 parentid=5 +DEAL:2:3d::cellid=5_1:2 parentid=5 +DEAL:2:3d::cellid=5_1:3 parentid=5 +DEAL:2:3d::cellid=5_1:4 parentid=5 +DEAL:2:3d::cellid=5_1:5 parentid=5 +DEAL:2:3d::cellid=5_1:6 parentid=5 +DEAL:2:3d::cellid=5_1:7 parentid=5 +DEAL:2:3d::cellid=6_1:0 parentid=6 +DEAL:2:3d::cellid=6_1:1 parentid=6 +DEAL:2:3d::cellid=6_1:2 parentid=6 +DEAL:2:3d::cellid=6_1:3 parentid=6 +DEAL:2:3d::cellid=6_1:4 parentid=6 +DEAL:2:3d::cellid=6_1:5 parentid=6 +DEAL:2:3d::cellid=6_1:6 parentid=6 +DEAL:2:3d::cellid=6_1:7 parentid=6 +DEAL:2:3d::cellid=7_1:0 parentid=7 +DEAL:2:3d::cellid=7_1:1 parentid=7 +DEAL:2:3d::cellid=7_1:2 parentid=7 +DEAL:2:3d::cellid=7_1:3 parentid=7 +DEAL:2:3d::cellid=7_1:4 parentid=7 +DEAL:2:3d::cellid=7_1:5 parentid=7 +DEAL:2:3d::cellid=7_1:6 parentid=7 +DEAL:2:3d::cellid=7_1:7 parentid=7 +DEAL:2:3d::reading with 5 +DEAL:2:3d::cellid=3_1:0 parentid=3 +DEAL:2:3d::cellid=3_1:1 parentid=3 +DEAL:2:3d::cellid=3_1:2 parentid=3 +DEAL:2:3d::cellid=3_1:3 parentid=3 +DEAL:2:3d::cellid=3_1:4 parentid=3 +DEAL:2:3d::cellid=3_1:5 parentid=3 +DEAL:2:3d::cellid=3_1:6 parentid=3 +DEAL:2:3d::cellid=3_1:7 parentid=3 +DEAL:2:3d::cellid=4_1:0 parentid=4 +DEAL:2:3d::cellid=4_1:1 parentid=4 +DEAL:2:3d::cellid=4_1:2 parentid=4 +DEAL:2:3d::cellid=4_1:3 parentid=4 +DEAL:2:3d::cellid=4_1:4 parentid=4 +DEAL:2:3d::cellid=4_1:5 parentid=4 +DEAL:2:3d::cellid=4_1:6 parentid=4 +DEAL:2:3d::cellid=4_1:7 parentid=4 +DEAL:2:3d::OK + + +DEAL:3:2d::reading with 5 +DEAL:3:2d::cellid=2_1:0 parentid=2 +DEAL:3:2d::cellid=2_1:1 parentid=2 +DEAL:3:2d::cellid=2_1:2 parentid=2 +DEAL:3:2d::cellid=2_1:3 parentid=2 +DEAL:3:2d::OK +DEAL:3:3d::reading with 5 +DEAL:3:3d::cellid=5_1:0 parentid=5 +DEAL:3:3d::cellid=5_1:1 parentid=5 +DEAL:3:3d::cellid=5_1:2 parentid=5 +DEAL:3:3d::cellid=5_1:3 parentid=5 +DEAL:3:3d::cellid=5_1:4 parentid=5 +DEAL:3:3d::cellid=5_1:5 parentid=5 +DEAL:3:3d::cellid=5_1:6 parentid=5 +DEAL:3:3d::cellid=5_1:7 parentid=5 +DEAL:3:3d::OK + + +DEAL:4:2d::reading with 5 +DEAL:4:2d::cellid=3_1:0 parentid=3 +DEAL:4:2d::cellid=3_1:1 parentid=3 +DEAL:4:2d::cellid=3_1:2 parentid=3 +DEAL:4:2d::cellid=3_1:3 parentid=3 +DEAL:4:2d::OK +DEAL:4:3d::reading with 5 +DEAL:4:3d::cellid=6_1:0 parentid=6 +DEAL:4:3d::cellid=6_1:1 parentid=6 +DEAL:4:3d::cellid=6_1:2 parentid=6 +DEAL:4:3d::cellid=6_1:3 parentid=6 +DEAL:4:3d::cellid=6_1:4 parentid=6 +DEAL:4:3d::cellid=6_1:5 parentid=6 +DEAL:4:3d::cellid=6_1:6 parentid=6 +DEAL:4:3d::cellid=6_1:7 parentid=6 +DEAL:4:3d::cellid=7_1:0 parentid=7 +DEAL:4:3d::cellid=7_1:1 parentid=7 +DEAL:4:3d::cellid=7_1:2 parentid=7 +DEAL:4:3d::cellid=7_1:3 parentid=7 +DEAL:4:3d::cellid=7_1:4 parentid=7 +DEAL:4:3d::cellid=7_1:5 parentid=7 +DEAL:4:3d::cellid=7_1:6 parentid=7 +DEAL:4:3d::cellid=7_1:7 parentid=7 +DEAL:4:3d::OK + diff --git a/tests/mpi/hp_active_fe_indices_transfer_04.cc b/tests/mpi/hp_active_fe_indices_transfer_04.cc new file mode 100644 index 0000000000..2f8ed1e372 --- /dev/null +++ b/tests/mpi/hp_active_fe_indices_transfer_04.cc @@ -0,0 +1,137 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2020 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + + + +// active fe indices serialization with a different number of cpus + + +#include + +#include + +#include + +#include + +#include "../tests.h" + + +template +void +test() +{ + unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD); + MPI_Comm com_all = MPI_COMM_WORLD; + MPI_Comm com_small; + + // split the communicator in proc 0,1,2 and 3,4 + MPI_Comm_split(com_all, (myid < 3) ? 0 : 1, myid, &com_small); + + // prepare FECollection with arbitrary number of entries + hp::FECollection fe_collection; + const unsigned int max_degree = 1 + Utilities::pow(2, dim); + for (unsigned int i = 0; i < max_degree; ++i) + fe_collection.push_back(FE_Q(max_degree - i)); + + // write with small com + if (myid < 3) + { + deallog << "writing with " << Utilities::MPI::n_mpi_processes(com_small) + << std::endl; + + // ------ setup ------ + parallel::distributed::Triangulation tria(com_small); + GridGenerator::subdivided_hyper_cube(tria, 2); + tria.refine_global(1); + + hp::DoFHandler dh(tria); + // we need to introduce dof_handler to its fe_collection first + dh.set_fe(fe_collection); + + unsigned int i = 0; + for (auto &cell : dh.active_cell_iterators()) + if (cell->is_locally_owned()) + { + // set active fe index + if (i >= fe_collection.size()) + i = 0; + cell->set_active_fe_index(i++); + + deallog << "cellid=" << cell->id() + << " fe_index=" << cell->active_fe_index() << std::endl; + } + + // ----- transfer ----- + dh.prepare_for_serialization_of_active_fe_indices(); + tria.save("file"); + } + + // make sure no processor is hanging + MPI_Barrier(MPI_COMM_WORLD); + + { + deallog << "reading with " << Utilities::MPI::n_mpi_processes(com_all) + << std::endl; + + // ------ setup ------ + parallel::distributed::Triangulation tria(com_all); + GridGenerator::subdivided_hyper_cube(tria, 2); + // triangulation has to be initialized with correct coarse cells + + // we need to introduce dof_handler to its fe_collection first + hp::DoFHandler dh(tria); + dh.set_fe(fe_collection); + + // ----- transfer ----- + tria.load("file"); + dh.deserialize_active_fe_indices(); + + // ------ verify ------ + // check if all children adopted the correct id + for (auto &cell : dh.active_cell_iterators()) + if (!cell->is_artificial()) + { + deallog << "cellid=" << cell->id() + << " fe_index=" << cell->active_fe_index(); + if (cell->is_ghost()) + deallog << " ghost"; + deallog << std::endl; + } + + // distribute dofs again for further calculations, i.e. + // dh.distribute_dofs(fe_collection); + } + + // make sure no processor is hanging + MPI_Barrier(MPI_COMM_WORLD); + + deallog << "OK" << std::endl; +} + + +int +main(int argc, char *argv[]) +{ + Utilities::MPI::MPI_InitFinalize mpi_initialization(argc, argv, 1); + MPILogInitAll log; + + deallog.push("2d"); + test<2>(); + deallog.pop(); + deallog.push("3d"); + test<3>(); + deallog.pop(); +} diff --git a/tests/mpi/hp_active_fe_indices_transfer_04.with_p4est=true.mpirun=5.output b/tests/mpi/hp_active_fe_indices_transfer_04.with_p4est=true.mpirun=5.output new file mode 100644 index 0000000000..3809f23b43 --- /dev/null +++ b/tests/mpi/hp_active_fe_indices_transfer_04.with_p4est=true.mpirun=5.output @@ -0,0 +1,323 @@ + +DEAL:0:2d::writing with 3 +DEAL:0:2d::cellid=0_1:0 fe_index=0 +DEAL:0:2d::cellid=0_1:1 fe_index=1 +DEAL:0:2d::cellid=0_1:2 fe_index=2 +DEAL:0:2d::cellid=0_1:3 fe_index=3 +DEAL:0:2d::reading with 5 +DEAL:0:2d::cellid=0_1:0 fe_index=0 +DEAL:0:2d::cellid=0_1:1 fe_index=1 +DEAL:0:2d::cellid=0_1:2 fe_index=2 +DEAL:0:2d::cellid=0_1:3 fe_index=3 +DEAL:0:2d::cellid=1_1:0 fe_index=0 ghost +DEAL:0:2d::cellid=1_1:2 fe_index=2 ghost +DEAL:0:2d::cellid=2_1:0 fe_index=4 ghost +DEAL:0:2d::cellid=2_1:1 fe_index=0 ghost +DEAL:0:2d::cellid=3_1:0 fe_index=0 ghost +DEAL:0:2d::OK +DEAL:0:3d::writing with 3 +DEAL:0:3d::cellid=0_1:0 fe_index=0 +DEAL:0:3d::cellid=0_1:1 fe_index=1 +DEAL:0:3d::cellid=0_1:2 fe_index=2 +DEAL:0:3d::cellid=0_1:3 fe_index=3 +DEAL:0:3d::cellid=0_1:4 fe_index=4 +DEAL:0:3d::cellid=0_1:5 fe_index=5 +DEAL:0:3d::cellid=0_1:6 fe_index=6 +DEAL:0:3d::cellid=0_1:7 fe_index=7 +DEAL:0:3d::cellid=1_1:0 fe_index=8 +DEAL:0:3d::cellid=1_1:1 fe_index=0 +DEAL:0:3d::cellid=1_1:2 fe_index=1 +DEAL:0:3d::cellid=1_1:3 fe_index=2 +DEAL:0:3d::cellid=1_1:4 fe_index=3 +DEAL:0:3d::cellid=1_1:5 fe_index=4 +DEAL:0:3d::cellid=1_1:6 fe_index=5 +DEAL:0:3d::cellid=1_1:7 fe_index=6 +DEAL:0:3d::cellid=2_1:0 fe_index=7 +DEAL:0:3d::cellid=2_1:1 fe_index=8 +DEAL:0:3d::cellid=2_1:2 fe_index=0 +DEAL:0:3d::cellid=2_1:3 fe_index=1 +DEAL:0:3d::cellid=2_1:4 fe_index=2 +DEAL:0:3d::cellid=2_1:5 fe_index=3 +DEAL:0:3d::cellid=2_1:6 fe_index=4 +DEAL:0:3d::cellid=2_1:7 fe_index=5 +DEAL:0:3d::reading with 5 +DEAL:0:3d::cellid=0_1:0 fe_index=0 +DEAL:0:3d::cellid=0_1:1 fe_index=1 +DEAL:0:3d::cellid=0_1:2 fe_index=2 +DEAL:0:3d::cellid=0_1:3 fe_index=3 +DEAL:0:3d::cellid=0_1:4 fe_index=4 +DEAL:0:3d::cellid=0_1:5 fe_index=5 +DEAL:0:3d::cellid=0_1:6 fe_index=6 +DEAL:0:3d::cellid=0_1:7 fe_index=7 +DEAL:0:3d::cellid=1_1:0 fe_index=8 +DEAL:0:3d::cellid=1_1:1 fe_index=0 +DEAL:0:3d::cellid=1_1:2 fe_index=1 +DEAL:0:3d::cellid=1_1:3 fe_index=2 +DEAL:0:3d::cellid=1_1:4 fe_index=3 +DEAL:0:3d::cellid=1_1:5 fe_index=4 +DEAL:0:3d::cellid=1_1:6 fe_index=5 +DEAL:0:3d::cellid=1_1:7 fe_index=6 +DEAL:0:3d::cellid=2_1:0 fe_index=7 ghost +DEAL:0:3d::cellid=2_1:1 fe_index=8 ghost +DEAL:0:3d::cellid=2_1:4 fe_index=2 ghost +DEAL:0:3d::cellid=2_1:5 fe_index=3 ghost +DEAL:0:3d::cellid=3_1:0 fe_index=0 ghost +DEAL:0:3d::cellid=3_1:1 fe_index=1 ghost +DEAL:0:3d::cellid=3_1:4 fe_index=4 ghost +DEAL:0:3d::cellid=3_1:5 fe_index=5 ghost +DEAL:0:3d::cellid=4_1:0 fe_index=8 ghost +DEAL:0:3d::cellid=4_1:1 fe_index=0 ghost +DEAL:0:3d::cellid=4_1:2 fe_index=1 ghost +DEAL:0:3d::cellid=4_1:3 fe_index=2 ghost +DEAL:0:3d::cellid=5_1:0 fe_index=0 ghost +DEAL:0:3d::cellid=5_1:1 fe_index=1 ghost +DEAL:0:3d::cellid=5_1:2 fe_index=2 ghost +DEAL:0:3d::cellid=5_1:3 fe_index=3 ghost +DEAL:0:3d::cellid=6_1:0 fe_index=8 ghost +DEAL:0:3d::cellid=6_1:1 fe_index=0 ghost +DEAL:0:3d::cellid=7_1:0 fe_index=7 ghost +DEAL:0:3d::cellid=7_1:1 fe_index=8 ghost +DEAL:0:3d::OK + +DEAL:1:2d::writing with 3 +DEAL:1:2d::cellid=1_1:0 fe_index=0 +DEAL:1:2d::cellid=1_1:1 fe_index=1 +DEAL:1:2d::cellid=1_1:2 fe_index=2 +DEAL:1:2d::cellid=1_1:3 fe_index=3 +DEAL:1:2d::cellid=2_1:0 fe_index=4 +DEAL:1:2d::cellid=2_1:1 fe_index=0 +DEAL:1:2d::cellid=2_1:2 fe_index=1 +DEAL:1:2d::cellid=2_1:3 fe_index=2 +DEAL:1:2d::reading with 5 +DEAL:1:2d::cellid=0_1:1 fe_index=1 ghost +DEAL:1:2d::cellid=0_1:3 fe_index=3 ghost +DEAL:1:2d::cellid=1_1:0 fe_index=0 +DEAL:1:2d::cellid=1_1:1 fe_index=1 +DEAL:1:2d::cellid=1_1:2 fe_index=2 +DEAL:1:2d::cellid=1_1:3 fe_index=3 +DEAL:1:2d::cellid=2_1:1 fe_index=0 ghost +DEAL:1:2d::cellid=3_1:0 fe_index=0 ghost +DEAL:1:2d::cellid=3_1:1 fe_index=1 ghost +DEAL:1:2d::OK +DEAL:1:3d::writing with 3 +DEAL:1:3d::cellid=3_1:0 fe_index=0 +DEAL:1:3d::cellid=3_1:1 fe_index=1 +DEAL:1:3d::cellid=3_1:2 fe_index=2 +DEAL:1:3d::cellid=3_1:3 fe_index=3 +DEAL:1:3d::cellid=3_1:4 fe_index=4 +DEAL:1:3d::cellid=3_1:5 fe_index=5 +DEAL:1:3d::cellid=3_1:6 fe_index=6 +DEAL:1:3d::cellid=3_1:7 fe_index=7 +DEAL:1:3d::cellid=4_1:0 fe_index=8 +DEAL:1:3d::cellid=4_1:1 fe_index=0 +DEAL:1:3d::cellid=4_1:2 fe_index=1 +DEAL:1:3d::cellid=4_1:3 fe_index=2 +DEAL:1:3d::cellid=4_1:4 fe_index=3 +DEAL:1:3d::cellid=4_1:5 fe_index=4 +DEAL:1:3d::cellid=4_1:6 fe_index=5 +DEAL:1:3d::cellid=4_1:7 fe_index=6 +DEAL:1:3d::reading with 5 +DEAL:1:3d::cellid=0_1:2 fe_index=2 ghost +DEAL:1:3d::cellid=0_1:3 fe_index=3 ghost +DEAL:1:3d::cellid=0_1:6 fe_index=6 ghost +DEAL:1:3d::cellid=0_1:7 fe_index=7 ghost +DEAL:1:3d::cellid=1_1:2 fe_index=1 ghost +DEAL:1:3d::cellid=1_1:6 fe_index=5 ghost +DEAL:1:3d::cellid=2_1:0 fe_index=7 +DEAL:1:3d::cellid=2_1:1 fe_index=8 +DEAL:1:3d::cellid=2_1:2 fe_index=0 +DEAL:1:3d::cellid=2_1:3 fe_index=1 +DEAL:1:3d::cellid=2_1:4 fe_index=2 +DEAL:1:3d::cellid=2_1:5 fe_index=3 +DEAL:1:3d::cellid=2_1:6 fe_index=4 +DEAL:1:3d::cellid=2_1:7 fe_index=5 +DEAL:1:3d::cellid=3_1:0 fe_index=0 ghost +DEAL:1:3d::cellid=3_1:2 fe_index=2 ghost +DEAL:1:3d::cellid=3_1:4 fe_index=4 ghost +DEAL:1:3d::cellid=3_1:6 fe_index=6 ghost +DEAL:1:3d::cellid=4_1:2 fe_index=1 ghost +DEAL:1:3d::cellid=4_1:3 fe_index=2 ghost +DEAL:1:3d::cellid=5_1:2 fe_index=2 ghost +DEAL:1:3d::cellid=6_1:0 fe_index=8 ghost +DEAL:1:3d::cellid=6_1:1 fe_index=0 ghost +DEAL:1:3d::cellid=6_1:2 fe_index=1 ghost +DEAL:1:3d::cellid=6_1:3 fe_index=2 ghost +DEAL:1:3d::cellid=7_1:0 fe_index=7 ghost +DEAL:1:3d::cellid=7_1:2 fe_index=0 ghost +DEAL:1:3d::OK + + +DEAL:2:2d::writing with 3 +DEAL:2:2d::cellid=3_1:0 fe_index=0 +DEAL:2:2d::cellid=3_1:1 fe_index=1 +DEAL:2:2d::cellid=3_1:2 fe_index=2 +DEAL:2:2d::cellid=3_1:3 fe_index=3 +DEAL:2:2d::reading with 5 +DEAL:2:2d::OK +DEAL:2:3d::writing with 3 +DEAL:2:3d::cellid=5_1:0 fe_index=0 +DEAL:2:3d::cellid=5_1:1 fe_index=1 +DEAL:2:3d::cellid=5_1:2 fe_index=2 +DEAL:2:3d::cellid=5_1:3 fe_index=3 +DEAL:2:3d::cellid=5_1:4 fe_index=4 +DEAL:2:3d::cellid=5_1:5 fe_index=5 +DEAL:2:3d::cellid=5_1:6 fe_index=6 +DEAL:2:3d::cellid=5_1:7 fe_index=7 +DEAL:2:3d::cellid=6_1:0 fe_index=8 +DEAL:2:3d::cellid=6_1:1 fe_index=0 +DEAL:2:3d::cellid=6_1:2 fe_index=1 +DEAL:2:3d::cellid=6_1:3 fe_index=2 +DEAL:2:3d::cellid=6_1:4 fe_index=3 +DEAL:2:3d::cellid=6_1:5 fe_index=4 +DEAL:2:3d::cellid=6_1:6 fe_index=5 +DEAL:2:3d::cellid=6_1:7 fe_index=6 +DEAL:2:3d::cellid=7_1:0 fe_index=7 +DEAL:2:3d::cellid=7_1:1 fe_index=8 +DEAL:2:3d::cellid=7_1:2 fe_index=0 +DEAL:2:3d::cellid=7_1:3 fe_index=1 +DEAL:2:3d::cellid=7_1:4 fe_index=2 +DEAL:2:3d::cellid=7_1:5 fe_index=3 +DEAL:2:3d::cellid=7_1:6 fe_index=4 +DEAL:2:3d::cellid=7_1:7 fe_index=5 +DEAL:2:3d::reading with 5 +DEAL:2:3d::cellid=0_1:3 fe_index=3 ghost +DEAL:2:3d::cellid=0_1:4 fe_index=4 ghost +DEAL:2:3d::cellid=0_1:5 fe_index=5 ghost +DEAL:2:3d::cellid=0_1:6 fe_index=6 ghost +DEAL:2:3d::cellid=0_1:7 fe_index=7 ghost +DEAL:2:3d::cellid=1_1:2 fe_index=1 ghost +DEAL:2:3d::cellid=1_1:3 fe_index=2 ghost +DEAL:2:3d::cellid=1_1:4 fe_index=3 ghost +DEAL:2:3d::cellid=1_1:6 fe_index=5 ghost +DEAL:2:3d::cellid=1_1:7 fe_index=6 ghost +DEAL:2:3d::cellid=2_1:1 fe_index=8 ghost +DEAL:2:3d::cellid=2_1:3 fe_index=1 ghost +DEAL:2:3d::cellid=2_1:4 fe_index=2 ghost +DEAL:2:3d::cellid=2_1:5 fe_index=3 ghost +DEAL:2:3d::cellid=2_1:7 fe_index=5 ghost +DEAL:2:3d::cellid=3_1:0 fe_index=0 +DEAL:2:3d::cellid=3_1:1 fe_index=1 +DEAL:2:3d::cellid=3_1:2 fe_index=2 +DEAL:2:3d::cellid=3_1:3 fe_index=3 +DEAL:2:3d::cellid=3_1:4 fe_index=4 +DEAL:2:3d::cellid=3_1:5 fe_index=5 +DEAL:2:3d::cellid=3_1:6 fe_index=6 +DEAL:2:3d::cellid=3_1:7 fe_index=7 +DEAL:2:3d::cellid=4_1:0 fe_index=8 +DEAL:2:3d::cellid=4_1:1 fe_index=0 +DEAL:2:3d::cellid=4_1:2 fe_index=1 +DEAL:2:3d::cellid=4_1:3 fe_index=2 +DEAL:2:3d::cellid=4_1:4 fe_index=3 +DEAL:2:3d::cellid=4_1:5 fe_index=4 +DEAL:2:3d::cellid=4_1:6 fe_index=5 +DEAL:2:3d::cellid=4_1:7 fe_index=6 +DEAL:2:3d::cellid=5_1:0 fe_index=0 ghost +DEAL:2:3d::cellid=5_1:2 fe_index=2 ghost +DEAL:2:3d::cellid=5_1:3 fe_index=3 ghost +DEAL:2:3d::cellid=5_1:4 fe_index=4 ghost +DEAL:2:3d::cellid=5_1:6 fe_index=6 ghost +DEAL:2:3d::cellid=6_1:0 fe_index=8 ghost +DEAL:2:3d::cellid=6_1:1 fe_index=0 ghost +DEAL:2:3d::cellid=6_1:3 fe_index=2 ghost +DEAL:2:3d::cellid=6_1:4 fe_index=3 ghost +DEAL:2:3d::cellid=6_1:5 fe_index=4 ghost +DEAL:2:3d::cellid=7_1:0 fe_index=7 ghost +DEAL:2:3d::cellid=7_1:1 fe_index=8 ghost +DEAL:2:3d::cellid=7_1:2 fe_index=0 ghost +DEAL:2:3d::cellid=7_1:3 fe_index=1 ghost +DEAL:2:3d::cellid=7_1:4 fe_index=2 ghost +DEAL:2:3d::OK + + +DEAL:3:2d::reading with 5 +DEAL:3:2d::cellid=0_1:2 fe_index=2 ghost +DEAL:3:2d::cellid=0_1:3 fe_index=3 ghost +DEAL:3:2d::cellid=1_1:2 fe_index=2 ghost +DEAL:3:2d::cellid=2_1:0 fe_index=4 +DEAL:3:2d::cellid=2_1:1 fe_index=0 +DEAL:3:2d::cellid=2_1:2 fe_index=1 +DEAL:3:2d::cellid=2_1:3 fe_index=2 +DEAL:3:2d::cellid=3_1:0 fe_index=0 ghost +DEAL:3:2d::cellid=3_1:2 fe_index=2 ghost +DEAL:3:2d::OK +DEAL:3:3d::reading with 5 +DEAL:3:3d::cellid=0_1:5 fe_index=5 ghost +DEAL:3:3d::cellid=0_1:7 fe_index=7 ghost +DEAL:3:3d::cellid=1_1:4 fe_index=3 ghost +DEAL:3:3d::cellid=1_1:5 fe_index=4 ghost +DEAL:3:3d::cellid=1_1:6 fe_index=5 ghost +DEAL:3:3d::cellid=1_1:7 fe_index=6 ghost +DEAL:3:3d::cellid=2_1:5 fe_index=3 ghost +DEAL:3:3d::cellid=3_1:4 fe_index=4 ghost +DEAL:3:3d::cellid=3_1:5 fe_index=5 ghost +DEAL:3:3d::cellid=4_1:1 fe_index=0 ghost +DEAL:3:3d::cellid=4_1:3 fe_index=2 ghost +DEAL:3:3d::cellid=4_1:5 fe_index=4 ghost +DEAL:3:3d::cellid=4_1:7 fe_index=6 ghost +DEAL:3:3d::cellid=5_1:0 fe_index=0 +DEAL:3:3d::cellid=5_1:1 fe_index=1 +DEAL:3:3d::cellid=5_1:2 fe_index=2 +DEAL:3:3d::cellid=5_1:3 fe_index=3 +DEAL:3:3d::cellid=5_1:4 fe_index=4 +DEAL:3:3d::cellid=5_1:5 fe_index=5 +DEAL:3:3d::cellid=5_1:6 fe_index=6 +DEAL:3:3d::cellid=5_1:7 fe_index=7 +DEAL:3:3d::cellid=6_1:1 fe_index=0 ghost +DEAL:3:3d::cellid=6_1:5 fe_index=4 ghost +DEAL:3:3d::cellid=7_1:0 fe_index=7 ghost +DEAL:3:3d::cellid=7_1:1 fe_index=8 ghost +DEAL:3:3d::cellid=7_1:4 fe_index=2 ghost +DEAL:3:3d::cellid=7_1:5 fe_index=3 ghost +DEAL:3:3d::OK + + +DEAL:4:2d::reading with 5 +DEAL:4:2d::cellid=0_1:3 fe_index=3 ghost +DEAL:4:2d::cellid=1_1:2 fe_index=2 ghost +DEAL:4:2d::cellid=1_1:3 fe_index=3 ghost +DEAL:4:2d::cellid=2_1:1 fe_index=0 ghost +DEAL:4:2d::cellid=2_1:3 fe_index=2 ghost +DEAL:4:2d::cellid=3_1:0 fe_index=0 +DEAL:4:2d::cellid=3_1:1 fe_index=1 +DEAL:4:2d::cellid=3_1:2 fe_index=2 +DEAL:4:2d::cellid=3_1:3 fe_index=3 +DEAL:4:2d::OK +DEAL:4:3d::reading with 5 +DEAL:4:3d::cellid=0_1:6 fe_index=6 ghost +DEAL:4:3d::cellid=0_1:7 fe_index=7 ghost +DEAL:4:3d::cellid=1_1:6 fe_index=5 ghost +DEAL:4:3d::cellid=1_1:7 fe_index=6 ghost +DEAL:4:3d::cellid=2_1:4 fe_index=2 ghost +DEAL:4:3d::cellid=2_1:5 fe_index=3 ghost +DEAL:4:3d::cellid=2_1:6 fe_index=4 ghost +DEAL:4:3d::cellid=2_1:7 fe_index=5 ghost +DEAL:4:3d::cellid=3_1:4 fe_index=4 ghost +DEAL:4:3d::cellid=3_1:5 fe_index=5 ghost +DEAL:4:3d::cellid=3_1:6 fe_index=6 ghost +DEAL:4:3d::cellid=3_1:7 fe_index=7 ghost +DEAL:4:3d::cellid=4_1:2 fe_index=1 ghost +DEAL:4:3d::cellid=4_1:3 fe_index=2 ghost +DEAL:4:3d::cellid=4_1:6 fe_index=5 ghost +DEAL:4:3d::cellid=4_1:7 fe_index=6 ghost +DEAL:4:3d::cellid=5_1:2 fe_index=2 ghost +DEAL:4:3d::cellid=5_1:3 fe_index=3 ghost +DEAL:4:3d::cellid=5_1:6 fe_index=6 ghost +DEAL:4:3d::cellid=5_1:7 fe_index=7 ghost +DEAL:4:3d::cellid=6_1:0 fe_index=8 +DEAL:4:3d::cellid=6_1:1 fe_index=0 +DEAL:4:3d::cellid=6_1:2 fe_index=1 +DEAL:4:3d::cellid=6_1:3 fe_index=2 +DEAL:4:3d::cellid=6_1:4 fe_index=3 +DEAL:4:3d::cellid=6_1:5 fe_index=4 +DEAL:4:3d::cellid=6_1:6 fe_index=5 +DEAL:4:3d::cellid=6_1:7 fe_index=6 +DEAL:4:3d::cellid=7_1:0 fe_index=7 +DEAL:4:3d::cellid=7_1:1 fe_index=8 +DEAL:4:3d::cellid=7_1:2 fe_index=0 +DEAL:4:3d::cellid=7_1:3 fe_index=1 +DEAL:4:3d::cellid=7_1:4 fe_index=2 +DEAL:4:3d::cellid=7_1:5 fe_index=3 +DEAL:4:3d::cellid=7_1:6 fe_index=4 +DEAL:4:3d::cellid=7_1:7 fe_index=5 +DEAL:4:3d::OK + -- 2.39.5