From 1799726e1ca6d9c70278a31f3f622d217ad91496 Mon Sep 17 00:00:00 2001 From: Pasquale Africa Date: Sat, 1 Jul 2023 19:08:49 +0000 Subject: [PATCH] Move CellAttachedData and DataTransfer to Triangulation base class --- examples/step-68/step-68.cc | 28 +- include/deal.II/base/quadrature_point_data.h | 46 +- .../deal.II/distributed/cell_data_transfer.h | 6 +- .../cell_data_transfer.templates.h | 32 +- include/deal.II/distributed/cell_weights.h | 4 +- .../distributed/fully_distributed_tria.h | 3 - .../distributed/repartitioning_policy_tools.h | 5 +- .../deal.II/distributed/solution_transfer.h | 4 +- include/deal.II/distributed/tria.h | 3 - include/deal.II/distributed/tria_base.h | 419 ------ include/deal.II/grid/cell_status.h | 53 + include/deal.II/grid/data_transfer.h | 1125 +++++++++++++++++ include/deal.II/grid/tria.h | 263 +++- include/deal.II/particles/particle_handler.h | 4 +- source/distributed/cell_weights.cc | 36 +- source/distributed/fully_distributed_tria.cc | 3 +- .../repartitioning_policy_tools.cc | 6 +- source/distributed/solution_transfer.cc | 26 +- source/distributed/tria.cc | 100 +- source/distributed/tria_base.cc | 1086 ---------------- source/grid/grid_tools.cc | 6 +- source/grid/tria.cc | 149 +++ source/particles/particle_handler.cc | 38 +- .../repartitioning_06.cc | 2 +- tests/mpi/attach_data_01.cc | 30 +- tests/mpi/attach_data_02.cc | 30 +- .../mpi/cell_weights_01_back_and_forth_01.cc | 4 +- .../mpi/cell_weights_01_back_and_forth_02.cc | 2 +- tests/mpi/cell_weights_02.cc | 2 +- tests/mpi/cell_weights_03.cc | 6 +- tests/mpi/cell_weights_04.cc | 6 +- tests/mpi/cell_weights_05.cc | 6 +- tests/mpi/cell_weights_06.cc | 6 +- tests/mpi/p4est_save_05.cc | 18 +- tests/mpi/p4est_save_07.cc | 18 +- tests/mpi/repartition_02.cc | 30 +- tests/particles/particle_weights_01.cc | 16 +- tests/particles/step-68.cc | 28 +- 38 files changed, 1790 insertions(+), 1859 deletions(-) create mode 100644 include/deal.II/grid/cell_status.h create mode 100644 include/deal.II/grid/data_transfer.h diff --git a/examples/step-68/step-68.cc b/examples/step-68/step-68.cc index d6273b9408..5622a782dd 100644 --- a/examples/step-68/step-68.cc +++ b/examples/step-68/step-68.cc @@ -253,9 +253,8 @@ namespace Step68 // `private`. unsigned int cell_weight( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) const; + & cell, + const CellStatus status) const; // The following two functions are responsible for outputting the simulation // results for the particles and for the velocity profile on the background @@ -329,9 +328,8 @@ namespace Step68 template unsigned int ParticleTracking::cell_weight( const typename parallel::distributed::Triangulation::cell_iterator - & cell, - const typename parallel::distributed::Triangulation::CellStatus status) - const + & cell, + const CellStatus status) const { // First, we introduce a base weight that will be assigned to every cell. const unsigned int base_weight = 1; @@ -353,15 +351,15 @@ namespace Step68 unsigned int n_particles_in_cell = 0; switch (status) { - case parallel::distributed::Triangulation::CELL_PERSIST: - case parallel::distributed::Triangulation::CELL_REFINE: + case CELL_PERSIST: + case CELL_REFINE: n_particles_in_cell = particle_handler.n_particles_in_cell(cell); break; - case parallel::distributed::Triangulation::CELL_INVALID: + case CELL_INVALID: break; - case parallel::distributed::Triangulation::CELL_COARSEN: + case CELL_COARSEN: for (const auto &child : cell->child_iterators()) n_particles_in_cell += particle_handler.n_particles_in_cell(child); break; @@ -399,11 +397,11 @@ namespace Step68 // of this class, but for the purpose of this example we want to group the // particle related instructions. background_triangulation.signals.weight.connect( - [&]( - const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) -> unsigned int { return this->cell_weight(cell, status); }); + [&](const typename parallel::distributed::Triangulation< + dim>::cell_iterator &cell, + const CellStatus status) -> unsigned int { + return this->cell_weight(cell, status); + }); // This initializes the background triangulation where the particles are // living and the number of properties of the particles. diff --git a/include/deal.II/base/quadrature_point_data.h b/include/deal.II/base/quadrature_point_data.h index 6e51a874a9..0aad2b2eb7 100644 --- a/include/deal.II/base/quadrature_point_data.h +++ b/include/deal.II/base/quadrature_point_data.h @@ -500,9 +500,8 @@ namespace parallel std::vector pack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status); + & cell, + const CellStatus status); /** * A callback function used to unpack the data on the current mesh that @@ -512,9 +511,8 @@ namespace parallel void unpack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, + & cell, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range); @@ -935,11 +933,11 @@ namespace parallel data_storage = &data_storage_; handle = triangulation->register_data_attach( - [this]( - const typename parallel::distributed::Triangulation< - dim>::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) { return this->pack_function(cell, status); }, + [this](const typename parallel::distributed::Triangulation< + dim>::cell_iterator &cell, + const CellStatus status) { + return this->pack_function(cell, status); + }, /*returns_variable_size_data=*/true); } @@ -951,13 +949,13 @@ namespace parallel { triangulation->notify_ready_to_unpack( handle, - [this]( - const typename parallel::distributed::Triangulation< - dim>::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, - const boost::iterator_range::const_iterator> - &data_range) { this->unpack_function(cell, status, data_range); }); + [this](const typename parallel::distributed::Triangulation< + dim>::cell_iterator &cell, + const CellStatus status, + const boost::iterator_range::const_iterator> + &data_range) { + this->unpack_function(cell, status, data_range); + }); // invalidate the pointers data_storage = nullptr; @@ -971,8 +969,7 @@ namespace parallel ContinuousQuadratureDataTransfer::pack_function( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation< - dim>::CellStatus /*status*/) + const CellStatus /*status*/) { pack_cell_data(cell, data_storage, matrix_quadrature); @@ -991,15 +988,12 @@ namespace parallel inline void ContinuousQuadratureDataTransfer::unpack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, + & cell, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { - Assert((status != - parallel::distributed::Triangulation::CELL_COARSEN), - ExcNotImplemented()); + Assert((status != CELL_COARSEN), ExcNotImplemented()); (void)status; matrix_dofs = diff --git a/include/deal.II/distributed/cell_data_transfer.h b/include/deal.II/distributed/cell_data_transfer.h index 115ff2edad..271adcb2d2 100644 --- a/include/deal.II/distributed/cell_data_transfer.h +++ b/include/deal.II/distributed/cell_data_transfer.h @@ -290,8 +290,7 @@ namespace parallel std::vector pack_callback(const typename parallel::distributed:: Triangulation::cell_iterator &cell, - const typename parallel::distributed:: - Triangulation::CellStatus status); + const CellStatus status); /** * A callback function used to unpack the data on the current mesh that @@ -302,8 +301,7 @@ namespace parallel unpack_callback( const typename parallel::distributed::Triangulation:: cell_iterator &cell, - const typename parallel::distributed::Triangulation:: - CellStatus status, + const CellStatus status, const boost::iterator_range::const_iterator> & data_range, std::vector &all_out); diff --git a/include/deal.II/distributed/cell_data_transfer.templates.h b/include/deal.II/distributed/cell_data_transfer.templates.h index 16fc5d86cc..26fd05b13a 100644 --- a/include/deal.II/distributed/cell_data_transfer.templates.h +++ b/include/deal.II/distributed/cell_data_transfer.templates.h @@ -110,8 +110,7 @@ namespace parallel handle = tria->register_data_attach( [this](const typename parallel::distributed:: Triangulation::cell_iterator &cell, - const typename parallel::distributed:: - Triangulation::CellStatus status) { + const CellStatus status) { return this->pack_callback(cell, status); }, /*returns_variable_size_data=*/transfer_variable_size_data); @@ -185,8 +184,7 @@ namespace parallel [this, &all_out]( const typename parallel::distributed::Triangulation:: cell_iterator &cell, - const typename parallel::distributed::Triangulation:: - CellStatus status, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { this->unpack_callback(cell, status, data_range, all_out); @@ -247,9 +245,7 @@ namespace parallel CellDataTransfer::pack_callback( const typename parallel::distributed::Triangulation:: cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) + const CellStatus status) { std::vector cell_data(input_vectors.size()); @@ -260,17 +256,14 @@ namespace parallel { switch (status) { - case parallel::distributed::Triangulation::CELL_PERSIST: - case parallel::distributed::Triangulation::CELL_REFINE: + case CELL_PERSIST: + case CELL_REFINE: // Cell either persists, or will be refined, and its children do // not exist yet in the latter case. *it_output = (**it_input)[cell->active_cell_index()]; break; - case parallel::distributed::Triangulation::CELL_COARSEN: + case CELL_COARSEN: { // Cell is parent whose children will get coarsened to. // Decide data to store on parent by provided strategy. @@ -314,9 +307,7 @@ namespace parallel CellDataTransfer::unpack_callback( const typename parallel::distributed::Triangulation:: cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, + const CellStatus status, const boost::iterator_range::const_iterator> & data_range, std::vector &all_out) @@ -344,17 +335,14 @@ namespace parallel for (; it_input != cell_data.cend(); ++it_input, ++it_output) switch (status) { - case parallel::distributed::Triangulation::CELL_PERSIST: - case parallel::distributed::Triangulation::CELL_COARSEN: + case CELL_PERSIST: + case CELL_COARSEN: // Cell either persists, or has been coarsened. // Thus, cell has no (longer) children. (**it_output)[cell->active_cell_index()] = *it_input; break; - case parallel::distributed::Triangulation::CELL_REFINE: + case CELL_REFINE: { // Cell has been refined, and is now parent of its children. // Thus, distribute parent's data on its children. diff --git a/include/deal.II/distributed/cell_weights.h b/include/deal.II/distributed/cell_weights.h index 351fb70db6..34ab6cef53 100644 --- a/include/deal.II/distributed/cell_weights.h +++ b/include/deal.II/distributed/cell_weights.h @@ -159,7 +159,7 @@ namespace parallel */ static std::function::cell_iterator &cell, - const typename dealii::Triangulation::CellStatus status)> + const CellStatus status)> make_weighting_callback(const DoFHandler &dof_handler, const WeightingFunction &weighting_function); @@ -219,7 +219,7 @@ namespace parallel static unsigned int weighting_callback( const typename dealii::Triangulation::cell_iterator &cell, - const typename dealii::Triangulation::CellStatus status, + const CellStatus status, const DoFHandler & dof_handler, const parallel::TriangulationBase &triangulation, const WeightingFunction & weighting_function); diff --git a/include/deal.II/distributed/fully_distributed_tria.h b/include/deal.II/distributed/fully_distributed_tria.h index ec67d6928b..4e4319ae25 100644 --- a/include/deal.II/distributed/fully_distributed_tria.h +++ b/include/deal.II/distributed/fully_distributed_tria.h @@ -120,9 +120,6 @@ namespace parallel using active_cell_iterator = typename dealii::Triangulation::active_cell_iterator; - using CellStatus = - typename dealii::Triangulation::CellStatus; - /** * Constructor. * diff --git a/include/deal.II/distributed/repartitioning_policy_tools.h b/include/deal.II/distributed/repartitioning_policy_tools.h index 2265b09651..ecc435b47e 100644 --- a/include/deal.II/distributed/repartitioning_policy_tools.h +++ b/include/deal.II/distributed/repartitioning_policy_tools.h @@ -163,8 +163,7 @@ namespace RepartitioningPolicyTools CellWeightPolicy( const std::function::cell_iterator &, - const typename Triangulation::CellStatus)> - &weighting_function); + const CellStatus)> &weighting_function); virtual LinearAlgebra::distributed::Vector partition(const Triangulation &tria_in) const override; @@ -175,7 +174,7 @@ namespace RepartitioningPolicyTools */ const std::function< unsigned int(const typename Triangulation::cell_iterator &, - const typename Triangulation::CellStatus)> + const CellStatus)> weighting_function; }; diff --git a/include/deal.II/distributed/solution_transfer.h b/include/deal.II/distributed/solution_transfer.h index 05d7a941ff..85cd95c63c 100644 --- a/include/deal.II/distributed/solution_transfer.h +++ b/include/deal.II/distributed/solution_transfer.h @@ -349,7 +349,7 @@ namespace parallel std::vector pack_callback( const typename Triangulation::cell_iterator &cell, - const typename Triangulation::CellStatus status); + const CellStatus status); /** * A callback function used to unpack the data on the current mesh that @@ -359,7 +359,7 @@ namespace parallel void unpack_callback( const typename Triangulation::cell_iterator &cell, - const typename Triangulation::CellStatus status, + const CellStatus status, const boost::iterator_range::const_iterator> & data_range, std::vector &all_out, diff --git a/include/deal.II/distributed/tria.h b/include/deal.II/distributed/tria.h index 1300a3b060..06583ab6f8 100644 --- a/include/deal.II/distributed/tria.h +++ b/include/deal.II/distributed/tria.h @@ -290,9 +290,6 @@ namespace parallel using active_cell_iterator = typename dealii::Triangulation::active_cell_iterator; - using CellStatus = - typename dealii::Triangulation::CellStatus; - /** * Configuration flags for distributed Triangulations to be set in the * constructor. Settings can be combined using bitwise OR. diff --git a/include/deal.II/distributed/tria_base.h b/include/deal.II/distributed/tria_base.h index 1920f63994..7818336260 100644 --- a/include/deal.II/distributed/tria_base.h +++ b/include/deal.II/distributed/tria_base.h @@ -384,7 +384,6 @@ namespace parallel }; - /** * A base class for distributed triangulations, i.e., triangulations that * do not store all cells on all processors. This implies that not @@ -459,9 +458,6 @@ namespace parallel using cell_iterator = typename dealii::Triangulation::cell_iterator; - using CellStatus = - typename dealii::Triangulation::CellStatus; - /** * Return true if the triangulation has hanging nodes. * @@ -507,421 +503,6 @@ namespace parallel DEAL_II_DEPRECATED virtual void load(const std::string &filename, const bool autopartition) = 0; - - /** - * Register a function that can be used to attach data of fixed size - * to cells. This is useful for two purposes: (i) Upon refinement and - * coarsening of a triangulation (@a e.g. in - * parallel::distributed::Triangulation::execute_coarsening_and_refinement()), - * one needs to be able to store one or more data vectors per cell that - * characterizes the solution values on the cell so that this data can - * then be transferred to the new owning processor of the cell (or - * its parent/children) when the mesh is re-partitioned; (ii) when - * serializing a computation to a file, it is necessary to attach - * data to cells so that it can be saved (@a e.g. in - * parallel::distributed::Triangulation::save()) along with the cell's - * other information and, if necessary, later be reloaded from disk - * with a different subdivision of cells among the processors. - * - * The way this function works is that it allows any number of interest - * parties to register their intent to attach data to cells. One example - * of classes that do this is parallel::distributed::SolutionTransfer - * where each parallel::distributed::SolutionTransfer object that works - * on the current Triangulation object then needs to register its intent. - * Each of these parties registers a callback function (the first - * argument here, @p pack_callback) that will be called whenever the - * triangulation's execute_coarsening_and_refinement() or save() - * functions are called. - * - * The current function then returns an integer handle that corresponds - * to the number of data set that the callback provided here will attach. - * While this number could be given a precise meaning, this is - * not important: You will never actually have to do anything with - * this number except return it to the notify_ready_to_unpack() function. - * In other words, each interested party (i.e., the caller of the current - * function) needs to store their respective returned handle for later use - * when unpacking data in the callback provided to - * notify_ready_to_unpack(). - * - * Whenever @p pack_callback is then called by - * execute_coarsening_and_refinement() or load() on a given cell, it - * receives a number of arguments. In particular, the first - * argument passed to the callback indicates the cell for which - * it is supposed to attach data. This is always an active cell. - * - * The second, CellStatus, argument provided to the callback function - * will tell you if the given cell will be coarsened, refined, or will - * persist as is. (This status may be different than the refinement - * or coarsening flags set on that cell, to accommodate things such as - * the "one hanging node per edge" rule.). These flags need to be - * read in context with the p4est quadrant they belong to, as their - * relations are gathered in local_cell_relations. - * - * Specifically, the values for this argument mean the following: - * - * - `CELL_PERSIST`: The cell won't be refined/coarsened, but might be - * moved to a different processor. If this is the case, the callback - * will want to pack up the data on this cell into an array and store - * it at the provided address for later unpacking wherever this cell - * may land. - * - `CELL_REFINE`: This cell will be refined into 4 or 8 cells (in 2d - * and 3d, respectively). However, because these children don't exist - * yet, you cannot access them at the time when the callback is - * called. Thus, in local_cell_relations, the corresponding - * p4est quadrants of the children cells are linked to the deal.II - * cell which is going to be refined. To be specific, only the very - * first child is marked with `CELL_REFINE`, whereas the others will be - * marked with `CELL_INVALID`, which indicates that these cells will be - * ignored by default during the packing or unpacking process. This - * ensures that data is only transferred once onto or from the parent - * cell. If the callback is called with `CELL_REFINE`, the callback - * will want to pack up the data on this cell into an array and store - * it at the provided address for later unpacking in a way so that - * it can then be transferred to the children of the cell that will - * then be available. In other words, if the data the callback - * will want to pack up corresponds to a finite element field, then - * the prolongation from parent to (new) children will have to happen - * during unpacking. - * - `CELL_COARSEN`: The children of this cell will be coarsened into the - * given cell. These children still exist, so if this is the value - * given to the callback as second argument, the callback will want - * to transfer data from the children to the current parent cell and - * pack it up so that it can later be unpacked again on a cell that - * then no longer has any children (and may also be located on a - * different processor). In other words, if the data the callback - * will want to pack up corresponds to a finite element field, then - * it will need to do the restriction from children to parent at - * this point. - * - `CELL_INVALID`: See `CELL_REFINE`. - * - * @note If this function is used for serialization of data - * using save() and load(), then the cell status argument with which - * the callback is called will always be `CELL_PERSIST`. - * - * The callback function is expected to return a memory chunk of the - * format `std::vector`, representing the packed data on a - * certain cell. - * - * The second parameter @p returns_variable_size_data indicates whether - * the returned size of the memory region from the callback function - * varies by cell (=true) or stays constant on each one - * throughout the whole domain (=false). - * - * @note The purpose of this function is to register intent to - * attach data for a single, subsequent call to - * execute_coarsening_and_refinement() and notify_ready_to_unpack(), - * save(), load(). Consequently, notify_ready_to_unpack(), save(), - * and load() all forget the registered callbacks once these - * callbacks have been called, and you will have to re-register - * them with a triangulation if you want them to be active for - * another call to these functions. - */ - unsigned int - register_data_attach( - const std::function(const cell_iterator &, - const CellStatus)> &pack_callback, - const bool returns_variable_size_data); - - /** - * This function is the opposite of register_data_attach(). It is called - * after the execute_coarsening_and_refinement() or save()/load() - * functions are done when classes and functions that have previously - * attached data to a triangulation for either transfer to other - * processors, across mesh refinement, or serialization of data to - * a file are ready to receive that data back. The important part about - * this process is that the triangulation cannot do this right away from - * the end of execute_coarsening_and_refinement() or load() via a - * previously attached callback function (as the register_data_attach() - * function does) because the classes that eventually want the data - * back may need to do some setup between the point in time where the - * mesh has been recreated and when the data can actually be received. - * An example is the parallel::distributed::SolutionTransfer class - * that can really only receive the data once not only the mesh is - * completely available again on the current processor, but only - * after a DoFHandler has been reinitialized and distributed - * degrees of freedom. In other words, there is typically a significant - * amount of set up that needs to happen in user space before the classes - * that can receive data attached to cell are ready to actually do so. - * When they are, they use the current function to tell the triangulation - * object that now is the time when they are ready by calling the - * current function. - * - * The supplied callback function is then called for each newly locally - * owned cell. The first argument to the callback is an iterator that - * designates the cell; the second argument indicates the status of the - * cell in question; and the third argument localizes a memory area by - * two iterators that contains the data that was previously saved from - * the callback provided to register_data_attach(). - * - * The CellStatus will indicate if the cell was refined, coarsened, or - * persisted unchanged. The @p cell_iterator argument to the callback - * will then either be an active, - * locally owned cell (if the cell was not refined), or the immediate - * parent if it was refined during execute_coarsening_and_refinement(). - * Therefore, contrary to during register_data_attach(), you can now - * access the children if the status is `CELL_REFINE` but no longer for - * callbacks with status `CELL_COARSEN`. - * - * The first argument to this function, `handle`, corresponds to - * the return value of register_data_attach(). (The precise - * meaning of what the numeric value of this handle is supposed - * to represent is neither important, nor should you try to use - * it for anything other than transmit information between a - * call to register_data_attach() to the corresponding call to - * notify_ready_to_unpack().) - */ - void - notify_ready_to_unpack( - const unsigned int handle, - const std::function< - void(const cell_iterator &, - const CellStatus, - const boost::iterator_range::const_iterator> &)> - &unpack_callback); - - protected: - /** - * Save additional cell-attached data into the given file. The first - * arguments are used to determine the offsets where to write buffers to. - * - * Called by - * @ref save. - */ - void - save_attached_data(const unsigned int global_first_cell, - const unsigned int global_num_cells, - const std::string &filename) const; - - /** - * Load additional cell-attached data from the given file, if any was saved. - * The first arguments are used to determine the offsets where to read - * buffers from. - * - * Called by - * @ref load. - */ - void - load_attached_data(const unsigned int global_first_cell, - const unsigned int global_num_cells, - const unsigned int local_num_cells, - const std::string &filename, - const unsigned int n_attached_deserialize_fixed, - const unsigned int n_attached_deserialize_variable); - - /** - * A function to record the CellStatus of currently active cells that - * are locally owned. This information is mandatory to transfer data - * between meshes during adaptation or serialization, e.g., using - * parallel::distributed::SolutionTransfer. - * - * Relations will be stored in the private member local_cell_relations. For - * an extensive description of CellStatus, see the documentation for the - * member function register_data_attach(). - */ - virtual void - update_cell_relations() = 0; - - /** - * Auxiliary data structure for assigning a CellStatus to a deal.II cell - * iterator. For an extensive description of the former, see the - * documentation for the member function register_data_attach(). - */ - using cell_relation_t = typename std::pair; - - /** - * Vector of pairs, each containing a deal.II cell iterator and its - * respective CellStatus. To update its contents, use the - * update_cell_relations() member function. - */ - std::vector local_cell_relations; - - /** - * A structure that stores information about the data that has been, or - * will be, attached to cells via the register_data_attach() function - * and later retrieved via notify_ready_to_unpack(). - */ - struct CellAttachedData - { - /** - * number of functions that get attached to the Triangulation through - * register_data_attach() for example SolutionTransfer. - */ - unsigned int n_attached_data_sets; - - /** - * number of functions that need to unpack their data after a call from - * load() - */ - unsigned int n_attached_deserialize; - - using pack_callback_t = std::function( - typename dealii::Triangulation::cell_iterator, - typename dealii::Triangulation::CellStatus)>; - - /** - * These callback functions will be stored in the order in which they - * have been registered with the register_data_attach() function. - */ - std::vector pack_callbacks_fixed; - std::vector pack_callbacks_variable; - }; - - CellAttachedData cell_attached_data; - - /** - * This class in the private scope of parallel::DistributedTriangulationBase - * is dedicated to the data transfer across repartitioned meshes - * and to/from the file system. - * - * It is designed to store all data buffers intended for transfer. - */ - class DataTransfer - { - public: - DataTransfer(const MPI_Comm mpi_communicator); - - /** - * Prepare data transfer by calling the pack callback functions on each - * cell - * in @p cell_relations. - * - * All registered callback functions in @p pack_callbacks_fixed will write - * into the fixed size buffer, whereas each entry of @p pack_callbacks_variable - * will write its data into the variable size buffer. - */ - void - pack_data(const std::vector &cell_relations, - const std::vector - &pack_callbacks_fixed, - const std::vector - &pack_callbacks_variable); - - - - /** - * Unpack the CellStatus information on each entry of - * @p cell_relations. - * - * Data has to be previously transferred with execute_transfer() - * or read from the file system via load(). - */ - void - unpack_cell_status(std::vector &cell_relations) const; - - /** - * Unpack previously transferred data on each cell registered in - * @p cell_relations with the provided @p unpack_callback function. - * - * The parameter @p handle corresponds to the position where the - * @p unpack_callback function is allowed to read from the memory. Its - * value needs to be in accordance with the corresponding pack_callback - * function that has been registered previously. - * - * Data has to be previously transferred with execute_transfer() - * or read from the file system via load(). - */ - void - unpack_data( - const std::vector &cell_relations, - const unsigned int handle, - const std::function::cell_iterator &, - const typename dealii::Triangulation::CellStatus &, - const boost::iterator_range::const_iterator> &)> - &unpack_callback) const; - - /** - * Transfer data to file system. - * - * The data will be written in a separate file, whose name - * consists of the stem @p filename and an attached identifier - * _fixed.data for fixed size data and _variable.data - * for variable size data. - * - * All processors write into these files simultaneously via MPIIO. - * Each processor's position to write to will be determined - * from the provided input parameters. - * - * Data has to be previously packed with pack_data(). - */ - void - save(const unsigned int global_first_cell, - const unsigned int global_num_cells, - const std::string &filename) const; - - /** - * Transfer data from file system. - * - * The data will be read from separate file, whose name - * consists of the stem @p filename and an attached identifier - * _fixed.data for fixed size data and _variable.data - * for variable size data. - * The @p n_attached_deserialize_fixed and @p n_attached_deserialize_variable - * parameters are required to gather the memory offsets for each - * callback. - * - * All processors read from these files simultaneously via MPIIO. - * Each processor's position to read from will be determined - * from the provided input arguments. - * - * After loading, unpack_data() needs to be called to finally - * distribute data across the associated triangulation. - */ - void - load(const unsigned int global_first_cell, - const unsigned int global_num_cells, - const unsigned int local_num_cells, - const std::string &filename, - const unsigned int n_attached_deserialize_fixed, - const unsigned int n_attached_deserialize_variable); - - /** - * Clears all containers and associated data, and resets member - * values to their default state. - * - * Frees memory completely. - */ - void - clear(); - - /** - * Flag that denotes if variable size data has been packed. - */ - bool variable_size_data_stored; - - /** - * Cumulative size in bytes that those functions that have called - * register_data_attach() want to attach to each cell. This number - * only pertains to fixed-sized buffers where the data attached to - * each cell has exactly the same size. - * - * The last entry of this container corresponds to the data size - * packed per cell in the fixed size buffer (which can be accessed - * calling sizes_fixed_cumulative.back()). - */ - std::vector sizes_fixed_cumulative; - - /** - * Consecutive buffers designed for the fixed size transfer - * functions of p4est. - */ - std::vector src_data_fixed; - std::vector dest_data_fixed; - - /** - * Consecutive buffers designed for the variable size transfer - * functions of p4est. - */ - std::vector src_sizes_variable; - std::vector dest_sizes_variable; - std::vector src_data_variable; - std::vector dest_data_variable; - - private: - MPI_Comm mpi_communicator; - }; - - DataTransfer data_transfer; }; } // namespace parallel diff --git a/include/deal.II/grid/cell_status.h b/include/deal.II/grid/cell_status.h new file mode 100644 index 0000000000..b10b3fc9c2 --- /dev/null +++ b/include/deal.II/grid/cell_status.h @@ -0,0 +1,53 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 1998 - 2023 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + +#ifndef dealii_cell_status_h +#define dealii_cell_status_h + +#include + +DEAL_II_NAMESPACE_OPEN + +/** + * Used to inform functions in derived classes how the cell with the given + * cell_iterator is going to change. Note that this may me different than + * the refine_flag() and coarsen_flag() in the cell_iterator in parallel + * calculations because of refinement constraints that this machine does not + * see. + */ +enum CellStatus +{ + /** + * The cell will not be refined or coarsened and might or might not move + * to a different processor. + */ + CELL_PERSIST, + /** + * The cell will be or was refined. + */ + CELL_REFINE, + /** + * The children of this cell will be or were coarsened into this cell. + */ + CELL_COARSEN, + /** + * Invalid status. Will not occur for the user. + */ + CELL_INVALID +}; + +DEAL_II_NAMESPACE_CLOSE + +#endif diff --git a/include/deal.II/grid/data_transfer.h b/include/deal.II/grid/data_transfer.h new file mode 100644 index 0000000000..14c2b4ec7c --- /dev/null +++ b/include/deal.II/grid/data_transfer.h @@ -0,0 +1,1125 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 1998 - 2023 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + +#ifndef dealii_tria_data_transfer_h +#define dealii_tria_data_transfer_h + +#include + +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include + +DEAL_II_NAMESPACE_OPEN + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +struct CellAttachedData +{ + using cell_iterator = TriaIterator>; + + /** + * Number of functions that get attached to the Triangulation through + * register_data_attach() for example SolutionTransfer. + */ + unsigned int n_attached_data_sets; + + /** + * Number of functions that need to unpack their data after a call from + * load(). + */ + unsigned int n_attached_deserialize; + + using pack_callback_t = + std::function(cell_iterator, CellStatus)>; + + /** + * These callback functions will be stored in the order in which they + * have been registered with the register_data_attach() function. + */ + std::vector pack_callbacks_fixed; + std::vector pack_callbacks_variable; +}; + +/** + * A structure that stores information about the data that has been, or + * will be, attached to cells via the register_data_attach() function + * and later retrieved via notify_ready_to_unpack(). + * + * This class in the private scope of parallel::DistributedTriangulationBase + * is dedicated to the data transfer across repartitioned meshes + * and to/from the file system. + * + * It is designed to store all data buffers intended for transfer. + */ +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +class DataTransfer +{ +public: + using cell_iterator = TriaIterator>; + + /** + * Auxiliary data structure for assigning a CellStatus to a deal.II cell + * iterator. For an extensive description of the former, see the + * documentation for the member function register_data_attach(). + */ + using cell_relation_t = typename std::pair; + + DataTransfer(const MPI_Comm mpi_communicator); + + /** + * Prepare data transfer by calling the pack callback functions on each + * cell in @p cell_relations. + * + * All registered callback functions in @p pack_callbacks_fixed will write + * into the fixed size buffer, whereas each entry of @p pack_callbacks_variable + * will write its data into the variable size buffer. + */ + void + pack_data( + const std::vector &cell_relations, + const std::vector::pack_callback_t> + &pack_callbacks_fixed, + const std::vector::pack_callback_t> + &pack_callbacks_variable); + + /** + * Unpack the CellStatus information on each entry of + * @p cell_relations. + * + * Data has to be previously transferred with execute_transfer() + * or read from the file system via load(). + */ + void + unpack_cell_status(std::vector &cell_relations) const; + + /** + * Unpack previously transferred data on each cell registered in + * @p cell_relations with the provided @p unpack_callback function. + * + * The parameter @p handle corresponds to the position where the + * @p unpack_callback function is allowed to read from the memory. Its + * value needs to be in accordance with the corresponding pack_callback + * function that has been registered previously. + * + * Data has to be previously transferred with execute_transfer() + * or read from the file system via load(). + */ + void + unpack_data( + const std::vector &cell_relations, + const unsigned int handle, + const std::function< + void(const cell_iterator &, + const CellStatus &, + const boost::iterator_range::const_iterator> &)> + &unpack_callback) const; + + /** + * Transfer data to file system. + * + * The data will be written in a separate file, whose name + * consists of the stem @p filename and an attached identifier + * _fixed.data for fixed size data and _variable.data + * for variable size data. + * + * All processors write into these files simultaneously via MPIIO. + * Each processor's position to write to will be determined + * from the provided input parameters. + * + * Data has to be previously packed with pack_data(). + */ + void + save(const unsigned int global_first_cell, + const unsigned int global_num_cells, + const std::string &filename) const; + + /** + * Transfer data from file system. + * + * The data will be read from separate file, whose name + * consists of the stem @p filename and an attached identifier + * _fixed.data for fixed size data and _variable.data + * for variable size data. + * The @p n_attached_deserialize_fixed and @p n_attached_deserialize_variable + * parameters are required to gather the memory offsets for each + * callback. + * + * All processors read from these files simultaneously via MPIIO. + * Each processor's position to read from will be determined + * from the provided input arguments. + * + * After loading, unpack_data() needs to be called to finally + * distribute data across the associated triangulation. + */ + void + load(const unsigned int global_first_cell, + const unsigned int global_num_cells, + const unsigned int local_num_cells, + const std::string &filename, + const unsigned int n_attached_deserialize_fixed, + const unsigned int n_attached_deserialize_variable); + + /** + * Clears all containers and associated data, and resets member + * values to their default state. + * + * Frees memory completely. + */ + void + clear(); + + /** + * Flag that denotes if variable size data has been packed. + */ + bool variable_size_data_stored; + + /** + * Cumulative size in bytes that those functions that have called + * register_data_attach() want to attach to each cell. This number + * only pertains to fixed-sized buffers where the data attached to + * each cell has exactly the same size. + * + * The last entry of this container corresponds to the data size + * packed per cell in the fixed size buffer (which can be accessed + * calling sizes_fixed_cumulative.back()). + */ + std::vector sizes_fixed_cumulative; + + /** + * Consecutive buffers designed for the fixed size transfer + * functions of p4est. + */ + std::vector src_data_fixed; + std::vector dest_data_fixed; + + /** + * Consecutive buffers designed for the variable size transfer + * functions of p4est. + */ + std::vector src_sizes_variable; + std::vector dest_sizes_variable; + std::vector src_data_variable; + std::vector dest_data_variable; + +private: + MPI_Comm mpi_communicator; +}; + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +DataTransfer::DataTransfer(const MPI_Comm mpi_communicator) + : variable_size_data_stored(false) + , mpi_communicator(mpi_communicator) +{} + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void DataTransfer::pack_data( + const std::vector &cell_relations, + const std::vector::pack_callback_t> + &pack_callbacks_fixed, + const std::vector::pack_callback_t> + &pack_callbacks_variable) +{ + Assert(src_data_fixed.size() == 0, + ExcMessage("Previously packed data has not been released yet!")); + Assert(src_sizes_variable.size() == 0, ExcInternalError()); + + const unsigned int n_callbacks_fixed = pack_callbacks_fixed.size(); + const unsigned int n_callbacks_variable = pack_callbacks_variable.size(); + + // Store information that we packed variable size data in + // a member variable for later. + variable_size_data_stored = (n_callbacks_variable > 0); + + // If variable transfer is scheduled, we will store the data size that + // each variable size callback function writes in this auxiliary + // container. The information will be stored by each cell in this vector + // temporarily. + std::vector cell_sizes_variable_cumulative( + n_callbacks_variable); + + // Prepare the buffer structure, in which each callback function will + // store its data for each active cell. + // The outmost shell in this container construct corresponds to the + // data packed per cell. The next layer resembles the data that + // each callback function packs on the corresponding cell. These + // buffers are chains of chars stored in an std::vector. + // A visualisation of the data structure: + /* clang-format off */ + // | cell_1 | | cell_2 | ... + // || callback_1 || callback_2 |...| || callback_1 || callback_2 |...| ... + // |||char|char|...|||char|char|...|...| |||char|char|...|||char|char|...|...| ... + /* clang-format on */ + std::vector>> packed_fixed_size_data( + cell_relations.size()); + std::vector>> packed_variable_size_data( + variable_size_data_stored ? cell_relations.size() : 0); + + // + // --------- Pack data for fixed and variable size transfer --------- + // + // Iterate over all cells, call all callback functions on each cell, + // and store their data in the corresponding buffer scope. + { + auto cell_rel_it = cell_relations.cbegin(); + auto data_cell_fixed_it = packed_fixed_size_data.begin(); + auto data_cell_variable_it = packed_variable_size_data.begin(); + for (; cell_rel_it != cell_relations.cend(); ++cell_rel_it) + { + const auto &dealii_cell = cell_rel_it->first; + const auto &cell_status = cell_rel_it->second; + + // Assertions about the tree structure. + switch (cell_status) + { + case CELL_PERSIST: + case CELL_REFINE: + // double check the condition that we will only ever attach + // data to active cells when we get here + Assert(dealii_cell->is_active(), ExcInternalError()); + break; + + case CELL_COARSEN: + // double check the condition that we will only ever attach + // data to cells with children when we get here. however, we + // can only tolerate one level of coarsening at a time, so + // check that the children are all active + Assert(dealii_cell->is_active() == false, ExcInternalError()); + for (unsigned int c = 0; + c < GeometryInfo::max_children_per_cell; + ++c) + Assert(dealii_cell->child(c)->is_active(), ExcInternalError()); + break; + + case CELL_INVALID: + // do nothing on invalid cells + break; + + default: + Assert(false, ExcInternalError()); + break; + } + + // Reserve memory corresponding to the number of callback + // functions that will be called. + // If variable size transfer is scheduled, we need to leave + // room for an array that holds information about how many + // bytes each of the variable size callback functions will + // write. + // On cells flagged with CELL_INVALID, only its CellStatus + // will be stored. + const unsigned int n_fixed_size_data_sets_on_cell = + 1 + ((cell_status == CELL_INVALID) ? + 0 : + ((variable_size_data_stored ? 1 : 0) + n_callbacks_fixed)); + data_cell_fixed_it->resize(n_fixed_size_data_sets_on_cell); + + // We continue with packing all data on this specific cell. + auto data_fixed_it = data_cell_fixed_it->begin(); + + // First, we pack the CellStatus information. + // to get consistent data sizes on each cell for the fixed size + // transfer, we won't allow compression + *data_fixed_it = + Utilities::pack(cell_status, /*allow_compression=*/false); + ++data_fixed_it; + + // Proceed with all registered callback functions. + // Skip cells with the CELL_INVALID flag. + if (cell_status != CELL_INVALID) + { + // Pack fixed size data. + for (auto callback_it = pack_callbacks_fixed.cbegin(); + callback_it != pack_callbacks_fixed.cend(); + ++callback_it, ++data_fixed_it) + { + *data_fixed_it = (*callback_it)(dealii_cell, cell_status); + } + + // Pack variable size data. + // If we store variable size data, we need to transfer + // the sizes of each corresponding callback function + // via fixed size transfer as well. + if (variable_size_data_stored) + { + const unsigned int n_variable_size_data_sets_on_cell = + ((cell_status == CELL_INVALID) ? 0 : n_callbacks_variable); + data_cell_variable_it->resize( + n_variable_size_data_sets_on_cell); + + auto callback_it = pack_callbacks_variable.cbegin(); + auto data_variable_it = data_cell_variable_it->begin(); + auto sizes_variable_it = cell_sizes_variable_cumulative.begin(); + for (; callback_it != pack_callbacks_variable.cend(); + ++callback_it, ++data_variable_it, ++sizes_variable_it) + { + *data_variable_it = + (*callback_it)(dealii_cell, cell_status); + + // Store data sizes for each callback function first. + // Make it cumulative below. + *sizes_variable_it = data_variable_it->size(); + } + + // Turn size vector into its cumulative representation. + std::partial_sum(cell_sizes_variable_cumulative.begin(), + cell_sizes_variable_cumulative.end(), + cell_sizes_variable_cumulative.begin()); + + // Serialize cumulative variable size vector + // value-by-value. This way we can circumvent the overhead + // of storing the container object as a whole, since we + // know its size by the number of registered callback + // functions. + data_fixed_it->resize(n_callbacks_variable * + sizeof(unsigned int)); + for (unsigned int i = 0; i < n_callbacks_variable; ++i) + std::memcpy(&(data_fixed_it->at(i * sizeof(unsigned int))), + &(cell_sizes_variable_cumulative.at(i)), + sizeof(unsigned int)); + + ++data_fixed_it; + } + + // Double check that we packed everything we wanted + // in the fixed size buffers. + Assert(data_fixed_it == data_cell_fixed_it->end(), + ExcInternalError()); + } + + ++data_cell_fixed_it; + + // Increment the variable size data iterator + // only if we actually pack this kind of data + // to avoid getting out of bounds. + if (variable_size_data_stored) + ++data_cell_variable_it; + } // loop over cell_relations + } + + // + // ----------- Gather data sizes for fixed size transfer ------------ + // + // Generate a vector which stores the sizes of each callback function, + // including the packed CellStatus transfer. + // Find the very first cell that we wrote to with all callback + // functions (i.e. a cell that was not flagged with CELL_INVALID) + // and store the sizes of each buffer. + // + // To deal with the case that at least one of the processors does not + // own any cell at all, we will exchange the information about the data + // sizes among them later. The code in between is still well-defined, + // since the following loops will be skipped. + std::vector local_sizes_fixed( + 1 + n_callbacks_fixed + (variable_size_data_stored ? 1 : 0)); + for (const auto &data_cell : packed_fixed_size_data) + { + if (data_cell.size() == local_sizes_fixed.size()) + { + auto sizes_fixed_it = local_sizes_fixed.begin(); + auto data_fixed_it = data_cell.cbegin(); + for (; data_fixed_it != data_cell.cend(); + ++data_fixed_it, ++sizes_fixed_it) + { + *sizes_fixed_it = data_fixed_it->size(); + } + + break; + } + } + + // Check if all cells have valid sizes. + for (auto data_cell_fixed_it = packed_fixed_size_data.cbegin(); + data_cell_fixed_it != packed_fixed_size_data.cend(); + ++data_cell_fixed_it) + { + Assert((data_cell_fixed_it->size() == 1) || + (data_cell_fixed_it->size() == local_sizes_fixed.size()), + ExcInternalError()); + } + + // Share information about the packed data sizes + // of all callback functions across all processors, in case one + // of them does not own any cells at all. + std::vector global_sizes_fixed(local_sizes_fixed.size()); + Utilities::MPI::max(local_sizes_fixed, + this->mpi_communicator, + global_sizes_fixed); + + // Construct cumulative sizes, since this is the only information + // we need from now on. + sizes_fixed_cumulative.resize(global_sizes_fixed.size()); + std::partial_sum(global_sizes_fixed.begin(), + global_sizes_fixed.end(), + sizes_fixed_cumulative.begin()); + + // + // ---------- Gather data sizes for variable size transfer ---------- + // + if (variable_size_data_stored) + { + src_sizes_variable.reserve(packed_variable_size_data.size()); + for (const auto &data_cell : packed_variable_size_data) + { + int variable_data_size_on_cell = 0; + + for (const auto &data : data_cell) + variable_data_size_on_cell += data.size(); + + src_sizes_variable.push_back(variable_data_size_on_cell); + } + } + + // + // ------------------------ Build buffers --------------------------- + // + const unsigned int expected_size_fixed = + cell_relations.size() * sizes_fixed_cumulative.back(); + const unsigned int expected_size_variable = + std::accumulate(src_sizes_variable.begin(), + src_sizes_variable.end(), + std::vector::size_type(0)); + + // Move every piece of packed fixed size data into the consecutive + // buffer. + src_data_fixed.reserve(expected_size_fixed); + for (const auto &data_cell_fixed : packed_fixed_size_data) + { + // Move every fraction of packed data into the buffer + // reserved for this particular cell. + for (const auto &data_fixed : data_cell_fixed) + std::move(data_fixed.begin(), + data_fixed.end(), + std::back_inserter(src_data_fixed)); + + // If we only packed the CellStatus information + // (i.e. encountered a cell flagged CELL_INVALID), + // fill the remaining space with invalid entries. + // We can skip this if there is nothing else to pack. + if ((data_cell_fixed.size() == 1) && (sizes_fixed_cumulative.size() > 1)) + { + const std::size_t bytes_skipped = + sizes_fixed_cumulative.back() - sizes_fixed_cumulative.front(); + + src_data_fixed.insert(src_data_fixed.end(), + bytes_skipped, + static_cast(-1)); // invalid_char + } + } + + // Move every piece of packed variable size data into the consecutive + // buffer. + if (variable_size_data_stored) + { + src_data_variable.reserve(expected_size_variable); + for (const auto &data_cell : packed_variable_size_data) + { + // Move every fraction of packed data into the buffer + // reserved for this particular cell. + for (const auto &data : data_cell) + std::move(data.begin(), + data.end(), + std::back_inserter(src_data_variable)); + } + } + + // Double check that we packed everything correctly. + Assert(src_data_fixed.size() == expected_size_fixed, ExcInternalError()); + Assert(src_data_variable.size() == expected_size_variable, + ExcInternalError()); +} + + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void DataTransfer::unpack_cell_status( + std::vector::cell_relation_t> + &cell_relations) const +{ + Assert(sizes_fixed_cumulative.size() > 0, + ExcMessage("No data has been packed!")); + if (cell_relations.size() > 0) + { + Assert(dest_data_fixed.size() > 0, + ExcMessage("No data has been received!")); + } + + // Size of CellStatus object that will be unpacked on each cell. + const unsigned int size = sizes_fixed_cumulative.front(); + + // Iterate over all cells and overwrite the CellStatus + // information from the transferred data. + // Proceed buffer iterator position to next cell after + // each iteration. + auto cell_rel_it = cell_relations.begin(); + auto dest_fixed_it = dest_data_fixed.cbegin(); + for (; cell_rel_it != cell_relations.end(); + ++cell_rel_it, dest_fixed_it += sizes_fixed_cumulative.back()) + { + cell_rel_it->second = // cell_status + Utilities::unpack(dest_fixed_it, + dest_fixed_it + size, + /*allow_compression=*/false); + } +} + + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void DataTransfer::unpack_data( + const std::vector::cell_relation_t> + & cell_relations, + const unsigned int handle, + const std::function< + void(const cell_iterator &, + const CellStatus &, + const boost::iterator_range::const_iterator> &)> + &unpack_callback) const +{ + // We decode the handle returned by register_data_attach() back into + // a format we can use. All even handles belong to those callback + // functions which write/read variable size data, all odd handles + // interact with fixed size buffers. + const bool callback_variable_transfer = (handle % 2 == 0); + const unsigned int callback_index = handle / 2; + + // Cells will always receive fixed size data (i.e., CellStatus + // information), but not necessarily variable size data (e.g., with a + // ParticleHandler a cell might not contain any particle at all). + // Thus it is sufficient to check if fixed size data has been received. + Assert(sizes_fixed_cumulative.size() > 0, + ExcMessage("No data has been packed!")); + if (cell_relations.size() > 0) + { + Assert(dest_data_fixed.size() > 0, + ExcMessage("No data has been received!")); + } + + std::vector::const_iterator dest_data_it; + std::vector::const_iterator dest_sizes_cell_it; + + // Depending on whether our callback function unpacks fixed or + // variable size data, we have to pursue different approaches + // to localize the correct fraction of the buffer from which + // we are allowed to read. + unsigned int offset = numbers::invalid_unsigned_int; + unsigned int size = numbers::invalid_unsigned_int; + unsigned int data_increment = numbers::invalid_unsigned_int; + + if (callback_variable_transfer) + { + // For the variable size data, we need to extract the + // data size from the fixed size buffer on each cell. + // + // We packed this information last, so the last packed + // object in the fixed size buffer corresponds to the + // variable data sizes. + // + // The last entry of sizes_fixed_cumulative corresponds + // to the size of all fixed size data packed on the cell. + // To get the offset for the last packed object, we need + // to get the next-to-last entry. + const unsigned int offset_variable_data_sizes = + sizes_fixed_cumulative[sizes_fixed_cumulative.size() - 2]; + + // This iterator points to the data size that the + // callback_function packed for each specific cell. + // Adjust buffer iterator to the offset of the callback + // function so that we only have to advance its position + // to the next cell after each iteration. + dest_sizes_cell_it = dest_data_fixed.cbegin() + + offset_variable_data_sizes + + callback_index * sizeof(unsigned int); + + // Let the data iterator point to the correct buffer. + dest_data_it = dest_data_variable.cbegin(); + } + else + { + // For the fixed size data, we can get the information about + // the buffer location on each cell directly from the + // sizes_fixed_cumulative vector. + offset = sizes_fixed_cumulative[callback_index]; + size = sizes_fixed_cumulative[callback_index + 1] - offset; + data_increment = sizes_fixed_cumulative.back(); + + // Let the data iterator point to the correct buffer. + // Adjust buffer iterator to the offset of the callback + // function so that we only have to advance its position + // to the next cell after each iteration. + if (cell_relations.begin() != cell_relations.end()) + dest_data_it = dest_data_fixed.cbegin() + offset; + } + + // Iterate over all cells and unpack the transferred data. + auto cell_rel_it = cell_relations.begin(); + auto dest_sizes_it = dest_sizes_variable.cbegin(); + for (; cell_rel_it != cell_relations.end(); ++cell_rel_it) + { + const auto &dealii_cell = cell_rel_it->first; + const auto &cell_status = cell_rel_it->second; + + if (callback_variable_transfer) + { + // Update the increment according to the whole data size + // of the current cell. + data_increment = *dest_sizes_it; + + if (cell_status != CELL_INVALID) + { + // Extract the corresponding values for offset and size from + // the cumulative sizes array stored in the fixed size + // buffer. + if (callback_index == 0) + offset = 0; + else + std::memcpy(&offset, + &(*(dest_sizes_cell_it - sizeof(unsigned int))), + sizeof(unsigned int)); + + std::memcpy(&size, &(*dest_sizes_cell_it), sizeof(unsigned int)); + + size -= offset; + + // Move the data iterator to the corresponding position + // of the callback function and adjust the increment + // accordingly. + dest_data_it += offset; + data_increment -= offset; + } + + // Advance data size iterators to the next cell, avoid iterating + // past the end of dest_sizes_cell_it + if (cell_rel_it != cell_relations.end() - 1) + dest_sizes_cell_it += sizes_fixed_cumulative.back(); + ++dest_sizes_it; + } + + switch (cell_status) + { + case CELL_PERSIST: + case CELL_COARSEN: + unpack_callback(dealii_cell, + cell_status, + boost::make_iterator_range(dest_data_it, + dest_data_it + size)); + break; + + case CELL_REFINE: + unpack_callback(dealii_cell->parent(), + cell_status, + boost::make_iterator_range(dest_data_it, + dest_data_it + size)); + break; + + case CELL_INVALID: + // Skip this cell. + break; + + default: + Assert(false, ExcInternalError()); + break; + } + + if (cell_rel_it != cell_relations.end() - 1) + dest_data_it += data_increment; + } +} + + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void DataTransfer::save(const unsigned int global_first_cell, + const unsigned int global_num_cells, + const std::string &filename) const +{ +#ifdef DEAL_II_WITH_MPI + // Large fractions of this function have been copied from + // DataOutInterface::write_vtu_in_parallel. + // TODO: Write general MPIIO interface. + + Assert(sizes_fixed_cumulative.size() > 0, + ExcMessage("No data has been packed!")); + + const int myrank = Utilities::MPI::this_mpi_process(mpi_communicator); + + const unsigned int bytes_per_cell = sizes_fixed_cumulative.back(); + + // + // ---------- Fixed size data ---------- + // + { + const std::string fname_fixed = std::string(filename) + "_fixed.data"; + + MPI_Info info; + int ierr = MPI_Info_create(&info); + AssertThrowMPI(ierr); + + MPI_File fh; + ierr = MPI_File_open(mpi_communicator, + fname_fixed.c_str(), + MPI_MODE_CREATE | MPI_MODE_WRONLY, + info, + &fh); + AssertThrowMPI(ierr); + + ierr = MPI_File_set_size(fh, 0); // delete the file contents + AssertThrowMPI(ierr); + // this barrier is necessary, because otherwise others might already + // write while one core is still setting the size to zero. + ierr = MPI_Barrier(mpi_communicator); + AssertThrowMPI(ierr); + ierr = MPI_Info_free(&info); + AssertThrowMPI(ierr); + // ------------------ + + // Write cumulative sizes to file. + // Since each processor owns the same information about the data + // sizes, it is sufficient to let only the first processor perform + // this task. + if (myrank == 0) + { + ierr = Utilities::MPI::LargeCount::File_write_at_c( + fh, + 0, + sizes_fixed_cumulative.data(), + sizes_fixed_cumulative.size(), + MPI_UNSIGNED, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + } + + // Write packed data to file simultaneously. + const MPI_Offset size_header = + sizes_fixed_cumulative.size() * sizeof(unsigned int); + + // Make sure we do the following computation in 64bit integers to be + // able to handle 4GB+ files: + const MPI_Offset my_global_file_position = + size_header + static_cast(global_first_cell) * bytes_per_cell; + + ierr = Utilities::MPI::LargeCount::File_write_at_c(fh, + my_global_file_position, + src_data_fixed.data(), + src_data_fixed.size(), + MPI_BYTE, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + + ierr = MPI_File_close(&fh); + AssertThrowMPI(ierr); + } + + + + // + // ---------- Variable size data ---------- + // + if (variable_size_data_stored) + { + const std::string fname_variable = + std::string(filename) + "_variable.data"; + + MPI_Info info; + int ierr = MPI_Info_create(&info); + AssertThrowMPI(ierr); + + MPI_File fh; + ierr = MPI_File_open(mpi_communicator, + fname_variable.c_str(), + MPI_MODE_CREATE | MPI_MODE_WRONLY, + info, + &fh); + AssertThrowMPI(ierr); + + ierr = MPI_File_set_size(fh, 0); // delete the file contents + AssertThrowMPI(ierr); + // this barrier is necessary, because otherwise others might already + // write while one core is still setting the size to zero. + ierr = MPI_Barrier(mpi_communicator); + AssertThrowMPI(ierr); + ierr = MPI_Info_free(&info); + AssertThrowMPI(ierr); + + // Write sizes of each cell into file simultaneously. + { + const MPI_Offset my_global_file_position = + static_cast(global_first_cell) * sizeof(unsigned int); + + // It is very unlikely that a single process has more than + // 2 billion cells, but we might as well check. + AssertThrow(src_sizes_variable.size() < + static_cast(std::numeric_limits::max()), + ExcNotImplemented()); + + ierr = + Utilities::MPI::LargeCount::File_write_at_c(fh, + my_global_file_position, + src_sizes_variable.data(), + src_sizes_variable.size(), + MPI_INT, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + } + + // Gather size of data in bytes we want to store from this + // processor and compute the prefix sum. We do this in 64 bit + // to avoid overflow for files larger than 4GB: + const std::uint64_t size_on_proc = src_data_variable.size(); + std::uint64_t prefix_sum = 0; + ierr = MPI_Exscan( + &size_on_proc, &prefix_sum, 1, MPI_UINT64_T, MPI_SUM, mpi_communicator); + AssertThrowMPI(ierr); + + const MPI_Offset my_global_file_position = + static_cast(global_num_cells) * sizeof(unsigned int) + + prefix_sum; + + // Write data consecutively into file. + ierr = + Utilities::MPI::LargeCount::File_write_at_c(fh, + my_global_file_position, + src_data_variable.data(), + src_data_variable.size(), + MPI_BYTE, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + + + ierr = MPI_File_close(&fh); + AssertThrowMPI(ierr); + } +#else + (void)global_first_cell; + (void)global_num_cells; + (void)filename; + + AssertThrow(false, ExcNeedsMPI()); +#endif +} + + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void DataTransfer::load( + const unsigned int global_first_cell, + const unsigned int global_num_cells, + const unsigned int local_num_cells, + const std::string &filename, + const unsigned int n_attached_deserialize_fixed, + const unsigned int n_attached_deserialize_variable) +{ +#ifdef DEAL_II_WITH_MPI + // Large fractions of this function have been copied from + // DataOutInterface::write_vtu_in_parallel. + // TODO: Write general MPIIO interface. + + Assert(dest_data_fixed.size() == 0, + ExcMessage("Previously loaded data has not been released yet!")); + + variable_size_data_stored = (n_attached_deserialize_variable > 0); + + // + // ---------- Fixed size data ---------- + // + { + const std::string fname_fixed = std::string(filename) + "_fixed.data"; + + MPI_Info info; + int ierr = MPI_Info_create(&info); + AssertThrowMPI(ierr); + + MPI_File fh; + ierr = MPI_File_open( + mpi_communicator, fname_fixed.c_str(), MPI_MODE_RDONLY, info, &fh); + AssertThrowMPI(ierr); + + ierr = MPI_Info_free(&info); + AssertThrowMPI(ierr); + + // Read cumulative sizes from file. + // Since all processors need the same information about the data + // sizes, let each of them retrieve it by reading from the same + // location in the file. + sizes_fixed_cumulative.resize(1 + n_attached_deserialize_fixed + + (variable_size_data_stored ? 1 : 0)); + ierr = + Utilities::MPI::LargeCount::File_read_at_c(fh, + 0, + sizes_fixed_cumulative.data(), + sizes_fixed_cumulative.size(), + MPI_UNSIGNED, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + + // Allocate sufficient memory. + const unsigned int bytes_per_cell = sizes_fixed_cumulative.back(); + dest_data_fixed.resize(static_cast(local_num_cells) * + bytes_per_cell); + + // Read packed data from file simultaneously. + const MPI_Offset size_header = + sizes_fixed_cumulative.size() * sizeof(unsigned int); + + // Make sure we do the following computation in 64bit integers to be + // able to handle 4GB+ files: + const MPI_Offset my_global_file_position = + size_header + static_cast(global_first_cell) * bytes_per_cell; + + ierr = Utilities::MPI::LargeCount::File_read_at_c(fh, + my_global_file_position, + dest_data_fixed.data(), + dest_data_fixed.size(), + MPI_BYTE, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + + + ierr = MPI_File_close(&fh); + AssertThrowMPI(ierr); + } + + // + // ---------- Variable size data ---------- + // + if (variable_size_data_stored) + { + const std::string fname_variable = + std::string(filename) + "_variable.data"; + + MPI_Info info; + int ierr = MPI_Info_create(&info); + AssertThrowMPI(ierr); + + MPI_File fh; + ierr = MPI_File_open( + mpi_communicator, fname_variable.c_str(), MPI_MODE_RDONLY, info, &fh); + AssertThrowMPI(ierr); + + ierr = MPI_Info_free(&info); + AssertThrowMPI(ierr); + + // Read sizes of all locally owned cells. + dest_sizes_variable.resize(local_num_cells); + + const MPI_Offset my_global_file_position_sizes = + static_cast(global_first_cell) * sizeof(unsigned int); + + ierr = Utilities::MPI::LargeCount::File_read_at_c( + fh, + my_global_file_position_sizes, + dest_sizes_variable.data(), + dest_sizes_variable.size(), + MPI_INT, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + + + // Compute my data size in bytes and compute prefix sum. We do this + // in 64 bit to avoid overflow for files larger than 4 GB: + const std::uint64_t size_on_proc = + std::accumulate(dest_sizes_variable.begin(), + dest_sizes_variable.end(), + 0ULL); + + std::uint64_t prefix_sum = 0; + ierr = MPI_Exscan( + &size_on_proc, &prefix_sum, 1, MPI_UINT64_T, MPI_SUM, mpi_communicator); + AssertThrowMPI(ierr); + + const MPI_Offset my_global_file_position = + static_cast(global_num_cells) * sizeof(unsigned int) + + prefix_sum; + + dest_data_variable.resize(size_on_proc); + + ierr = + Utilities::MPI::LargeCount::File_read_at_c(fh, + my_global_file_position, + dest_data_variable.data(), + dest_data_variable.size(), + MPI_BYTE, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + + ierr = MPI_File_close(&fh); + AssertThrowMPI(ierr); + } +#else + (void)global_first_cell; + (void)global_num_cells; + (void)local_num_cells; + (void)filename; + (void)n_attached_deserialize_fixed; + (void)n_attached_deserialize_variable; + + AssertThrow(false, ExcNeedsMPI()); +#endif +} + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void DataTransfer::clear() +{ + variable_size_data_stored = false; + + // free information about data sizes + sizes_fixed_cumulative.clear(); + sizes_fixed_cumulative.shrink_to_fit(); + + // free fixed size transfer data + src_data_fixed.clear(); + src_data_fixed.shrink_to_fit(); + + dest_data_fixed.clear(); + dest_data_fixed.shrink_to_fit(); + + // free variable size transfer data + src_sizes_variable.clear(); + src_sizes_variable.shrink_to_fit(); + + src_data_variable.clear(); + src_data_variable.shrink_to_fit(); + + dest_sizes_variable.clear(); + dest_sizes_variable.shrink_to_fit(); + + dest_data_variable.clear(); + dest_data_variable.shrink_to_fit(); +} + +DEAL_II_NAMESPACE_CLOSE + +#endif diff --git a/include/deal.II/grid/tria.h b/include/deal.II/grid/tria.h index 3566fbf641..81bc4a19a5 100644 --- a/include/deal.II/grid/tria.h +++ b/include/deal.II/grid/tria.h @@ -27,6 +27,8 @@ #include #include +#include +#include #include #include #include @@ -2032,34 +2034,6 @@ public: */ - /** - * Used to inform functions in derived classes how the cell with the given - * cell_iterator is going to change. Note that this may me different than - * the refine_flag() and coarsen_flag() in the cell_iterator in parallel - * calculations because of refinement constraints that this machine does not - * see. - */ - enum CellStatus - { - /** - * The cell will not be refined or coarsened and might or might not move - * to a different processor. - */ - CELL_PERSIST, - /** - * The cell will be or was refined. - */ - CELL_REFINE, - /** - * The children of this cell will be or were coarsened into this cell. - */ - CELL_COARSEN, - /** - * Invalid status. Will not occur for the user. - */ - CELL_INVALID - }; - /** * A structure used to accumulate the results of the `weight` signal slot * functions below. It takes an iterator range and returns the sum of @@ -3484,6 +3458,239 @@ public: BOOST_SERIALIZATION_SPLIT_MEMBER() #endif + /** + * @name Serialization facilities. + * @{ + */ +public: + /** + * Register a function that can be used to attach data of fixed size + * to cells. This is useful for two purposes: (i) Upon refinement and + * coarsening of a triangulation (@a e.g. in + * parallel::distributed::Triangulation::execute_coarsening_and_refinement()), + * one needs to be able to store one or more data vectors per cell that + * characterizes the solution values on the cell so that this data can + * then be transferred to the new owning processor of the cell (or + * its parent/children) when the mesh is re-partitioned; (ii) when + * serializing a computation to a file, it is necessary to attach + * data to cells so that it can be saved (@a e.g. in + * parallel::distributed::Triangulation::save()) along with the cell's + * other information and, if necessary, later be reloaded from disk + * with a different subdivision of cells among the processors. + * + * The way this function works is that it allows any number of interest + * parties to register their intent to attach data to cells. One example + * of classes that do this is parallel::distributed::SolutionTransfer + * where each parallel::distributed::SolutionTransfer object that works + * on the current Triangulation object then needs to register its intent. + * Each of these parties registers a callback function (the first + * argument here, @p pack_callback) that will be called whenever the + * triangulation's execute_coarsening_and_refinement() or save() + * functions are called. + * + * The current function then returns an integer handle that corresponds + * to the number of data set that the callback provided here will attach. + * While this number could be given a precise meaning, this is + * not important: You will never actually have to do anything with + * this number except return it to the notify_ready_to_unpack() function. + * In other words, each interested party (i.e., the caller of the current + * function) needs to store their respective returned handle for later use + * when unpacking data in the callback provided to + * notify_ready_to_unpack(). + * + * Whenever @p pack_callback is then called by + * execute_coarsening_and_refinement() or load() on a given cell, it + * receives a number of arguments. In particular, the first + * argument passed to the callback indicates the cell for which + * it is supposed to attach data. This is always an active cell. + * + * The second, CellStatus, argument provided to the callback function + * will tell you if the given cell will be coarsened, refined, or will + * persist as is. (This status may be different than the refinement + * or coarsening flags set on that cell, to accommodate things such as + * the "one hanging node per edge" rule.). These flags need to be + * read in context with the p4est quadrant they belong to, as their + * relations are gathered in local_cell_relations. + * + * Specifically, the values for this argument mean the following: + * + * - `CELL_PERSIST`: The cell won't be refined/coarsened, but might be + * moved to a different processor. If this is the case, the callback + * will want to pack up the data on this cell into an array and store + * it at the provided address for later unpacking wherever this cell + * may land. + * - `CELL_REFINE`: This cell will be refined into 4 or 8 cells (in 2d + * and 3d, respectively). However, because these children don't exist + * yet, you cannot access them at the time when the callback is + * called. Thus, in local_cell_relations, the corresponding + * p4est quadrants of the children cells are linked to the deal.II + * cell which is going to be refined. To be specific, only the very + * first child is marked with `CELL_REFINE`, whereas the others will be + * marked with `CELL_INVALID`, which indicates that these cells will be + * ignored by default during the packing or unpacking process. This + * ensures that data is only transferred once onto or from the parent + * cell. If the callback is called with `CELL_REFINE`, the callback + * will want to pack up the data on this cell into an array and store + * it at the provided address for later unpacking in a way so that + * it can then be transferred to the children of the cell that will + * then be available. In other words, if the data the callback + * will want to pack up corresponds to a finite element field, then + * the prolongation from parent to (new) children will have to happen + * during unpacking. + * - `CELL_COARSEN`: The children of this cell will be coarsened into the + * given cell. These children still exist, so if this is the value + * given to the callback as second argument, the callback will want + * to transfer data from the children to the current parent cell and + * pack it up so that it can later be unpacked again on a cell that + * then no longer has any children (and may also be located on a + * different processor). In other words, if the data the callback + * will want to pack up corresponds to a finite element field, then + * it will need to do the restriction from children to parent at + * this point. + * - `CELL_INVALID`: See `CELL_REFINE`. + * + * @note If this function is used for serialization of data + * using save() and load(), then the cell status argument with which + * the callback is called will always be `CELL_PERSIST`. + * + * The callback function is expected to return a memory chunk of the + * format `std::vector`, representing the packed data on a + * certain cell. + * + * The second parameter @p returns_variable_size_data indicates whether + * the returned size of the memory region from the callback function + * varies by cell (=true) or stays constant on each one + * throughout the whole domain (=false). + * + * @note The purpose of this function is to register intent to + * attach data for a single, subsequent call to + * execute_coarsening_and_refinement() and notify_ready_to_unpack(), + * save(), load(). Consequently, notify_ready_to_unpack(), save(), + * and load() all forget the registered callbacks once these + * callbacks have been called, and you will have to re-register + * them with a triangulation if you want them to be active for + * another call to these functions. + */ + unsigned int + register_data_attach( + const std::function(const cell_iterator &, + const CellStatus)> &pack_callback, + const bool returns_variable_size_data); + + /** + * This function is the opposite of register_data_attach(). It is called + * after the execute_coarsening_and_refinement() or save()/load() + * functions are done when classes and functions that have previously + * attached data to a triangulation for either transfer to other + * processors, across mesh refinement, or serialization of data to + * a file are ready to receive that data back. The important part about + * this process is that the triangulation cannot do this right away from + * the end of execute_coarsening_and_refinement() or load() via a + * previously attached callback function (as the register_data_attach() + * function does) because the classes that eventually want the data + * back may need to do some setup between the point in time where the + * mesh has been recreated and when the data can actually be received. + * An example is the parallel::distributed::SolutionTransfer class + * that can really only receive the data once not only the mesh is + * completely available again on the current processor, but only + * after a DoFHandler has been reinitialized and distributed + * degrees of freedom. In other words, there is typically a significant + * amount of set up that needs to happen in user space before the classes + * that can receive data attached to cell are ready to actually do so. + * When they are, they use the current function to tell the triangulation + * object that now is the time when they are ready by calling the + * current function. + * + * The supplied callback function is then called for each newly locally + * owned cell. The first argument to the callback is an iterator that + * designates the cell; the second argument indicates the status of the + * cell in question; and the third argument localizes a memory area by + * two iterators that contains the data that was previously saved from + * the callback provided to register_data_attach(). + * + * The CellStatus will indicate if the cell was refined, coarsened, or + * persisted unchanged. The @p cell_iterator argument to the callback + * will then either be an active, + * locally owned cell (if the cell was not refined), or the immediate + * parent if it was refined during execute_coarsening_and_refinement(). + * Therefore, contrary to during register_data_attach(), you can now + * access the children if the status is `CELL_REFINE` but no longer for + * callbacks with status `CELL_COARSEN`. + * + * The first argument to this function, `handle`, corresponds to + * the return value of register_data_attach(). (The precise + * meaning of what the numeric value of this handle is supposed + * to represent is neither important, nor should you try to use + * it for anything other than transmit information between a + * call to register_data_attach() to the corresponding call to + * notify_ready_to_unpack().) + */ + void + notify_ready_to_unpack( + const unsigned int handle, + const std::function< + void(const cell_iterator &, + const CellStatus, + const boost::iterator_range::const_iterator> &)> + &unpack_callback); + + CellAttachedData cell_attached_data; + +protected: + /** + * Save additional cell-attached data into the given file. The first + * arguments are used to determine the offsets where to write buffers to. + * + * Called by @ref save. + */ + void + save_attached_data(const unsigned int global_first_cell, + const unsigned int global_num_cells, + const std::string &filename) const; + + /** + * Load additional cell-attached data from the given file, if any was saved. + * The first arguments are used to determine the offsets where to read + * buffers from. + * + * Called by @ref load. + */ + void + load_attached_data(const unsigned int global_first_cell, + const unsigned int global_num_cells, + const unsigned int local_num_cells, + const std::string &filename, + const unsigned int n_attached_deserialize_fixed, + const unsigned int n_attached_deserialize_variable); + + /** + * A function to record the CellStatus of currently active cells that + * are locally owned. This information is mandatory to transfer data + * between meshes during adaptation or serialization, e.g., using + * parallel::distributed::SolutionTransfer. + * + * Relations will be stored in the private member local_cell_relations. For + * an extensive description of CellStatus, see the documentation for the + * member function register_data_attach(). + */ + virtual void + update_cell_relations() + {} + + /** + * Vector of pairs, each containing a deal.II cell iterator and its + * respective CellStatus. To update its contents, use the + * update_cell_relations() member function. + */ + std::vector::cell_relation_t> + local_cell_relations; + + std::unique_ptr> data_transfer; + /** + * @} + */ + +public: /** * @name Exceptions * @{ diff --git a/include/deal.II/particles/particle_handler.h b/include/deal.II/particles/particle_handler.h index 49a0def316..b29eadd3f6 100644 --- a/include/deal.II/particles/particle_handler.h +++ b/include/deal.II/particles/particle_handler.h @@ -1143,7 +1143,7 @@ namespace Particles std::vector pack_callback( const typename Triangulation::cell_iterator &cell, - const typename Triangulation::CellStatus status) const; + const CellStatus status) const; /** * Called by listener functions after a refinement step for each cell @@ -1152,7 +1152,7 @@ namespace Particles void unpack_callback( const typename Triangulation::cell_iterator &cell, - const typename Triangulation::CellStatus status, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range); diff --git a/source/distributed/cell_weights.cc b/source/distributed/cell_weights.cc index 3d92c5ce3c..e54a17c4f5 100644 --- a/source/distributed/cell_weights.cc +++ b/source/distributed/cell_weights.cc @@ -128,7 +128,7 @@ namespace parallel template std::function::cell_iterator &cell, - const typename dealii::Triangulation::CellStatus status)> + const CellStatus status)> CellWeights::make_weighting_callback( const DoFHandler &dof_handler, const typename CellWeights::WeightingFunction @@ -143,19 +143,17 @@ namespace parallel ExcMessage( "parallel::CellWeights requires a parallel::TriangulationBase object.")); - return - [&dof_handler, tria, weighting_function]( - const typename dealii::Triangulation::cell_iterator - & cell, - const typename dealii::Triangulation::CellStatus status) - -> unsigned int { - return CellWeights::weighting_callback( - cell, - status, - std::cref(dof_handler), - std::cref(*tria), - weighting_function); - }; + return [&dof_handler, tria, weighting_function]( + const typename dealii::Triangulation::cell_iterator + & cell, + const CellStatus status) -> unsigned int { + return CellWeights::weighting_callback(cell, + status, + std::cref( + dof_handler), + std::cref(*tria), + weighting_function); + }; } @@ -164,7 +162,7 @@ namespace parallel unsigned int CellWeights::weighting_callback( const typename dealii::Triangulation::cell_iterator &cell_, - const typename dealii::Triangulation::CellStatus status, + const CellStatus status, const DoFHandler & dof_handler, const parallel::TriangulationBase &triangulation, const typename CellWeights::WeightingFunction @@ -191,13 +189,13 @@ namespace parallel unsigned int fe_index = numbers::invalid_unsigned_int; switch (status) { - case Triangulation::CELL_PERSIST: - case Triangulation::CELL_REFINE: - case Triangulation::CELL_INVALID: + case CELL_PERSIST: + case CELL_REFINE: + case CELL_INVALID: fe_index = cell->future_fe_index(); break; - case Triangulation::CELL_COARSEN: + case CELL_COARSEN: #ifdef DEBUG for (const auto &child : cell->child_iterators()) Assert(child->is_active() && child->coarsen_flag_set(), diff --git a/source/distributed/fully_distributed_tria.cc b/source/distributed/fully_distributed_tria.cc index c120fe867f..ee1ad3f816 100644 --- a/source/distributed/fully_distributed_tria.cc +++ b/source/distributed/fully_distributed_tria.cc @@ -445,8 +445,7 @@ namespace parallel for (const auto &cell : this->active_cell_iterators()) if (cell->is_locally_owned()) - this->local_cell_relations.emplace_back( - cell, Triangulation::CELL_PERSIST); + this->local_cell_relations.emplace_back(cell, CELL_PERSIST); } diff --git a/source/distributed/repartitioning_policy_tools.cc b/source/distributed/repartitioning_policy_tools.cc index 4e77977e71..73fdb4f0b2 100644 --- a/source/distributed/repartitioning_policy_tools.cc +++ b/source/distributed/repartitioning_policy_tools.cc @@ -262,8 +262,7 @@ namespace RepartitioningPolicyTools CellWeightPolicy::CellWeightPolicy( const std::function< unsigned int(const typename Triangulation::cell_iterator &, - const typename Triangulation::CellStatus)> - &weighting_function) + const CellStatus)> &weighting_function) : weighting_function(weighting_function) {} @@ -297,8 +296,7 @@ namespace RepartitioningPolicyTools for (const auto &cell : tria->active_cell_iterators() | IteratorFilters::LocallyOwnedCell()) weights[partitioner->global_to_local(cell->global_active_cell_index())] = - weighting_function( - cell, Triangulation::CellStatus::CELL_PERSIST); + weighting_function(cell, CELL_PERSIST); // determine weight of all the cells locally owned by this process std::uint64_t process_local_weight = 0; diff --git a/source/distributed/solution_transfer.cc b/source/distributed/solution_transfer.cc index 74dd379c64..107ce34ff7 100644 --- a/source/distributed/solution_transfer.cc +++ b/source/distributed/solution_transfer.cc @@ -164,7 +164,7 @@ namespace parallel handle = tria->register_data_attach( [this]( const typename Triangulation::cell_iterator &cell_, - const typename Triangulation::CellStatus status) { + const CellStatus status) { return this->pack_callback(cell_, status); }, /*returns_variable_size_data=*/dof_handler->has_hp_capabilities()); @@ -260,7 +260,7 @@ namespace parallel handle, [this, &all_out, &valence]( const typename Triangulation::cell_iterator &cell_, - const typename Triangulation::CellStatus status, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { this->unpack_callback(cell_, status, data_range, all_out, valence); @@ -310,7 +310,7 @@ namespace parallel std::vector SolutionTransfer::pack_callback( const typename Triangulation::cell_iterator &cell_, - const typename Triangulation::CellStatus status) + const CellStatus status) { typename DoFHandler::cell_iterator cell(*cell_, dof_handler); @@ -324,17 +324,14 @@ namespace parallel { switch (status) { - case parallel::distributed::Triangulation::CELL_PERSIST: - case parallel::distributed::Triangulation::CELL_REFINE: + case CELL_PERSIST: + case CELL_REFINE: { fe_index = cell->future_fe_index(); break; } - case parallel::distributed::Triangulation::CELL_COARSEN: + case CELL_COARSEN: { // In case of coarsening, we need to find a suitable FE index // for the parent cell. We choose the 'least dominant fe' @@ -381,7 +378,7 @@ namespace parallel void SolutionTransfer::unpack_callback( const typename Triangulation::cell_iterator &cell_, - const typename Triangulation::CellStatus status, + const CellStatus status, const boost::iterator_range::const_iterator> & data_range, std::vector &all_out, @@ -395,17 +392,14 @@ namespace parallel { switch (status) { - case parallel::distributed::Triangulation::CELL_PERSIST: - case parallel::distributed::Triangulation::CELL_COARSEN: + case CELL_PERSIST: + case CELL_COARSEN: { fe_index = cell->active_fe_index(); break; } - case parallel::distributed::Triangulation::CELL_REFINE: + case CELL_REFINE: { // After refinement, this particular cell is no longer active, // and its children have inherited its FE index. However, to diff --git a/source/distributed/tria.cc b/source/distributed/tria.cc index 261c20261d..9640ba8d0d 100644 --- a/source/distributed/tria.cc +++ b/source/distributed/tria.cc @@ -1522,7 +1522,7 @@ namespace template using cell_relation_t = typename std::pair< typename dealii::Triangulation::cell_iterator, - typename dealii::Triangulation::CellStatus>; + CellStatus>; /** * Adds a pair of a @p dealii_cell and its @p status @@ -1540,7 +1540,7 @@ namespace const typename dealii::internal::p4est::types::tree & tree, const unsigned int idx, const typename Triangulation::cell_iterator &dealii_cell, - const typename Triangulation::CellStatus status) + const CellStatus status) { const unsigned int local_quadrant_index = tree.quadrants_offset + idx; @@ -1620,11 +1620,7 @@ namespace // this active cell didn't change // save pair into corresponding position add_single_cell_relation( - cell_rel, - tree, - idx, - dealii_cell, - Triangulation::CELL_PERSIST); + cell_rel, tree, idx, dealii_cell, CELL_PERSIST); } else if (p4est_has_children) // based on the conditions above, we know that // dealii_cell has no children @@ -1656,8 +1652,8 @@ namespace // mark first child with CELL_REFINE and the remaining children with // CELL_INVALID, but associate them all with the parent cell unpack // algorithm will be called only on CELL_REFINE flagged quadrant - int child_idx; - typename Triangulation::CellStatus cell_status; + int child_idx; + CellStatus cell_status; for (unsigned int i = 0; i < GeometryInfo::max_children_per_cell; ++i) { @@ -1666,8 +1662,7 @@ namespace &p4est_child[i], dealii::internal::p4est::functions::quadrant_compare); - cell_status = (i == 0) ? Triangulation::CELL_REFINE : - Triangulation::CELL_INVALID; + cell_status = (i == 0) ? CELL_REFINE : CELL_INVALID; add_single_cell_relation( cell_rel, tree, child_idx, dealii_cell, cell_status); @@ -1679,11 +1674,7 @@ namespace // its children got coarsened into this cell in p4est, // but the dealii_cell still has its children add_single_cell_relation( - cell_rel, - tree, - idx, - dealii_cell, - Triangulation::CELL_COARSEN); + cell_rel, tree, idx, dealii_cell, CELL_COARSEN); } } } // namespace @@ -1877,13 +1868,13 @@ namespace parallel const typename dealii::internal::p4est::types::gloidx *previous_global_first_quadrant) { - Assert(this->data_transfer.sizes_fixed_cumulative.size() > 0, + Assert(this->data_transfer->sizes_fixed_cumulative.size() > 0, ExcMessage("No data has been packed!")); // Resize memory according to the data that we will receive. - this->data_transfer.dest_data_fixed.resize( + this->data_transfer->dest_data_fixed.resize( parallel_forest->local_num_quadrants * - this->data_transfer.sizes_fixed_cumulative.back()); + this->data_transfer->sizes_fixed_cumulative.back()); // Execute non-blocking fixed size transfer. typename dealii::internal::p4est::types::transfer_context @@ -1894,14 +1885,14 @@ namespace parallel previous_global_first_quadrant, parallel_forest->mpicomm, 0, - this->data_transfer.dest_data_fixed.data(), - this->data_transfer.src_data_fixed.data(), - this->data_transfer.sizes_fixed_cumulative.back()); + this->data_transfer->dest_data_fixed.data(), + this->data_transfer->src_data_fixed.data(), + this->data_transfer->sizes_fixed_cumulative.back()); - if (this->data_transfer.variable_size_data_stored) + if (this->data_transfer->variable_size_data_stored) { // Resize memory according to the data that we will receive. - this->data_transfer.dest_sizes_variable.resize( + this->data_transfer->dest_sizes_variable.resize( parallel_forest->local_num_quadrants); // Execute fixed size transfer of data sizes for variable size @@ -1911,23 +1902,23 @@ namespace parallel previous_global_first_quadrant, parallel_forest->mpicomm, 1, - this->data_transfer.dest_sizes_variable.data(), - this->data_transfer.src_sizes_variable.data(), + this->data_transfer->dest_sizes_variable.data(), + this->data_transfer->src_sizes_variable.data(), sizeof(unsigned int)); } dealii::internal::p4est::functions::transfer_fixed_end(tf_context); // Release memory of previously packed data. - this->data_transfer.src_data_fixed.clear(); - this->data_transfer.src_data_fixed.shrink_to_fit(); + this->data_transfer->src_data_fixed.clear(); + this->data_transfer->src_data_fixed.shrink_to_fit(); - if (this->data_transfer.variable_size_data_stored) + if (this->data_transfer->variable_size_data_stored) { // Resize memory according to the data that we will receive. - this->data_transfer.dest_data_variable.resize( - std::accumulate(this->data_transfer.dest_sizes_variable.begin(), - this->data_transfer.dest_sizes_variable.end(), + this->data_transfer->dest_data_variable.resize( + std::accumulate(this->data_transfer->dest_sizes_variable.begin(), + this->data_transfer->dest_sizes_variable.end(), std::vector::size_type(0))); # if DEAL_II_P4EST_VERSION_GTE(2, 0, 65, 0) @@ -1937,10 +1928,10 @@ namespace parallel // at all, which is mandatory if one of our processes does not own // any quadrant. This bypasses the assertion from being triggered. // - see: https://github.com/cburstedde/p4est/issues/48 - if (this->data_transfer.src_sizes_variable.size() == 0) - this->data_transfer.src_sizes_variable.resize(1); - if (this->data_transfer.dest_sizes_variable.size() == 0) - this->data_transfer.dest_sizes_variable.resize(1); + if (this->data_transfer->src_sizes_variable.size() == 0) + this->data_transfer->src_sizes_variable.resize(1); + if (this->data_transfer->dest_sizes_variable.size() == 0) + this->data_transfer->dest_sizes_variable.resize(1); # endif // Execute variable size transfer. @@ -1949,16 +1940,16 @@ namespace parallel previous_global_first_quadrant, parallel_forest->mpicomm, 1, - this->data_transfer.dest_data_variable.data(), - this->data_transfer.dest_sizes_variable.data(), - this->data_transfer.src_data_variable.data(), - this->data_transfer.src_sizes_variable.data()); + this->data_transfer->dest_data_variable.data(), + this->data_transfer->dest_sizes_variable.data(), + this->data_transfer->src_data_variable.data(), + this->data_transfer->src_sizes_variable.data()); // Release memory of previously packed data. - this->data_transfer.src_sizes_variable.clear(); - this->data_transfer.src_sizes_variable.shrink_to_fit(); - this->data_transfer.src_data_variable.clear(); - this->data_transfer.src_data_variable.shrink_to_fit(); + this->data_transfer->src_sizes_variable.clear(); + this->data_transfer->src_sizes_variable.shrink_to_fit(); + this->data_transfer->src_data_variable.clear(); + this->data_transfer->src_data_variable.shrink_to_fit(); } } @@ -2035,10 +2026,9 @@ namespace parallel for (const auto &cell_rel : this->local_cell_relations) { (void)cell_rel; - Assert( - (cell_rel.second == // cell_status - parallel::distributed::Triangulation::CELL_PERSIST), - ExcInternalError()); + Assert((cell_rel.second == // cell_status + CELL_PERSIST), + ExcInternalError()); } // Save cell attached data. @@ -3395,7 +3385,7 @@ namespace parallel // pack data before triangulation gets updated if (this->cell_attached_data.n_attached_data_sets > 0) { - this->data_transfer.pack_data( + this->data_transfer->pack_data( this->local_cell_relations, this->cell_attached_data.pack_callbacks_fixed, this->cell_attached_data.pack_callbacks_variable); @@ -3428,7 +3418,7 @@ namespace parallel previous_global_first_quadrant.data()); // also update the CellStatus information on the new mesh - this->data_transfer.unpack_cell_status(this->local_cell_relations); + this->data_transfer->unpack_cell_status(this->local_cell_relations); } # ifdef DEBUG @@ -3568,7 +3558,7 @@ namespace parallel // pack data before triangulation gets updated if (this->cell_attached_data.n_attached_data_sets > 0) { - this->data_transfer.pack_data( + this->data_transfer->pack_data( this->local_cell_relations, this->cell_attached_data.pack_callbacks_fixed, this->cell_attached_data.pack_callbacks_variable); @@ -4244,19 +4234,19 @@ namespace parallel switch (status) { - case dealii::Triangulation::CELL_PERSIST: + case CELL_PERSIST: // cell remains unchanged cell->clear_refine_flag(); cell->clear_coarsen_flag(); break; - case dealii::Triangulation::CELL_REFINE: + case CELL_REFINE: // cell will be refined cell->clear_coarsen_flag(); cell->set_refine_flag(); break; - case dealii::Triangulation::CELL_COARSEN: + case CELL_COARSEN: // children of this cell will be coarsened for (const auto &child : cell->child_iterators()) { @@ -4265,7 +4255,7 @@ namespace parallel } break; - case dealii::Triangulation::CELL_INVALID: + case CELL_INVALID: // do nothing as cell does not exist yet break; diff --git a/source/distributed/tria_base.cc b/source/distributed/tria_base.cc index 72a318013e..622ca0ac26 100644 --- a/source/distributed/tria_base.cc +++ b/source/distributed/tria_base.cc @@ -678,8 +678,6 @@ namespace parallel mpi_communicator, smooth_grid, check_for_distorted_cells) - , cell_attached_data({0, 0, {}, {}}) - , data_transfer(mpi_communicator) {} @@ -688,51 +686,11 @@ namespace parallel DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) void DistributedTriangulationBase::clear() { - cell_attached_data = {0, 0, {}, {}}; - data_transfer.clear(); - dealii::Triangulation::clear(); } - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::save_attached_data( - const unsigned int global_first_cell, - const unsigned int global_num_cells, - const std::string &filename) const - { - // cast away constness - auto tria = const_cast< - dealii::parallel::DistributedTriangulationBase *>(this); - - if (this->cell_attached_data.n_attached_data_sets > 0) - { - // pack attached data first - tria->data_transfer.pack_data( - tria->local_cell_relations, - tria->cell_attached_data.pack_callbacks_fixed, - tria->cell_attached_data.pack_callbacks_variable); - - // then store buffers in file - tria->data_transfer.save(global_first_cell, global_num_cells, filename); - - // and release the memory afterwards - tria->data_transfer.clear(); - } - - // clear all of the callback data, as explained in the documentation of - // register_data_attach() - { - tria->cell_attached_data.n_attached_data_sets = 0; - tria->cell_attached_data.pack_callbacks_fixed.clear(); - tria->cell_attached_data.pack_callbacks_variable.clear(); - } - } - - - template DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) bool DistributedTriangulationBase::has_hanging_nodes() const @@ -759,1050 +717,6 @@ namespace parallel } - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::load_attached_data( - const unsigned int global_first_cell, - const unsigned int global_num_cells, - const unsigned int local_num_cells, - const std::string &filename, - const unsigned int n_attached_deserialize_fixed, - const unsigned int n_attached_deserialize_variable) - { - // load saved data, if any was stored - if (this->cell_attached_data.n_attached_deserialize > 0) - { - this->data_transfer.load(global_first_cell, - global_num_cells, - local_num_cells, - filename, - n_attached_deserialize_fixed, - n_attached_deserialize_variable); - - this->data_transfer.unpack_cell_status(this->local_cell_relations); - - // the CellStatus of all stored cells should always be CELL_PERSIST. - for (const auto &cell_rel : this->local_cell_relations) - { - (void)cell_rel; - Assert( - (cell_rel.second == // cell_status - parallel::DistributedTriangulationBase::CELL_PERSIST), - ExcInternalError()); - } - } - } - - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - unsigned int DistributedTriangulationBase:: - register_data_attach( - const std::function(const cell_iterator &, - const CellStatus)> &pack_callback, - const bool returns_variable_size_data) - { - unsigned int handle = numbers::invalid_unsigned_int; - - // Add new callback function to the corresponding register. - // Encode handles according to returns_variable_size_data. - if (returns_variable_size_data) - { - handle = 2 * cell_attached_data.pack_callbacks_variable.size(); - cell_attached_data.pack_callbacks_variable.push_back(pack_callback); - } - else - { - handle = 2 * cell_attached_data.pack_callbacks_fixed.size() + 1; - cell_attached_data.pack_callbacks_fixed.push_back(pack_callback); - } - - // Increase overall counter. - ++cell_attached_data.n_attached_data_sets; - - return handle; - } - - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::notify_ready_to_unpack( - const unsigned int handle, - const std::function< - void(const cell_iterator &, - const CellStatus, - const boost::iterator_range::const_iterator> &)> - &unpack_callback) - { - // perform unpacking - data_transfer.unpack_data(local_cell_relations, handle, unpack_callback); - - // decrease counters - --cell_attached_data.n_attached_data_sets; - if (cell_attached_data.n_attached_deserialize > 0) - --cell_attached_data.n_attached_deserialize; - - // important: only remove data if we are not in the deserialization - // process. There, each SolutionTransfer registers and unpacks before - // the next one does this, so n_attached_data_sets is only 1 here. This - // would destroy the saved data before the second SolutionTransfer can - // get it. This created a bug that is documented in - // tests/mpi/p4est_save_03 with more than one SolutionTransfer. - - if (cell_attached_data.n_attached_data_sets == 0 && - cell_attached_data.n_attached_deserialize == 0) - { - // everybody got their data, time for cleanup! - cell_attached_data.pack_callbacks_fixed.clear(); - cell_attached_data.pack_callbacks_variable.clear(); - data_transfer.clear(); - - // reset all cell_status entries after coarsening/refinement - for (auto &cell_rel : local_cell_relations) - cell_rel.second = - parallel::DistributedTriangulationBase::CELL_PERSIST; - } - } - - - - /* ------------------ class DataTransfer ----------------- */ - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - DistributedTriangulationBase::DataTransfer::DataTransfer( - const MPI_Comm mpi_communicator) - : variable_size_data_stored(false) - , mpi_communicator(mpi_communicator) - {} - - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::DataTransfer::pack_data( - const std::vector &cell_relations, - const std::vector - &pack_callbacks_fixed, - const std::vector - &pack_callbacks_variable) - { - Assert(src_data_fixed.size() == 0, - ExcMessage("Previously packed data has not been released yet!")); - Assert(src_sizes_variable.size() == 0, ExcInternalError()); - - const unsigned int n_callbacks_fixed = pack_callbacks_fixed.size(); - const unsigned int n_callbacks_variable = pack_callbacks_variable.size(); - - // Store information that we packed variable size data in - // a member variable for later. - variable_size_data_stored = (n_callbacks_variable > 0); - - // If variable transfer is scheduled, we will store the data size that - // each variable size callback function writes in this auxiliary - // container. The information will be stored by each cell in this vector - // temporarily. - std::vector cell_sizes_variable_cumulative( - n_callbacks_variable); - - // Prepare the buffer structure, in which each callback function will - // store its data for each active cell. - // The outmost shell in this container construct corresponds to the - // data packed per cell. The next layer resembles the data that - // each callback function packs on the corresponding cell. These - // buffers are chains of chars stored in an std::vector. - // A visualisation of the data structure: - /* clang-format off */ - // | cell_1 | | cell_2 | ... - // || callback_1 || callback_2 |...| || callback_1 || callback_2 |...| ... - // |||char|char|...|||char|char|...|...| |||char|char|...|||char|char|...|...| ... - /* clang-format on */ - std::vector>> packed_fixed_size_data( - cell_relations.size()); - std::vector>> packed_variable_size_data( - variable_size_data_stored ? cell_relations.size() : 0); - - // - // --------- Pack data for fixed and variable size transfer --------- - // - // Iterate over all cells, call all callback functions on each cell, - // and store their data in the corresponding buffer scope. - { - auto cell_rel_it = cell_relations.cbegin(); - auto data_cell_fixed_it = packed_fixed_size_data.begin(); - auto data_cell_variable_it = packed_variable_size_data.begin(); - for (; cell_rel_it != cell_relations.cend(); ++cell_rel_it) - { - const auto &dealii_cell = cell_rel_it->first; - const auto &cell_status = cell_rel_it->second; - - // Assertions about the tree structure. - switch (cell_status) - { - case parallel::DistributedTriangulationBase:: - CELL_PERSIST: - case parallel::DistributedTriangulationBase:: - CELL_REFINE: - // double check the condition that we will only ever attach - // data to active cells when we get here - Assert(dealii_cell->is_active(), ExcInternalError()); - break; - - case parallel::DistributedTriangulationBase:: - CELL_COARSEN: - // double check the condition that we will only ever attach - // data to cells with children when we get here. however, we - // can only tolerate one level of coarsening at a time, so - // check that the children are all active - Assert(dealii_cell->is_active() == false, ExcInternalError()); - for (unsigned int c = 0; - c < GeometryInfo::max_children_per_cell; - ++c) - Assert(dealii_cell->child(c)->is_active(), - ExcInternalError()); - break; - - case parallel::DistributedTriangulationBase:: - CELL_INVALID: - // do nothing on invalid cells - break; - - default: - Assert(false, ExcInternalError()); - break; - } - - // Reserve memory corresponding to the number of callback - // functions that will be called. - // If variable size transfer is scheduled, we need to leave - // room for an array that holds information about how many - // bytes each of the variable size callback functions will - // write. - // On cells flagged with CELL_INVALID, only its CellStatus - // will be stored. - const unsigned int n_fixed_size_data_sets_on_cell = - 1 + - ((cell_status == - parallel::DistributedTriangulationBase::CELL_INVALID) ? - 0 : - ((variable_size_data_stored ? 1 : 0) + n_callbacks_fixed)); - data_cell_fixed_it->resize(n_fixed_size_data_sets_on_cell); - - // We continue with packing all data on this specific cell. - auto data_fixed_it = data_cell_fixed_it->begin(); - - // First, we pack the CellStatus information. - // to get consistent data sizes on each cell for the fixed size - // transfer, we won't allow compression - *data_fixed_it = - Utilities::pack(cell_status, /*allow_compression=*/false); - ++data_fixed_it; - - // Proceed with all registered callback functions. - // Skip cells with the CELL_INVALID flag. - if (cell_status != - parallel::DistributedTriangulationBase::CELL_INVALID) - { - // Pack fixed size data. - for (auto callback_it = pack_callbacks_fixed.cbegin(); - callback_it != pack_callbacks_fixed.cend(); - ++callback_it, ++data_fixed_it) - { - *data_fixed_it = (*callback_it)(dealii_cell, cell_status); - } - - // Pack variable size data. - // If we store variable size data, we need to transfer - // the sizes of each corresponding callback function - // via fixed size transfer as well. - if (variable_size_data_stored) - { - const unsigned int n_variable_size_data_sets_on_cell = - ((cell_status == - parallel::DistributedTriangulationBase:: - CELL_INVALID) ? - 0 : - n_callbacks_variable); - data_cell_variable_it->resize( - n_variable_size_data_sets_on_cell); - - auto callback_it = pack_callbacks_variable.cbegin(); - auto data_variable_it = data_cell_variable_it->begin(); - auto sizes_variable_it = - cell_sizes_variable_cumulative.begin(); - for (; callback_it != pack_callbacks_variable.cend(); - ++callback_it, ++data_variable_it, ++sizes_variable_it) - { - *data_variable_it = - (*callback_it)(dealii_cell, cell_status); - - // Store data sizes for each callback function first. - // Make it cumulative below. - *sizes_variable_it = data_variable_it->size(); - } - - // Turn size vector into its cumulative representation. - std::partial_sum(cell_sizes_variable_cumulative.begin(), - cell_sizes_variable_cumulative.end(), - cell_sizes_variable_cumulative.begin()); - - // Serialize cumulative variable size vector - // value-by-value. This way we can circumvent the overhead - // of storing the container object as a whole, since we - // know its size by the number of registered callback - // functions. - data_fixed_it->resize(n_callbacks_variable * - sizeof(unsigned int)); - for (unsigned int i = 0; i < n_callbacks_variable; ++i) - std::memcpy(&(data_fixed_it->at(i * sizeof(unsigned int))), - &(cell_sizes_variable_cumulative.at(i)), - sizeof(unsigned int)); - - ++data_fixed_it; - } - - // Double check that we packed everything we wanted - // in the fixed size buffers. - Assert(data_fixed_it == data_cell_fixed_it->end(), - ExcInternalError()); - } - - ++data_cell_fixed_it; - - // Increment the variable size data iterator - // only if we actually pack this kind of data - // to avoid getting out of bounds. - if (variable_size_data_stored) - ++data_cell_variable_it; - } // loop over cell_relations - } - - // - // ----------- Gather data sizes for fixed size transfer ------------ - // - // Generate a vector which stores the sizes of each callback function, - // including the packed CellStatus transfer. - // Find the very first cell that we wrote to with all callback - // functions (i.e. a cell that was not flagged with CELL_INVALID) - // and store the sizes of each buffer. - // - // To deal with the case that at least one of the processors does not - // own any cell at all, we will exchange the information about the data - // sizes among them later. The code in between is still well-defined, - // since the following loops will be skipped. - std::vector local_sizes_fixed( - 1 + n_callbacks_fixed + (variable_size_data_stored ? 1 : 0)); - for (const auto &data_cell : packed_fixed_size_data) - { - if (data_cell.size() == local_sizes_fixed.size()) - { - auto sizes_fixed_it = local_sizes_fixed.begin(); - auto data_fixed_it = data_cell.cbegin(); - for (; data_fixed_it != data_cell.cend(); - ++data_fixed_it, ++sizes_fixed_it) - { - *sizes_fixed_it = data_fixed_it->size(); - } - - break; - } - } - - // Check if all cells have valid sizes. - for (auto data_cell_fixed_it = packed_fixed_size_data.cbegin(); - data_cell_fixed_it != packed_fixed_size_data.cend(); - ++data_cell_fixed_it) - { - Assert((data_cell_fixed_it->size() == 1) || - (data_cell_fixed_it->size() == local_sizes_fixed.size()), - ExcInternalError()); - } - - // Share information about the packed data sizes - // of all callback functions across all processors, in case one - // of them does not own any cells at all. - std::vector global_sizes_fixed(local_sizes_fixed.size()); - Utilities::MPI::max(local_sizes_fixed, - this->mpi_communicator, - global_sizes_fixed); - - // Construct cumulative sizes, since this is the only information - // we need from now on. - sizes_fixed_cumulative.resize(global_sizes_fixed.size()); - std::partial_sum(global_sizes_fixed.begin(), - global_sizes_fixed.end(), - sizes_fixed_cumulative.begin()); - - // - // ---------- Gather data sizes for variable size transfer ---------- - // - if (variable_size_data_stored) - { - src_sizes_variable.reserve(packed_variable_size_data.size()); - for (const auto &data_cell : packed_variable_size_data) - { - int variable_data_size_on_cell = 0; - - for (const auto &data : data_cell) - variable_data_size_on_cell += data.size(); - - src_sizes_variable.push_back(variable_data_size_on_cell); - } - } - - // - // ------------------------ Build buffers --------------------------- - // - const unsigned int expected_size_fixed = - cell_relations.size() * sizes_fixed_cumulative.back(); - const unsigned int expected_size_variable = - std::accumulate(src_sizes_variable.begin(), - src_sizes_variable.end(), - std::vector::size_type(0)); - - // Move every piece of packed fixed size data into the consecutive - // buffer. - src_data_fixed.reserve(expected_size_fixed); - for (const auto &data_cell_fixed : packed_fixed_size_data) - { - // Move every fraction of packed data into the buffer - // reserved for this particular cell. - for (const auto &data_fixed : data_cell_fixed) - std::move(data_fixed.begin(), - data_fixed.end(), - std::back_inserter(src_data_fixed)); - - // If we only packed the CellStatus information - // (i.e. encountered a cell flagged CELL_INVALID), - // fill the remaining space with invalid entries. - // We can skip this if there is nothing else to pack. - if ((data_cell_fixed.size() == 1) && - (sizes_fixed_cumulative.size() > 1)) - { - const std::size_t bytes_skipped = - sizes_fixed_cumulative.back() - sizes_fixed_cumulative.front(); - - src_data_fixed.insert(src_data_fixed.end(), - bytes_skipped, - static_cast(-1)); // invalid_char - } - } - - // Move every piece of packed variable size data into the consecutive - // buffer. - if (variable_size_data_stored) - { - src_data_variable.reserve(expected_size_variable); - for (const auto &data_cell : packed_variable_size_data) - { - // Move every fraction of packed data into the buffer - // reserved for this particular cell. - for (const auto &data : data_cell) - std::move(data.begin(), - data.end(), - std::back_inserter(src_data_variable)); - } - } - - // Double check that we packed everything correctly. - Assert(src_data_fixed.size() == expected_size_fixed, ExcInternalError()); - Assert(src_data_variable.size() == expected_size_variable, - ExcInternalError()); - } - - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::DataTransfer:: - unpack_cell_status(std::vector &cell_relations) const - { - Assert(sizes_fixed_cumulative.size() > 0, - ExcMessage("No data has been packed!")); - if (cell_relations.size() > 0) - { - Assert(dest_data_fixed.size() > 0, - ExcMessage("No data has been received!")); - } - - // Size of CellStatus object that will be unpacked on each cell. - const unsigned int size = sizes_fixed_cumulative.front(); - - // Iterate over all cells and overwrite the CellStatus - // information from the transferred data. - // Proceed buffer iterator position to next cell after - // each iteration. - auto cell_rel_it = cell_relations.begin(); - auto dest_fixed_it = dest_data_fixed.cbegin(); - for (; cell_rel_it != cell_relations.end(); - ++cell_rel_it, dest_fixed_it += sizes_fixed_cumulative.back()) - { - cell_rel_it->second = // cell_status - Utilities::unpack::CellStatus>(dest_fixed_it, - dest_fixed_it + size, - /*allow_compression=*/false); - } - } - - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::DataTransfer::unpack_data( - const std::vector &cell_relations, - const unsigned int handle, - const std::function< - void(const typename dealii::Triangulation::cell_iterator &, - const typename dealii::Triangulation::CellStatus &, - const boost::iterator_range::const_iterator> &)> - &unpack_callback) const - { - // We decode the handle returned by register_data_attach() back into - // a format we can use. All even handles belong to those callback - // functions which write/read variable size data, all odd handles - // interact with fixed size buffers. - const bool callback_variable_transfer = (handle % 2 == 0); - const unsigned int callback_index = handle / 2; - - // Cells will always receive fixed size data (i.e., CellStatus - // information), but not necessarily variable size data (e.g., with a - // ParticleHandler a cell might not contain any particle at all). - // Thus it is sufficient to check if fixed size data has been received. - Assert(sizes_fixed_cumulative.size() > 0, - ExcMessage("No data has been packed!")); - if (cell_relations.size() > 0) - { - Assert(dest_data_fixed.size() > 0, - ExcMessage("No data has been received!")); - } - - std::vector::const_iterator dest_data_it; - std::vector::const_iterator dest_sizes_cell_it; - - // Depending on whether our callback function unpacks fixed or - // variable size data, we have to pursue different approaches - // to localize the correct fraction of the buffer from which - // we are allowed to read. - unsigned int offset = numbers::invalid_unsigned_int; - unsigned int size = numbers::invalid_unsigned_int; - unsigned int data_increment = numbers::invalid_unsigned_int; - - if (callback_variable_transfer) - { - // For the variable size data, we need to extract the - // data size from the fixed size buffer on each cell. - // - // We packed this information last, so the last packed - // object in the fixed size buffer corresponds to the - // variable data sizes. - // - // The last entry of sizes_fixed_cumulative corresponds - // to the size of all fixed size data packed on the cell. - // To get the offset for the last packed object, we need - // to get the next-to-last entry. - const unsigned int offset_variable_data_sizes = - sizes_fixed_cumulative[sizes_fixed_cumulative.size() - 2]; - - // This iterator points to the data size that the - // callback_function packed for each specific cell. - // Adjust buffer iterator to the offset of the callback - // function so that we only have to advance its position - // to the next cell after each iteration. - dest_sizes_cell_it = dest_data_fixed.cbegin() + - offset_variable_data_sizes + - callback_index * sizeof(unsigned int); - - // Let the data iterator point to the correct buffer. - dest_data_it = dest_data_variable.cbegin(); - } - else - { - // For the fixed size data, we can get the information about - // the buffer location on each cell directly from the - // sizes_fixed_cumulative vector. - offset = sizes_fixed_cumulative[callback_index]; - size = sizes_fixed_cumulative[callback_index + 1] - offset; - data_increment = sizes_fixed_cumulative.back(); - - // Let the data iterator point to the correct buffer. - // Adjust buffer iterator to the offset of the callback - // function so that we only have to advance its position - // to the next cell after each iteration. - if (cell_relations.begin() != cell_relations.end()) - dest_data_it = dest_data_fixed.cbegin() + offset; - } - - // Iterate over all cells and unpack the transferred data. - auto cell_rel_it = cell_relations.begin(); - auto dest_sizes_it = dest_sizes_variable.cbegin(); - for (; cell_rel_it != cell_relations.end(); ++cell_rel_it) - { - const auto &dealii_cell = cell_rel_it->first; - const auto &cell_status = cell_rel_it->second; - - if (callback_variable_transfer) - { - // Update the increment according to the whole data size - // of the current cell. - data_increment = *dest_sizes_it; - - if (cell_status != - parallel::DistributedTriangulationBase::CELL_INVALID) - { - // Extract the corresponding values for offset and size from - // the cumulative sizes array stored in the fixed size - // buffer. - if (callback_index == 0) - offset = 0; - else - std::memcpy(&offset, - &(*(dest_sizes_cell_it - sizeof(unsigned int))), - sizeof(unsigned int)); - - std::memcpy(&size, - &(*dest_sizes_cell_it), - sizeof(unsigned int)); - - size -= offset; - - // Move the data iterator to the corresponding position - // of the callback function and adjust the increment - // accordingly. - dest_data_it += offset; - data_increment -= offset; - } - - // Advance data size iterators to the next cell, avoid iterating - // past the end of dest_sizes_cell_it - if (cell_rel_it != cell_relations.end() - 1) - dest_sizes_cell_it += sizes_fixed_cumulative.back(); - ++dest_sizes_it; - } - - switch (cell_status) - { - case parallel::DistributedTriangulationBase::CELL_PERSIST: - case parallel::DistributedTriangulationBase::CELL_COARSEN: - unpack_callback(dealii_cell, - cell_status, - boost::make_iterator_range(dest_data_it, - dest_data_it + size)); - break; - - case parallel::DistributedTriangulationBase::CELL_REFINE: - unpack_callback(dealii_cell->parent(), - cell_status, - boost::make_iterator_range(dest_data_it, - dest_data_it + size)); - break; - - case parallel::DistributedTriangulationBase::CELL_INVALID: - // Skip this cell. - break; - - default: - Assert(false, ExcInternalError()); - break; - } - - if (cell_rel_it != cell_relations.end() - 1) - dest_data_it += data_increment; - } - } - - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::DataTransfer::save( - const unsigned int global_first_cell, - const unsigned int global_num_cells, - const std::string &filename) const - { -#ifdef DEAL_II_WITH_MPI - // Large fractions of this function have been copied from - // DataOutInterface::write_vtu_in_parallel. - // TODO: Write general MPIIO interface. - - Assert(sizes_fixed_cumulative.size() > 0, - ExcMessage("No data has been packed!")); - - const int myrank = Utilities::MPI::this_mpi_process(mpi_communicator); - - const unsigned int bytes_per_cell = sizes_fixed_cumulative.back(); - - // - // ---------- Fixed size data ---------- - // - { - const std::string fname_fixed = std::string(filename) + "_fixed.data"; - - MPI_Info info; - int ierr = MPI_Info_create(&info); - AssertThrowMPI(ierr); - - MPI_File fh; - ierr = MPI_File_open(mpi_communicator, - fname_fixed.c_str(), - MPI_MODE_CREATE | MPI_MODE_WRONLY, - info, - &fh); - AssertThrowMPI(ierr); - - ierr = MPI_File_set_size(fh, 0); // delete the file contents - AssertThrowMPI(ierr); - // this barrier is necessary, because otherwise others might already - // write while one core is still setting the size to zero. - ierr = MPI_Barrier(mpi_communicator); - AssertThrowMPI(ierr); - ierr = MPI_Info_free(&info); - AssertThrowMPI(ierr); - // ------------------ - - // Write cumulative sizes to file. - // Since each processor owns the same information about the data - // sizes, it is sufficient to let only the first processor perform - // this task. - if (myrank == 0) - { - ierr = Utilities::MPI::LargeCount::File_write_at_c( - fh, - 0, - sizes_fixed_cumulative.data(), - sizes_fixed_cumulative.size(), - MPI_UNSIGNED, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); - } - - // Write packed data to file simultaneously. - const MPI_Offset size_header = - sizes_fixed_cumulative.size() * sizeof(unsigned int); - - // Make sure we do the following computation in 64bit integers to be - // able to handle 4GB+ files: - const MPI_Offset my_global_file_position = - size_header + - static_cast(global_first_cell) * bytes_per_cell; - - ierr = - Utilities::MPI::LargeCount::File_write_at_c(fh, - my_global_file_position, - src_data_fixed.data(), - src_data_fixed.size(), - MPI_BYTE, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); - - ierr = MPI_File_close(&fh); - AssertThrowMPI(ierr); - } - - - - // - // ---------- Variable size data ---------- - // - if (variable_size_data_stored) - { - const std::string fname_variable = - std::string(filename) + "_variable.data"; - - MPI_Info info; - int ierr = MPI_Info_create(&info); - AssertThrowMPI(ierr); - - MPI_File fh; - ierr = MPI_File_open(mpi_communicator, - fname_variable.c_str(), - MPI_MODE_CREATE | MPI_MODE_WRONLY, - info, - &fh); - AssertThrowMPI(ierr); - - ierr = MPI_File_set_size(fh, 0); // delete the file contents - AssertThrowMPI(ierr); - // this barrier is necessary, because otherwise others might already - // write while one core is still setting the size to zero. - ierr = MPI_Barrier(mpi_communicator); - AssertThrowMPI(ierr); - ierr = MPI_Info_free(&info); - AssertThrowMPI(ierr); - - // Write sizes of each cell into file simultaneously. - { - const MPI_Offset my_global_file_position = - static_cast(global_first_cell) * sizeof(unsigned int); - - // It is very unlikely that a single process has more than - // 2 billion cells, but we might as well check. - AssertThrow(src_sizes_variable.size() < - static_cast( - std::numeric_limits::max()), - ExcNotImplemented()); - - ierr = Utilities::MPI::LargeCount::File_write_at_c( - fh, - my_global_file_position, - src_sizes_variable.data(), - src_sizes_variable.size(), - MPI_INT, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); - } - - // Gather size of data in bytes we want to store from this - // processor and compute the prefix sum. We do this in 64 bit - // to avoid overflow for files larger than 4GB: - const std::uint64_t size_on_proc = src_data_variable.size(); - std::uint64_t prefix_sum = 0; - ierr = MPI_Exscan(&size_on_proc, - &prefix_sum, - 1, - MPI_UINT64_T, - MPI_SUM, - mpi_communicator); - AssertThrowMPI(ierr); - - const MPI_Offset my_global_file_position = - static_cast(global_num_cells) * sizeof(unsigned int) + - prefix_sum; - - // Write data consecutively into file. - ierr = - Utilities::MPI::LargeCount::File_write_at_c(fh, - my_global_file_position, - src_data_variable.data(), - src_data_variable.size(), - MPI_BYTE, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); - - - ierr = MPI_File_close(&fh); - AssertThrowMPI(ierr); - } -#else - (void)global_first_cell; - (void)global_num_cells; - (void)filename; - - AssertThrow(false, ExcNeedsMPI()); -#endif - } - - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::DataTransfer::load( - const unsigned int global_first_cell, - const unsigned int global_num_cells, - const unsigned int local_num_cells, - const std::string &filename, - const unsigned int n_attached_deserialize_fixed, - const unsigned int n_attached_deserialize_variable) - { -#ifdef DEAL_II_WITH_MPI - // Large fractions of this function have been copied from - // DataOutInterface::write_vtu_in_parallel. - // TODO: Write general MPIIO interface. - - Assert(dest_data_fixed.size() == 0, - ExcMessage("Previously loaded data has not been released yet!")); - - variable_size_data_stored = (n_attached_deserialize_variable > 0); - - // - // ---------- Fixed size data ---------- - // - { - const std::string fname_fixed = std::string(filename) + "_fixed.data"; - - MPI_Info info; - int ierr = MPI_Info_create(&info); - AssertThrowMPI(ierr); - - MPI_File fh; - ierr = MPI_File_open( - mpi_communicator, fname_fixed.c_str(), MPI_MODE_RDONLY, info, &fh); - AssertThrowMPI(ierr); - - ierr = MPI_Info_free(&info); - AssertThrowMPI(ierr); - - // Read cumulative sizes from file. - // Since all processors need the same information about the data - // sizes, let each of them retrieve it by reading from the same - // location in the file. - sizes_fixed_cumulative.resize(1 + n_attached_deserialize_fixed + - (variable_size_data_stored ? 1 : 0)); - ierr = Utilities::MPI::LargeCount::File_read_at_c( - fh, - 0, - sizes_fixed_cumulative.data(), - sizes_fixed_cumulative.size(), - MPI_UNSIGNED, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); - - // Allocate sufficient memory. - const unsigned int bytes_per_cell = sizes_fixed_cumulative.back(); - dest_data_fixed.resize(static_cast(local_num_cells) * - bytes_per_cell); - - // Read packed data from file simultaneously. - const MPI_Offset size_header = - sizes_fixed_cumulative.size() * sizeof(unsigned int); - - // Make sure we do the following computation in 64bit integers to be - // able to handle 4GB+ files: - const MPI_Offset my_global_file_position = - size_header + - static_cast(global_first_cell) * bytes_per_cell; - - ierr = Utilities::MPI::LargeCount::File_read_at_c(fh, - my_global_file_position, - dest_data_fixed.data(), - dest_data_fixed.size(), - MPI_BYTE, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); - - - ierr = MPI_File_close(&fh); - AssertThrowMPI(ierr); - } - - // - // ---------- Variable size data ---------- - // - if (variable_size_data_stored) - { - const std::string fname_variable = - std::string(filename) + "_variable.data"; - - MPI_Info info; - int ierr = MPI_Info_create(&info); - AssertThrowMPI(ierr); - - MPI_File fh; - ierr = MPI_File_open( - mpi_communicator, fname_variable.c_str(), MPI_MODE_RDONLY, info, &fh); - AssertThrowMPI(ierr); - - ierr = MPI_Info_free(&info); - AssertThrowMPI(ierr); - - // Read sizes of all locally owned cells. - dest_sizes_variable.resize(local_num_cells); - - const MPI_Offset my_global_file_position_sizes = - static_cast(global_first_cell) * sizeof(unsigned int); - - ierr = Utilities::MPI::LargeCount::File_read_at_c( - fh, - my_global_file_position_sizes, - dest_sizes_variable.data(), - dest_sizes_variable.size(), - MPI_INT, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); - - - // Compute my data size in bytes and compute prefix sum. We do this - // in 64 bit to avoid overflow for files larger than 4 GB: - const std::uint64_t size_on_proc = - std::accumulate(dest_sizes_variable.begin(), - dest_sizes_variable.end(), - 0ULL); - - std::uint64_t prefix_sum = 0; - ierr = MPI_Exscan(&size_on_proc, - &prefix_sum, - 1, - MPI_UINT64_T, - MPI_SUM, - mpi_communicator); - AssertThrowMPI(ierr); - - const MPI_Offset my_global_file_position = - static_cast(global_num_cells) * sizeof(unsigned int) + - prefix_sum; - - dest_data_variable.resize(size_on_proc); - - ierr = - Utilities::MPI::LargeCount::File_read_at_c(fh, - my_global_file_position, - dest_data_variable.data(), - dest_data_variable.size(), - MPI_BYTE, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); - - ierr = MPI_File_close(&fh); - AssertThrowMPI(ierr); - } -#else - (void)global_first_cell; - (void)global_num_cells; - (void)local_num_cells; - (void)filename; - (void)n_attached_deserialize_fixed; - (void)n_attached_deserialize_variable; - - AssertThrow(false, ExcNeedsMPI()); -#endif - } - - - - template - DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) - void DistributedTriangulationBase::DataTransfer::clear() - { - variable_size_data_stored = false; - - // free information about data sizes - sizes_fixed_cumulative.clear(); - sizes_fixed_cumulative.shrink_to_fit(); - - // free fixed size transfer data - src_data_fixed.clear(); - src_data_fixed.shrink_to_fit(); - - dest_data_fixed.clear(); - dest_data_fixed.shrink_to_fit(); - - // free variable size transfer data - src_sizes_variable.clear(); - src_sizes_variable.shrink_to_fit(); - - src_data_variable.clear(); - src_data_variable.shrink_to_fit(); - - dest_sizes_variable.clear(); - dest_sizes_variable.shrink_to_fit(); - - dest_data_variable.clear(); - dest_data_variable.shrink_to_fit(); - } - } // end namespace parallel diff --git a/source/grid/grid_tools.cc b/source/grid/grid_tools.cc index 12175f1403..79cc106903 100644 --- a/source/grid/grid_tools.cc +++ b/source/grid/grid_tools.cc @@ -3914,8 +3914,7 @@ namespace GridTools for (const auto &cell : triangulation.active_cell_iterators()) if (cell->is_locally_owned()) cell_weights[cell->active_cell_index()] = - triangulation.signals.weight( - cell, Triangulation::CellStatus::CELL_PERSIST); + triangulation.signals.weight(cell, CELL_PERSIST); // If this is a parallel triangulation, we then need to also // get the weights for all other cells. We have asserted above @@ -4015,8 +4014,7 @@ namespace GridTools for (const auto &cell : triangulation.active_cell_iterators() | IteratorFilters::LocallyOwnedCell()) cell_weights[cell->active_cell_index()] = - triangulation.signals.weight( - cell, Triangulation::CellStatus::CELL_PERSIST); + triangulation.signals.weight(cell, CELL_PERSIST); // If this is a parallel triangulation, we then need to also // get the weights for all other cells. We have asserted above diff --git a/source/grid/tria.cc b/source/grid/tria.cc index e512e95cc9..22695f336f 100644 --- a/source/grid/tria.cc +++ b/source/grid/tria.cc @@ -11081,6 +11081,10 @@ Triangulation::Triangulation( signals.post_refinement.connect(signals.any_change); signals.clear.connect(signals.any_change); signals.mesh_movement.connect(signals.any_change); + + cell_attached_data = {0, 0, {}, {}}; + data_transfer = + std::make_unique>(get_communicator()); } @@ -11184,6 +11188,9 @@ void Triangulation::clear() periodic_face_pairs_level_0.clear(); periodic_face_map.clear(); reference_cells.clear(); + + cell_attached_data = {0, 0, {}, {}}; + data_transfer->clear(); } @@ -14816,6 +14823,148 @@ bool Triangulation::is_mixed_mesh() const +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +unsigned int Triangulation::register_data_attach( + const std::function(const cell_iterator &, + const CellStatus)> &pack_callback, + const bool returns_variable_size_data) +{ + unsigned int handle = numbers::invalid_unsigned_int; + + // Add new callback function to the corresponding register. + // Encode handles according to returns_variable_size_data. + if (returns_variable_size_data) + { + handle = 2 * this->cell_attached_data.pack_callbacks_variable.size(); + this->cell_attached_data.pack_callbacks_variable.push_back(pack_callback); + } + else + { + handle = 2 * this->cell_attached_data.pack_callbacks_fixed.size() + 1; + this->cell_attached_data.pack_callbacks_fixed.push_back(pack_callback); + } + + // Increase overall counter. + ++this->cell_attached_data.n_attached_data_sets; + + return handle; +} + + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void Triangulation::notify_ready_to_unpack( + const unsigned int handle, + const std::function< + void(const cell_iterator &, + const CellStatus, + const boost::iterator_range::const_iterator> &)> + &unpack_callback) +{ + // perform unpacking + this->data_transfer->unpack_data(this->local_cell_relations, + handle, + unpack_callback); + + // decrease counters + --this->cell_attached_data.n_attached_data_sets; + if (this->cell_attached_data.n_attached_deserialize > 0) + --this->cell_attached_data.n_attached_deserialize; + + // important: only remove data if we are not in the deserialization + // process. There, each SolutionTransfer registers and unpacks before + // the next one does this, so n_attached_data_sets is only 1 here. This + // would destroy the saved data before the second SolutionTransfer can + // get it. This created a bug that is documented in + // tests/mpi/p4est_save_03 with more than one SolutionTransfer. + + if (this->cell_attached_data.n_attached_data_sets == 0 && + this->cell_attached_data.n_attached_deserialize == 0) + { + // everybody got their data, time for cleanup! + this->cell_attached_data.pack_callbacks_fixed.clear(); + this->cell_attached_data.pack_callbacks_variable.clear(); + this->data_transfer->clear(); + + // reset all cell_status entries after coarsening/refinement + for (auto &cell_rel : this->local_cell_relations) + cell_rel.second = CELL_PERSIST; + } +} + + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void Triangulation::save_attached_data( + const unsigned int global_first_cell, + const unsigned int global_num_cells, + const std::string &filename) const +{ + // cast away constness + auto tria = const_cast *>(this); + + if (this->cell_attached_data.n_attached_data_sets > 0) + { + // pack attached data first + tria->data_transfer->pack_data( + tria->local_cell_relations, + tria->cell_attached_data.pack_callbacks_fixed, + tria->cell_attached_data.pack_callbacks_variable); + + // then store buffers in file + tria->data_transfer->save(global_first_cell, global_num_cells, filename); + + // and release the memory afterwards + tria->data_transfer->clear(); + } + + // clear all of the callback data, as explained in the documentation of + // register_data_attach() + { + tria->cell_attached_data.n_attached_data_sets = 0; + tria->cell_attached_data.pack_callbacks_fixed.clear(); + tria->cell_attached_data.pack_callbacks_variable.clear(); + } +} + + +template +DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) +void Triangulation::load_attached_data( + const unsigned int global_first_cell, + const unsigned int global_num_cells, + const unsigned int local_num_cells, + const std::string &filename, + const unsigned int n_attached_deserialize_fixed, + const unsigned int n_attached_deserialize_variable) +{ + // load saved data, if any was stored + if (this->cell_attached_data.n_attached_deserialize > 0) + { + this->data_transfer->load(global_first_cell, + global_num_cells, + local_num_cells, + filename, + n_attached_deserialize_fixed, + n_attached_deserialize_variable); + + this->data_transfer->unpack_cell_status(this->local_cell_relations); + + // the CellStatus of all stored cells should always be CELL_PERSIST. + for (const auto &cell_rel : this->local_cell_relations) + { + (void)cell_rel; + Assert((cell_rel.second == // cell_status + CELL_PERSIST), + ExcInternalError()); + } + } +} + + template DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim)) void Triangulation::clear_despite_subscriptions() diff --git a/source/particles/particle_handler.cc b/source/particles/particle_handler.cc index b671032dae..5bad34b832 100644 --- a/source/particles/particle_handler.cc +++ b/source/particles/particle_handler.cc @@ -2195,10 +2195,9 @@ namespace Particles "creation and do not refine afterwards, or use a distributed triangulation.")); const auto callback_function = - [this]( - const typename Triangulation::cell_iterator - & cell_iterator, - const typename Triangulation::CellStatus cell_status) { + [this](const typename Triangulation::cell_iterator + & cell_iterator, + const CellStatus cell_status) { return this->pack_callback(cell_iterator, cell_status); }; @@ -2271,12 +2270,11 @@ namespace Particles if (handle != numbers::invalid_unsigned_int) { const auto callback_function = - [this]( - const typename Triangulation::cell_iterator - &cell_iterator, - const typename Triangulation::CellStatus cell_status, - const boost::iterator_range::const_iterator> - &range_iterator) { + [this](const typename Triangulation::cell_iterator + & cell_iterator, + const CellStatus cell_status, + const boost::iterator_range::const_iterator> + &range_iterator) { this->unpack_callback(cell_iterator, cell_status, range_iterator); }; @@ -2295,14 +2293,14 @@ namespace Particles std::vector ParticleHandler::pack_callback( const typename Triangulation::cell_iterator &cell, - const typename Triangulation::CellStatus status) const + const CellStatus status) const { std::vector stored_particles_on_cell; switch (status) { - case parallel::TriangulationBase::CELL_PERSIST: - case parallel::TriangulationBase::CELL_REFINE: + case CELL_PERSIST: + case CELL_REFINE: // If the cell persist or is refined store all particles of the // current cell. { @@ -2317,7 +2315,7 @@ namespace Particles } break; - case parallel::TriangulationBase::CELL_COARSEN: + case CELL_COARSEN: // If this cell is the parent of children that will be coarsened, // collect the particles of all children. { @@ -2351,16 +2349,14 @@ namespace Particles void ParticleHandler::unpack_callback( const typename Triangulation::cell_iterator & cell, - const typename Triangulation::CellStatus status, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { if (data_range.begin() == data_range.end()) return; const auto cell_to_store_particles = - (status != parallel::TriangulationBase::CELL_REFINE) ? - cell : - cell->child(0); + (status != CELL_REFINE) ? cell : cell->child(0); // deserialize particles and insert into local storage if (data_range.begin() != data_range.end()) @@ -2384,13 +2380,13 @@ namespace Particles // now update particle storage location and properties if necessary switch (status) { - case parallel::TriangulationBase::CELL_PERSIST: + case CELL_PERSIST: { // all particles are correctly inserted } break; - case parallel::TriangulationBase::CELL_COARSEN: + case CELL_COARSEN: { // all particles are in correct cell, but their reference location // has changed @@ -2404,7 +2400,7 @@ namespace Particles } break; - case parallel::TriangulationBase::CELL_REFINE: + case CELL_REFINE: { // we need to find the correct child to store the particles and // their reference location has changed diff --git a/tests/fullydistributed_grids/repartitioning_06.cc b/tests/fullydistributed_grids/repartitioning_06.cc index 4aebe0fa11..6b6ffe95a2 100644 --- a/tests/fullydistributed_grids/repartitioning_06.cc +++ b/tests/fullydistributed_grids/repartitioning_06.cc @@ -46,7 +46,7 @@ namespace dealii template std::function< unsigned int(const typename Triangulation::cell_iterator &, - const typename Triangulation::CellStatus)> + const CellStatus)> hanging_nodes_weighting(const double weight) { return [weight](const auto &cell, const auto &) -> unsigned int { diff --git a/tests/mpi/attach_data_01.cc b/tests/mpi/attach_data_01.cc index d16751b355..a844f2c269 100644 --- a/tests/mpi/attach_data_01.cc +++ b/tests/mpi/attach_data_01.cc @@ -36,24 +36,21 @@ template std::vector pack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) + & cell, + const CellStatus status) { static int some_number = 0; deallog << "packing cell " << cell->id() << " with data=" << some_number << " status="; - if (status == parallel::distributed::Triangulation::CELL_PERSIST) + if (status == CELL_PERSIST) deallog << "PERSIST"; - else if (status == - parallel::distributed::Triangulation::CELL_REFINE) + else if (status == CELL_REFINE) deallog << "REFINE"; - else if (status == - parallel::distributed::Triangulation::CELL_COARSEN) + else if (status == CELL_COARSEN) deallog << "COARSEN"; deallog << std::endl; - if (status == parallel::distributed::Triangulation::CELL_COARSEN) + if (status == CELL_COARSEN) { Assert(cell->has_children(), ExcInternalError()); } @@ -69,9 +66,8 @@ template void unpack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, + & cell, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { const int intdata = Utilities::unpack(data_range.begin(), @@ -80,17 +76,15 @@ unpack_function( deallog << "unpacking cell " << cell->id() << " with data=" << intdata << " status="; - if (status == parallel::distributed::Triangulation::CELL_PERSIST) + if (status == CELL_PERSIST) deallog << "PERSIST"; - else if (status == - parallel::distributed::Triangulation::CELL_REFINE) + else if (status == CELL_REFINE) deallog << "REFINE"; - else if (status == - parallel::distributed::Triangulation::CELL_COARSEN) + else if (status == CELL_COARSEN) deallog << "COARSEN"; deallog << std::endl; - if (status == parallel::distributed::Triangulation::CELL_REFINE) + if (status == CELL_REFINE) { Assert(cell->has_children(), ExcInternalError()); } diff --git a/tests/mpi/attach_data_02.cc b/tests/mpi/attach_data_02.cc index dcbc19ff16..ff1c2895e2 100644 --- a/tests/mpi/attach_data_02.cc +++ b/tests/mpi/attach_data_02.cc @@ -37,9 +37,8 @@ template std::vector pack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) + & cell, + const CellStatus status) { static unsigned int some_number = 1; std::vector some_vector(some_number); @@ -58,17 +57,15 @@ pack_function( << " with data size =" << buffer.size() << " accumulated data=" << std::accumulate(some_vector.begin(), some_vector.end(), 0) << " status="; - if (status == parallel::distributed::Triangulation::CELL_PERSIST) + if (status == CELL_PERSIST) deallog << "PERSIST"; - else if (status == - parallel::distributed::Triangulation::CELL_REFINE) + else if (status == CELL_REFINE) deallog << "REFINE"; - else if (status == - parallel::distributed::Triangulation::CELL_COARSEN) + else if (status == CELL_COARSEN) deallog << "COARSEN"; deallog << std::endl; - if (status == parallel::distributed::Triangulation::CELL_COARSEN) + if (status == CELL_COARSEN) { Assert(cell->has_children(), ExcInternalError()); } @@ -85,9 +82,8 @@ template void unpack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, + & cell, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { const unsigned int data_in_bytes = @@ -111,17 +107,15 @@ unpack_function( << " accumulated data=" << std::accumulate(intdatavector.begin(), intdatavector.end(), 0) << " status="; - if (status == parallel::distributed::Triangulation::CELL_PERSIST) + if (status == CELL_PERSIST) deallog << "PERSIST"; - else if (status == - parallel::distributed::Triangulation::CELL_REFINE) + else if (status == CELL_REFINE) deallog << "REFINE"; - else if (status == - parallel::distributed::Triangulation::CELL_COARSEN) + else if (status == CELL_COARSEN) deallog << "COARSEN"; deallog << std::endl; - if (status == parallel::distributed::Triangulation::CELL_REFINE) + if (status == CELL_REFINE) { Assert(cell->has_children(), ExcInternalError()); } diff --git a/tests/mpi/cell_weights_01_back_and_forth_01.cc b/tests/mpi/cell_weights_01_back_and_forth_01.cc index 4de2a6a8b3..dba36f2794 100644 --- a/tests/mpi/cell_weights_01_back_and_forth_01.cc +++ b/tests/mpi/cell_weights_01_back_and_forth_01.cc @@ -41,7 +41,7 @@ template unsigned int cell_weight_1( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus status) + const CellStatus status) { return current_cell_weight++; } @@ -50,7 +50,7 @@ template unsigned int cell_weight_2( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus status) + const CellStatus status) { return 1; } diff --git a/tests/mpi/cell_weights_01_back_and_forth_02.cc b/tests/mpi/cell_weights_01_back_and_forth_02.cc index 0068ce9657..e73d3a1433 100644 --- a/tests/mpi/cell_weights_01_back_and_forth_02.cc +++ b/tests/mpi/cell_weights_01_back_and_forth_02.cc @@ -40,7 +40,7 @@ template unsigned int cell_weight_1( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus status) + const CellStatus status) { return current_cell_weight++; } diff --git a/tests/mpi/cell_weights_02.cc b/tests/mpi/cell_weights_02.cc index b55f488d0f..1889902f25 100644 --- a/tests/mpi/cell_weights_02.cc +++ b/tests/mpi/cell_weights_02.cc @@ -38,7 +38,7 @@ template unsigned int cell_weight( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus status) + const CellStatus status) { return 1; } diff --git a/tests/mpi/cell_weights_03.cc b/tests/mpi/cell_weights_03.cc index 1313a8589e..ad2b8ae2c1 100644 --- a/tests/mpi/cell_weights_03.cc +++ b/tests/mpi/cell_weights_03.cc @@ -39,7 +39,7 @@ template unsigned int cell_weight( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus status) + const CellStatus status) { const unsigned int cell_weight = (cell->center()[0] < 0.5 || cell->center()[1] < 0.5 ? 1 : 4); @@ -80,9 +80,7 @@ test() std::vector integrated_weights(numproc, 0); for (const auto &cell : tr.active_cell_iterators() | IteratorFilters::LocallyOwnedCell()) - integrated_weights[myid] += - cell_weight(cell, - parallel::distributed::Triangulation::CELL_PERSIST); + integrated_weights[myid] += cell_weight(cell, CELL_PERSIST); Utilities::MPI::sum(integrated_weights, MPI_COMM_WORLD, integrated_weights); if (myid == 0) diff --git a/tests/mpi/cell_weights_04.cc b/tests/mpi/cell_weights_04.cc index 9305ba654d..2c46ce8ba3 100644 --- a/tests/mpi/cell_weights_04.cc +++ b/tests/mpi/cell_weights_04.cc @@ -37,7 +37,7 @@ template unsigned int cell_weight( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus status) + const CellStatus status) { const unsigned int cell_weight = (cell->center()[0] < 0.5 || cell->center()[1] < 0.5 ? 1 : 1000); @@ -74,9 +74,7 @@ test() std::vector integrated_weights(numproc, 0); for (const auto &cell : tr.active_cell_iterators() | IteratorFilters::LocallyOwnedCell()) - integrated_weights[myid] += - cell_weight(cell, - parallel::distributed::Triangulation::CELL_PERSIST); + integrated_weights[myid] += cell_weight(cell, CELL_PERSIST); Utilities::MPI::sum(integrated_weights, MPI_COMM_WORLD, integrated_weights); if (myid == 0) diff --git a/tests/mpi/cell_weights_05.cc b/tests/mpi/cell_weights_05.cc index c0d027b0e5..0a942ed7c2 100644 --- a/tests/mpi/cell_weights_05.cc +++ b/tests/mpi/cell_weights_05.cc @@ -48,7 +48,7 @@ template unsigned int cell_weight( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus status) + const CellStatus status) { unsigned int weight = 1; @@ -94,9 +94,7 @@ test() std::vector integrated_weights(numproc, 0); for (const auto &cell : tr.active_cell_iterators() | IteratorFilters::LocallyOwnedCell()) - integrated_weights[myid] += - cell_weight(cell, - parallel::distributed::Triangulation::CELL_PERSIST); + integrated_weights[myid] += cell_weight(cell, CELL_PERSIST); Utilities::MPI::sum(integrated_weights, MPI_COMM_WORLD, integrated_weights); if (myid == 0) diff --git a/tests/mpi/cell_weights_06.cc b/tests/mpi/cell_weights_06.cc index f4865b0fa5..aa3f764589 100644 --- a/tests/mpi/cell_weights_06.cc +++ b/tests/mpi/cell_weights_06.cc @@ -47,7 +47,7 @@ template unsigned int cell_weight( const typename parallel::distributed::Triangulation::cell_iterator &cell, - const typename parallel::distributed::Triangulation::CellStatus status) + const CellStatus status) { unsigned int weight = 1; @@ -93,9 +93,7 @@ test() std::vector integrated_weights(numproc, 0); for (const auto &cell : tr.active_cell_iterators() | IteratorFilters::LocallyOwnedCell()) - integrated_weights[myid] += - cell_weight(cell, - parallel::distributed::Triangulation::CELL_PERSIST); + integrated_weights[myid] += cell_weight(cell, CELL_PERSIST); Utilities::MPI::sum(integrated_weights, MPI_COMM_WORLD, integrated_weights); if (myid == 0) diff --git a/tests/mpi/p4est_save_05.cc b/tests/mpi/p4est_save_05.cc index b8c9567254..7a7352407a 100644 --- a/tests/mpi/p4est_save_05.cc +++ b/tests/mpi/p4est_save_05.cc @@ -38,9 +38,8 @@ template std::vector pack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) + & cell, + const CellStatus status) { static unsigned int some_number = 1; std::vector some_vector(some_number); @@ -60,9 +59,7 @@ pack_function( << std::accumulate(some_vector.begin(), some_vector.end(), 0) << std::endl; - Assert((status == - parallel::distributed::Triangulation::CELL_PERSIST), - ExcInternalError()); + Assert((status == CELL_PERSIST), ExcInternalError()); ++some_number; return buffer; @@ -74,9 +71,8 @@ template void unpack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, + & cell, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { const unsigned int data_in_bytes = @@ -101,9 +97,7 @@ unpack_function( << std::accumulate(intdatavector.begin(), intdatavector.end(), 0) << std::endl; - Assert((status == - parallel::distributed::Triangulation::CELL_PERSIST), - ExcInternalError()); + Assert((status == CELL_PERSIST), ExcInternalError()); } diff --git a/tests/mpi/p4est_save_07.cc b/tests/mpi/p4est_save_07.cc index 4e63f04986..972a1b5158 100644 --- a/tests/mpi/p4est_save_07.cc +++ b/tests/mpi/p4est_save_07.cc @@ -41,9 +41,8 @@ template std::vector pack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) + & cell, + const CellStatus status) { static unsigned int some_number = 1; std::vector some_vector(some_number); @@ -63,9 +62,7 @@ pack_function( << std::accumulate(some_vector.begin(), some_vector.end(), 0) << std::endl; - Assert((status == - parallel::distributed::Triangulation::CELL_PERSIST), - ExcInternalError()); + Assert((status == CELL_PERSIST), ExcInternalError()); ++some_number; return buffer; @@ -77,9 +74,8 @@ template void unpack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, + & cell, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { const unsigned int data_in_bytes = @@ -104,9 +100,7 @@ unpack_function( << std::accumulate(intdatavector.begin(), intdatavector.end(), 0) << std::endl; - Assert((status == - parallel::distributed::Triangulation::CELL_PERSIST), - ExcInternalError()); + Assert((status == CELL_PERSIST), ExcInternalError()); } diff --git a/tests/mpi/repartition_02.cc b/tests/mpi/repartition_02.cc index d0fc26cccf..953d09ad87 100644 --- a/tests/mpi/repartition_02.cc +++ b/tests/mpi/repartition_02.cc @@ -36,24 +36,21 @@ template std::vector pack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) + & cell, + const CellStatus status) { static int some_number = cell->index(); deallog << "packing cell " << cell->id() << " with data=" << some_number << " status="; - if (status == parallel::distributed::Triangulation::CELL_PERSIST) + if (status == CELL_PERSIST) deallog << "PERSIST"; - else if (status == - parallel::distributed::Triangulation::CELL_REFINE) + else if (status == CELL_REFINE) deallog << "REFINE"; - else if (status == - parallel::distributed::Triangulation::CELL_COARSEN) + else if (status == CELL_COARSEN) deallog << "COARSEN"; deallog << std::endl; - if (status == parallel::distributed::Triangulation::CELL_COARSEN) + if (status == CELL_COARSEN) { Assert(cell->has_children(), ExcInternalError()); } @@ -69,9 +66,8 @@ template void unpack_function( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status, + & cell, + const CellStatus status, const boost::iterator_range::const_iterator> &data_range) { const int number = Utilities::unpack(data_range.begin(), @@ -80,17 +76,15 @@ unpack_function( deallog << "unpacking cell " << cell->id() << " with data=" << number << " status="; - if (status == parallel::distributed::Triangulation::CELL_PERSIST) + if (status == CELL_PERSIST) deallog << "PERSIST"; - else if (status == - parallel::distributed::Triangulation::CELL_REFINE) + else if (status == CELL_REFINE) deallog << "REFINE"; - else if (status == - parallel::distributed::Triangulation::CELL_COARSEN) + else if (status == CELL_COARSEN) deallog << "COARSEN"; deallog << std::endl; - if (status == parallel::distributed::Triangulation::CELL_REFINE) + if (status == CELL_REFINE) { Assert(cell->has_children(), ExcInternalError()); } diff --git a/tests/particles/particle_weights_01.cc b/tests/particles/particle_weights_01.cc index 8d4825d48e..884408207d 100644 --- a/tests/particles/particle_weights_01.cc +++ b/tests/particles/particle_weights_01.cc @@ -72,23 +72,21 @@ test() // register weighting function that returns the number of particles per cell triangulation.signals.weight.connect( - [&particle_handler]( - const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) -> unsigned int { + [&particle_handler](const typename parallel::distributed::Triangulation< + dim>::cell_iterator &cell, + const CellStatus status) -> unsigned int { unsigned int n_particles_in_cell = 0; switch (status) { - case parallel::distributed::Triangulation::CELL_PERSIST: - case parallel::distributed::Triangulation::CELL_REFINE: + case CELL_PERSIST: + case CELL_REFINE: n_particles_in_cell = particle_handler.n_particles_in_cell(cell); break; - case parallel::distributed::Triangulation::CELL_INVALID: + case CELL_INVALID: break; - case parallel::distributed::Triangulation::CELL_COARSEN: + case CELL_COARSEN: for (const auto &child : cell->child_iterators()) n_particles_in_cell += particle_handler.n_particles_in_cell(child); diff --git a/tests/particles/step-68.cc b/tests/particles/step-68.cc index 654a27572c..69210d0f99 100644 --- a/tests/particles/step-68.cc +++ b/tests/particles/step-68.cc @@ -172,9 +172,8 @@ namespace Step68 unsigned int cell_weight( const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) const; + & cell, + const CellStatus status) const; // The following two functions are responsible for outputting the simulation // results for the particles and for the velocity profile on the background @@ -257,9 +256,8 @@ namespace Step68 unsigned int ParticleTracking::cell_weight( const typename parallel::distributed::Triangulation::cell_iterator - & cell, - const typename parallel::distributed::Triangulation::CellStatus status) - const + & cell, + const CellStatus status) const { // First, we introduce a base weight that will be assigned to every cell. const unsigned int base_weight = 1; @@ -281,15 +279,15 @@ namespace Step68 unsigned int n_particles_in_cell = 0; switch (status) { - case parallel::distributed::Triangulation::CELL_PERSIST: - case parallel::distributed::Triangulation::CELL_REFINE: + case CELL_PERSIST: + case CELL_REFINE: n_particles_in_cell = particle_handler.n_particles_in_cell(cell); break; - case parallel::distributed::Triangulation::CELL_INVALID: + case CELL_INVALID: break; - case parallel::distributed::Triangulation::CELL_COARSEN: + case CELL_COARSEN: for (const auto &child : cell->child_iterators()) n_particles_in_cell += particle_handler.n_particles_in_cell(child); break; @@ -328,11 +326,11 @@ namespace Step68 // of this class, but for the purpose of this example we want to group the // particle related instructions. background_triangulation.signals.weight.connect( - [&]( - const typename parallel::distributed::Triangulation::cell_iterator - &cell, - const typename parallel::distributed::Triangulation::CellStatus - status) -> unsigned int { return this->cell_weight(cell, status); }); + [&](const typename parallel::distributed::Triangulation< + dim>::cell_iterator &cell, + const CellStatus status) -> unsigned int { + return this->cell_weight(cell, status); + }); // This initializes the background triangulation where the particles are // living and the number of properties of the particles. -- 2.39.5