]> https://gitweb.dealii.org/ - dealii.git/commitdiff
Move CellAttachedDataSerializer to internal namespace 15576/head
authorPasquale Africa <pafrica@sissa.it>
Tue, 4 Jul 2023 16:02:14 +0000 (16:02 +0000)
committerPasquale Africa <pafrica@sissa.it>
Fri, 7 Jul 2023 23:17:06 +0000 (23:17 +0000)
include/deal.II/grid/data_serializer.h [deleted file]
include/deal.II/grid/tria.h
source/grid/CMakeLists.txt
source/grid/data_serializer.cc [deleted file]
source/grid/data_serializer.inst.in [deleted file]
source/grid/tria.cc

diff --git a/include/deal.II/grid/data_serializer.h b/include/deal.II/grid/data_serializer.h
deleted file mode 100644 (file)
index 9291d0b..0000000
+++ /dev/null
@@ -1,242 +0,0 @@
-// ---------------------------------------------------------------------
-//
-// Copyright (C) 1998 - 2023 by the deal.II authors
-//
-// This file is part of the deal.II library.
-//
-// The deal.II library is free software; you can use it, redistribute
-// it, and/or modify it under the terms of the GNU Lesser General
-// Public License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-// The full text of the license can be found in the file LICENSE.md at
-// the top level directory of deal.II.
-//
-// ---------------------------------------------------------------------
-
-#ifndef dealii_tria_data_serializer_h
-#define dealii_tria_data_serializer_h
-
-#include <deal.II/base/config.h>
-
-#include <deal.II/base/mpi.h>
-#include <deal.II/base/mpi_large_count.h>
-#include <deal.II/base/utilities.h>
-
-#include <deal.II/grid/cell_status.h>
-#include <deal.II/grid/tria_iterator_selector.h>
-
-#include <boost/range/iterator_range.hpp>
-
-#include <algorithm>
-#include <fstream>
-#include <functional>
-#include <utility>
-#include <vector>
-
-DEAL_II_NAMESPACE_OPEN
-
-namespace internal
-{
-  /**
-   * A structure that binds information about data attached to cells.
-   */
-  template <int dim, int spacedim = dim>
-  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-  struct CellAttachedData
-  {
-    using cell_iterator = TriaIterator<CellAccessor<dim, spacedim>>;
-
-    /**
-     * Number of functions that get attached to the Triangulation through
-     * register_data_attach() for example SolutionTransfer.
-     */
-    unsigned int n_attached_data_sets;
-
-    /**
-     * Number of functions that need to unpack their data after a call from
-     * load().
-     */
-    unsigned int n_attached_deserialize;
-
-    using pack_callback_t =
-      std::function<std::vector<char>(cell_iterator, CellStatus)>;
-
-    /**
-     * These callback functions will be stored in the order in which they
-     * have been registered with the register_data_attach() function.
-     */
-    std::vector<pack_callback_t> pack_callbacks_fixed;
-    std::vector<pack_callback_t> pack_callbacks_variable;
-  };
-} // namespace internal
-
-/**
- * A structure that stores information about the data that has been, or
- * will be, attached to cells via the register_data_attach() function
- * and later retrieved via notify_ready_to_unpack().
- *
- * This internalclass is dedicated to the data serialization and transfer across
- * repartitioned meshes and to/from the file system.
- *
- * It is designed to store all data buffers intended for serialization.
- */
-template <int dim, int spacedim = dim>
-DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-class CellAttachedDataSerializer
-{
-public:
-  using cell_iterator = TriaIterator<CellAccessor<dim, spacedim>>;
-
-  /**
-   * Auxiliary data structure for assigning a CellStatus to a deal.II cell
-   * iterator. For an extensive description of the former, see the
-   * documentation for the member function register_data_attach().
-   */
-  using cell_relation_t = typename std::pair<cell_iterator, CellStatus>;
-
-  CellAttachedDataSerializer();
-
-  /**
-   * Prepare data serialization by calling the pack callback functions on each
-   * cell in @p cell_relations.
-   *
-   * All registered callback functions in @p pack_callbacks_fixed will write
-   * into the fixed size buffer, whereas each entry of @p pack_callbacks_variable
-   * will write its data into the variable size buffer.
-   */
-  void
-  pack_data(
-    const std::vector<cell_relation_t> &cell_relations,
-    const std::vector<
-      typename internal::CellAttachedData<dim, spacedim>::pack_callback_t>
-      &pack_callbacks_fixed,
-    const std::vector<
-      typename internal::CellAttachedData<dim, spacedim>::pack_callback_t>
-      &             pack_callbacks_variable,
-    const MPI_Comm &mpi_communicator);
-
-  /**
-   * Unpack the CellStatus information on each entry of
-   * @p cell_relations.
-   *
-   * Data has to be previously transferred with execute_transfer()
-   * or deserialized from the file system via load().
-   */
-  void
-  unpack_cell_status(std::vector<cell_relation_t> &cell_relations) const;
-
-  /**
-   * Unpack previously serialized data on each cell registered in
-   * @p cell_relations with the provided @p unpack_callback function.
-   *
-   * The parameter @p handle corresponds to the position where the
-   * @p unpack_callback function is allowed to read from the memory. Its
-   * value needs to be in accordance with the corresponding pack_callback
-   * function that has been registered previously.
-   *
-   * Data has to be previously transferred with execute_transfer()
-   * or deserialized from the file system via load().
-   */
-  void
-  unpack_data(
-    const std::vector<cell_relation_t> &cell_relations,
-    const unsigned int                  handle,
-    const std::function<
-      void(const cell_iterator &,
-           const CellStatus &,
-           const boost::iterator_range<std::vector<char>::const_iterator> &)>
-      &unpack_callback) const;
-
-  /**
-   * Serialize data to file system.
-   *
-   * The data will be written in a separate file, whose name
-   * consists of the stem @p filename and an attached identifier
-   * <tt>_fixed.data</tt> for fixed size data and <tt>_variable.data</tt>
-   * for variable size data.
-   *
-   * If MPI support is enabled, all processors write into these files
-   * simultaneously via MPIIO. Each processor's position to write to will be
-   * determined from the provided input parameters.
-   *
-   * Data has to be previously packed with pack_data().
-   */
-  void
-  save(const unsigned int global_first_cell,
-       const unsigned int global_num_cells,
-       const std::string &filename,
-       const MPI_Comm &   mpi_communicator) const;
-
-  /**
-   * Deserialize data from file system.
-   *
-   * The data will be read from separate file, whose name
-   * consists of the stem @p filename and an attached identifier
-   * <tt>_fixed.data</tt> for fixed size data and <tt>_variable.data</tt>
-   * for variable size data.
-   * The @p n_attached_deserialize_fixed and @p n_attached_deserialize_variable
-   * parameters are required to gather the memory offsets for each
-   * callback.
-   *
-   * If MPI support is enabled, all processors read from these files
-   * simultaneously via MPIIO. Each processor's position to read from will be
-   * determined from the provided input arguments.
-   *
-   * After loading, unpack_data() needs to be called to finally
-   * distribute data across the associated triangulation.
-   */
-  void
-  load(const unsigned int global_first_cell,
-       const unsigned int global_num_cells,
-       const unsigned int local_num_cells,
-       const std::string &filename,
-       const unsigned int n_attached_deserialize_fixed,
-       const unsigned int n_attached_deserialize_variable,
-       const MPI_Comm &   mpi_communicator);
-
-  /**
-   * Clears all containers and associated data, and resets member
-   * values to their default state.
-   *
-   * Frees memory completely.
-   */
-  void
-  clear();
-
-  /**
-   * Flag that denotes if variable size data has been packed.
-   */
-  bool variable_size_data_stored;
-
-  /**
-   * Cumulative size in bytes that those functions that have called
-   * register_data_attach() want to attach to each cell. This number
-   * only pertains to fixed-sized buffers where the data attached to
-   * each cell has exactly the same size.
-   *
-   * The last entry of this container corresponds to the data size
-   * packed per cell in the fixed size buffer (which can be accessed
-   * calling <tt>sizes_fixed_cumulative.back()</tt>).
-   */
-  std::vector<unsigned int> sizes_fixed_cumulative;
-
-  /**
-   * Consecutive buffers designed for the fixed size serialization
-   * functions.
-   */
-  std::vector<char> src_data_fixed;
-  std::vector<char> dest_data_fixed;
-
-  /**
-   * Consecutive buffers designed for the variable size serialization
-   * functions.
-   */
-  std::vector<int>  src_sizes_variable;
-  std::vector<int>  dest_sizes_variable;
-  std::vector<char> src_data_variable;
-  std::vector<char> dest_data_variable;
-};
-
-DEAL_II_NAMESPACE_CLOSE
-
-#endif
index 79be01f4d734658dff5ef76f6ba366f67b40f414..c73d0600ce3afe0429c65118b01e9613c36c4ce7 100644 (file)
@@ -28,7 +28,6 @@
 
 #include <deal.II/grid/cell_id.h>
 #include <deal.II/grid/cell_status.h>
-#include <deal.II/grid/data_serializer.h>
 #include <deal.II/grid/tria_description.h>
 #include <deal.II/grid/tria_iterator_selector.h>
 #include <deal.II/grid/tria_levels.h>
@@ -330,6 +329,206 @@ namespace internal
       serialize(Archive &ar, const unsigned int version);
     };
   } // namespace TriangulationImplementation
+
+
+  /**
+   * A structure that binds information about data attached to cells.
+   */
+  template <int dim, int spacedim = dim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  struct CellAttachedData
+  {
+    using cell_iterator = TriaIterator<CellAccessor<dim, spacedim>>;
+
+    /**
+     * Number of functions that get attached to the Triangulation through
+     * register_data_attach() for example SolutionTransfer.
+     */
+    unsigned int n_attached_data_sets;
+
+    /**
+     * Number of functions that need to unpack their data after a call from
+     * load().
+     */
+    unsigned int n_attached_deserialize;
+
+    using pack_callback_t =
+      std::function<std::vector<char>(cell_iterator, CellStatus)>;
+
+    /**
+     * These callback functions will be stored in the order in which they
+     * have been registered with the register_data_attach() function.
+     */
+    std::vector<pack_callback_t> pack_callbacks_fixed;
+    std::vector<pack_callback_t> pack_callbacks_variable;
+  };
+
+  /**
+   * A structure that stores information about the data that has been, or
+   * will be, attached to cells via the register_data_attach() function
+   * and later retrieved via notify_ready_to_unpack().
+   *
+   * This internalclass is dedicated to the data serialization and transfer
+   * across repartitioned meshes and to/from the file system.
+   *
+   * It is designed to store all data buffers intended for serialization.
+   */
+  template <int dim, int spacedim = dim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  class CellAttachedDataSerializer
+  {
+  public:
+    using cell_iterator = TriaIterator<CellAccessor<dim, spacedim>>;
+
+    /**
+     * Auxiliary data structure for assigning a CellStatus to a deal.II cell
+     * iterator. For an extensive description of the former, see the
+     * documentation for the member function register_data_attach().
+     */
+    using cell_relation_t = typename std::pair<cell_iterator, CellStatus>;
+
+    CellAttachedDataSerializer();
+
+    /**
+     * Prepare data serialization by calling the pack callback functions on each
+     * cell in @p cell_relations.
+     *
+     * All registered callback functions in @p pack_callbacks_fixed will write
+     * into the fixed size buffer, whereas each entry of @p pack_callbacks_variable
+     * will write its data into the variable size buffer.
+     */
+    void
+    pack_data(
+      const std::vector<cell_relation_t> &cell_relations,
+      const std::vector<
+        typename internal::CellAttachedData<dim, spacedim>::pack_callback_t>
+        &pack_callbacks_fixed,
+      const std::vector<
+        typename internal::CellAttachedData<dim, spacedim>::pack_callback_t>
+        &             pack_callbacks_variable,
+      const MPI_Comm &mpi_communicator);
+
+    /**
+     * Unpack the CellStatus information on each entry of
+     * @p cell_relations.
+     *
+     * Data has to be previously transferred with execute_transfer()
+     * or deserialized from the file system via load().
+     */
+    void
+    unpack_cell_status(std::vector<cell_relation_t> &cell_relations) const;
+
+    /**
+     * Unpack previously serialized data on each cell registered in
+     * @p cell_relations with the provided @p unpack_callback function.
+     *
+     * The parameter @p handle corresponds to the position where the
+     * @p unpack_callback function is allowed to read from the memory. Its
+     * value needs to be in accordance with the corresponding pack_callback
+     * function that has been registered previously.
+     *
+     * Data has to be previously transferred with execute_transfer()
+     * or deserialized from the file system via load().
+     */
+    void
+    unpack_data(
+      const std::vector<cell_relation_t> &cell_relations,
+      const unsigned int                  handle,
+      const std::function<
+        void(const cell_iterator &,
+             const CellStatus &,
+             const boost::iterator_range<std::vector<char>::const_iterator> &)>
+        &unpack_callback) const;
+
+    /**
+     * Serialize data to file system.
+     *
+     * The data will be written in a separate file, whose name
+     * consists of the stem @p filename and an attached identifier
+     * <tt>_fixed.data</tt> for fixed size data and <tt>_variable.data</tt>
+     * for variable size data.
+     *
+     * If MPI support is enabled, all processors write into these files
+     * simultaneously via MPIIO. Each processor's position to write to will be
+     * determined from the provided input parameters.
+     *
+     * Data has to be previously packed with pack_data().
+     */
+    void
+    save(const unsigned int global_first_cell,
+         const unsigned int global_num_cells,
+         const std::string &filename,
+         const MPI_Comm &   mpi_communicator) const;
+
+    /**
+     * Deserialize data from file system.
+     *
+     * The data will be read from separate file, whose name
+     * consists of the stem @p filename and an attached identifier
+     * <tt>_fixed.data</tt> for fixed size data and <tt>_variable.data</tt>
+     * for variable size data.
+     * The @p n_attached_deserialize_fixed and @p n_attached_deserialize_variable
+     * parameters are required to gather the memory offsets for each
+     * callback.
+     *
+     * If MPI support is enabled, all processors read from these files
+     * simultaneously via MPIIO. Each processor's position to read from will be
+     * determined from the provided input arguments.
+     *
+     * After loading, unpack_data() needs to be called to finally
+     * distribute data across the associated triangulation.
+     */
+    void
+    load(const unsigned int global_first_cell,
+         const unsigned int global_num_cells,
+         const unsigned int local_num_cells,
+         const std::string &filename,
+         const unsigned int n_attached_deserialize_fixed,
+         const unsigned int n_attached_deserialize_variable,
+         const MPI_Comm &   mpi_communicator);
+
+    /**
+     * Clears all containers and associated data, and resets member
+     * values to their default state.
+     *
+     * Frees memory completely.
+     */
+    void
+    clear();
+
+    /**
+     * Flag that denotes if variable size data has been packed.
+     */
+    bool variable_size_data_stored;
+
+    /**
+     * Cumulative size in bytes that those functions that have called
+     * register_data_attach() want to attach to each cell. This number
+     * only pertains to fixed-sized buffers where the data attached to
+     * each cell has exactly the same size.
+     *
+     * The last entry of this container corresponds to the data size
+     * packed per cell in the fixed size buffer (which can be accessed
+     * calling <tt>sizes_fixed_cumulative.back()</tt>).
+     */
+    std::vector<unsigned int> sizes_fixed_cumulative;
+
+    /**
+     * Consecutive buffers designed for the fixed size serialization
+     * functions.
+     */
+    std::vector<char> src_data_fixed;
+    std::vector<char> dest_data_fixed;
+
+    /**
+     * Consecutive buffers designed for the variable size serialization
+     * functions.
+     */
+    std::vector<int>  src_sizes_variable;
+    std::vector<int>  dest_sizes_variable;
+    std::vector<char> src_data_variable;
+    std::vector<char> dest_data_variable;
+  };
 } // namespace internal
 
 
@@ -3705,11 +3904,11 @@ protected:
    * respective CellStatus. To update its contents, use the
    * update_cell_relations() member function.
    */
-  std::vector<
-    typename CellAttachedDataSerializer<dim, spacedim>::cell_relation_t>
+  std::vector<typename internal::CellAttachedDataSerializer<dim, spacedim>::
+                cell_relation_t>
     local_cell_relations;
 
-  CellAttachedDataSerializer<dim, spacedim> data_serializer;
+  internal::CellAttachedDataSerializer<dim, spacedim> data_serializer;
   /**
    * @}
    */
index ef3e81893f3ead86b0fdee81dd0607f5314347f5..3d70ba8095710f43f9019e8eb33da461829b2e92 100644 (file)
@@ -32,7 +32,6 @@ endif()
 
 set(_unity_include_src
   cell_id.cc
-  data_serializer.cc
   grid_refinement.cc
   intergrid_map.cc
   manifold.cc
@@ -71,7 +70,6 @@ setup_source_list("${_unity_include_src}"
 
 set(_inst
   cell_id.inst.in
-  data_serializer.inst.in
   grid_generator_cgal.inst.in
   grid_generator_from_name.inst.in
   grid_generator.inst.in
diff --git a/source/grid/data_serializer.cc b/source/grid/data_serializer.cc
deleted file mode 100644 (file)
index ad6f61c..0000000
+++ /dev/null
@@ -1,1036 +0,0 @@
-// ---------------------------------------------------------------------
-//
-// Copyright (C) 1998 - 2023 by the deal.II authors
-//
-// This file is part of the deal.II library.
-//
-// The deal.II library is free software; you can use it, redistribute
-// it, and/or modify it under the terms of the GNU Lesser General
-// Public License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-// The full text of the license can be found in the file LICENSE.md at
-// the top level directory of deal.II.
-//
-// ---------------------------------------------------------------------
-
-#include <deal.II/base/config.h>
-
-#include <deal.II/base/geometry_info.h>
-
-#include <deal.II/grid/data_serializer.h>
-#include <deal.II/grid/tria_accessor.h>
-
-#include <utility>
-#include <vector>
-
-DEAL_II_NAMESPACE_OPEN
-
-template <int dim, int spacedim>
-DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-CellAttachedDataSerializer<dim, spacedim>::CellAttachedDataSerializer()
-  : variable_size_data_stored(false)
-{}
-
-
-template <int dim, int spacedim>
-DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-void CellAttachedDataSerializer<dim, spacedim>::pack_data(
-  const std::vector<cell_relation_t> &cell_relations,
-  const std::vector<
-    typename internal::CellAttachedData<dim, spacedim>::pack_callback_t>
-    &pack_callbacks_fixed,
-  const std::vector<
-    typename internal::CellAttachedData<dim, spacedim>::pack_callback_t>
-    &             pack_callbacks_variable,
-  const MPI_Comm &mpi_communicator)
-{
-  Assert(src_data_fixed.size() == 0,
-         ExcMessage("Previously packed data has not been released yet!"));
-  Assert(src_sizes_variable.size() == 0, ExcInternalError());
-
-  const unsigned int n_callbacks_fixed    = pack_callbacks_fixed.size();
-  const unsigned int n_callbacks_variable = pack_callbacks_variable.size();
-
-  // Store information that we packed variable size data in
-  // a member variable for later.
-  variable_size_data_stored = (n_callbacks_variable > 0);
-
-  // If variable transfer is scheduled, we will store the data size that
-  // each variable size callback function writes in this auxiliary
-  // container. The information will be stored by each cell in this vector
-  // temporarily.
-  std::vector<unsigned int> cell_sizes_variable_cumulative(
-    n_callbacks_variable);
-
-  // Prepare the buffer structure, in which each callback function will
-  // store its data for each active cell.
-  // The outmost shell in this container construct corresponds to the
-  // data packed per cell. The next layer resembles the data that
-  // each callback function packs on the corresponding cell. These
-  // buffers are chains of chars stored in an std::vector<char>.
-  // A visualisation of the data structure:
-  /* clang-format off */
-      // |             cell_1                | |             cell_2                | ...
-      // ||  callback_1  ||  callback_2  |...| ||  callback_1  ||  callback_2  |...| ...
-      // |||char|char|...|||char|char|...|...| |||char|char|...|||char|char|...|...| ...
-  /* clang-format on */
-  std::vector<std::vector<std::vector<char>>> packed_fixed_size_data(
-    cell_relations.size());
-  std::vector<std::vector<std::vector<char>>> packed_variable_size_data(
-    variable_size_data_stored ? cell_relations.size() : 0);
-
-  //
-  // --------- Pack data for fixed and variable size transfer ---------
-  //
-  // Iterate over all cells, call all callback functions on each cell,
-  // and store their data in the corresponding buffer scope.
-  {
-    auto cell_rel_it           = cell_relations.cbegin();
-    auto data_cell_fixed_it    = packed_fixed_size_data.begin();
-    auto data_cell_variable_it = packed_variable_size_data.begin();
-    for (; cell_rel_it != cell_relations.cend(); ++cell_rel_it)
-      {
-        const auto &dealii_cell = cell_rel_it->first;
-        const auto &cell_status = cell_rel_it->second;
-
-        // Assertions about the tree structure.
-        switch (cell_status)
-          {
-            case CellStatus::cell_will_persist:
-            case CellStatus::cell_will_be_refined:
-              // double check the condition that we will only ever attach
-              // data to active cells when we get here
-              Assert(dealii_cell->is_active(), ExcInternalError());
-              break;
-
-            case CellStatus::children_will_be_coarsened:
-              // double check the condition that we will only ever attach
-              // data to cells with children when we get here. however, we
-              // can only tolerate one level of coarsening at a time, so
-              // check that the children are all active
-              Assert(dealii_cell->is_active() == false, ExcInternalError());
-              for (unsigned int c = 0;
-                   c < GeometryInfo<dim>::max_children_per_cell;
-                   ++c)
-                Assert(dealii_cell->child(c)->is_active(), ExcInternalError());
-              break;
-
-            case CellStatus::cell_invalid:
-              // do nothing on invalid cells
-              break;
-
-            default:
-              Assert(false, ExcInternalError());
-              break;
-          }
-
-        // Reserve memory corresponding to the number of callback
-        // functions that will be called.
-        // If variable size transfer is scheduled, we need to leave
-        // room for an array that holds information about how many
-        // bytes each of the variable size callback functions will
-        // write.
-        // On cells flagged with CellStatus::cell_invalid, only its CellStatus
-        // will be stored.
-        const unsigned int n_fixed_size_data_sets_on_cell =
-          1 + ((cell_status == CellStatus::cell_invalid) ?
-                 0 :
-                 ((variable_size_data_stored ? 1 : 0) + n_callbacks_fixed));
-        data_cell_fixed_it->resize(n_fixed_size_data_sets_on_cell);
-
-        // We continue with packing all data on this specific cell.
-        auto data_fixed_it = data_cell_fixed_it->begin();
-
-        // First, we pack the CellStatus information.
-        // to get consistent data sizes on each cell for the fixed size
-        // transfer, we won't allow compression
-        *data_fixed_it =
-          Utilities::pack(cell_status, /*allow_compression=*/false);
-        ++data_fixed_it;
-
-        // Proceed with all registered callback functions.
-        // Skip cells with the CellStatus::cell_invalid flag.
-        if (cell_status != CellStatus::cell_invalid)
-          {
-            // Pack fixed size data.
-            for (auto callback_it = pack_callbacks_fixed.cbegin();
-                 callback_it != pack_callbacks_fixed.cend();
-                 ++callback_it, ++data_fixed_it)
-              {
-                *data_fixed_it = (*callback_it)(dealii_cell, cell_status);
-              }
-
-            // Pack variable size data.
-            // If we store variable size data, we need to transfer
-            // the sizes of each corresponding callback function
-            // via fixed size transfer as well.
-            if (variable_size_data_stored)
-              {
-                const unsigned int n_variable_size_data_sets_on_cell =
-                  ((cell_status == CellStatus::cell_invalid) ?
-                     0 :
-                     n_callbacks_variable);
-                data_cell_variable_it->resize(
-                  n_variable_size_data_sets_on_cell);
-
-                auto callback_it       = pack_callbacks_variable.cbegin();
-                auto data_variable_it  = data_cell_variable_it->begin();
-                auto sizes_variable_it = cell_sizes_variable_cumulative.begin();
-                for (; callback_it != pack_callbacks_variable.cend();
-                     ++callback_it, ++data_variable_it, ++sizes_variable_it)
-                  {
-                    *data_variable_it =
-                      (*callback_it)(dealii_cell, cell_status);
-
-                    // Store data sizes for each callback function first.
-                    // Make it cumulative below.
-                    *sizes_variable_it = data_variable_it->size();
-                  }
-
-                // Turn size vector into its cumulative representation.
-                std::partial_sum(cell_sizes_variable_cumulative.begin(),
-                                 cell_sizes_variable_cumulative.end(),
-                                 cell_sizes_variable_cumulative.begin());
-
-                // Serialize cumulative variable size vector
-                // value-by-value. This way we can circumvent the overhead
-                // of storing the container object as a whole, since we
-                // know its size by the number of registered callback
-                // functions.
-                data_fixed_it->resize(n_callbacks_variable *
-                                      sizeof(unsigned int));
-                for (unsigned int i = 0; i < n_callbacks_variable; ++i)
-                  std::memcpy(&(data_fixed_it->at(i * sizeof(unsigned int))),
-                              &(cell_sizes_variable_cumulative.at(i)),
-                              sizeof(unsigned int));
-
-                ++data_fixed_it;
-              }
-
-            // Double check that we packed everything we wanted
-            // in the fixed size buffers.
-            Assert(data_fixed_it == data_cell_fixed_it->end(),
-                   ExcInternalError());
-          }
-
-        ++data_cell_fixed_it;
-
-        // Increment the variable size data iterator
-        // only if we actually pack this kind of data
-        // to avoid getting out of bounds.
-        if (variable_size_data_stored)
-          ++data_cell_variable_it;
-      } // loop over cell_relations
-  }
-
-  //
-  // ----------- Gather data sizes for fixed size transfer ------------
-  //
-  // Generate a vector which stores the sizes of each callback function,
-  // including the packed CellStatus transfer.
-  // Find the very first cell that we wrote to with all callback
-  // functions (i.e. a cell that was not flagged with CellStatus::cell_invalid)
-  // and store the sizes of each buffer.
-  //
-  // To deal with the case that at least one of the processors does not
-  // own any cell at all, we will exchange the information about the data
-  // sizes among them later. The code in between is still well-defined,
-  // since the following loops will be skipped.
-  std::vector<unsigned int> local_sizes_fixed(
-    1 + n_callbacks_fixed + (variable_size_data_stored ? 1 : 0));
-  for (const auto &data_cell : packed_fixed_size_data)
-    {
-      if (data_cell.size() == local_sizes_fixed.size())
-        {
-          auto sizes_fixed_it = local_sizes_fixed.begin();
-          auto data_fixed_it  = data_cell.cbegin();
-          for (; data_fixed_it != data_cell.cend();
-               ++data_fixed_it, ++sizes_fixed_it)
-            {
-              *sizes_fixed_it = data_fixed_it->size();
-            }
-
-          break;
-        }
-    }
-
-  // Check if all cells have valid sizes.
-  for (auto data_cell_fixed_it = packed_fixed_size_data.cbegin();
-       data_cell_fixed_it != packed_fixed_size_data.cend();
-       ++data_cell_fixed_it)
-    {
-      Assert((data_cell_fixed_it->size() == 1) ||
-               (data_cell_fixed_it->size() == local_sizes_fixed.size()),
-             ExcInternalError());
-    }
-
-  // Share information about the packed data sizes
-  // of all callback functions across all processors, in case one
-  // of them does not own any cells at all.
-  std::vector<unsigned int> global_sizes_fixed(local_sizes_fixed.size());
-  Utilities::MPI::max(local_sizes_fixed, mpi_communicator, global_sizes_fixed);
-
-  // Construct cumulative sizes, since this is the only information
-  // we need from now on.
-  sizes_fixed_cumulative.resize(global_sizes_fixed.size());
-  std::partial_sum(global_sizes_fixed.begin(),
-                   global_sizes_fixed.end(),
-                   sizes_fixed_cumulative.begin());
-
-  //
-  // ---------- Gather data sizes for variable size transfer ----------
-  //
-  if (variable_size_data_stored)
-    {
-      src_sizes_variable.reserve(packed_variable_size_data.size());
-      for (const auto &data_cell : packed_variable_size_data)
-        {
-          int variable_data_size_on_cell = 0;
-
-          for (const auto &data : data_cell)
-            variable_data_size_on_cell += data.size();
-
-          src_sizes_variable.push_back(variable_data_size_on_cell);
-        }
-    }
-
-  //
-  // ------------------------ Build buffers ---------------------------
-  //
-  const unsigned int expected_size_fixed =
-    cell_relations.size() * sizes_fixed_cumulative.back();
-  const unsigned int expected_size_variable =
-    std::accumulate(src_sizes_variable.begin(),
-                    src_sizes_variable.end(),
-                    std::vector<int>::size_type(0));
-
-  // Move every piece of packed fixed size data into the consecutive
-  // buffer.
-  src_data_fixed.reserve(expected_size_fixed);
-  for (const auto &data_cell_fixed : packed_fixed_size_data)
-    {
-      // Move every fraction of packed data into the buffer
-      // reserved for this particular cell.
-      for (const auto &data_fixed : data_cell_fixed)
-        std::move(data_fixed.begin(),
-                  data_fixed.end(),
-                  std::back_inserter(src_data_fixed));
-
-      // If we only packed the CellStatus information
-      // (i.e. encountered a cell flagged CellStatus::cell_invalid),
-      // fill the remaining space with invalid entries.
-      // We can skip this if there is nothing else to pack.
-      if ((data_cell_fixed.size() == 1) && (sizes_fixed_cumulative.size() > 1))
-        {
-          const std::size_t bytes_skipped =
-            sizes_fixed_cumulative.back() - sizes_fixed_cumulative.front();
-
-          src_data_fixed.insert(src_data_fixed.end(),
-                                bytes_skipped,
-                                static_cast<char>(-1)); // invalid_char
-        }
-    }
-
-  // Move every piece of packed variable size data into the consecutive
-  // buffer.
-  if (variable_size_data_stored)
-    {
-      src_data_variable.reserve(expected_size_variable);
-      for (const auto &data_cell : packed_variable_size_data)
-        {
-          // Move every fraction of packed data into the buffer
-          // reserved for this particular cell.
-          for (const auto &data : data_cell)
-            std::move(data.begin(),
-                      data.end(),
-                      std::back_inserter(src_data_variable));
-        }
-    }
-
-  // Double check that we packed everything correctly.
-  Assert(src_data_fixed.size() == expected_size_fixed, ExcInternalError());
-  Assert(src_data_variable.size() == expected_size_variable,
-         ExcInternalError());
-}
-
-
-
-template <int dim, int spacedim>
-DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-void CellAttachedDataSerializer<dim, spacedim>::unpack_cell_status(
-  std::vector<
-    typename CellAttachedDataSerializer<dim, spacedim>::cell_relation_t>
-    &cell_relations) const
-{
-  Assert(sizes_fixed_cumulative.size() > 0,
-         ExcMessage("No data has been packed!"));
-  if (cell_relations.size() > 0)
-    {
-      Assert(dest_data_fixed.size() > 0,
-             ExcMessage("No data has been received!"));
-    }
-
-  // Size of CellStatus object that will be unpacked on each cell.
-  const unsigned int size = sizes_fixed_cumulative.front();
-
-  // Iterate over all cells and overwrite the CellStatus
-  // information from the transferred data.
-  // Proceed buffer iterator position to next cell after
-  // each iteration.
-  auto cell_rel_it   = cell_relations.begin();
-  auto dest_fixed_it = dest_data_fixed.cbegin();
-  for (; cell_rel_it != cell_relations.end();
-       ++cell_rel_it, dest_fixed_it += sizes_fixed_cumulative.back())
-    {
-      cell_rel_it->second = // cell_status
-        Utilities::unpack<CellStatus>(dest_fixed_it,
-                                      dest_fixed_it + size,
-                                      /*allow_compression=*/false);
-    }
-}
-
-
-
-template <int dim, int spacedim>
-DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-void CellAttachedDataSerializer<dim, spacedim>::unpack_data(
-  const std::vector<
-    typename CellAttachedDataSerializer<dim, spacedim>::cell_relation_t>
-    &                cell_relations,
-  const unsigned int handle,
-  const std::function<
-    void(const cell_iterator &,
-         const CellStatus &,
-         const boost::iterator_range<std::vector<char>::const_iterator> &)>
-    &unpack_callback) const
-{
-  // We decode the handle returned by register_data_attach() back into
-  // a format we can use. All even handles belong to those callback
-  // functions which write/read variable size data, all odd handles
-  // interact with fixed size buffers.
-  const bool         callback_variable_transfer = (handle % 2 == 0);
-  const unsigned int callback_index             = handle / 2;
-
-  // Cells will always receive fixed size data (i.e., CellStatus
-  // information), but not necessarily variable size data (e.g., with a
-  // ParticleHandler a cell might not contain any particle at all).
-  // Thus it is sufficient to check if fixed size data has been received.
-  Assert(sizes_fixed_cumulative.size() > 0,
-         ExcMessage("No data has been packed!"));
-  if (cell_relations.size() > 0)
-    {
-      Assert(dest_data_fixed.size() > 0,
-             ExcMessage("No data has been received!"));
-    }
-
-  std::vector<char>::const_iterator dest_data_it;
-  std::vector<char>::const_iterator dest_sizes_cell_it;
-
-  // Depending on whether our callback function unpacks fixed or
-  // variable size data, we have to pursue different approaches
-  // to localize the correct fraction of the buffer from which
-  // we are allowed to read.
-  unsigned int offset         = numbers::invalid_unsigned_int;
-  unsigned int size           = numbers::invalid_unsigned_int;
-  unsigned int data_increment = numbers::invalid_unsigned_int;
-
-  if (callback_variable_transfer)
-    {
-      // For the variable size data, we need to extract the
-      // data size from the fixed size buffer on each cell.
-      //
-      // We packed this information last, so the last packed
-      // object in the fixed size buffer corresponds to the
-      // variable data sizes.
-      //
-      // The last entry of sizes_fixed_cumulative corresponds
-      // to the size of all fixed size data packed on the cell.
-      // To get the offset for the last packed object, we need
-      // to get the next-to-last entry.
-      const unsigned int offset_variable_data_sizes =
-        sizes_fixed_cumulative[sizes_fixed_cumulative.size() - 2];
-
-      // This iterator points to the data size that the
-      // callback_function packed for each specific cell.
-      // Adjust buffer iterator to the offset of the callback
-      // function so that we only have to advance its position
-      // to the next cell after each iteration.
-      dest_sizes_cell_it = dest_data_fixed.cbegin() +
-                           offset_variable_data_sizes +
-                           callback_index * sizeof(unsigned int);
-
-      // Let the data iterator point to the correct buffer.
-      dest_data_it = dest_data_variable.cbegin();
-    }
-  else
-    {
-      // For the fixed size data, we can get the information about
-      // the buffer location on each cell directly from the
-      // sizes_fixed_cumulative vector.
-      offset         = sizes_fixed_cumulative[callback_index];
-      size           = sizes_fixed_cumulative[callback_index + 1] - offset;
-      data_increment = sizes_fixed_cumulative.back();
-
-      // Let the data iterator point to the correct buffer.
-      // Adjust buffer iterator to the offset of the callback
-      // function so that we only have to advance its position
-      // to the next cell after each iteration.
-      if (cell_relations.begin() != cell_relations.end())
-        dest_data_it = dest_data_fixed.cbegin() + offset;
-    }
-
-  // Iterate over all cells and unpack the transferred data.
-  auto cell_rel_it   = cell_relations.begin();
-  auto dest_sizes_it = dest_sizes_variable.cbegin();
-  for (; cell_rel_it != cell_relations.end(); ++cell_rel_it)
-    {
-      const auto &dealii_cell = cell_rel_it->first;
-      const auto &cell_status = cell_rel_it->second;
-
-      if (callback_variable_transfer)
-        {
-          // Update the increment according to the whole data size
-          // of the current cell.
-          data_increment = *dest_sizes_it;
-
-          if (cell_status != CellStatus::cell_invalid)
-            {
-              // Extract the corresponding values for offset and size from
-              // the cumulative sizes array stored in the fixed size
-              // buffer.
-              if (callback_index == 0)
-                offset = 0;
-              else
-                std::memcpy(&offset,
-                            &(*(dest_sizes_cell_it - sizeof(unsigned int))),
-                            sizeof(unsigned int));
-
-              std::memcpy(&size, &(*dest_sizes_cell_it), sizeof(unsigned int));
-
-              size -= offset;
-
-              // Move the data iterator to the corresponding position
-              // of the callback function and adjust the increment
-              // accordingly.
-              dest_data_it += offset;
-              data_increment -= offset;
-            }
-
-          // Advance data size iterators to the next cell, avoid iterating
-          // past the end of dest_sizes_cell_it
-          if (cell_rel_it != cell_relations.end() - 1)
-            dest_sizes_cell_it += sizes_fixed_cumulative.back();
-          ++dest_sizes_it;
-        }
-
-      switch (cell_status)
-        {
-          case CellStatus::cell_will_persist:
-          case CellStatus::children_will_be_coarsened:
-            unpack_callback(dealii_cell,
-                            cell_status,
-                            boost::make_iterator_range(dest_data_it,
-                                                       dest_data_it + size));
-            break;
-
-          case CellStatus::cell_will_be_refined:
-            unpack_callback(dealii_cell->parent(),
-                            cell_status,
-                            boost::make_iterator_range(dest_data_it,
-                                                       dest_data_it + size));
-            break;
-
-          case CellStatus::cell_invalid:
-            // Skip this cell.
-            break;
-
-          default:
-            Assert(false, ExcInternalError());
-            break;
-        }
-
-      if (cell_rel_it != cell_relations.end() - 1)
-        dest_data_it += data_increment;
-    }
-}
-
-
-
-template <int dim, int spacedim>
-DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-void CellAttachedDataSerializer<dim, spacedim>::save(
-  const unsigned int global_first_cell,
-  const unsigned int global_num_cells,
-  const std::string &filename,
-  const MPI_Comm &   mpi_communicator) const
-{
-  Assert(sizes_fixed_cumulative.size() > 0,
-         ExcMessage("No data has been packed!"));
-
-#ifdef DEAL_II_WITH_MPI
-  // Large fractions of this function have been copied from
-  // DataOutInterface::write_vtu_in_parallel.
-  // TODO: Write general MPIIO interface.
-
-  const int myrank  = Utilities::MPI::this_mpi_process(mpi_communicator);
-  const int mpisize = Utilities::MPI::n_mpi_processes(mpi_communicator);
-
-  if (mpisize > 1)
-    {
-      const unsigned int bytes_per_cell = sizes_fixed_cumulative.back();
-
-      //
-      // ---------- Fixed size data ----------
-      //
-      {
-        const std::string fname_fixed = std::string(filename) + "_fixed.data";
-
-        MPI_Info info;
-        int      ierr = MPI_Info_create(&info);
-        AssertThrowMPI(ierr);
-
-        MPI_File fh;
-        ierr = MPI_File_open(mpi_communicator,
-                             fname_fixed.c_str(),
-                             MPI_MODE_CREATE | MPI_MODE_WRONLY,
-                             info,
-                             &fh);
-        AssertThrowMPI(ierr);
-
-        ierr = MPI_File_set_size(fh, 0); // delete the file contents
-        AssertThrowMPI(ierr);
-        // this barrier is necessary, because otherwise others might already
-        // write while one core is still setting the size to zero.
-        ierr = MPI_Barrier(mpi_communicator);
-        AssertThrowMPI(ierr);
-        ierr = MPI_Info_free(&info);
-        AssertThrowMPI(ierr);
-        // ------------------
-
-        // Write cumulative sizes to file.
-        // Since each processor owns the same information about the data
-        // sizes, it is sufficient to let only the first processor perform
-        // this task.
-        if (myrank == 0)
-          {
-            ierr = Utilities::MPI::LargeCount::File_write_at_c(
-              fh,
-              0,
-              sizes_fixed_cumulative.data(),
-              sizes_fixed_cumulative.size(),
-              MPI_UNSIGNED,
-              MPI_STATUS_IGNORE);
-            AssertThrowMPI(ierr);
-          }
-
-        // Write packed data to file simultaneously.
-        const MPI_Offset size_header =
-          sizes_fixed_cumulative.size() * sizeof(unsigned int);
-
-        // Make sure we do the following computation in 64bit integers to be
-        // able to handle 4GB+ files:
-        const MPI_Offset my_global_file_position =
-          size_header +
-          static_cast<MPI_Offset>(global_first_cell) * bytes_per_cell;
-
-        ierr =
-          Utilities::MPI::LargeCount::File_write_at_c(fh,
-                                                      my_global_file_position,
-                                                      src_data_fixed.data(),
-                                                      src_data_fixed.size(),
-                                                      MPI_BYTE,
-                                                      MPI_STATUS_IGNORE);
-        AssertThrowMPI(ierr);
-
-        ierr = MPI_File_close(&fh);
-        AssertThrowMPI(ierr);
-      }
-
-
-
-      //
-      // ---------- Variable size data ----------
-      //
-      if (variable_size_data_stored)
-        {
-          const std::string fname_variable =
-            std::string(filename) + "_variable.data";
-
-          MPI_Info info;
-          int      ierr = MPI_Info_create(&info);
-          AssertThrowMPI(ierr);
-
-          MPI_File fh;
-          ierr = MPI_File_open(mpi_communicator,
-                               fname_variable.c_str(),
-                               MPI_MODE_CREATE | MPI_MODE_WRONLY,
-                               info,
-                               &fh);
-          AssertThrowMPI(ierr);
-
-          ierr = MPI_File_set_size(fh, 0); // delete the file contents
-          AssertThrowMPI(ierr);
-          // this barrier is necessary, because otherwise others might already
-          // write while one core is still setting the size to zero.
-          ierr = MPI_Barrier(mpi_communicator);
-          AssertThrowMPI(ierr);
-          ierr = MPI_Info_free(&info);
-          AssertThrowMPI(ierr);
-
-          // Write sizes of each cell into file simultaneously.
-          {
-            const MPI_Offset my_global_file_position =
-              static_cast<MPI_Offset>(global_first_cell) * sizeof(unsigned int);
-
-            // It is very unlikely that a single process has more than
-            // 2 billion cells, but we might as well check.
-            AssertThrow(src_sizes_variable.size() <
-                          static_cast<std::size_t>(
-                            std::numeric_limits<int>::max()),
-                        ExcNotImplemented());
-
-            ierr = Utilities::MPI::LargeCount::File_write_at_c(
-              fh,
-              my_global_file_position,
-              src_sizes_variable.data(),
-              src_sizes_variable.size(),
-              MPI_INT,
-              MPI_STATUS_IGNORE);
-            AssertThrowMPI(ierr);
-          }
-
-          // Gather size of data in bytes we want to store from this
-          // processor and compute the prefix sum. We do this in 64 bit
-          // to avoid overflow for files larger than 4GB:
-          const std::uint64_t size_on_proc = src_data_variable.size();
-          std::uint64_t       prefix_sum   = 0;
-          ierr                             = MPI_Exscan(&size_on_proc,
-                            &prefix_sum,
-                            1,
-                            MPI_UINT64_T,
-                            MPI_SUM,
-                            mpi_communicator);
-          AssertThrowMPI(ierr);
-
-          const MPI_Offset my_global_file_position =
-            static_cast<MPI_Offset>(global_num_cells) * sizeof(unsigned int) +
-            prefix_sum;
-
-          // Write data consecutively into file.
-          ierr = Utilities::MPI::LargeCount::File_write_at_c(
-            fh,
-            my_global_file_position,
-            src_data_variable.data(),
-            src_data_variable.size(),
-            MPI_BYTE,
-            MPI_STATUS_IGNORE);
-          AssertThrowMPI(ierr);
-
-
-          ierr = MPI_File_close(&fh);
-          AssertThrowMPI(ierr);
-        }
-    } // if (mpisize > 1)
-  else
-#endif
-    {
-      (void)global_first_cell;
-      (void)global_num_cells;
-
-      //
-      // ---------- Fixed size data ----------
-      //
-      {
-        const std::string fname_fixed = std::string(filename) + "_fixed.data";
-
-        std::ofstream file(fname_fixed, std::ios::binary | std::ios::out);
-
-        // Write header data.
-        file.write(reinterpret_cast<const char *>(
-                     sizes_fixed_cumulative.data()),
-                   sizes_fixed_cumulative.size() * sizeof(unsigned int));
-
-        // Write packed data.
-        file.write(reinterpret_cast<const char *>(src_data_fixed.data()),
-                   src_data_fixed.size() * sizeof(char));
-
-        file.close();
-      }
-
-      //
-      // ---------- Variable size data ----------
-      //
-      if (variable_size_data_stored)
-        {
-          const std::string fname_variable =
-            std::string(filename) + "_variable.data";
-
-          std::ofstream file(fname_variable, std::ios::binary | std::ios::out);
-
-          // Write header data.
-          file.write(reinterpret_cast<const char *>(src_sizes_variable.data()),
-                     src_sizes_variable.size() * sizeof(int));
-
-          // Write packed data.
-          file.write(reinterpret_cast<const char *>(src_data_variable.data()),
-                     src_data_variable.size() * sizeof(char));
-
-          file.close();
-        }
-    }
-}
-
-
-
-template <int dim, int spacedim>
-DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-void CellAttachedDataSerializer<dim, spacedim>::load(
-  const unsigned int global_first_cell,
-  const unsigned int global_num_cells,
-  const unsigned int local_num_cells,
-  const std::string &filename,
-  const unsigned int n_attached_deserialize_fixed,
-  const unsigned int n_attached_deserialize_variable,
-  const MPI_Comm &   mpi_communicator)
-{
-  Assert(dest_data_fixed.size() == 0,
-         ExcMessage("Previously loaded data has not been released yet!"));
-
-  variable_size_data_stored = (n_attached_deserialize_variable > 0);
-
-#ifdef DEAL_II_WITH_MPI
-  // Large fractions of this function have been copied from
-  // DataOutInterface::write_vtu_in_parallel.
-  // TODO: Write general MPIIO interface.
-
-  const int mpisize = Utilities::MPI::n_mpi_processes(mpi_communicator);
-
-  if (mpisize > 1)
-    {
-      //
-      // ---------- Fixed size data ----------
-      //
-      {
-        const std::string fname_fixed = std::string(filename) + "_fixed.data";
-
-        MPI_Info info;
-        int      ierr = MPI_Info_create(&info);
-        AssertThrowMPI(ierr);
-
-        MPI_File fh;
-        ierr = MPI_File_open(
-          mpi_communicator, fname_fixed.c_str(), MPI_MODE_RDONLY, info, &fh);
-        AssertThrowMPI(ierr);
-
-        ierr = MPI_Info_free(&info);
-        AssertThrowMPI(ierr);
-
-        // Read cumulative sizes from file.
-        // Since all processors need the same information about the data
-        // sizes, let each of them retrieve it by reading from the same
-        // location in the file.
-        sizes_fixed_cumulative.resize(1 + n_attached_deserialize_fixed +
-                                      (variable_size_data_stored ? 1 : 0));
-        ierr = Utilities::MPI::LargeCount::File_read_at_c(
-          fh,
-          0,
-          sizes_fixed_cumulative.data(),
-          sizes_fixed_cumulative.size(),
-          MPI_UNSIGNED,
-          MPI_STATUS_IGNORE);
-        AssertThrowMPI(ierr);
-
-        // Allocate sufficient memory.
-        const unsigned int bytes_per_cell = sizes_fixed_cumulative.back();
-        dest_data_fixed.resize(static_cast<size_t>(local_num_cells) *
-                               bytes_per_cell);
-
-        // Read packed data from file simultaneously.
-        const MPI_Offset size_header =
-          sizes_fixed_cumulative.size() * sizeof(unsigned int);
-
-        // Make sure we do the following computation in 64bit integers to be
-        // able to handle 4GB+ files:
-        const MPI_Offset my_global_file_position =
-          size_header +
-          static_cast<MPI_Offset>(global_first_cell) * bytes_per_cell;
-
-        ierr =
-          Utilities::MPI::LargeCount::File_read_at_c(fh,
-                                                     my_global_file_position,
-                                                     dest_data_fixed.data(),
-                                                     dest_data_fixed.size(),
-                                                     MPI_BYTE,
-                                                     MPI_STATUS_IGNORE);
-        AssertThrowMPI(ierr);
-
-
-        ierr = MPI_File_close(&fh);
-        AssertThrowMPI(ierr);
-      }
-
-      //
-      // ---------- Variable size data ----------
-      //
-      if (variable_size_data_stored)
-        {
-          const std::string fname_variable =
-            std::string(filename) + "_variable.data";
-
-          MPI_Info info;
-          int      ierr = MPI_Info_create(&info);
-          AssertThrowMPI(ierr);
-
-          MPI_File fh;
-          ierr = MPI_File_open(mpi_communicator,
-                               fname_variable.c_str(),
-                               MPI_MODE_RDONLY,
-                               info,
-                               &fh);
-          AssertThrowMPI(ierr);
-
-          ierr = MPI_Info_free(&info);
-          AssertThrowMPI(ierr);
-
-          // Read sizes of all locally owned cells.
-          dest_sizes_variable.resize(local_num_cells);
-
-          const MPI_Offset my_global_file_position_sizes =
-            static_cast<MPI_Offset>(global_first_cell) * sizeof(unsigned int);
-
-          ierr = Utilities::MPI::LargeCount::File_read_at_c(
-            fh,
-            my_global_file_position_sizes,
-            dest_sizes_variable.data(),
-            dest_sizes_variable.size(),
-            MPI_INT,
-            MPI_STATUS_IGNORE);
-          AssertThrowMPI(ierr);
-
-
-          // Compute my data size in bytes and compute prefix sum. We do this
-          // in 64 bit to avoid overflow for files larger than 4 GB:
-          const std::uint64_t size_on_proc =
-            std::accumulate(dest_sizes_variable.begin(),
-                            dest_sizes_variable.end(),
-                            0ULL);
-
-          std::uint64_t prefix_sum = 0;
-          ierr                     = MPI_Exscan(&size_on_proc,
-                            &prefix_sum,
-                            1,
-                            MPI_UINT64_T,
-                            MPI_SUM,
-                            mpi_communicator);
-          AssertThrowMPI(ierr);
-
-          const MPI_Offset my_global_file_position =
-            static_cast<MPI_Offset>(global_num_cells) * sizeof(unsigned int) +
-            prefix_sum;
-
-          dest_data_variable.resize(size_on_proc);
-
-          ierr = Utilities::MPI::LargeCount::File_read_at_c(
-            fh,
-            my_global_file_position,
-            dest_data_variable.data(),
-            dest_data_variable.size(),
-            MPI_BYTE,
-            MPI_STATUS_IGNORE);
-          AssertThrowMPI(ierr);
-
-          ierr = MPI_File_close(&fh);
-          AssertThrowMPI(ierr);
-        }
-    }
-  else // if (mpisize > 1)
-#endif
-    {
-      (void)global_first_cell;
-      (void)global_num_cells;
-      (void)local_num_cells;
-
-      //
-      // ---------- Fixed size data ----------
-      //
-      {
-        const std::string fname_fixed = std::string(filename) + "_fixed.data";
-
-        std::ifstream file(fname_fixed, std::ios::binary | std::ios::in);
-        sizes_fixed_cumulative.resize(1 + n_attached_deserialize_fixed +
-                                      (variable_size_data_stored ? 1 : 0));
-
-        // Read header data.
-        file.read(reinterpret_cast<char *>(sizes_fixed_cumulative.data()),
-                  sizes_fixed_cumulative.size() * sizeof(unsigned int));
-
-        const unsigned int bytes_per_cell = sizes_fixed_cumulative.back();
-        dest_data_fixed.resize(static_cast<size_t>(local_num_cells) *
-                               bytes_per_cell);
-
-        // Read packed data.
-        file.read(reinterpret_cast<char *>(dest_data_fixed.data()),
-                  dest_data_fixed.size() * sizeof(char));
-
-        file.close();
-      }
-
-      //
-      // ---------- Variable size data ----------
-      //
-      if (variable_size_data_stored)
-        {
-          const std::string fname_variable =
-            std::string(filename) + "_variable.data";
-
-          std::ifstream file(fname_variable, std::ios::binary | std::ios::in);
-
-          // Read header data.
-          file.read(reinterpret_cast<char *>(dest_sizes_variable.data()),
-                    dest_sizes_variable.size() * sizeof(int));
-
-          // Read packed data.
-          file.read(reinterpret_cast<char *>(dest_data_variable.data()),
-                    dest_data_variable.size() * sizeof(char));
-
-          file.close();
-        }
-    }
-}
-
-
-template <int dim, int spacedim>
-DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
-void CellAttachedDataSerializer<dim, spacedim>::clear()
-{
-  variable_size_data_stored = false;
-
-  // free information about data sizes
-  sizes_fixed_cumulative.clear();
-  sizes_fixed_cumulative.shrink_to_fit();
-
-  // free fixed size transfer data
-  src_data_fixed.clear();
-  src_data_fixed.shrink_to_fit();
-
-  dest_data_fixed.clear();
-  dest_data_fixed.shrink_to_fit();
-
-  // free variable size transfer data
-  src_sizes_variable.clear();
-  src_sizes_variable.shrink_to_fit();
-
-  src_data_variable.clear();
-  src_data_variable.shrink_to_fit();
-
-  dest_sizes_variable.clear();
-  dest_sizes_variable.shrink_to_fit();
-
-  dest_data_variable.clear();
-  dest_data_variable.shrink_to_fit();
-}
-
-// explicit instantiations
-#include "data_serializer.inst"
-
-DEAL_II_NAMESPACE_CLOSE
diff --git a/source/grid/data_serializer.inst.in b/source/grid/data_serializer.inst.in
deleted file mode 100644 (file)
index d99a5d4..0000000
+++ /dev/null
@@ -1,24 +0,0 @@
-// ---------------------------------------------------------------------
-//
-// Copyright (C) 2010 - 2023 by the deal.II authors
-//
-// This file is part of the deal.II library.
-//
-// The deal.II library is free software; you can use it, redistribute
-// it, and/or modify it under the terms of the GNU Lesser General
-// Public License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-// The full text of the license can be found in the file LICENSE.md at
-// the top level directory of deal.II.
-//
-// ---------------------------------------------------------------------
-
-
-
-for (deal_II_dimension : DIMENSIONS; deal_II_space_dimension : SPACE_DIMENSIONS)
-  {
-#if deal_II_dimension <= deal_II_space_dimension
-    template class CellAttachedDataSerializer<deal_II_dimension,
-                                              deal_II_space_dimension>;
-#endif
-  }
index b46325717932fedd5377c0f856b67180d93a54da..bb6e2717733776ccd384862d24669cfe319ecbb0 100644 (file)
 
 #include <deal.II/base/geometry_info.h>
 #include <deal.II/base/memory_consumption.h>
+#include <deal.II/base/mpi.templates.h>
+#include <deal.II/base/mpi_large_count.h>
+#include <deal.II/base/mpi_stub.h>
 #include <deal.II/base/thread_management.h>
+#include <deal.II/base/utilities.h>
 
 #include <deal.II/grid/connectivity.h>
 #include <deal.II/grid/grid_tools.h>
 #include <algorithm>
 #include <array>
 #include <cmath>
+#include <cstdint>
 #include <fstream>
 #include <functional>
+#include <limits>
 #include <list>
 #include <map>
 #include <memory>
@@ -120,6 +126,1029 @@ namespace internal
               MemoryConsumption::memory_consumption(n_active_hexes_level));
     }
   } // namespace TriangulationImplementation
+
+
+  template <int dim, int spacedim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  CellAttachedDataSerializer<dim, spacedim>::CellAttachedDataSerializer()
+    : variable_size_data_stored(false)
+  {}
+
+
+  template <int dim, int spacedim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  void CellAttachedDataSerializer<dim, spacedim>::pack_data(
+    const std::vector<cell_relation_t> &cell_relations,
+    const std::vector<
+      typename internal::CellAttachedData<dim, spacedim>::pack_callback_t>
+      &pack_callbacks_fixed,
+    const std::vector<
+      typename internal::CellAttachedData<dim, spacedim>::pack_callback_t>
+      &             pack_callbacks_variable,
+    const MPI_Comm &mpi_communicator)
+  {
+    Assert(src_data_fixed.size() == 0,
+           ExcMessage("Previously packed data has not been released yet!"));
+    Assert(src_sizes_variable.size() == 0, ExcInternalError());
+
+    const unsigned int n_callbacks_fixed    = pack_callbacks_fixed.size();
+    const unsigned int n_callbacks_variable = pack_callbacks_variable.size();
+
+    // Store information that we packed variable size data in
+    // a member variable for later.
+    variable_size_data_stored = (n_callbacks_variable > 0);
+
+    // If variable transfer is scheduled, we will store the data size that
+    // each variable size callback function writes in this auxiliary
+    // container. The information will be stored by each cell in this vector
+    // temporarily.
+    std::vector<unsigned int> cell_sizes_variable_cumulative(
+      n_callbacks_variable);
+
+    // Prepare the buffer structure, in which each callback function will
+    // store its data for each active cell.
+    // The outmost shell in this container construct corresponds to the
+    // data packed per cell. The next layer resembles the data that
+    // each callback function packs on the corresponding cell. These
+    // buffers are chains of chars stored in an std::vector<char>.
+    // A visualisation of the data structure:
+    /* clang-format off */
+      // |             cell_1                | |             cell_2                | ...
+      // ||  callback_1  ||  callback_2  |...| ||  callback_1  ||  callback_2  |...| ...
+      // |||char|char|...|||char|char|...|...| |||char|char|...|||char|char|...|...| ...
+    /* clang-format on */
+    std::vector<std::vector<std::vector<char>>> packed_fixed_size_data(
+      cell_relations.size());
+    std::vector<std::vector<std::vector<char>>> packed_variable_size_data(
+      variable_size_data_stored ? cell_relations.size() : 0);
+
+    //
+    // --------- Pack data for fixed and variable size transfer ---------
+    //
+    // Iterate over all cells, call all callback functions on each cell,
+    // and store their data in the corresponding buffer scope.
+    {
+      auto cell_rel_it           = cell_relations.cbegin();
+      auto data_cell_fixed_it    = packed_fixed_size_data.begin();
+      auto data_cell_variable_it = packed_variable_size_data.begin();
+      for (; cell_rel_it != cell_relations.cend(); ++cell_rel_it)
+        {
+          const auto &dealii_cell = cell_rel_it->first;
+          const auto &cell_status = cell_rel_it->second;
+
+          // Assertions about the tree structure.
+          switch (cell_status)
+            {
+              case CellStatus::cell_will_persist:
+              case CellStatus::cell_will_be_refined:
+                // double check the condition that we will only ever attach
+                // data to active cells when we get here
+                Assert(dealii_cell->is_active(), ExcInternalError());
+                break;
+
+              case CellStatus::children_will_be_coarsened:
+                // double check the condition that we will only ever attach
+                // data to cells with children when we get here. however, we
+                // can only tolerate one level of coarsening at a time, so
+                // check that the children are all active
+                Assert(dealii_cell->is_active() == false, ExcInternalError());
+                for (unsigned int c = 0;
+                     c < GeometryInfo<dim>::max_children_per_cell;
+                     ++c)
+                  Assert(dealii_cell->child(c)->is_active(),
+                         ExcInternalError());
+                break;
+
+              case CellStatus::cell_invalid:
+                // do nothing on invalid cells
+                break;
+
+              default:
+                Assert(false, ExcInternalError());
+                break;
+            }
+
+          // Reserve memory corresponding to the number of callback
+          // functions that will be called.
+          // If variable size transfer is scheduled, we need to leave
+          // room for an array that holds information about how many
+          // bytes each of the variable size callback functions will
+          // write.
+          // On cells flagged with CellStatus::cell_invalid, only its CellStatus
+          // will be stored.
+          const unsigned int n_fixed_size_data_sets_on_cell =
+            1 + ((cell_status == CellStatus::cell_invalid) ?
+                   0 :
+                   ((variable_size_data_stored ? 1 : 0) + n_callbacks_fixed));
+          data_cell_fixed_it->resize(n_fixed_size_data_sets_on_cell);
+
+          // We continue with packing all data on this specific cell.
+          auto data_fixed_it = data_cell_fixed_it->begin();
+
+          // First, we pack the CellStatus information.
+          // to get consistent data sizes on each cell for the fixed size
+          // transfer, we won't allow compression
+          *data_fixed_it =
+            Utilities::pack(cell_status, /*allow_compression=*/false);
+          ++data_fixed_it;
+
+          // Proceed with all registered callback functions.
+          // Skip cells with the CellStatus::cell_invalid flag.
+          if (cell_status != CellStatus::cell_invalid)
+            {
+              // Pack fixed size data.
+              for (auto callback_it = pack_callbacks_fixed.cbegin();
+                   callback_it != pack_callbacks_fixed.cend();
+                   ++callback_it, ++data_fixed_it)
+                {
+                  *data_fixed_it = (*callback_it)(dealii_cell, cell_status);
+                }
+
+              // Pack variable size data.
+              // If we store variable size data, we need to transfer
+              // the sizes of each corresponding callback function
+              // via fixed size transfer as well.
+              if (variable_size_data_stored)
+                {
+                  const unsigned int n_variable_size_data_sets_on_cell =
+                    ((cell_status == CellStatus::cell_invalid) ?
+                       0 :
+                       n_callbacks_variable);
+                  data_cell_variable_it->resize(
+                    n_variable_size_data_sets_on_cell);
+
+                  auto callback_it      = pack_callbacks_variable.cbegin();
+                  auto data_variable_it = data_cell_variable_it->begin();
+                  auto sizes_variable_it =
+                    cell_sizes_variable_cumulative.begin();
+                  for (; callback_it != pack_callbacks_variable.cend();
+                       ++callback_it, ++data_variable_it, ++sizes_variable_it)
+                    {
+                      *data_variable_it =
+                        (*callback_it)(dealii_cell, cell_status);
+
+                      // Store data sizes for each callback function first.
+                      // Make it cumulative below.
+                      *sizes_variable_it = data_variable_it->size();
+                    }
+
+                  // Turn size vector into its cumulative representation.
+                  std::partial_sum(cell_sizes_variable_cumulative.begin(),
+                                   cell_sizes_variable_cumulative.end(),
+                                   cell_sizes_variable_cumulative.begin());
+
+                  // Serialize cumulative variable size vector
+                  // value-by-value. This way we can circumvent the overhead
+                  // of storing the container object as a whole, since we
+                  // know its size by the number of registered callback
+                  // functions.
+                  data_fixed_it->resize(n_callbacks_variable *
+                                        sizeof(unsigned int));
+                  for (unsigned int i = 0; i < n_callbacks_variable; ++i)
+                    std::memcpy(&(data_fixed_it->at(i * sizeof(unsigned int))),
+                                &(cell_sizes_variable_cumulative.at(i)),
+                                sizeof(unsigned int));
+
+                  ++data_fixed_it;
+                }
+
+              // Double check that we packed everything we wanted
+              // in the fixed size buffers.
+              Assert(data_fixed_it == data_cell_fixed_it->end(),
+                     ExcInternalError());
+            }
+
+          ++data_cell_fixed_it;
+
+          // Increment the variable size data iterator
+          // only if we actually pack this kind of data
+          // to avoid getting out of bounds.
+          if (variable_size_data_stored)
+            ++data_cell_variable_it;
+        } // loop over cell_relations
+    }
+
+    //
+    // ----------- Gather data sizes for fixed size transfer ------------
+    //
+    // Generate a vector which stores the sizes of each callback function,
+    // including the packed CellStatus transfer.
+    // Find the very first cell that we wrote to with all callback
+    // functions (i.e. a cell that was not flagged with
+    // CellStatus::cell_invalid) and store the sizes of each buffer.
+    //
+    // To deal with the case that at least one of the processors does not
+    // own any cell at all, we will exchange the information about the data
+    // sizes among them later. The code in between is still well-defined,
+    // since the following loops will be skipped.
+    std::vector<unsigned int> local_sizes_fixed(
+      1 + n_callbacks_fixed + (variable_size_data_stored ? 1 : 0));
+    for (const auto &data_cell : packed_fixed_size_data)
+      {
+        if (data_cell.size() == local_sizes_fixed.size())
+          {
+            auto sizes_fixed_it = local_sizes_fixed.begin();
+            auto data_fixed_it  = data_cell.cbegin();
+            for (; data_fixed_it != data_cell.cend();
+                 ++data_fixed_it, ++sizes_fixed_it)
+              {
+                *sizes_fixed_it = data_fixed_it->size();
+              }
+
+            break;
+          }
+      }
+
+    // Check if all cells have valid sizes.
+    for (auto data_cell_fixed_it = packed_fixed_size_data.cbegin();
+         data_cell_fixed_it != packed_fixed_size_data.cend();
+         ++data_cell_fixed_it)
+      {
+        Assert((data_cell_fixed_it->size() == 1) ||
+                 (data_cell_fixed_it->size() == local_sizes_fixed.size()),
+               ExcInternalError());
+      }
+
+    // Share information about the packed data sizes
+    // of all callback functions across all processors, in case one
+    // of them does not own any cells at all.
+    std::vector<unsigned int> global_sizes_fixed(local_sizes_fixed.size());
+    Utilities::MPI::max(local_sizes_fixed,
+                        mpi_communicator,
+                        global_sizes_fixed);
+
+    // Construct cumulative sizes, since this is the only information
+    // we need from now on.
+    sizes_fixed_cumulative.resize(global_sizes_fixed.size());
+    std::partial_sum(global_sizes_fixed.begin(),
+                     global_sizes_fixed.end(),
+                     sizes_fixed_cumulative.begin());
+
+    //
+    // ---------- Gather data sizes for variable size transfer ----------
+    //
+    if (variable_size_data_stored)
+      {
+        src_sizes_variable.reserve(packed_variable_size_data.size());
+        for (const auto &data_cell : packed_variable_size_data)
+          {
+            int variable_data_size_on_cell = 0;
+
+            for (const auto &data : data_cell)
+              variable_data_size_on_cell += data.size();
+
+            src_sizes_variable.push_back(variable_data_size_on_cell);
+          }
+      }
+
+    //
+    // ------------------------ Build buffers ---------------------------
+    //
+    const unsigned int expected_size_fixed =
+      cell_relations.size() * sizes_fixed_cumulative.back();
+    const unsigned int expected_size_variable =
+      std::accumulate(src_sizes_variable.begin(),
+                      src_sizes_variable.end(),
+                      std::vector<int>::size_type(0));
+
+    // Move every piece of packed fixed size data into the consecutive
+    // buffer.
+    src_data_fixed.reserve(expected_size_fixed);
+    for (const auto &data_cell_fixed : packed_fixed_size_data)
+      {
+        // Move every fraction of packed data into the buffer
+        // reserved for this particular cell.
+        for (const auto &data_fixed : data_cell_fixed)
+          std::move(data_fixed.begin(),
+                    data_fixed.end(),
+                    std::back_inserter(src_data_fixed));
+
+        // If we only packed the CellStatus information
+        // (i.e. encountered a cell flagged CellStatus::cell_invalid),
+        // fill the remaining space with invalid entries.
+        // We can skip this if there is nothing else to pack.
+        if ((data_cell_fixed.size() == 1) &&
+            (sizes_fixed_cumulative.size() > 1))
+          {
+            const std::size_t bytes_skipped =
+              sizes_fixed_cumulative.back() - sizes_fixed_cumulative.front();
+
+            src_data_fixed.insert(src_data_fixed.end(),
+                                  bytes_skipped,
+                                  static_cast<char>(-1)); // invalid_char
+          }
+      }
+
+    // Move every piece of packed variable size data into the consecutive
+    // buffer.
+    if (variable_size_data_stored)
+      {
+        src_data_variable.reserve(expected_size_variable);
+        for (const auto &data_cell : packed_variable_size_data)
+          {
+            // Move every fraction of packed data into the buffer
+            // reserved for this particular cell.
+            for (const auto &data : data_cell)
+              std::move(data.begin(),
+                        data.end(),
+                        std::back_inserter(src_data_variable));
+          }
+      }
+
+    // Double check that we packed everything correctly.
+    Assert(src_data_fixed.size() == expected_size_fixed, ExcInternalError());
+    Assert(src_data_variable.size() == expected_size_variable,
+           ExcInternalError());
+  }
+
+
+
+  template <int dim, int spacedim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  void CellAttachedDataSerializer<dim, spacedim>::unpack_cell_status(
+    std::vector<
+      typename CellAttachedDataSerializer<dim, spacedim>::cell_relation_t>
+      &cell_relations) const
+  {
+    Assert(sizes_fixed_cumulative.size() > 0,
+           ExcMessage("No data has been packed!"));
+    if (cell_relations.size() > 0)
+      {
+        Assert(dest_data_fixed.size() > 0,
+               ExcMessage("No data has been received!"));
+      }
+
+    // Size of CellStatus object that will be unpacked on each cell.
+    const unsigned int size = sizes_fixed_cumulative.front();
+
+    // Iterate over all cells and overwrite the CellStatus
+    // information from the transferred data.
+    // Proceed buffer iterator position to next cell after
+    // each iteration.
+    auto cell_rel_it   = cell_relations.begin();
+    auto dest_fixed_it = dest_data_fixed.cbegin();
+    for (; cell_rel_it != cell_relations.end();
+         ++cell_rel_it, dest_fixed_it += sizes_fixed_cumulative.back())
+      {
+        cell_rel_it->second = // cell_status
+          Utilities::unpack<CellStatus>(dest_fixed_it,
+                                        dest_fixed_it + size,
+                                        /*allow_compression=*/false);
+      }
+  }
+
+
+
+  template <int dim, int spacedim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  void CellAttachedDataSerializer<dim, spacedim>::unpack_data(
+    const std::vector<
+      typename CellAttachedDataSerializer<dim, spacedim>::cell_relation_t>
+      &                cell_relations,
+    const unsigned int handle,
+    const std::function<
+      void(const cell_iterator &,
+           const CellStatus &,
+           const boost::iterator_range<std::vector<char>::const_iterator> &)>
+      &unpack_callback) const
+  {
+    // We decode the handle returned by register_data_attach() back into
+    // a format we can use. All even handles belong to those callback
+    // functions which write/read variable size data, all odd handles
+    // interact with fixed size buffers.
+    const bool         callback_variable_transfer = (handle % 2 == 0);
+    const unsigned int callback_index             = handle / 2;
+
+    // Cells will always receive fixed size data (i.e., CellStatus
+    // information), but not necessarily variable size data (e.g., with a
+    // ParticleHandler a cell might not contain any particle at all).
+    // Thus it is sufficient to check if fixed size data has been received.
+    Assert(sizes_fixed_cumulative.size() > 0,
+           ExcMessage("No data has been packed!"));
+    if (cell_relations.size() > 0)
+      {
+        Assert(dest_data_fixed.size() > 0,
+               ExcMessage("No data has been received!"));
+      }
+
+    std::vector<char>::const_iterator dest_data_it;
+    std::vector<char>::const_iterator dest_sizes_cell_it;
+
+    // Depending on whether our callback function unpacks fixed or
+    // variable size data, we have to pursue different approaches
+    // to localize the correct fraction of the buffer from which
+    // we are allowed to read.
+    unsigned int offset         = numbers::invalid_unsigned_int;
+    unsigned int size           = numbers::invalid_unsigned_int;
+    unsigned int data_increment = numbers::invalid_unsigned_int;
+
+    if (callback_variable_transfer)
+      {
+        // For the variable size data, we need to extract the
+        // data size from the fixed size buffer on each cell.
+        //
+        // We packed this information last, so the last packed
+        // object in the fixed size buffer corresponds to the
+        // variable data sizes.
+        //
+        // The last entry of sizes_fixed_cumulative corresponds
+        // to the size of all fixed size data packed on the cell.
+        // To get the offset for the last packed object, we need
+        // to get the next-to-last entry.
+        const unsigned int offset_variable_data_sizes =
+          sizes_fixed_cumulative[sizes_fixed_cumulative.size() - 2];
+
+        // This iterator points to the data size that the
+        // callback_function packed for each specific cell.
+        // Adjust buffer iterator to the offset of the callback
+        // function so that we only have to advance its position
+        // to the next cell after each iteration.
+        dest_sizes_cell_it = dest_data_fixed.cbegin() +
+                             offset_variable_data_sizes +
+                             callback_index * sizeof(unsigned int);
+
+        // Let the data iterator point to the correct buffer.
+        dest_data_it = dest_data_variable.cbegin();
+      }
+    else
+      {
+        // For the fixed size data, we can get the information about
+        // the buffer location on each cell directly from the
+        // sizes_fixed_cumulative vector.
+        offset         = sizes_fixed_cumulative[callback_index];
+        size           = sizes_fixed_cumulative[callback_index + 1] - offset;
+        data_increment = sizes_fixed_cumulative.back();
+
+        // Let the data iterator point to the correct buffer.
+        // Adjust buffer iterator to the offset of the callback
+        // function so that we only have to advance its position
+        // to the next cell after each iteration.
+        if (cell_relations.begin() != cell_relations.end())
+          dest_data_it = dest_data_fixed.cbegin() + offset;
+      }
+
+    // Iterate over all cells and unpack the transferred data.
+    auto cell_rel_it   = cell_relations.begin();
+    auto dest_sizes_it = dest_sizes_variable.cbegin();
+    for (; cell_rel_it != cell_relations.end(); ++cell_rel_it)
+      {
+        const auto &dealii_cell = cell_rel_it->first;
+        const auto &cell_status = cell_rel_it->second;
+
+        if (callback_variable_transfer)
+          {
+            // Update the increment according to the whole data size
+            // of the current cell.
+            data_increment = *dest_sizes_it;
+
+            if (cell_status != CellStatus::cell_invalid)
+              {
+                // Extract the corresponding values for offset and size from
+                // the cumulative sizes array stored in the fixed size
+                // buffer.
+                if (callback_index == 0)
+                  offset = 0;
+                else
+                  std::memcpy(&offset,
+                              &(*(dest_sizes_cell_it - sizeof(unsigned int))),
+                              sizeof(unsigned int));
+
+                std::memcpy(&size,
+                            &(*dest_sizes_cell_it),
+                            sizeof(unsigned int));
+
+                size -= offset;
+
+                // Move the data iterator to the corresponding position
+                // of the callback function and adjust the increment
+                // accordingly.
+                dest_data_it += offset;
+                data_increment -= offset;
+              }
+
+            // Advance data size iterators to the next cell, avoid iterating
+            // past the end of dest_sizes_cell_it
+            if (cell_rel_it != cell_relations.end() - 1)
+              dest_sizes_cell_it += sizes_fixed_cumulative.back();
+            ++dest_sizes_it;
+          }
+
+        switch (cell_status)
+          {
+            case CellStatus::cell_will_persist:
+            case CellStatus::children_will_be_coarsened:
+              unpack_callback(dealii_cell,
+                              cell_status,
+                              boost::make_iterator_range(dest_data_it,
+                                                         dest_data_it + size));
+              break;
+
+            case CellStatus::cell_will_be_refined:
+              unpack_callback(dealii_cell->parent(),
+                              cell_status,
+                              boost::make_iterator_range(dest_data_it,
+                                                         dest_data_it + size));
+              break;
+
+            case CellStatus::cell_invalid:
+              // Skip this cell.
+              break;
+
+            default:
+              Assert(false, ExcInternalError());
+              break;
+          }
+
+        if (cell_rel_it != cell_relations.end() - 1)
+          dest_data_it += data_increment;
+      }
+  }
+
+
+
+  template <int dim, int spacedim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  void CellAttachedDataSerializer<dim, spacedim>::save(
+    const unsigned int global_first_cell,
+    const unsigned int global_num_cells,
+    const std::string &filename,
+    const MPI_Comm &   mpi_communicator) const
+  {
+    Assert(sizes_fixed_cumulative.size() > 0,
+           ExcMessage("No data has been packed!"));
+
+#ifdef DEAL_II_WITH_MPI
+    // Large fractions of this function have been copied from
+    // DataOutInterface::write_vtu_in_parallel.
+    // TODO: Write general MPIIO interface.
+
+    const int myrank  = Utilities::MPI::this_mpi_process(mpi_communicator);
+    const int mpisize = Utilities::MPI::n_mpi_processes(mpi_communicator);
+
+    if (mpisize > 1)
+      {
+        const unsigned int bytes_per_cell = sizes_fixed_cumulative.back();
+
+        //
+        // ---------- Fixed size data ----------
+        //
+        {
+          const std::string fname_fixed = std::string(filename) + "_fixed.data";
+
+          MPI_Info info;
+          int      ierr = MPI_Info_create(&info);
+          AssertThrowMPI(ierr);
+
+          MPI_File fh;
+          ierr = MPI_File_open(mpi_communicator,
+                               fname_fixed.c_str(),
+                               MPI_MODE_CREATE | MPI_MODE_WRONLY,
+                               info,
+                               &fh);
+          AssertThrowMPI(ierr);
+
+          ierr = MPI_File_set_size(fh, 0); // delete the file contents
+          AssertThrowMPI(ierr);
+          // this barrier is necessary, because otherwise others might already
+          // write while one core is still setting the size to zero.
+          ierr = MPI_Barrier(mpi_communicator);
+          AssertThrowMPI(ierr);
+          ierr = MPI_Info_free(&info);
+          AssertThrowMPI(ierr);
+          // ------------------
+
+          // Write cumulative sizes to file.
+          // Since each processor owns the same information about the data
+          // sizes, it is sufficient to let only the first processor perform
+          // this task.
+          if (myrank == 0)
+            {
+              ierr = Utilities::MPI::LargeCount::File_write_at_c(
+                fh,
+                0,
+                sizes_fixed_cumulative.data(),
+                sizes_fixed_cumulative.size(),
+                MPI_UNSIGNED,
+                MPI_STATUS_IGNORE);
+              AssertThrowMPI(ierr);
+            }
+
+          // Write packed data to file simultaneously.
+          const MPI_Offset size_header =
+            sizes_fixed_cumulative.size() * sizeof(unsigned int);
+
+          // Make sure we do the following computation in 64bit integers to be
+          // able to handle 4GB+ files:
+          const MPI_Offset my_global_file_position =
+            size_header +
+            static_cast<MPI_Offset>(global_first_cell) * bytes_per_cell;
+
+          ierr =
+            Utilities::MPI::LargeCount::File_write_at_c(fh,
+                                                        my_global_file_position,
+                                                        src_data_fixed.data(),
+                                                        src_data_fixed.size(),
+                                                        MPI_BYTE,
+                                                        MPI_STATUS_IGNORE);
+          AssertThrowMPI(ierr);
+
+          ierr = MPI_File_close(&fh);
+          AssertThrowMPI(ierr);
+        }
+
+
+
+        //
+        // ---------- Variable size data ----------
+        //
+        if (variable_size_data_stored)
+          {
+            const std::string fname_variable =
+              std::string(filename) + "_variable.data";
+
+            MPI_Info info;
+            int      ierr = MPI_Info_create(&info);
+            AssertThrowMPI(ierr);
+
+            MPI_File fh;
+            ierr = MPI_File_open(mpi_communicator,
+                                 fname_variable.c_str(),
+                                 MPI_MODE_CREATE | MPI_MODE_WRONLY,
+                                 info,
+                                 &fh);
+            AssertThrowMPI(ierr);
+
+            ierr = MPI_File_set_size(fh, 0); // delete the file contents
+            AssertThrowMPI(ierr);
+            // this barrier is necessary, because otherwise others might already
+            // write while one core is still setting the size to zero.
+            ierr = MPI_Barrier(mpi_communicator);
+            AssertThrowMPI(ierr);
+            ierr = MPI_Info_free(&info);
+            AssertThrowMPI(ierr);
+
+            // Write sizes of each cell into file simultaneously.
+            {
+              const MPI_Offset my_global_file_position =
+                static_cast<MPI_Offset>(global_first_cell) *
+                sizeof(unsigned int);
+
+              // It is very unlikely that a single process has more than
+              // 2 billion cells, but we might as well check.
+              AssertThrow(src_sizes_variable.size() <
+                            static_cast<std::size_t>(
+                              std::numeric_limits<int>::max()),
+                          ExcNotImplemented());
+
+              ierr = Utilities::MPI::LargeCount::File_write_at_c(
+                fh,
+                my_global_file_position,
+                src_sizes_variable.data(),
+                src_sizes_variable.size(),
+                MPI_INT,
+                MPI_STATUS_IGNORE);
+              AssertThrowMPI(ierr);
+            }
+
+            // Gather size of data in bytes we want to store from this
+            // processor and compute the prefix sum. We do this in 64 bit
+            // to avoid overflow for files larger than 4GB:
+            const std::uint64_t size_on_proc = src_data_variable.size();
+            std::uint64_t       prefix_sum   = 0;
+            ierr                             = MPI_Exscan(&size_on_proc,
+                              &prefix_sum,
+                              1,
+                              MPI_UINT64_T,
+                              MPI_SUM,
+                              mpi_communicator);
+            AssertThrowMPI(ierr);
+
+            const MPI_Offset my_global_file_position =
+              static_cast<MPI_Offset>(global_num_cells) * sizeof(unsigned int) +
+              prefix_sum;
+
+            // Write data consecutively into file.
+            ierr = Utilities::MPI::LargeCount::File_write_at_c(
+              fh,
+              my_global_file_position,
+              src_data_variable.data(),
+              src_data_variable.size(),
+              MPI_BYTE,
+              MPI_STATUS_IGNORE);
+            AssertThrowMPI(ierr);
+
+
+            ierr = MPI_File_close(&fh);
+            AssertThrowMPI(ierr);
+          }
+      } // if (mpisize > 1)
+    else
+#endif
+      {
+        (void)global_first_cell;
+        (void)global_num_cells;
+        (void)mpi_communicator;
+
+        //
+        // ---------- Fixed size data ----------
+        //
+        {
+          const std::string fname_fixed = std::string(filename) + "_fixed.data";
+
+          std::ofstream file(fname_fixed, std::ios::binary | std::ios::out);
+
+          // Write header data.
+          file.write(reinterpret_cast<const char *>(
+                       sizes_fixed_cumulative.data()),
+                     sizes_fixed_cumulative.size() * sizeof(unsigned int));
+
+          // Write packed data.
+          file.write(reinterpret_cast<const char *>(src_data_fixed.data()),
+                     src_data_fixed.size() * sizeof(char));
+
+          file.close();
+        }
+
+        //
+        // ---------- Variable size data ----------
+        //
+        if (variable_size_data_stored)
+          {
+            const std::string fname_variable =
+              std::string(filename) + "_variable.data";
+
+            std::ofstream file(fname_variable,
+                               std::ios::binary | std::ios::out);
+
+            // Write header data.
+            file.write(reinterpret_cast<const char *>(
+                         src_sizes_variable.data()),
+                       src_sizes_variable.size() * sizeof(int));
+
+            // Write packed data.
+            file.write(reinterpret_cast<const char *>(src_data_variable.data()),
+                       src_data_variable.size() * sizeof(char));
+
+            file.close();
+          }
+      }
+  }
+
+
+  template <int dim, int spacedim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  void CellAttachedDataSerializer<dim, spacedim>::load(
+    const unsigned int global_first_cell,
+    const unsigned int global_num_cells,
+    const unsigned int local_num_cells,
+    const std::string &filename,
+    const unsigned int n_attached_deserialize_fixed,
+    const unsigned int n_attached_deserialize_variable,
+    const MPI_Comm &   mpi_communicator)
+  {
+    Assert(dest_data_fixed.size() == 0,
+           ExcMessage("Previously loaded data has not been released yet!"));
+
+    variable_size_data_stored = (n_attached_deserialize_variable > 0);
+
+#ifdef DEAL_II_WITH_MPI
+    // Large fractions of this function have been copied from
+    // DataOutInterface::write_vtu_in_parallel.
+    // TODO: Write general MPIIO interface.
+
+    const int mpisize = Utilities::MPI::n_mpi_processes(mpi_communicator);
+
+    if (mpisize > 1)
+      {
+        //
+        // ---------- Fixed size data ----------
+        //
+        {
+          const std::string fname_fixed = std::string(filename) + "_fixed.data";
+
+          MPI_Info info;
+          int      ierr = MPI_Info_create(&info);
+          AssertThrowMPI(ierr);
+
+          MPI_File fh;
+          ierr = MPI_File_open(
+            mpi_communicator, fname_fixed.c_str(), MPI_MODE_RDONLY, info, &fh);
+          AssertThrowMPI(ierr);
+
+          ierr = MPI_Info_free(&info);
+          AssertThrowMPI(ierr);
+
+          // Read cumulative sizes from file.
+          // Since all processors need the same information about the data
+          // sizes, let each of them retrieve it by reading from the same
+          // location in the file.
+          sizes_fixed_cumulative.resize(1 + n_attached_deserialize_fixed +
+                                        (variable_size_data_stored ? 1 : 0));
+          ierr = Utilities::MPI::LargeCount::File_read_at_c(
+            fh,
+            0,
+            sizes_fixed_cumulative.data(),
+            sizes_fixed_cumulative.size(),
+            MPI_UNSIGNED,
+            MPI_STATUS_IGNORE);
+          AssertThrowMPI(ierr);
+
+          // Allocate sufficient memory.
+          const unsigned int bytes_per_cell = sizes_fixed_cumulative.back();
+          dest_data_fixed.resize(static_cast<size_t>(local_num_cells) *
+                                 bytes_per_cell);
+
+          // Read packed data from file simultaneously.
+          const MPI_Offset size_header =
+            sizes_fixed_cumulative.size() * sizeof(unsigned int);
+
+          // Make sure we do the following computation in 64bit integers to be
+          // able to handle 4GB+ files:
+          const MPI_Offset my_global_file_position =
+            size_header +
+            static_cast<MPI_Offset>(global_first_cell) * bytes_per_cell;
+
+          ierr =
+            Utilities::MPI::LargeCount::File_read_at_c(fh,
+                                                       my_global_file_position,
+                                                       dest_data_fixed.data(),
+                                                       dest_data_fixed.size(),
+                                                       MPI_BYTE,
+                                                       MPI_STATUS_IGNORE);
+          AssertThrowMPI(ierr);
+
+
+          ierr = MPI_File_close(&fh);
+          AssertThrowMPI(ierr);
+        }
+
+        //
+        // ---------- Variable size data ----------
+        //
+        if (variable_size_data_stored)
+          {
+            const std::string fname_variable =
+              std::string(filename) + "_variable.data";
+
+            MPI_Info info;
+            int      ierr = MPI_Info_create(&info);
+            AssertThrowMPI(ierr);
+
+            MPI_File fh;
+            ierr = MPI_File_open(mpi_communicator,
+                                 fname_variable.c_str(),
+                                 MPI_MODE_RDONLY,
+                                 info,
+                                 &fh);
+            AssertThrowMPI(ierr);
+
+            ierr = MPI_Info_free(&info);
+            AssertThrowMPI(ierr);
+
+            // Read sizes of all locally owned cells.
+            dest_sizes_variable.resize(local_num_cells);
+
+            const MPI_Offset my_global_file_position_sizes =
+              static_cast<MPI_Offset>(global_first_cell) * sizeof(unsigned int);
+
+            ierr = Utilities::MPI::LargeCount::File_read_at_c(
+              fh,
+              my_global_file_position_sizes,
+              dest_sizes_variable.data(),
+              dest_sizes_variable.size(),
+              MPI_INT,
+              MPI_STATUS_IGNORE);
+            AssertThrowMPI(ierr);
+
+
+            // Compute my data size in bytes and compute prefix sum. We do this
+            // in 64 bit to avoid overflow for files larger than 4 GB:
+            const std::uint64_t size_on_proc =
+              std::accumulate(dest_sizes_variable.begin(),
+                              dest_sizes_variable.end(),
+                              0ULL);
+
+            std::uint64_t prefix_sum = 0;
+            ierr                     = MPI_Exscan(&size_on_proc,
+                              &prefix_sum,
+                              1,
+                              MPI_UINT64_T,
+                              MPI_SUM,
+                              mpi_communicator);
+            AssertThrowMPI(ierr);
+
+            const MPI_Offset my_global_file_position =
+              static_cast<MPI_Offset>(global_num_cells) * sizeof(unsigned int) +
+              prefix_sum;
+
+            dest_data_variable.resize(size_on_proc);
+
+            ierr = Utilities::MPI::LargeCount::File_read_at_c(
+              fh,
+              my_global_file_position,
+              dest_data_variable.data(),
+              dest_data_variable.size(),
+              MPI_BYTE,
+              MPI_STATUS_IGNORE);
+            AssertThrowMPI(ierr);
+
+            ierr = MPI_File_close(&fh);
+            AssertThrowMPI(ierr);
+          }
+      }
+    else // if (mpisize > 1)
+#endif
+      {
+        (void)global_first_cell;
+        (void)global_num_cells;
+        (void)mpi_communicator;
+
+        //
+        // ---------- Fixed size data ----------
+        //
+        {
+          const std::string fname_fixed = std::string(filename) + "_fixed.data";
+
+          std::ifstream file(fname_fixed, std::ios::binary | std::ios::in);
+          sizes_fixed_cumulative.resize(1 + n_attached_deserialize_fixed +
+                                        (variable_size_data_stored ? 1 : 0));
+
+          // Read header data.
+          file.read(reinterpret_cast<char *>(sizes_fixed_cumulative.data()),
+                    sizes_fixed_cumulative.size() * sizeof(unsigned int));
+
+          const unsigned int bytes_per_cell = sizes_fixed_cumulative.back();
+          dest_data_fixed.resize(static_cast<size_t>(local_num_cells) *
+                                 bytes_per_cell);
+
+          // Read packed data.
+          file.read(reinterpret_cast<char *>(dest_data_fixed.data()),
+                    dest_data_fixed.size() * sizeof(char));
+
+          file.close();
+        }
+
+        //
+        // ---------- Variable size data ----------
+        //
+        if (variable_size_data_stored)
+          {
+            const std::string fname_variable =
+              std::string(filename) + "_variable.data";
+
+            std::ifstream file(fname_variable, std::ios::binary | std::ios::in);
+
+            // Read header data.
+            dest_sizes_variable.resize(local_num_cells);
+            file.read(reinterpret_cast<char *>(dest_sizes_variable.data()),
+                      dest_sizes_variable.size() * sizeof(int));
+
+            // Read packed data.
+            const std::uint64_t size =
+              std::accumulate(dest_sizes_variable.begin(),
+                              dest_sizes_variable.end(),
+                              0ULL);
+            dest_data_variable.resize(size);
+            file.read(reinterpret_cast<char *>(dest_data_variable.data()),
+                      dest_data_variable.size() * sizeof(char));
+
+            file.close();
+          }
+      }
+  }
+
+
+  template <int dim, int spacedim>
+  DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>))
+  void CellAttachedDataSerializer<dim, spacedim>::clear()
+  {
+    variable_size_data_stored = false;
+
+    // free information about data sizes
+    sizes_fixed_cumulative.clear();
+    sizes_fixed_cumulative.shrink_to_fit();
+
+    // free fixed size transfer data
+    src_data_fixed.clear();
+    src_data_fixed.shrink_to_fit();
+
+    dest_data_fixed.clear();
+    dest_data_fixed.shrink_to_fit();
+
+    // free variable size transfer data
+    src_sizes_variable.clear();
+    src_sizes_variable.shrink_to_fit();
+
+    src_data_variable.clear();
+    src_data_variable.shrink_to_fit();
+
+    dest_sizes_variable.clear();
+    dest_sizes_variable.shrink_to_fit();
+
+    dest_data_variable.clear();
+    dest_data_variable.shrink_to_fit();
+  }
+
 } // namespace internal
 
 // anonymous namespace for internal helper functions

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.