From: Wolfgang Bangerth Date: Mon, 19 Jun 2017 11:46:40 +0000 (-0600) Subject: Simplify the logic of CellDataTransferBuffer::(un)pack_data(). X-Git-Tag: v9.0.0-rc1~1485^2 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F4528%2Fhead;p=dealii.git Simplify the logic of CellDataTransferBuffer::(un)pack_data(). Specifically, instead of serializing by hand with memcpy etc, use the BOOST facilities for serialization. This works for 2 out of 3 members of this class without problem, and the last one is easy to deal with as well. Secondly, serialize into a gzip compressed stream to reduce the amount of data to transfer. This reduces the size of messages significantly; for example, in the last cycle of step-40, we get the following data -- old packet size on the left, new (compressed) data size on the right: 6400 1694 8440 2200 8440 2412 6400 1925 7828 2042 6060 1589 1504 583 1504 520 6060 1849 7828 2280 6060 1947 7828 2375 1504 612 76 102 76 104 6400 2025 8440 2537 I did also try to run things with BOOST's bzip2 compressor instead of gzip, given that bzip2 often does a better job than gzip, but apparently that isn't the case here as I get the following: 6400 1912 8440 2381 8440 2712 6400 2230 7828 2197 1504 681 6060 2177 6060 1803 1504 556 7828 2678 6060 2285 7828 2738 1504 720 76 122 76 123 6400 2392 8440 2896 --- diff --git a/source/dofs/dof_handler_policy.cc b/source/dofs/dof_handler_policy.cc index c105f5b648..8bf34a0918 100644 --- a/source/dofs/dof_handler_policy.cc +++ b/source/dofs/dof_handler_policy.cc @@ -28,6 +28,15 @@ #include #include +DEAL_II_DISABLE_EXTRA_DIAGNOSTICS +#include +#include +#include +#include +#include +#include +DEAL_II_ENABLE_EXTRA_DIAGNOSTICS + #include #include #include @@ -1252,82 +1261,104 @@ namespace internal } - std::vector - pack_data () const + /** + * Write the data of this object to a stream for the purpose of + * serialization. + */ + template + void save (Archive &ar, + const unsigned int /*version*/) const { - std::vector buffer (bytes_for_buffer()); + // we would like to directly serialize the 'quadrants' vector, + // but the element type is internal to p4est and does not + // know how to serialize itself. consequently, first copy it over + // to an array of bytes, and then serialize that + std::vector quadrants_as_chars (sizeof(quadrants[0]) * quadrants.size()); + std::memcpy(&quadrants_as_chars[0], + &quadrants[0], + quadrants_as_chars.size()); - char *ptr = &buffer[0]; + // now serialize everything + ar &quadrants_as_chars + &tree_index + &dof_numbers_and_indices; + } - const unsigned int num_cells = tree_index.size(); - std::memcpy(ptr, &num_cells, sizeof(unsigned int)); - ptr += sizeof(unsigned int); + /** + * Read the data of this object from a stream for the purpose of + * serialization. Throw away the previous content. + */ + template + void load (Archive &ar, + const unsigned int /*version*/) + { + // undo the copying trick from the 'save' function + std::vector quadrants_as_chars; + ar &quadrants_as_chars + &tree_index + &dof_numbers_and_indices; - const unsigned int num_dofs = dof_numbers_and_indices.size(); - std::memcpy(ptr, &num_dofs, sizeof(unsigned int)); - ptr += sizeof(unsigned int); + quadrants.resize (quadrants_as_chars.size() / sizeof(quadrants[0])); + std::memcpy(&quadrants[0], + &quadrants_as_chars[0], + quadrants_as_chars.size()); + } - std::memcpy(ptr, - &tree_index[0], - num_cells*sizeof(unsigned int)); - ptr += num_cells*sizeof(unsigned int); + BOOST_SERIALIZATION_SPLIT_MEMBER() - std::memcpy(ptr, - &quadrants[0], - num_cells * sizeof(typename dealii::internal::p4est:: - types::quadrant)); - ptr += num_cells*sizeof(typename dealii::internal::p4est::types:: - quadrant); - if (num_dofs>0) - std::memcpy(ptr, - &dof_numbers_and_indices[0], - dof_numbers_and_indices.size() * sizeof(dealii::types::global_dof_index)); - ptr += dof_numbers_and_indices.size() * sizeof(dealii::types::global_dof_index); + /** + * Pack the data that corresponds to this object into a buffer in + * the form of a vector of chars and return it. + */ + std::vector + pack_data () const + { + // set up a buffer and then use it as the target of a compressing + // stream into which we serialize the current object + std::vector buffer; + { + boost::iostreams::filtering_ostream out; + out.push(boost::iostreams::gzip_compressor + (boost::iostreams::gzip_params + (boost::iostreams::gzip::best_compression))); + out.push(boost::iostreams::back_inserter(buffer)); + + boost::archive::binary_oarchive archive(out); - Assert (ptr == &buffer[0]+buffer.size(), - ExcInternalError()); + archive << *this; + out.flush(); + } return buffer; } - void unpack_data(const std::vector &buffer) + /** + * Given a buffer in the form of an array of chars, unpack it and + * restore the current object to the state that it was when + * it was packed into said buffer by the pack_data() function. + */ + void unpack_data (const std::vector &buffer) { - const char *ptr = &buffer[0]; - unsigned int num_cells; - memcpy(&num_cells, ptr, sizeof(unsigned int)); - ptr+=sizeof(unsigned int); - - unsigned int num_dofs; - memcpy(&num_dofs, ptr, sizeof(unsigned int)); - ptr+=sizeof(unsigned int); - - tree_index.resize(num_cells); - std::memcpy(&tree_index[0], - ptr, - num_cells*sizeof(unsigned int)); - ptr += num_cells*sizeof(unsigned int); - - quadrants.resize(num_cells); - std::memcpy(&quadrants[0], - ptr, - num_cells * sizeof(typename dealii::internal::p4est:: - types::quadrant)); - ptr += num_cells*sizeof(typename dealii::internal::p4est::types:: - quadrant); - - dof_numbers_and_indices.resize(num_dofs); - if (num_dofs>0) - std::memcpy(&dof_numbers_and_indices[0], - ptr, - dof_numbers_and_indices.size() * sizeof(dealii::types::global_dof_index)); - ptr += dof_numbers_and_indices.size() * sizeof(dealii::types::global_dof_index); - - Assert (ptr == &buffer[0]+buffer.size(), - ExcInternalError()); - } + std::string decompressed_buffer; + // first decompress the buffer + { + boost::iostreams::filtering_ostream decompressing_stream; + decompressing_stream.push(boost::iostreams::gzip_decompressor()); + decompressing_stream.push(boost::iostreams::back_inserter(decompressed_buffer)); + + for (const auto p : buffer) + decompressing_stream << p; + } + + // then restore the object from the buffer + std::istringstream in(decompressed_buffer); + boost::archive::binary_iarchive archive(in); + + archive >> *this; + } };