From 54d6eb8db947caf29650a20e5f5eee61dbf341b2 Mon Sep 17 00:00:00 2001 From: Marc Fehling Date: Fri, 13 Apr 2018 16:39:45 -0600 Subject: [PATCH] Allow Utilities::pack to append data to existing buffer. Allow Utitilities::unpack to read from iterator range. Add test utilities_pack_unpack_05. --- include/deal.II/base/utilities.h | 110 ++++++++++++++++----- tests/base/utilities_pack_unpack_05.cc | 89 +++++++++++++++++ tests/base/utilities_pack_unpack_05.output | 6 ++ 3 files changed, 183 insertions(+), 22 deletions(-) create mode 100644 tests/base/utilities_pack_unpack_05.cc create mode 100644 tests/base/utilities_pack_unpack_05.output diff --git a/include/deal.II/base/utilities.h b/include/deal.II/base/utilities.h index 051662d6ee..6f0babfc1c 100644 --- a/include/deal.II/base/utilities.h +++ b/include/deal.II/base/utilities.h @@ -429,12 +429,27 @@ namespace Utilities /** * Given an arbitrary object of type T, use boost::serialization utilities - * to pack the object into a vector of characters. The object can be unpacked - * using the Utilities::unpack function below. + * to pack the object into a vector of characters and append it to the + * given buffer. The number of elements that have been added to the buffer + * will be returned. The object can be unpacked using the Utilities::unpack + * function below. * * If the library has been compiled with ZLIB enabled, then the output buffer * is compressed. * + * If many consecutive calls with the same buffer are considered, it is + * recommended for reasons of performance to ensure that its capacity is + * sufficient. + * + * @author Timo Heister, Wolfgang Bangerth, 2017. + */ + template + size_t pack (const T &object, std::vector &dest_buffer); + + /** + * Creates and returns a buffer solely for the given object, using the + * above mentioned pack function. + * * @author Timo Heister, Wolfgang Bangerth, 2017. */ template @@ -471,6 +486,16 @@ namespace Utilities template T unpack (const std::vector &buffer); + /** + * Same unpack function as above, but takes constant iterators on + * (a fraction of) a given packed buffer of type std::vector instead. + * + * @author Timo Heister, Wolfgang Bangerth, 2017. + */ + template + T unpack (const std::vector::iterator &begin, + const std::vector::iterator &end); + /** * Given a vector of characters, obtained through a call to the function * Utilities::pack, restore its content in an array of type T. @@ -505,6 +530,17 @@ namespace Utilities void unpack (const std::vector &buffer, T (&unpacked_object)[N]); + /** + * Same unpack function as above, but takes constant iterators on + * (a fraction of) a given packed buffer of type std::vector instead. + * + * @author Timo Heister, Wolfgang Bangerth, 2017. + */ + template + void unpack (const std::vector::iterator &begin, + const std::vector::iterator &end, + T (&unpacked_object)[N]); + /** * Convert an object of type `std::unique_ptr` to an object of * type `std::unique_ptr`, where it is assumed that we can cast @@ -926,9 +962,10 @@ namespace Utilities } +// --------------------- non-inline functions template - std::vector pack(const T &object) + size_t pack (const T &object, std::vector &dest_buffer) { // see if the object is small and copyable via memcpy. if so, use // this fast path. otherwise, we have to go through the BOOST @@ -946,22 +983,25 @@ namespace Utilities && sizeof(T)<256) { - std::vector buffer (sizeof(T)); - std::memcpy (buffer.data(), &object, sizeof(T)); - return buffer; + const size_t previous_size = dest_buffer.size(); + dest_buffer.resize (previous_size + sizeof(T)); + + std::memcpy (dest_buffer.data() + previous_size, &object, sizeof(T)); + + return sizeof(T); } else { - // set up a buffer and then use it as the target of a compressing + // use buffer as the target of a compressing // stream into which we serialize the current object - std::vector buffer; + const size_t previous_size = dest_buffer.size(); { #ifdef DEAL_II_WITH_ZLIB boost::iostreams::filtering_ostream out; out.push(boost::iostreams::gzip_compressor (boost::iostreams::gzip_params (boost::iostreams::gzip::best_compression))); - out.push(boost::iostreams::back_inserter(buffer)); + out.push(boost::iostreams::back_inserter(dest_buffer)); boost::archive::binary_oarchive archive(out); archive << object; @@ -970,18 +1010,29 @@ namespace Utilities std::ostringstream out; boost::archive::binary_oarchive archive(out); archive << object; + const std::string &s = out.str(); - buffer.reserve(s.size()); - buffer.assign(s.begin(), s.end()); + dest_buffer.reserve (dest_buffer.size() + s.size()); + std::move (s.begin(), s.end(), std::back_inserter(dest_buffer)); #endif } - return buffer; + return (dest_buffer.size() - previous_size); } } template - T unpack(const std::vector &buffer) + std::vector pack (const T &object) + { + std::vector buffer; + pack (object, buffer); + return buffer; + } + + + template + T unpack (const std::vector::const_iterator &cbegin, + const std::vector::const_iterator &cend) { // see if the object is small and copyable via memcpy. if so, use // this fast path. otherwise, we have to go through the BOOST @@ -999,9 +1050,9 @@ namespace Utilities && sizeof(T)<256) { - Assert (buffer.size() == sizeof(T), ExcInternalError()); + Assert (std::distance(cbegin, cend) == sizeof(T), ExcInternalError()); T object; - std::memcpy (&object, buffer.data(), sizeof(T)); + std::memcpy (&object, &*cbegin, sizeof(T)); return object; } else @@ -1015,9 +1066,9 @@ namespace Utilities boost::iostreams::filtering_ostream decompressing_stream; decompressing_stream.push(boost::iostreams::gzip_decompressor()); decompressing_stream.push(boost::iostreams::back_inserter(decompressed_buffer)); - decompressing_stream.write (buffer.data(), buffer.size()); + decompressing_stream.write (&*cbegin, std::distance(cbegin, cend)); #else - decompressed_buffer.assign (buffer.begin(), buffer.end()); + decompressed_buffer.assign (cbegin, cend); #endif } @@ -1031,9 +1082,16 @@ namespace Utilities } + template + T unpack(const std::vector &buffer) + { + return unpack (buffer.cbegin(), buffer.cend()); + } + template - void unpack (const std::vector &buffer, + void unpack (const std::vector::const_iterator &cbegin, + const std::vector::const_iterator &cend, T (&unpacked_object)[N]) { // see if the object is small and copyable via memcpy. if so, use @@ -1052,8 +1110,8 @@ namespace Utilities && sizeof(T)*N<256) { - Assert (buffer.size() == sizeof(T)*N, ExcInternalError()); - std::memcpy (unpacked_object, buffer.data(), sizeof(T)*N); + Assert (std::distance(cbegin, cend) == sizeof(T)*N, ExcInternalError()); + std::memcpy (unpacked_object, &*cbegin, sizeof(T)*N); } else { @@ -1065,9 +1123,9 @@ namespace Utilities boost::iostreams::filtering_ostream decompressing_stream; decompressing_stream.push(boost::iostreams::gzip_decompressor()); decompressing_stream.push(boost::iostreams::back_inserter(decompressed_buffer)); - decompressing_stream.write (buffer.data(), buffer.size()); + decompressing_stream.write (&*cbegin, std::distance(cbegin, cend)); #else - decompressed_buffer.assign (buffer.begin(), buffer.end()); + decompressed_buffer.assign (cbegin, cend); #endif } @@ -1079,6 +1137,14 @@ namespace Utilities } } + + template + void unpack (const std::vector &buffer, + T (&unpacked_object)[N]) + { + unpack (buffer.cbegin(), buffer.cend(), unpacked_object); + } + } diff --git a/tests/base/utilities_pack_unpack_05.cc b/tests/base/utilities_pack_unpack_05.cc new file mode 100644 index 0000000000..a80eb1c2f0 --- /dev/null +++ b/tests/base/utilities_pack_unpack_05.cc @@ -0,0 +1,89 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2017 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE at +// the top level of the deal.II distribution. +// +// --------------------------------------------------------------------- + + +// test Utilities::pack/unpack on a consecutively built buffer of +// different types, here an array of doubles and a dealii::Point object. +// (based upon "utilities_pack_unpack_04") + + +#include "../tests.h" + +#include +#include + + +using namespace dealii; + + +template +void check (const double (&array)[N], const Point (&point)) +{ + std::vector buffer; + + // PACK BUFFER + // add first object to buffer and store buffer size for later separation + const size_t buffer_separator = Utilities::pack (array, buffer); + // add second object to buffer + Utilities::pack (point, buffer); + + // UNPACK BUFFER + double unpacked_array[N]; + Utilities::unpack (buffer.cbegin(), + buffer.cbegin()+buffer_separator, + unpacked_array); + Point unpacked_point = + Utilities::unpack> (buffer.cbegin()+buffer_separator, + buffer.cend()); + + // TEST RESULTS + bool equal_array = true; + for (unsigned int i=0; i p1 = random_point<3>(); + double x1[3] = { 1, 2, 3 }; + check (x1, p1); + + // now try much larger arrays that will actually be serialized + // using BOOST + const unsigned int N=10000; + Point p2 = random_point(); + double x2[N]; + for (unsigned int i=0; i