From fab39f18fa6bdad46be385e882c038d4f5b9cf3f Mon Sep 17 00:00:00 2001 From: heister Date: Thu, 1 Mar 2012 00:10:18 +0000 Subject: [PATCH] New: DataOut::write_vtu_in_parallel(). This routine uses MPI I/O to write a big vtu file in parallel. git-svn-id: https://svn.dealii.org/trunk@25195 0785d39b-7218-0410-832d-ea1e28bc413d --- deal.II/doc/news/changes.h | 5 + deal.II/include/deal.II/base/data_out_base.h | 48 ++++++ deal.II/source/base/data_out_base.cc | 161 ++++++++++++++----- 3 files changed, 173 insertions(+), 41 deletions(-) diff --git a/deal.II/doc/news/changes.h b/deal.II/doc/news/changes.h index d30b5b3b8b..177f59bbe2 100644 --- a/deal.II/doc/news/changes.h +++ b/deal.II/doc/news/changes.h @@ -145,6 +145,11 @@ enabled due to a missing include file in file

Specific improvements

    +
  1. New: DataOut::write_vtu_in_parallel(). This routine uses MPI I/O to write +a big vtu file in parallel. +
    +(Timo Heister, 2012/02/29) +
  2. Fixed: parallel::distributed::Triangulation::clear() forgot to update the number cache of this class, leading to wrong results when calling functions like diff --git a/deal.II/include/deal.II/base/data_out_base.h b/deal.II/include/deal.II/base/data_out_base.h index 887a23dd43..077a3e5622 100644 --- a/deal.II/include/deal.II/base/data_out_base.h +++ b/deal.II/include/deal.II/base/data_out_base.h @@ -22,6 +22,8 @@ #include #include +#include + // Only include the Tecplot API header if the appropriate files // were detected by configure #ifdef DEAL_II_HAVE_TECPLOT @@ -1723,6 +1725,35 @@ class DataOutBase const VtkFlags &flags, std::ostream &out); +/** + * This writes the header for the xml based vtu file format. This + * routine is used internally together with + * DataOutInterface::write_vtu_footer() and DataOutInterface::write_vtu_main() + * by DataOutBase::write_vtu(). + */ + static void write_vtu_header (std::ostream &out); + +/** + * This writes the footer for the xml based vtu file format. This + * routine is used internally together with + * DataOutInterface::write_vtu_header() and DataOutInterface::write_vtu_main() + * by DataOutBase::write_vtu(). + */ + static void write_vtu_footer (std::ostream &out); + +/** + * This writes the main part for the xml based vtu file format. This + * routine is used internally together with + * DataOutInterface::write_vtu_header() and DataOutInterface::write_vtu_footer() + * by DataOutBase::write_vtu(). + */ + template + static void write_vtu_main (const std::vector > &patches, + const std::vector &data_names, + const std::vector > &vector_data_ranges, + const VtkFlags &flags, + std::ostream &out); + /** * Write the given list of patches to the output stream in deal.II * intermediate format. This is not a format understood by any other @@ -1995,6 +2026,8 @@ class DataOutBase write_gmv_reorder_data_vectors (const std::vector > &patches, Table<2,double> &data_vectors); + + }; @@ -2253,6 +2286,21 @@ class DataOutInterface : private DataOutBase */ void write_vtu (std::ostream &out) const; + /** + * Collective MPI call to write the + * solution from all participating nodes + * (those in the given communicator) to a + * single compressed .vtu file on a + * shared file system. The communicator + * can be a sub communicator of the one + * used by the computation. This routine + * uses MPI I/O to achieve high + * performance on parallel filesystems. + * Also see + * DataOutInterface::write_vtu(). + */ + void write_vtu_in_parallel (const char* filename, MPI_Comm comm) const; + /** * Some visualization programs, such as * ParaView, can read several separate diff --git a/deal.II/source/base/data_out_base.cc b/deal.II/source/base/data_out_base.cc index 70290c412f..ebbf91020f 100644 --- a/deal.II/source/base/data_out_base.cc +++ b/deal.II/source/base/data_out_base.cc @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -41,8 +42,8 @@ #include #include #include - #include +#include #ifdef HAVE_LIBZ # include @@ -4710,13 +4711,66 @@ DataOutBase::write_vtk (const std::vector > &patches, } +void DataOutBase::write_vtu_header (std::ostream &out) +{ + AssertThrow (out, ExcIO()); + std::time_t time1= std::time (0); + std::tm *time = std::localtime(&time1); + out << " \n"; + out << "\n"; + + out << ""; + out << '\n'; + out << ""; + out << '\n'; +} + +void DataOutBase::write_vtu_footer (std::ostream &out) +{ + AssertThrow (out, ExcIO()); + out << " \n"; + out << "\n"; +} + template void DataOutBase::write_vtu (const std::vector > &patches, - const std::vector &data_names, - const std::vector > &vector_data_ranges, - const VtkFlags &flags, - std::ostream &out) + const std::vector &data_names, + const std::vector > &vector_data_ranges, + const VtkFlags &flags, + std::ostream &out) +{ + write_vtu_header(out); + write_vtu_main (patches, data_names, vector_data_ranges, flags, out); + write_vtu_footer(out); +} + + +template +void DataOutBase::write_vtu_main (const std::vector > &patches, + const std::vector &data_names, + const std::vector > &vector_data_ranges, + const VtkFlags &flags, + std::ostream &out) { AssertThrow (out, ExcIO()); @@ -4756,40 +4810,6 @@ DataOutBase::write_vtu (const std::vector > &patches, n_data_sets, patches[0].data.n_rows())); - /////////////////////// - // preamble - if (true) - { - std::time_t time1= std::time (0); - std::tm *time = std::localtime(&time1); - out << " \n"; - out << "\n"; - - out << ""; - out << '\n'; - out << ""; - out << '\n'; - } - #ifdef HAVE_LIBZ const char *ascii_or_binary = "binary"; @@ -5019,8 +5039,7 @@ DataOutBase::write_vtu (const std::vector > &patches, // Finish up writing a valid XML file out << " \n"; - out << " \n"; - out << "\n"; + // make sure everything now gets to // disk out.flush (); @@ -5254,6 +5273,66 @@ void DataOutInterface::write_vtu (std::ostream &out) const vtk_flags, out); } +template +void DataOutInterface::write_vtu_in_parallel (const char* filename, MPI_Comm comm) const +{ +#ifndef DEAL_II_COMPILER_SUPPORTS_MPI + //without MPI fall back to the normal way to write a vtu file: + (void)comm; + + std::ofstream f(filename); + write_vtu (f); +#else + + int myrank, nproc; + MPI_Comm_rank(comm, &myrank); + MPI_Comm_size(comm, &nproc); + + MPI_Info info; + MPI_Info_create(&info); + MPI_File fh; + MPI_File_open(MPI_COMM_WORLD, const_cast(filename), + MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &fh); + MPI_File_set_size(fh, 0); // delete the file contents + // this barrier is necessary, because otherwise others might already + // write while one core is still setting the size to zero. + MPI_Barrier(comm); + MPI_Info_free(&info); + + unsigned int header_size; + + //write header + if (myrank==0) + { + std::stringstream ss; + DataOutBase::write_vtu_header(ss); + header_size = ss.str().size(); + MPI_File_write(fh, const_cast(ss.str().c_str()), header_size, MPI_CHAR, NULL); + } + + MPI_Bcast(&header_size, 1, MPI_INT, 0, comm); + + MPI_File_seek_shared( fh, header_size, MPI_SEEK_SET ); + { + std::stringstream ss; + DataOutBase::write_vtu_main (get_patches(), get_dataset_names(), + get_vector_data_ranges(), + vtk_flags, ss); + MPI_File_write_ordered(fh, const_cast(ss.str().c_str()), ss.str().size(), MPI_CHAR, NULL); + } + + //write footer + if (myrank==0) + { + std::stringstream ss; + DataOutBase::write_vtu_footer(ss); + unsigned int footer_size = ss.str().size(); + MPI_File_write_shared(fh, const_cast(ss.str().c_str()), footer_size, MPI_CHAR, NULL); + } + MPI_File_close( &fh ); +#endif +} + template -- 2.39.5