From: Timo Heister Date: Sat, 20 Nov 2021 03:02:51 +0000 (-0500) Subject: fix checkpointing with >2GB per process X-Git-Tag: v9.4.0-rc1~807^2 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F12973%2Fhead;p=dealii.git fix checkpointing with >2GB per process This fixes save/load of fixed and variable checkpointing where individual ranks write more than 2GBs of data. Part of #12873 and #12752 --- diff --git a/source/distributed/tria_base.cc b/source/distributed/tria_base.cc index 971288f4b1..2831cae40a 100644 --- a/source/distributed/tria_base.cc +++ b/source/distributed/tria_base.cc @@ -1451,13 +1451,34 @@ namespace parallel size_header + static_cast(global_first_cell) * bytes_per_cell; - ierr = MPI_File_write_at(fh, - my_global_file_position, - DEAL_II_MPI_CONST_CAST(src_data_fixed.data()), - src_data_fixed.size(), - MPI_CHAR, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); + if (src_data_fixed.size() <= + static_cast(std::numeric_limits::max())) + { + ierr = + MPI_File_write_at(fh, + my_global_file_position, + DEAL_II_MPI_CONST_CAST(src_data_fixed.data()), + src_data_fixed.size(), + MPI_BYTE, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + } + else + { + // Writes bigger than 2GB require some extra care: + MPI_Datatype bigtype = + Utilities::MPI::create_mpi_data_type_n_bytes(src_data_fixed.size()); + ierr = + MPI_File_write_at(fh, + my_global_file_position, + DEAL_II_MPI_CONST_CAST(src_data_fixed.data()), + 1, + bigtype, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + ierr = MPI_Type_free(&bigtype); + AssertThrowMPI(ierr); + } ierr = MPI_File_close(&fh); AssertThrowMPI(ierr); @@ -1497,6 +1518,13 @@ namespace parallel const MPI_Offset my_global_file_position = static_cast(global_first_cell) * sizeof(unsigned int); + // It is very unlikely that a single process has more than 2 billion + // cells, but we might as well check. + AssertThrow(src_sizes_variable.size() < + static_cast( + std::numeric_limits::max()), + ExcNotImplemented()); + ierr = MPI_File_write_at(fh, my_global_file_position, @@ -1525,14 +1553,35 @@ namespace parallel prefix_sum; // Write data consecutively into file. - ierr = - MPI_File_write_at(fh, - my_global_file_position, - DEAL_II_MPI_CONST_CAST(src_data_variable.data()), - src_data_variable.size(), - MPI_CHAR, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); + if (src_data_variable.size() <= + static_cast(std::numeric_limits::max())) + { + ierr = MPI_File_write_at(fh, + my_global_file_position, + DEAL_II_MPI_CONST_CAST( + src_data_variable.data()), + src_data_variable.size(), + MPI_BYTE, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + } + else + { + // Writes bigger than 2GB require some extra care: + MPI_Datatype bigtype = Utilities::MPI::create_mpi_data_type_n_bytes( + src_data_variable.size()); + ierr = MPI_File_write_at(fh, + my_global_file_position, + DEAL_II_MPI_CONST_CAST( + src_data_variable.data()), + 1, + bigtype, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + ierr = MPI_Type_free(&bigtype); + AssertThrowMPI(ierr); + } + ierr = MPI_File_close(&fh); AssertThrowMPI(ierr); @@ -1618,13 +1667,32 @@ namespace parallel size_header + static_cast(global_first_cell) * bytes_per_cell; - ierr = MPI_File_read_at(fh, - my_global_file_position, - dest_data_fixed.data(), - dest_data_fixed.size(), // local buffer - MPI_CHAR, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); + if (dest_data_fixed.size() <= + static_cast(std::numeric_limits::max())) + { + ierr = MPI_File_read_at(fh, + my_global_file_position, + dest_data_fixed.data(), + dest_data_fixed.size(), + MPI_BYTE, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + } + else + { + // Reads bigger than 2GB require some extra care: + MPI_Datatype bigtype = Utilities::MPI::create_mpi_data_type_n_bytes( + dest_data_fixed.size()); + ierr = MPI_File_read_at(fh, + my_global_file_position, + dest_data_fixed.data(), + 1, + bigtype, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + ierr = MPI_Type_free(&bigtype); + AssertThrowMPI(ierr); + } ierr = MPI_File_close(&fh); AssertThrowMPI(ierr); @@ -1673,7 +1741,7 @@ namespace parallel const std::uint64_t size_on_proc = std::accumulate(dest_sizes_variable.begin(), dest_sizes_variable.end(), - 0); + 0ULL); std::uint64_t prefix_sum = 0; ierr = MPI_Exscan(DEAL_II_MPI_CONST_CAST(&size_on_proc), @@ -1689,13 +1757,34 @@ namespace parallel prefix_sum; dest_data_variable.resize(size_on_proc); - ierr = MPI_File_read_at(fh, - my_global_file_position, - dest_data_variable.data(), - dest_data_variable.size(), - MPI_CHAR, - MPI_STATUS_IGNORE); - AssertThrowMPI(ierr); + + if (dest_data_variable.size() <= + static_cast(std::numeric_limits::max())) + { + ierr = MPI_File_read_at(fh, + my_global_file_position, + dest_data_variable.data(), + dest_data_variable.size(), + MPI_BYTE, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + } + else + { + // Reads bigger than 2GB require some extra care: + MPI_Datatype bigtype = Utilities::MPI::create_mpi_data_type_n_bytes( + src_data_fixed.size()); + ierr = MPI_File_read_at(fh, + my_global_file_position, + dest_data_variable.data(), + 1, + bigtype, + MPI_STATUS_IGNORE); + AssertThrowMPI(ierr); + ierr = MPI_Type_free(&bigtype); + AssertThrowMPI(ierr); + } + ierr = MPI_File_close(&fh); AssertThrowMPI(ierr); diff --git a/tests/distributed_grids/checkpointing_02.cc b/tests/distributed_grids/checkpointing_02.cc index e3704d7ad9..ec5cb6d2dc 100644 --- a/tests/distributed_grids/checkpointing_02.cc +++ b/tests/distributed_grids/checkpointing_02.cc @@ -17,7 +17,11 @@ // Test (de)serialization of Trilinos vectors with checkpointing files >4GB. The // test here is of course simplified to run quickly. -// set to true to run a test that generates a 5GB file: +// Set this to true to run a test that generates an 8GB file. Run with +// 5 MPI ranks to check correctness with a total file size above 4GB. +// Run with 2 MPI ranks to test a single process above 2GB. Both cases were +// broken before this test was made. Warning, you probably need in the +// order of 32GB of RAM to run this. const bool big = false; #include @@ -156,7 +160,7 @@ LaplaceProblem::run(unsigned int n_cycles_global, setup_system(); - const unsigned int n_vectors = (big) ? 50 : 2; + const unsigned int n_vectors = (big) ? 70 : 2; { deallog << "checkpointing..." << std::endl; std::vector vectors(n_vectors);