From 9558b2c5f1da6a50cd63789e041f23c2c2092135 Mon Sep 17 00:00:00 2001
From: Benjamin Brands <benjamin.brands@fau.de>
Date: Mon, 27 Aug 2018 19:25:22 +0200
Subject: [PATCH] follow-up to #7044

---
 tests/scalapack/scalapack_10_a.cc             | 119 ++++++++++++++++++
 ...alapack_10_a.with_hdf5=on.mpirun=11.output |   8 ++
 ...calapack_10_a.with_hdf5=on.mpirun=4.output |   8 ++
 3 files changed, 135 insertions(+)
 create mode 100644 tests/scalapack/scalapack_10_a.cc
 create mode 100644 tests/scalapack/scalapack_10_a.with_hdf5=on.mpirun=11.output
 create mode 100644 tests/scalapack/scalapack_10_a.with_hdf5=on.mpirun=4.output

diff --git a/tests/scalapack/scalapack_10_a.cc b/tests/scalapack/scalapack_10_a.cc
new file mode 100644
index 0000000000..4a63413178
--- /dev/null
+++ b/tests/scalapack/scalapack_10_a.cc
@@ -0,0 +1,119 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2017 - 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+#include "../lapack/create_matrix.h"
+#include "../tests.h"
+
+// A test for saving and loading of distributed ScaLAPACKMatrices.
+// By using a 1x1 grid a call to save_serial() and load_serial() is mimicked.
+// The ScaLAPACKMatrix is saved in parallel/serial and afterwards loaded
+// in serial/parallel and the content is compared.
+// A former bug (removed in #7044) in save_parallel and load_parallel caused
+// an inconsistency so that matrices saved in serial could not be read
+// correctly in parallel. This test checks for that interoperability.
+
+#include <deal.II/base/conditional_ostream.h>
+#include <deal.II/base/logstream.h>
+#include <deal.II/base/multithread_info.h>
+#include <deal.II/base/timer.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/scalapack.h>
+
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+
+
+template <typename NumberType>
+void
+test(const unsigned int size, const unsigned int block_size)
+{
+  const std::string filename_serial("scalapack_10_a_s_test.h5");
+  const std::string filename_parallel("scalapack_10_a_p_test.h5");
+
+  MPI_Comm           mpi_communicator(MPI_COMM_WORLD);
+  const unsigned int this_mpi_process(
+    Utilities::MPI::this_mpi_process(mpi_communicator));
+  ConditionalOStream pcout(std::cout, (this_mpi_process == 0));
+
+  // create FullMatrix and fill it
+  FullMatrix<NumberType> full(size);
+  unsigned int           count = 0;
+  for (unsigned int i = 0; i < size; ++i)
+    for (unsigned int j = 0; j < size; ++j, ++count)
+      full(i, j) = count;
+
+  // create 2d process grid
+  std::shared_ptr<Utilities::MPI::ProcessGrid> grid =
+    std::make_shared<Utilities::MPI::ProcessGrid>(
+      mpi_communicator, size, size, block_size, block_size);
+
+  // create process grid containing only one process
+  std::shared_ptr<Utilities::MPI::ProcessGrid> grid_one =
+    std::make_shared<Utilities::MPI::ProcessGrid>(mpi_communicator, 1, 1);
+
+  ScaLAPACKMatrix<NumberType> matrix(size, size, grid, block_size, block_size);
+  ScaLAPACKMatrix<NumberType> matrix_one(
+    size, size, grid, block_size, block_size);
+
+  matrix     = full;
+  matrix_one = full;
+
+  matrix.save(filename_parallel.c_str());
+  matrix_one.save(filename_serial.c_str());
+
+  matrix.load(filename_serial.c_str());
+  matrix_one.load(filename_parallel.c_str());
+
+  std::remove(filename_serial.c_str());
+  std::remove(filename_parallel.c_str());
+
+  FullMatrix<NumberType> copy_I(size), copy_II(size);
+  matrix.copy_to(copy_I);
+  matrix_one.copy_to(copy_II);
+  copy_I.add(-1, full);
+  copy_II.add(-1, full);
+
+  pcout << size << " " << block_size << std::endl;
+
+  if (copy_I.frobenius_norm() > 1e-12)
+    pcout << "norm of difference I: " << copy_I.frobenius_norm() << std::endl;
+  AssertThrow(copy_I.frobenius_norm() < 1e-12, ExcInternalError());
+
+  if (copy_II.frobenius_norm() > 1e-12)
+    pcout << "norm of difference II: " << copy_II.frobenius_norm() << std::endl;
+  AssertThrow(copy_II.frobenius_norm() < 1e-12, ExcInternalError());
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, numbers::invalid_unsigned_int);
+
+  const std::vector<unsigned int> sizes       = {{200, 300}};
+  const std::vector<unsigned int> block_sizes = {{16, 32}};
+
+  for (const auto &s : sizes)
+    for (const auto &b : block_sizes)
+      test<double>(s, b);
+
+  for (const auto &s : sizes)
+    for (const auto &b : block_sizes)
+      test<float>(s, b);
+}
diff --git a/tests/scalapack/scalapack_10_a.with_hdf5=on.mpirun=11.output b/tests/scalapack/scalapack_10_a.with_hdf5=on.mpirun=11.output
new file mode 100644
index 0000000000..355f7c822d
--- /dev/null
+++ b/tests/scalapack/scalapack_10_a.with_hdf5=on.mpirun=11.output
@@ -0,0 +1,8 @@
+200 16
+200 32
+300 16
+300 32
+200 16
+200 32
+300 16
+300 32
diff --git a/tests/scalapack/scalapack_10_a.with_hdf5=on.mpirun=4.output b/tests/scalapack/scalapack_10_a.with_hdf5=on.mpirun=4.output
new file mode 100644
index 0000000000..355f7c822d
--- /dev/null
+++ b/tests/scalapack/scalapack_10_a.with_hdf5=on.mpirun=4.output
@@ -0,0 +1,8 @@
+200 16
+200 32
+300 16
+300 32
+200 16
+200 32
+300 16
+300 32
-- 
2.39.5