]> https://gitweb.dealii.org/ - dealii.git/commitdiff
Tests compiling an mostly working
authorDaniel Arndt <arndtd@ornl.gov>
Wed, 7 Dec 2022 20:04:54 +0000 (20:04 +0000)
committerDaniel Arndt <arndtd@ornl.gov>
Thu, 8 Dec 2022 22:13:15 +0000 (17:13 -0500)
44 files changed:
include/deal.II/lac/read_write_vector.templates.h
source/grid/grid_tools.cc
source/lac/read_write_vector.cc
source/particles/particle_handler.cc
tests/lac/parallel_vector_01.cc [new file with mode: 0644]
tests/lac/parallel_vector_01.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_02.cc [new file with mode: 0644]
tests/lac/parallel_vector_02.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_03.cc [new file with mode: 0644]
tests/lac/parallel_vector_03.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_03a.cc [new file with mode: 0644]
tests/lac/parallel_vector_03a.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_04.cc [new file with mode: 0644]
tests/lac/parallel_vector_04.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_05.cc [new file with mode: 0644]
tests/lac/parallel_vector_05.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_06.cc [new file with mode: 0644]
tests/lac/parallel_vector_06.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_08.cc [new file with mode: 0644]
tests/lac/parallel_vector_08.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_10.cc [new file with mode: 0644]
tests/lac/parallel_vector_10.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_11.cc [new file with mode: 0644]
tests/lac/parallel_vector_11.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_12.cc [new file with mode: 0644]
tests/lac/parallel_vector_12.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_13.cc [new file with mode: 0644]
tests/lac/parallel_vector_13.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_14.cc [new file with mode: 0644]
tests/lac/parallel_vector_14.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_15.cc [new file with mode: 0644]
tests/lac/parallel_vector_15.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_16.cc [new file with mode: 0644]
tests/lac/parallel_vector_16.with_64bit_indices=on.mpirun=4.output [new file with mode: 0644]
tests/lac/parallel_vector_21.cc [new file with mode: 0644]
tests/lac/parallel_vector_21.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_22.cc [new file with mode: 0644]
tests/lac/parallel_vector_22.mpirun=4.output [new file with mode: 0644]
tests/lac/parallel_vector_23.cc [new file with mode: 0644]
tests/lac/parallel_vector_23.mpirun=4.output [new file with mode: 0644]
tests/lac/parallel_vector_25.cc [new file with mode: 0644]
tests/lac/parallel_vector_25.mpirun=2.output [new file with mode: 0644]
tests/lac/parallel_vector_26.cc [new file with mode: 0644]
tests/lac/parallel_vector_26.mpirun=2.output [new file with mode: 0644]

index d1b9d6c070666daf6b5d31af1fe114ee7b066489..040ccd4ee24a90bd28526e1f48185ad6fbb3aede 100644 (file)
@@ -109,8 +109,8 @@ namespace LinearAlgebra
 
         static_assert(
           std::is_same<MemorySpace, ::dealii::MemorySpace::Host>::value ||
-            std::is_same<MemorySpace, ::dealii::MemorySpace::CUDA>::value,
-          "MemorySpace should be Host or CUDA");
+            std::is_same<MemorySpace, ::dealii::MemorySpace::Device>::value,
+          "MemorySpace should be Host or Device");
       }
     };
 
@@ -160,9 +160,8 @@ namespace LinearAlgebra
 
 
 
-#ifdef DEAL_II_COMPILER_CUDA_AWARE
     template <typename Number>
-    struct read_write_vector_functions<Number, ::dealii::MemorySpace::CUDA>
+    struct read_write_vector_functions<Number, ::dealii::MemorySpace::Device>
     {
       using size_type = types::global_dof_index;
 
@@ -178,11 +177,7 @@ namespace LinearAlgebra
 
         const unsigned int n_elements =
           communication_pattern->locally_owned_size();
-        cudaError_t cuda_error_code = cudaMemcpy(tmp_vector.begin(),
-                                                 values,
-                                                 n_elements * sizeof(Number),
-                                                 cudaMemcpyDeviceToHost);
-        AssertCuda(cuda_error_code);
+       Kokkos::deep_copy(Kokkos::View<Number*, Kokkos::HostSpace>(tmp_vector.begin(), n_elements), Kokkos::View<const Number*, ::dealii::MemorySpace::Device::kokkos_space>(values, n_elements));
         tmp_vector.update_ghost_values();
 
         const IndexSet &stored = rw_vector.get_stored_elements();
@@ -205,7 +200,6 @@ namespace LinearAlgebra
             rw_vector.local_element(i) = tmp_vector(stored.nth_index_in_set(i));
       }
     };
-#endif
   } // namespace internal
 
 
@@ -894,7 +888,7 @@ namespace LinearAlgebra
 
 
 
-#ifdef DEAL_II_COMPILER_CUDA_AWARE
+#ifdef DEAL_II_WITH_CUDA
   template <typename Number>
   void
   ReadWriteVector<Number>::import(
index dc2aa6242ae79d109a3b25b279ca7d145c9e66cd..6821c57fe52099163a8879c86a753bbc068aa69a 100644 (file)
@@ -2963,30 +2963,7 @@ namespace GridTools
           {
             if (!used_vertices_rtree.empty())
               {
-                // If we have an rtree at our disposal, use it.
-                using ValueType = std::pair<Point<spacedim>, unsigned int>;
-                std::function<bool(const ValueType &)> marked;
-                if (marked_vertices.size() == mesh.n_vertices())
-                  marked = [&marked_vertices](const ValueType &value) -> bool {
-                    return marked_vertices[value.second];
-                  };
-                else
-                  marked = [](const ValueType &) -> bool { return true; };
-
-                std::vector<std::pair<Point<spacedim>, unsigned int>> res;
-                used_vertices_rtree.query(
-                  boost::geometry::index::nearest(p, 1) &&
-                    boost::geometry::index::satisfies(marked),
-                  std::back_inserter(res));
-
-                // Searching for a point which is located outside the
-                // triangulation results in res.size() = 0
-                Assert(res.size() < 2,
-                       dealii::ExcMessage("There can not be multiple results"));
-
-                if (res.size() > 0)
-                  if (any_cell_marked(vertex_to_cells[res[0].second]))
-                    closest_vertex_index = res[0].second;
+               Assert(false, ExcInternalError());
               }
             else
               {
index f1ca4fee7ce39a71acb04e44e265a0c18c9ecf93..3b77c05a6bfab9e5e214591cb2b0a7245c3a9e86 100644 (file)
@@ -46,12 +46,22 @@ namespace LinearAlgebra
     const distributed::Vector<float, ::dealii::MemorySpace::Host> &,
     VectorOperation::values,
     const std::shared_ptr<const Utilities::MPI::CommunicationPatternBase> &);
+  template void
+  ReadWriteVector<float>::import(
+    const distributed::Vector<float, ::dealii::MemorySpace::CUDA> &,
+    VectorOperation::values,
+    const std::shared_ptr<const Utilities::MPI::CommunicationPatternBase> &);
 
   template void
   ReadWriteVector<double>::import(
     const distributed::Vector<double, ::dealii::MemorySpace::Host> &,
     VectorOperation::values,
     const std::shared_ptr<const Utilities::MPI::CommunicationPatternBase> &);
+  template void
+  ReadWriteVector<double>::import(
+    const distributed::Vector<double, ::dealii::MemorySpace::CUDA> &,
+    VectorOperation::values,
+    const std::shared_ptr<const Utilities::MPI::CommunicationPatternBase> &);
 #  ifdef DEAL_II_WITH_COMPLEX_VALUES
   template void
   ReadWriteVector<std::complex<float>>::import(
index e55fe4da7f58ecf8c38a163df108106a4f20c441..ffa7c14d7a00919f0dc6db58c6f3f0dfbb210b0b 100644 (file)
@@ -1395,37 +1395,7 @@ namespace Particles
 
           if (!found_cell)
             {
-              // The particle is not in a neighbor of the old cell.
-              // Look for the new cell in the whole local domain.
-              // This case is rare.
-              std::vector<std::pair<Point<spacedim>, unsigned int>>
-                closest_vertex_in_domain;
-              triangulation_cache->get_used_vertices_rtree().query(
-                boost::geometry::index::nearest(out_particle->get_location(),
-                                                1),
-                std::back_inserter(closest_vertex_in_domain));
-
-              // We should have one and only one result
-              AssertDimension(closest_vertex_in_domain.size(), 1);
-              const unsigned int closest_vertex_index_in_domain =
-                closest_vertex_in_domain[0].second;
-
-              // Search all of the cells adjacent to the closest vertex of the
-              // domain. Most likely we will find the particle in them.
-              for (const auto &cell :
-                   vertex_to_cells[closest_vertex_index_in_domain])
-                {
-                  mapping->transform_points_real_to_unit_cell(
-                    cell, real_locations, reference_locations);
-
-                  if (GeometryInfo<dim>::is_inside_unit_cell(
-                        reference_locations[0]))
-                    {
-                      current_cell = cell;
-                      found_cell   = true;
-                      break;
-                    }
-                }
+                   Assert(false, ExcInternalError());
             }
 
           if (!found_cell)
diff --git a/tests/lac/parallel_vector_01.cc b/tests/lac/parallel_vector_01.cc
new file mode 100644 (file)
index 0000000..3a6b820
--- /dev/null
@@ -0,0 +1,97 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check correct initialization of parallel vector without any ghosts
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_owned, MPI_COMM_WORLD);
+
+  // set local values
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  rw_vector(myid * 2)     = myid * 2.0;
+  rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+  v.import(rw_vector, VectorOperation::insert);
+
+  v *= 2.0;
+
+  rw_vector.import(v, VectorOperation::insert);
+  if (myid == 0)
+    {
+      deallog << myid * 2 << ":" << rw_vector(myid * 2) << std::endl;
+      deallog << myid * 2 + 1 << ":" << rw_vector(myid * 2 + 1) << std::endl;
+    }
+
+  Assert(rw_vector(myid * 2) == myid * 4.0, ExcInternalError());
+  Assert(rw_vector(myid * 2 + 1) == myid * 4.0 + 2.0, ExcInternalError());
+
+  // check l2 norm
+  const double l2_norm = v.l2_norm();
+  if (myid == 0)
+    deallog << "L2 norm: " << l2_norm << std::endl;
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+  
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_01.mpirun=2.output b/tests/lac/parallel_vector_01.mpirun=2.output
new file mode 100644 (file)
index 0000000..5f15965
--- /dev/null
@@ -0,0 +1,6 @@
+
+DEAL:0::numproc=2
+DEAL:0::0:0
+DEAL:0::1:2.000
+DEAL:0::L2 norm: 7.483
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_02.cc b/tests/lac/parallel_vector_02.cc
new file mode 100644 (file)
index 0000000..afd643c
--- /dev/null
@@ -0,0 +1,109 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check addition into ghosts for parallel vector
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+
+  // set local values and check them
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  rw_vector(myid * 2)     = myid * 2.0;
+  rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+  v.import(rw_vector, VectorOperation::add);
+
+  v *= 2.0;
+
+  rw_vector.import(v, VectorOperation::insert);
+  AssertThrow(rw_vector(myid * 2) == myid * 4.0, ExcInternalError());
+  AssertThrow(rw_vector(myid * 2 + 1) == myid * 4.0 + 2.0, ExcInternalError());
+
+  // set ghost dof, compress
+  LinearAlgebra::ReadWriteVector<double> rw_relevant_vector(numproc * 2);
+  rw_relevant_vector(1) = 7;
+  v.import(rw_relevant_vector, VectorOperation::add);
+
+  rw_vector.import(v, VectorOperation::insert);
+  if (myid == 0)
+    {
+      deallog << myid * 2 << ":" << rw_vector(myid * 2) << std::endl;
+      deallog << myid * 2 + 1 << ":" << rw_vector(myid * 2 + 1) << std::endl;
+    }
+
+  rw_relevant_vector.import(v, VectorOperation::insert);
+  AssertThrow(rw_relevant_vector(1) == 7. * numproc + 2, ExcInternalError());
+
+  // check l2 norm
+  const double l2_norm = v.l2_norm();
+  if (myid == 0)
+    deallog << "L2 norm: " << l2_norm << std::endl;
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_02.mpirun=2.output b/tests/lac/parallel_vector_02.mpirun=2.output
new file mode 100644 (file)
index 0000000..9e9690e
--- /dev/null
@@ -0,0 +1,6 @@
+
+DEAL:0::numproc=2
+DEAL:0::0:0
+DEAL:0::1:16.00
+DEAL:0::L2 norm: 17.55
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_03.cc b/tests/lac/parallel_vector_03.cc
new file mode 100644 (file)
index 0000000..5465361
--- /dev/null
@@ -0,0 +1,114 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check set of ghosts for parallel vector
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+
+  // set local values and check them
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  rw_vector(myid * 2)     = myid * 2.0;
+  rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+  v.import(rw_vector, VectorOperation::insert);
+
+  v *= 2.0;
+
+  rw_vector.import(v, VectorOperation::insert);
+  AssertThrow(rw_vector(myid * 2) == myid * 4.0, ExcInternalError());
+  AssertThrow(rw_vector(myid * 2 + 1) == myid * 4.0 + 2.0, ExcInternalError());
+
+  // set ghost dof on all processors, compress
+  // (insert mode)
+  IndexSet ghost_entry(numproc * 2);
+  ghost_entry.add_index(1);
+  LinearAlgebra::ReadWriteVector<double> rw_ghost_entry(ghost_entry);
+  rw_ghost_entry(1) = 7.;
+  v.import(rw_ghost_entry, VectorOperation::insert);
+
+  rw_vector.import(v, VectorOperation::insert);
+  if (myid == 0)
+    {
+      deallog << myid * 2 << ":" << rw_vector(myid * 2) << std::endl;
+      deallog << myid * 2 + 1 << ":" << rw_vector(myid * 2 + 1) << std::endl;
+    }
+  // import ghosts onto all procs
+  v.update_ghost_values();
+  rw_ghost_entry(1) = 0.;
+  rw_ghost_entry.import(v, VectorOperation::insert);
+  AssertThrow(rw_ghost_entry(1) == 7.0, ExcInternalError());
+
+  // check l2 norm
+  const double l2_norm = v.l2_norm();
+  if (myid == 0)
+    deallog << "L2 norm: " << l2_norm << std::endl;
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_03.mpirun=2.output b/tests/lac/parallel_vector_03.mpirun=2.output
new file mode 100644 (file)
index 0000000..94ec6d8
--- /dev/null
@@ -0,0 +1,6 @@
+
+DEAL:0::numproc=2
+DEAL:0::0:0
+DEAL:0::1:7.000
+DEAL:0::L2 norm: 10.05
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_03a.cc b/tests/lac/parallel_vector_03a.cc
new file mode 100644 (file)
index 0000000..ed1ebfa
--- /dev/null
@@ -0,0 +1,168 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// similar to parallel_sparse_vector_03.cc, but make sure
+// compress(insert) zeroes out ghosts in Release mode
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <fstream>
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+void
+check(const unsigned int myid, const LinearAlgebra::ReadWriteVector<double> &v)
+{
+  if (myid == 0)
+    {
+      AssertThrow(v(10) == 10.0, ExcInternalError());
+      AssertThrow(v(11) == 0., ExcInternalError());
+      AssertThrow(v(12) == 0., ExcInternalError());
+      AssertThrow(v(14) == 14., ExcInternalError());
+
+      AssertThrow(v(5) == 55., ExcInternalError());
+    }
+  else
+    {
+      AssertThrow(v(4) == 0., ExcInternalError());
+      AssertThrow(v(5) == 55., ExcInternalError());
+      AssertThrow(v(6) == 66., ExcInternalError());
+    }
+}
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  Assert(numproc == 2, ExcNotImplemented());
+
+  const unsigned int size = 20;
+  IndexSet           local_owned(size);
+  IndexSet           local_nonzero(size);
+  IndexSet           local_relevant(size);
+  if (myid == 0)
+    {
+      local_owned.add_range(0, 10);
+      local_nonzero.add_range(5, 10);
+      local_relevant = local_owned;
+      local_relevant.add_range(10, 13);
+      local_relevant.add_range(14, 15);
+    }
+  else
+    {
+      local_owned.add_range(10, size);
+      local_nonzero.add_range(10, 11);
+      local_nonzero.add_range(13, 15);
+      local_relevant = local_owned;
+      local_relevant.add_range(4, 7);
+    }
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+  v = 0.;
+
+  // set local values
+  IndexSet indexset_1(local_owned);
+  indexset_1.add_index(5);
+  indexset_1.compress();
+  LinearAlgebra::ReadWriteVector<double> rw_vector(indexset_1);
+  for (unsigned int i = 0; i < local_nonzero.n_elements(); ++i)
+    rw_vector(local_nonzero.nth_index_in_set(i)) =
+      local_nonzero.nth_index_in_set(i);
+
+  // set value from processor which does not own it:
+  rw_vector(5) = 55.;
+  v.import(rw_vector, VectorOperation::insert);
+
+  // add to value from processor which has it as a ghost
+  // Because of limitation in import, the ReadWriteVector needs to have locally
+  // owned elements
+  IndexSet                               indexset_2(local_owned.size());
+  LinearAlgebra::ReadWriteVector<double> rw_add;
+  if (myid == 1)
+    {
+      indexset_2.add_index(6);
+      rw_add.reinit(indexset_2);
+      rw_add(6) = 60;
+    }
+  else
+    {
+      rw_add.reinit(indexset_2);
+    }
+  v.import(rw_add, VectorOperation::add); // 60 + 6
+  // compress(insert) used to leave ghosts un-touched which resulted in
+  // the wrong 55+55 for this compress(add) operation.
+
+  v.update_ghost_values();
+
+  IndexSet indexset_3(size);
+  if (myid == 0)
+    {
+      indexset_3.add_index(10);
+      indexset_3.add_index(11);
+      indexset_3.add_index(12);
+      indexset_3.add_index(14);
+
+      indexset_3.add_index(5);
+    }
+  else
+    {
+      indexset_3.add_index(4);
+      indexset_3.add_index(5);
+      indexset_3.add_index(6);
+    }
+
+  rw_vector.reinit(indexset_3);
+  rw_vector.import(v, VectorOperation::insert);
+
+  check(myid, rw_vector);
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      std::ofstream logfile("output");
+      deallog.attach(logfile);
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_03a.mpirun=2.output b/tests/lac/parallel_vector_03a.mpirun=2.output
new file mode 100644 (file)
index 0000000..be8d055
--- /dev/null
@@ -0,0 +1,2 @@
+
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_04.cc b/tests/lac/parallel_vector_04.cc
new file mode 100644 (file)
index 0000000..a3ccdb1
--- /dev/null
@@ -0,0 +1,163 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check that operator= resets ghosts, both if they have been set and if they
+// have not been set
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::Device> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+
+  // set local values and check them
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  rw_vector(myid * 2)     = myid * 2.0;
+  rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+
+  v.import(rw_vector, VectorOperation::insert);
+  v *= 2.0;
+
+  rw_vector.import(v, VectorOperation::insert);
+  AssertThrow(rw_vector(myid * 2) == myid * 4.0, ExcInternalError());
+  AssertThrow(rw_vector(myid * 2 + 1) == myid * 4.0 + 2.0, ExcInternalError());
+
+  // set ghost dof on remote process, no compress called. Since we don't want to
+  // call compress we cannot use import
+  auto partitioner = v.get_partitioner();
+  if (myid > 0)
+    {
+      unsigned int local_index = partitioner->global_to_local(1);
+      double *     values_dev  = v.get_values();
+      Kokkos::deep_copy(Kokkos::View<double, MemorySpace::Device::kokkos_space>(values_dev+local_index), 7);
+    }
+
+  unsigned int        allocated_size = local_relevant.n_elements();
+  Kokkos::View<double*, MemorySpace::Device::kokkos_space> v_device(v.get_values(), allocated_size);
+  Kokkos::View<double*, Kokkos::HostSpace> v_host("v_host", allocated_size);
+  Kokkos::deep_copy(v_host, v_device);
+
+  AssertThrow(v_host[partitioner->global_to_local(myid * 2)] == myid * 4.0,
+              ExcInternalError());
+  AssertThrow(v_host[partitioner->global_to_local(myid * 2 + 1)] ==
+                myid * 4.0 + 2.0,
+              ExcInternalError());
+
+  if (myid > 0)
+    AssertThrow(v_host[partitioner->global_to_local(1)] == 7.0,
+                ExcInternalError());
+
+  // reset to zero
+  v = 0;
+
+  Kokkos::deep_copy(v_host, v_device);
+  AssertThrow(v_host[partitioner->global_to_local(myid * 2)] == 0.,
+              ExcInternalError());
+  AssertThrow(v_host[partitioner->global_to_local(myid * 2 + 1)] == 0.,
+              ExcInternalError());
+
+  // check that everything remains zero also
+  // after compress
+  v.compress(VectorOperation::add);
+
+  Kokkos::deep_copy(v_host, v_device);
+  AssertThrow(v_host[partitioner->global_to_local(myid * 2)] == 0.,
+              ExcInternalError());
+  AssertThrow(v_host[partitioner->global_to_local(myid * 2 + 1)] == 0.,
+              ExcInternalError());
+
+  // set element 1 on owning process to
+  // something nonzero
+  if (myid == 0)
+    {
+      unsigned int local_index = partitioner->global_to_local(1);
+      double *     values_dev  = v.get_values();
+      Kokkos::deep_copy(Kokkos::subview(v_device, local_index), 2);
+    }
+  if (myid > 0)
+    {
+      Kokkos::deep_copy(v_host, v_device);
+      AssertThrow(v_host[partitioner->global_to_local(1)] == 0.,
+                  ExcInternalError());
+    }
+
+  // check that all processors get the correct
+  // value again, and that it is erased by
+  // operator=
+  v.update_ghost_values();
+
+  Kokkos::deep_copy(v_host, v_device);
+  AssertThrow(v_host[partitioner->global_to_local(1)] == 2.,
+              ExcInternalError());
+
+  v = 0;
+  Kokkos::deep_copy(v_host, v_device);
+  AssertThrow(v_host[partitioner->global_to_local(1)] == 0.,
+              ExcInternalError());
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_04.mpirun=2.output b/tests/lac/parallel_vector_04.mpirun=2.output
new file mode 100644 (file)
index 0000000..1cf39ab
--- /dev/null
@@ -0,0 +1,3 @@
+
+DEAL:0::numproc=2
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_05.cc b/tests/lac/parallel_vector_05.cc
new file mode 100644 (file)
index 0000000..7c42d01
--- /dev/null
@@ -0,0 +1,110 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check that compress(add) with zero add does not change the vector entry
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+
+  // set local values and check them
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  rw_vector(myid * 2)     = myid * 2.0;
+  rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+
+  v.import(rw_vector, VectorOperation::insert);
+  v *= 2.0;
+
+  rw_vector.import(v, VectorOperation::insert);
+  Assert(rw_vector(myid * 2) == myid * 4.0, ExcInternalError());
+  Assert(rw_vector(myid * 2 + 1) == myid * 4.0 + 2.0, ExcInternalError());
+
+  // set ghost dof to zero on remote processors,
+  // compress
+  IndexSet                               ghost_set(numproc * 2);
+  LinearAlgebra::ReadWriteVector<double> ghost_vector;
+  if (myid > 0)
+    {
+      ghost_set.add_index(1);
+      ghost_vector.reinit(ghost_set);
+      ghost_vector(1) = 0;
+    }
+  else
+    ghost_vector.reinit(ghost_set);
+
+
+  v.import(ghost_vector, VectorOperation::add);
+
+  // check that nothing has changed
+  rw_vector.import(v, VectorOperation::insert);
+  Assert(rw_vector(myid * 2) == myid * 4.0, ExcInternalError());
+  Assert(rw_vector(myid * 2 + 1) == myid * 4.0 + 2.0, ExcInternalError());
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_05.mpirun=2.output b/tests/lac/parallel_vector_05.mpirun=2.output
new file mode 100644 (file)
index 0000000..1cf39ab
--- /dev/null
@@ -0,0 +1,3 @@
+
+DEAL:0::numproc=2
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_06.cc b/tests/lac/parallel_vector_06.cc
new file mode 100644 (file)
index 0000000..3ac6c1b
--- /dev/null
@@ -0,0 +1,155 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check global reduction operation (norms, operator==, operator!=) on
+// parallel vector
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all processors are
+  // ghosting element 1 (the second)
+  IndexSet local_owned(std::min(16U, numproc * 2));
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_owned, MPI_COMM_WORLD);
+
+  // set local values
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  {
+    rw_vector(myid * 2)     = myid * 2.0;
+    rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+  }
+  v.import(rw_vector, VectorOperation::insert);
+  v *= 2.0;
+  {
+    rw_vector.import(v, VectorOperation::insert);
+    AssertThrow(rw_vector(myid * 2) == myid * 4.0, ExcInternalError());
+    AssertThrow(rw_vector(myid * 2 + 1) == myid * 4.0 + 2.0,
+                ExcInternalError());
+  }
+
+  // check l2 norm
+  {
+    const double l2_norm = v.l2_norm();
+    if (myid == 0)
+      deallog << "l2 norm: " << l2_norm << std::endl;
+  }
+
+  // check l1 norm
+  {
+    const double l1_norm = v.l1_norm();
+    if (myid == 0)
+      deallog << "l1 norm: " << l1_norm << std::endl;
+  }
+
+  // check linfty norm
+  {
+    const double linfty_norm = v.linfty_norm();
+    if (myid == 0)
+      deallog << "linfty norm: " << linfty_norm << std::endl;
+  }
+
+  // check mean value (should be equal to l1
+  // norm divided by vector size here since we
+  // have no negative entries)
+  {
+    const double mean = v.mean_value();
+    if (myid == 0)
+      deallog << "Mean value: " << mean << std::endl;
+
+    Assert(std::fabs(mean * v.size() - v.l1_norm()) < 1e-15,
+           ExcInternalError());
+  }
+  // check inner product
+  {
+    const double norm_sqr = v.l2_norm() * v.l2_norm();
+    AssertThrow(std::fabs(v * v - norm_sqr) < 1e-15, ExcInternalError());
+    LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v2;
+    v2 = v;
+    AssertThrow(std::fabs(v2 * v - norm_sqr) < 1e-15, ExcInternalError());
+
+    const double inner_prod = v * v2;
+    if (myid == 0)
+      deallog << "Inner product: " << inner_prod << std::endl;
+  }
+
+  // check all_zero
+  {
+    bool allzero = v.all_zero();
+    if (myid == 0)
+      deallog << " v==0 ? " << allzero << std::endl;
+    LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v2;
+    v2.reinit(v);
+    allzero = v2.all_zero();
+    if (myid == 0)
+      deallog << " v2==0 ? " << allzero << std::endl;
+
+    v2.import(rw_vector, VectorOperation::insert);
+    allzero = v2.all_zero();
+    if (myid == 0)
+      deallog << " v2==0 ? " << allzero << std::endl;
+  }
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_06.mpirun=2.output b/tests/lac/parallel_vector_06.mpirun=2.output
new file mode 100644 (file)
index 0000000..61a7663
--- /dev/null
@@ -0,0 +1,11 @@
+
+DEAL:0::numproc=2
+DEAL:0::l2 norm: 7.483
+DEAL:0::l1 norm: 12.00
+DEAL:0::linfty norm: 6.00
+DEAL:0::Mean value:3.000
+DEAL:0::Inner product: 56.00
+DEAL:0:: v==0 ? 0
+DEAL:0:: v2==0 ? 1
+DEAL:0:: v2==0 ? 0
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_08.cc b/tests/lac/parallel_vector_08.cc
new file mode 100644 (file)
index 0000000..75da428
--- /dev/null
@@ -0,0 +1,138 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check parallel_vector::copy_from to update ghost values. Same vector layout
+// as in parallel_vector_07.cc
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+  const unsigned int set = 200;
+  AssertIndexRange(numproc, set - 2);
+  const unsigned int local_size  = set - myid;
+  unsigned int       global_size = 0;
+  unsigned int       my_start    = 0;
+  for (unsigned int i = 0; i < numproc; ++i)
+    {
+      global_size += set - i;
+      if (i < myid)
+        my_start += set - i;
+    }
+  // each processor owns some indices and all
+  // are ghosting elements from three
+  // processors (the second). some entries
+  // are right around the border between two
+  // processors
+  IndexSet local_owned(global_size);
+  local_owned.add_range(my_start, my_start + local_size);
+  IndexSet local_relevant(global_size);
+  local_relevant                 = local_owned;
+  unsigned int ghost_indices[10] = {
+    1, 2, 13, set - 3, set - 2, set - 1, set, set + 1, set + 2, set + 3};
+  local_relevant.add_indices(&ghost_indices[0], &ghost_indices[0] + 10);
+
+  // v has ghosts, w has none. set some entries
+  // on w, copy into v and check if they are
+  // there
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> w(
+    local_owned, local_owned, MPI_COMM_WORLD);
+
+  // set a few of the local elements
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  for (unsigned i = 0; i < local_size; ++i)
+    rw_vector.local_element(i) = 2.0 * (i + my_start);
+  w.import(rw_vector, VectorOperation::insert);
+
+  v = w;
+  v.update_ghost_values();
+
+  // check local values for correctness
+  rw_vector.import(v, VectorOperation::insert);
+  for (unsigned int i = 0; i < local_size; ++i)
+    AssertThrow(rw_vector.local_element(i) == 2.0 * (i + my_start),
+                ExcInternalError());
+
+  // check non-local entries on all processors
+  LinearAlgebra::ReadWriteVector<double> ghost_vector(local_relevant);
+  ghost_vector.import(v, VectorOperation::insert);
+  for (unsigned int i = 0; i < 10; ++i)
+    AssertThrow(ghost_vector(ghost_indices[i]) == 2. * ghost_indices[i],
+                ExcInternalError());
+
+  // now the same again, but import ghosts automatically because v had ghosts
+  // set before calling operator =
+  v.reinit(local_owned, local_relevant, MPI_COMM_WORLD);
+  v.update_ghost_values();
+  v = w;
+
+  // check local values for correctness
+  rw_vector.import(v, VectorOperation::insert);
+  for (unsigned int i = 0; i < local_size; ++i)
+    AssertThrow(rw_vector.local_element(i) == 2.0 * (i + my_start),
+                ExcInternalError());
+
+  // check non-local entries on all processors
+  ghost_vector.import(v, VectorOperation::insert);
+  for (unsigned int i = 0; i < 10; ++i)
+    AssertThrow(ghost_vector(ghost_indices[i]) == 2. * ghost_indices[i],
+                ExcInternalError());
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_08.mpirun=2.output b/tests/lac/parallel_vector_08.mpirun=2.output
new file mode 100644 (file)
index 0000000..1cf39ab
--- /dev/null
@@ -0,0 +1,3 @@
+
+DEAL:0::numproc=2
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_10.cc b/tests/lac/parallel_vector_10.cc
new file mode 100644 (file)
index 0000000..3ef5938
--- /dev/null
@@ -0,0 +1,104 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check operator= when we do some operations with ghosts
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> w(v);
+
+  // set local values and check them
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  rw_vector(myid * 2)     = myid * 2.0;
+  rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+  v.import(rw_vector, VectorOperation::insert);
+
+  v.update_ghost_values();
+
+  // check that the value of the ghost is 1.0
+  IndexSet ghost_set(numproc * 2);
+  ghost_set.add_index(1);
+  LinearAlgebra::ReadWriteVector<double> ghost_vector(ghost_set);
+  ghost_vector.import(v, VectorOperation::insert);
+  AssertThrow(ghost_vector(1) == 1., ExcInternalError());
+
+  // copy vector
+  w = v;
+  v *= 2.0;
+
+  v.update_ghost_values();
+  w.update_ghost_values();
+  ghost_vector.import(v, VectorOperation::insert);
+  AssertThrow(ghost_vector(1) == 2., ExcInternalError());
+  ghost_vector.import(w, VectorOperation::insert);
+  AssertThrow(ghost_vector(1) == 1., ExcInternalError());
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_10.mpirun=2.output b/tests/lac/parallel_vector_10.mpirun=2.output
new file mode 100644 (file)
index 0000000..1cf39ab
--- /dev/null
@@ -0,0 +1,3 @@
+
+DEAL:0::numproc=2
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_11.cc b/tests/lac/parallel_vector_11.cc
new file mode 100644 (file)
index 0000000..592f1a0
--- /dev/null
@@ -0,0 +1,249 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check that add, sadd, equ, scale work correctly on a vector where some
+// processor do not own any degrees of freedom
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+  // global size: 20, local_size: 3 as long as
+  // less than 20
+  const unsigned int local_size  = 3;
+  const unsigned int global_size = std::min(20U, local_size * numproc);
+  const int          my_start    = std::min(local_size * myid, global_size);
+  const int          my_end = std::min(local_size * (myid + 1), global_size);
+  const int          actual_local_size = my_end - my_start;
+
+  IndexSet local_owned(global_size);
+  if (my_end > my_start)
+    local_owned.add_range(static_cast<unsigned int>(my_start),
+                          static_cast<unsigned int>(my_end));
+  IndexSet local_relevant(global_size);
+  local_relevant = local_owned;
+  local_relevant.add_index(2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+  AssertDimension(static_cast<unsigned int>(actual_local_size), v.local_size());
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> w(v), x(v),
+    y(v);
+
+  // set local elements
+  LinearAlgebra::ReadWriteVector<double> v_rw(local_owned);
+  LinearAlgebra::ReadWriteVector<double> w_rw(local_owned);
+  LinearAlgebra::ReadWriteVector<double> x_rw(local_owned);
+  for (int i = 0; i < actual_local_size; ++i)
+    {
+      v_rw.local_element(i) = i + my_start;
+      w_rw.local_element(i) = 1000 + 2 * (my_start + i);
+      x_rw.local_element(i) = 10000;
+    }
+  v.import(v_rw, VectorOperation::insert);
+  w.import(w_rw, VectorOperation::insert);
+  x.import(x_rw, VectorOperation::insert);
+
+  y = v;
+  LinearAlgebra::ReadWriteVector<double> y_rw(local_owned);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == i + my_start, ExcInternalError());
+
+  if (myid == 0)
+    deallog << "Check add (scalar): ";
+  y.add(42);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == i + my_start + 42, ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check add (vector): ";
+  y.add(1., w);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == 3 * (i + my_start) + 1042,
+                ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check add (factor, vector): ";
+  y.add(-1., w);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == i + my_start + 42, ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check add (factor, vector, factor, vector): ";
+  y.add(2., w, -0.5, x);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == 5 * (i + my_start) + 2042 - 5000,
+                ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check sadd (factor, factor, vector): ";
+  y = v;
+  y.sadd(-3., 2., v);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == (-i - my_start), ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check sadd (factor, factor, vector, factor, vector): ";
+  y.sadd(2., 3., v);
+  y.add(2., w);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    {
+      AssertThrow(y_rw.local_element(i) == 5 * (i + my_start) + 2000,
+                  ExcInternalError());
+    }
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog
+      << "Check sadd (factor, factor, vector, factor, vector, factor, vector): ";
+  y.sadd(-1., 1., v);
+  y.add(2., w);
+  y.add(2., x);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == 20000, ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check add (factor, vector_1, factor, vector_1): ";
+  y = 0;
+  y.add(1., v, 3., v);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == 4 * (i + my_start),
+                ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check operator * (scalar): ";
+  x *= 2.;
+  x_rw.import(x, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(x_rw.local_element(i) == 20000., ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check operator / (scalar): ";
+  x /= 2.;
+  x_rw.import(x, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(x_rw.local_element(i) == 10000., ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check scale (vector): ";
+  y.scale(x);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == 40000. * (i + my_start),
+                ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check equ (factor, vector): ";
+  y.equ(10., x);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == 100000., ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check equ (factor, vector, factor, vector): ";
+  y.equ(10., v);
+  y.add(-2., w);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == 6. * (i + my_start) - 2000,
+                ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+
+  if (myid == 0)
+    deallog << "Check equ (factor, vector, factor, vector, factor, vector): ";
+  y.equ(10., v);
+  y.add(-2., w);
+  y.add(3., x);
+  y_rw.import(y, VectorOperation::insert);
+  for (int i = 0; i < actual_local_size; ++i)
+    AssertThrow(y_rw.local_element(i) == 6. * (i + my_start) + 28000,
+                ExcInternalError());
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_11.mpirun=2.output b/tests/lac/parallel_vector_11.mpirun=2.output
new file mode 100644 (file)
index 0000000..631cc4d
--- /dev/null
@@ -0,0 +1,16 @@
+
+DEAL:0::numproc=2
+DEAL:0::Check add (scalar): OK
+DEAL:0::Check add (vector): OK
+DEAL:0::Check add (factor, vector): OK
+DEAL:0::Check add (factor, vector, factor, vector): OK
+DEAL:0::Check sadd (factor, factor, vector): OK
+DEAL:0::Check sadd (factor, factor, vector, factor, vector): OK
+DEAL:0::Check sadd (factor, factor, vector, factor, vector, factor, vector): OK
+DEAL:0::Check add (factor, vector_1, factor, vector_1): OK
+DEAL:0::Check operator * (scalar): OK
+DEAL:0::Check operator / (scalar): OK
+DEAL:0::Check scale (vector): OK
+DEAL:0::Check equ (factor, vector): OK
+DEAL:0::Check equ (factor, vector, factor, vector): OK
+DEAL:0::Check equ (factor, vector, factor, vector, factor, vector): OK
diff --git a/tests/lac/parallel_vector_12.cc b/tests/lac/parallel_vector_12.cc
new file mode 100644 (file)
index 0000000..9236cc0
--- /dev/null
@@ -0,0 +1,228 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check LinearAlgebra::distributed::Vector::swap
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+DeclException2(ExcNonEqual,
+               double,
+               double,
+               << "Left compare: " << arg1 << ", right compare: " << arg2);
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+  // vector 0:
+  // global size: 20, local_size: 3 as long as
+  // less than 20
+  const unsigned int local_size0  = 3;
+  const unsigned int global_size0 = std::min(20U, local_size0 * numproc);
+  const unsigned int my_start0    = std::min(local_size0 * myid, global_size0);
+  const unsigned int my_end0 = std::min(local_size0 * (myid + 1), global_size0);
+  const unsigned int actual_local_size0 = my_end0 - my_start0;
+
+  IndexSet local_owned0(global_size0);
+  if (my_end0 > my_start0)
+    local_owned0.add_range(static_cast<unsigned int>(my_start0),
+                           static_cast<unsigned int>(my_end0));
+  IndexSet local_relevant0(global_size0);
+  local_relevant0 = local_owned0;
+  local_relevant0.add_index(2);
+  if (numproc > 2)
+    local_relevant0.add_index(8);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v0(
+    local_owned0, local_relevant0, MPI_COMM_WORLD);
+
+  // vector1: local size 4
+  const unsigned int local_size1  = 4;
+  const unsigned int global_size1 = local_size1 * numproc;
+  const int          my_start1    = local_size1 * myid;
+  const int          my_end1      = local_size1 * (myid + 1);
+
+  IndexSet local_owned1(global_size1);
+  local_owned1.add_range(static_cast<unsigned int>(my_start1),
+                         static_cast<unsigned int>(my_end1));
+  IndexSet local_relevant1(global_size1);
+  local_relevant1 = local_owned1;
+  local_relevant1.add_index(0);
+  local_relevant1.add_index(2);
+  if (numproc > 2)
+    {
+      local_relevant1.add_index(8);
+      local_relevant1.add_index(10);
+    }
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v1(
+    local_owned1, local_relevant1, MPI_COMM_WORLD);
+
+  v0 = 1;
+  v1 = 2;
+  // check assignment in initial state
+  LinearAlgebra::ReadWriteVector<double> v0_rw(local_owned0);
+  v0_rw.import(v0, VectorOperation::insert);
+  LinearAlgebra::ReadWriteVector<double> v1_rw(local_owned1);
+  v1_rw.import(v1, VectorOperation::insert);
+  for (unsigned int i = 0; i < v0.local_size(); ++i)
+    AssertThrow(v0_rw.local_element(i) == 1.,
+                ExcNonEqual(v0_rw.local_element(i), 1.));
+  for (unsigned int i = 0; i < v1.local_size(); ++i)
+    AssertThrow(v1_rw.local_element(i) == 2.,
+                ExcNonEqual(v1_rw.local_element(i), 2.));
+
+  // check ghost elements in initial state
+  v0.update_ghost_values();
+  v1.update_ghost_values();
+  LinearAlgebra::ReadWriteVector<double> v0_ghost_rw(local_relevant0);
+  v0_ghost_rw.import(v0, VectorOperation::insert);
+  LinearAlgebra::ReadWriteVector<double> v1_ghost_rw(local_relevant1);
+  v1_ghost_rw.import(v1, VectorOperation::insert);
+  AssertThrow(v0_ghost_rw(2) == 1., ExcNonEqual(v0_ghost_rw(2), 1.));
+  if (numproc > 2)
+    AssertThrow(v0_ghost_rw(8) == 1., ExcNonEqual(v0_ghost_rw(8), 2.));
+  AssertThrow(v1_ghost_rw(0) == 2., ExcNonEqual(v1_ghost_rw(0), 2.));
+  AssertThrow(v1_ghost_rw(2) == 2., ExcNonEqual(v1_ghost_rw(2), 2.));
+  if (numproc > 2)
+    {
+      AssertThrow(v1_ghost_rw(8) == 2., ExcNonEqual(v1_ghost_rw(8), 2.));
+      AssertThrow(v1_ghost_rw(10) == 2., ExcNonEqual(v1_ghost_rw(10), 2.));
+    }
+  MPI_Barrier(MPI_COMM_WORLD);
+  if (myid == 0)
+    deallog << "Initial set and ghost update OK" << std::endl;
+
+  // now swap v1 and v0
+  v0.swap(v1);
+  AssertDimension(v0.local_size(), local_size1);
+  AssertDimension(v1.local_size(), actual_local_size0);
+  AssertDimension(v0.size(), global_size1);
+  AssertDimension(v1.size(), global_size0);
+  v1_rw.import(v0, VectorOperation::insert);
+  for (unsigned int i = 0; i < local_size1; ++i)
+    AssertThrow(v1_rw.local_element(i) == 2.,
+                ExcNonEqual(v1_rw.local_element(i), 2.));
+  v0_rw.import(v1, VectorOperation::insert);
+  for (unsigned int i = 0; i < actual_local_size0; ++i)
+    AssertThrow(v0_rw.local_element(i) == 1.,
+                ExcNonEqual(v0_rw.local_element(i), 1.));
+  MPI_Barrier(MPI_COMM_WORLD);
+  if (myid == 0)
+    deallog << "First swap OK" << std::endl;
+  v0.update_ghost_values();
+  v1.update_ghost_values();
+  v0_ghost_rw.import(v1, VectorOperation::insert);
+  AssertThrow(v0_ghost_rw(2) == 1., ExcNonEqual(v0_ghost_rw(2), 1.));
+  if (numproc > 2)
+    AssertThrow(v0_ghost_rw(8) == 1., ExcNonEqual(v0_ghost_rw(8), 1.));
+  v1_ghost_rw.import(v0, VectorOperation::insert);
+  AssertThrow(v1_ghost_rw(0) == 2., ExcNonEqual(v1_ghost_rw(0), 2.));
+  AssertThrow(v1_ghost_rw(2) == 2., ExcNonEqual(v1_ghost_rw(2), 2.));
+  if (numproc > 2)
+    {
+      AssertThrow(v1_ghost_rw(8) == 2., ExcNonEqual(v1_ghost_rw(8), 2.));
+      AssertThrow(v1_ghost_rw(10) == 2., ExcNonEqual(v1_ghost_rw(10), 2.));
+    }
+  MPI_Barrier(MPI_COMM_WORLD);
+  if (myid == 0)
+    deallog << "Ghost values after first swap OK" << std::endl;
+
+  // now set the vectors to some different
+  // values and check the ghost values again
+  v0 = 7.;
+  v1 = 42.;
+  v0.update_ghost_values();
+  v1.update_ghost_values();
+  v0_ghost_rw.import(v1, VectorOperation::insert);
+  AssertThrow(v0_ghost_rw(2) == 42., ExcNonEqual(v0_ghost_rw(2), 42.));
+  if (numproc > 2)
+    AssertThrow(v0_ghost_rw(8) == 42., ExcNonEqual(v0_ghost_rw(8), 42.));
+  v1_ghost_rw.import(v0, VectorOperation::insert);
+  AssertThrow(v1_ghost_rw(0) == 7., ExcNonEqual(v1_ghost_rw(0), 7.));
+  AssertThrow(v1_ghost_rw(2) == 7., ExcNonEqual(v1_ghost_rw(2), 7.));
+  if (numproc > 2)
+    {
+      AssertThrow(v1_ghost_rw(8) == 7., ExcNonEqual(v1_ghost_rw(8), 7.));
+      AssertThrow(v1_ghost_rw(10) == 7., ExcNonEqual(v1_ghost_rw(10), 7.));
+    }
+  MPI_Barrier(MPI_COMM_WORLD);
+  if (myid == 0)
+    deallog << "Ghost values after re-set OK" << std::endl;
+
+  // swap with an empty vector
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v2;
+  v2.swap(v0);
+  AssertDimension(v0.size(), 0);
+  AssertDimension(v2.size(), global_size1);
+  AssertDimension(v2.local_size(), local_size1);
+  v1_rw.import(v2, VectorOperation::insert);
+  for (int i = my_start1; i < my_end1; ++i)
+    AssertThrow(v1_rw(i) == 7., ExcNonEqual(v1_rw(i), 7.));
+  MPI_Barrier(MPI_COMM_WORLD);
+  if (myid == 0)
+    deallog << "Second swap OK" << std::endl;
+  v2 = -1.;
+  v2.update_ghost_values();
+  v1_ghost_rw.import(v2, VectorOperation::insert);
+  AssertThrow(v1_ghost_rw(0) == -1., ExcNonEqual(v1_ghost_rw(0), -1.));
+  AssertThrow(v1_ghost_rw(2) == -1., ExcNonEqual(v1_ghost_rw(2), -1.));
+  if (numproc > 2)
+    {
+      AssertThrow(v1_ghost_rw(8) == -1., ExcNonEqual(v1_ghost_rw(8), -1.));
+      AssertThrow(v1_ghost_rw(10) == -1., ExcNonEqual(v1_ghost_rw(10), -1.));
+    }
+  MPI_Barrier(MPI_COMM_WORLD);
+  if (myid == 0)
+    deallog << "Ghost values after second swap OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_12.mpirun=2.output b/tests/lac/parallel_vector_12.mpirun=2.output
new file mode 100644 (file)
index 0000000..ad4bdea
--- /dev/null
@@ -0,0 +1,8 @@
+
+DEAL:0::numproc=2
+DEAL:0::Initial set and ghost update OK
+DEAL:0::First swap OK
+DEAL:0::Ghost values after first swap OK
+DEAL:0::Ghost values after re-set OK
+DEAL:0::Second swap OK
+DEAL:0::Ghost values after second swap OK
diff --git a/tests/lac/parallel_vector_13.cc b/tests/lac/parallel_vector_13.cc
new file mode 100644 (file)
index 0000000..5c4a9d2
--- /dev/null
@@ -0,0 +1,125 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check for ghosts on parallel vector: similar to parallel_vector_03, but
+// setting where one ghost is zero and should not have an effect on vector
+// entries
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+
+  // set local values and check them
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  rw_vector(myid * 2)     = myid * 2.0;
+  rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+
+  v.import(rw_vector, VectorOperation::insert);
+  v *= 2.0;
+  v.add(1.0);
+
+  rw_vector.import(v, VectorOperation::insert);
+  AssertThrow(rw_vector(myid * 2) == myid * 4.0 + 1, ExcInternalError());
+  AssertThrow(rw_vector(myid * 2 + 1) == myid * 4.0 + 3.0, ExcInternalError());
+
+  // set ghost dof on all processors, compress
+  // (insert mode)
+  IndexSet index(numproc * 2);
+  index.add_index(1);
+  LinearAlgebra::ReadWriteVector<double> local_rw_vector(index);
+  local_rw_vector(1) = 7;
+  v.import(local_rw_vector, VectorOperation::insert);
+
+  {
+    rw_vector.import(v, VectorOperation::insert);
+    deallog << myid * 2 << ":" << rw_vector(myid * 2) << std::endl;
+    deallog << myid * 2 + 1 << ":" << rw_vector(myid * 2 + 1) << std::endl;
+  }
+
+  local_rw_vector(1) = -7;
+  v.import(local_rw_vector, VectorOperation::insert);
+
+  {
+    rw_vector.import(v, VectorOperation::insert);
+    deallog << myid * 2 << ":" << rw_vector(myid * 2) << std::endl;
+    deallog << myid * 2 + 1 << ":" << rw_vector(myid * 2 + 1) << std::endl;
+  }
+
+  // import ghosts onto all procs
+  v.update_ghost_values();
+  local_rw_vector.import(v, VectorOperation::insert);
+  AssertThrow(local_rw_vector(1) == -7.0, ExcInternalError());
+
+  // check l2 norm
+  const double l2_norm = v.l2_norm();
+  if (myid == 0)
+    deallog << "L2 norm: " << l2_norm << std::endl;
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_13.mpirun=2.output b/tests/lac/parallel_vector_13.mpirun=2.output
new file mode 100644 (file)
index 0000000..848c89c
--- /dev/null
@@ -0,0 +1,8 @@
+
+DEAL:0::numproc=2
+DEAL:0::0:1.000
+DEAL:0::1:7.000
+DEAL:0::0:1.000
+DEAL:0::1:-7.000
+DEAL:0::L2 norm: 11.14
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_14.cc b/tests/lac/parallel_vector_14.cc
new file mode 100644 (file)
index 0000000..c0e05a2
--- /dev/null
@@ -0,0 +1,173 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check that handling of ghost elements in parallel distributed vectors works
+// appropriately when assigning from ghosted to non-ghosted vectors
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // processor 0 and 1 own 2 indices each, higher processors nothing, all are
+  // ghosting global elements 1 and 3
+  IndexSet local_owned(std::min(numproc * 2, 4U));
+  if (myid < 2)
+    local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(local_owned.size());
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+  if (numproc > 1)
+    local_relevant.add_range(3, 4);
+
+  // run this twice, once where the vectors have called update_ghost_values
+  // and once where they have not
+  for (unsigned int run = 0; run < 2; ++run)
+    {
+      LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+        local_owned, local_relevant, MPI_COMM_WORLD);
+
+      // set local values
+      LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+      if (myid < 2)
+        {
+          rw_vector(myid * 2)     = myid * 2.0;
+          rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+        }
+
+      v.import(rw_vector, VectorOperation::insert);
+
+      LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> w(v), u(v);
+      u = 0;
+
+      v *= 2.0;
+      v.add(1.0);
+
+      if (run == 1)
+        {
+          v.update_ghost_values();
+          w.update_ghost_values();
+          u.update_ghost_values();
+        }
+
+      rw_vector.import(v, VectorOperation::insert);
+      if (myid < 2)
+        {
+          Assert(rw_vector(myid * 2) == myid * 4.0 + 1, ExcInternalError());
+          Assert(rw_vector(myid * 2 + 1) == myid * 4.0 + 3.0,
+                 ExcInternalError());
+        }
+
+      // copy vector content to non-ghosted vectors, manually created.
+      LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v_dist(
+        local_owned, MPI_COMM_WORLD),
+        w_dist(v_dist), u_dist(v_dist);
+
+      v_dist = v;
+      w_dist = w;
+      u_dist = u;
+
+      u_dist.add(1.0, v_dist, -1.0, w);
+
+      // copy back to a ghosted vector and update ghost values there
+      u = u_dist;
+      u.update_ghost_values();
+
+      rw_vector.import(u_dist, VectorOperation::insert);
+      if (myid < 2)
+        {
+          Assert(rw_vector(myid * 2) == myid * 2.0 + 1, ExcInternalError());
+          Assert(rw_vector(myid * 2 + 1) == myid * 2.0 + 2.0,
+                 ExcInternalError());
+        }
+
+      rw_vector.import(u, VectorOperation::insert);
+      if (myid < 2)
+        {
+          Assert(rw_vector(myid * 2) == myid * 2.0 + 1, ExcInternalError());
+          Assert(rw_vector(myid * 2 + 1) == myid * 2.0 + 2.0,
+                 ExcInternalError());
+        }
+
+      IndexSet u_ghost_set(local_owned.size());
+      u_ghost_set.add_index(1);
+      u_ghost_set.add_index(3);
+      LinearAlgebra::ReadWriteVector<double> u_ghost_vector(u_ghost_set);
+      u_ghost_vector.import(u, VectorOperation::insert);
+      Assert(u_ghost_vector(1) == 2., ExcInternalError());
+      if (numproc > 1)
+        {
+          if (run == 1)
+            {
+              IndexSet v_ghost_set(local_owned.size());
+              v_ghost_set.add_index(3);
+              LinearAlgebra::ReadWriteVector<double> v_ghost_vector(
+                v_ghost_set);
+              v_ghost_vector.import(v, VectorOperation::insert);
+              Assert(v_ghost_vector(3) == 7., ExcInternalError());
+            }
+          Assert(u_ghost_vector(3) == 4., ExcInternalError());
+        }
+
+      // check l2 norm
+      const double l2_norm = u.l2_norm();
+      if (myid == 0 && run == 1)
+        deallog << "L2 norm: " << l2_norm << std::endl;
+    }
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_14.mpirun=2.output b/tests/lac/parallel_vector_14.mpirun=2.output
new file mode 100644 (file)
index 0000000..12e7a74
--- /dev/null
@@ -0,0 +1,4 @@
+
+DEAL:0::numproc=2
+DEAL:0::L2 norm: 5.477
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_15.cc b/tests/lac/parallel_vector_15.cc
new file mode 100644 (file)
index 0000000..ec64b62
--- /dev/null
@@ -0,0 +1,123 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check that handling of ghost elements in parallel distributed vectors works
+// appropriately when creating a vector from a non-ghosted source vector using
+// the assignment operator
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+  // processor 0 and 1 own 2 indices each, higher processors nothing, all are
+  // ghosting global elements 1 and 3
+  IndexSet local_owned(std::min(numproc * 2, 4U));
+  if (myid < 2)
+    local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(local_owned.size());
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+  if (numproc > 1)
+    local_relevant.add_range(3, 4);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+
+  // set local values
+  LinearAlgebra::ReadWriteVector<double> rw_vector(local_owned);
+  if (myid < 2)
+    {
+      rw_vector(myid * 2)     = myid * 2.0;
+      rw_vector(myid * 2 + 1) = myid * 2.0 + 1.0;
+    }
+
+  v.import(rw_vector, VectorOperation::insert);
+
+  if (myid == 0)
+    deallog << "v has ghost elements: " << v.has_ghost_elements() << std::endl;
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> w, x;
+  w = v;
+  if (myid == 0)
+    deallog << "w has ghost elements: " << w.has_ghost_elements() << std::endl;
+
+  v.update_ghost_values();
+  w = v;
+  if (myid == 0)
+    deallog << "w has ghost elements: " << w.has_ghost_elements() << std::endl;
+
+  v.zero_out_ghost_values();
+  w = v;
+  if (myid == 0)
+    deallog << "w has ghost elements: " << w.has_ghost_elements() << std::endl;
+
+  w.zero_out_ghost_values();
+  w = v;
+  if (myid == 0)
+    deallog << "w has ghost elements: " << w.has_ghost_elements() << std::endl;
+
+  v.update_ghost_values();
+  x = v;
+  if (myid == 0)
+    deallog << "x has ghost elements: " << x.has_ghost_elements() << std::endl;
+
+  x.zero_out_ghost_values();
+  if (myid == 0)
+    deallog << "x has ghost elements: " << x.has_ghost_elements() << std::endl;
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_15.mpirun=2.output b/tests/lac/parallel_vector_15.mpirun=2.output
new file mode 100644 (file)
index 0000000..39f1d15
--- /dev/null
@@ -0,0 +1,10 @@
+
+DEAL:0::numproc=2
+DEAL:0::v has ghost elements: 0
+DEAL:0::w has ghost elements: 0
+DEAL:0::w has ghost elements: 1
+DEAL:0::w has ghost elements: 1
+DEAL:0::w has ghost elements: 0
+DEAL:0::x has ghost elements: 1
+DEAL:0::x has ghost elements: 0
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_16.cc b/tests/lac/parallel_vector_16.cc
new file mode 100644 (file)
index 0000000..7ffc1ec
--- /dev/null
@@ -0,0 +1,116 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2011 - 2017 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// build a vector whose elements exceed the size of unsigned int in case of 64
+// bit indices. To avoid excessive memory consumption, let the vector start at
+// a number close to the maximum of unsigned int but extend past the last
+// index
+
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+__global__ void
+set_value(double *values_dev, unsigned int index, double val)
+{
+  values_dev[index] = val;
+}
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+  types::global_dof_index min_index  = 0xffffffffU - 39;
+  types::global_dof_index local_size = 42;
+  IndexSet                local_owned(min_index + numproc * local_size);
+  local_owned.add_range(min_index + myid * local_size,
+                        min_index + (myid + 1) * local_size);
+
+  // all processors ghost some entries around invalid_unsigned_int and on the
+  // border between two processors
+  IndexSet local_relevant(local_owned.size());
+  local_relevant = local_owned;
+  local_relevant.add_range(min_index + 38, min_index + 40);
+  local_relevant.add_range(min_index + 41, min_index + 43);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+
+  deallog << "Local range of proc 0: " << v.local_range().first << " "
+          << v.local_range().second << std::endl;
+
+  // set local values
+  for (types::global_dof_index i = 0; i < local_size; ++i)
+    {
+      double *values_dev = v.get_values();
+      set_value<<<1, 1>>>(values_dev, i, min_index + myid * local_size + i);
+    }
+
+  deallog << "vector norm: " << v.l2_norm() << std::endl;
+
+  // check ghost values
+  v.print(deallog.get_file_stream(), 12, false, false);
+  v.update_ghost_values();
+  v.print(deallog.get_file_stream(), 12, false, false);
+
+  v.zero_out_ghost_values();
+  double *    values_dev  = v.get_values();
+  const auto &partitioner = v.get_partitioner();
+  set_value<<<1, 1>>>(values_dev,
+                      partitioner->global_to_local(min_index + 38),
+                      min_index);
+  set_value<<<1, 1>>>(values_dev,
+                      partitioner->global_to_local(min_index + 39),
+                      min_index * 2);
+  set_value<<<1, 1>>>(values_dev,
+                      partitioner->global_to_local(min_index + 41),
+                      min_index + 7);
+  set_value<<<1, 1>>>(values_dev,
+                      partitioner->global_to_local(min_index + 42),
+                      -static_cast<double>(min_index));
+  v.compress(VectorOperation::add);
+  v.update_ghost_values();
+  v.print(deallog.get_file_stream(), 12, false, false);
+
+  deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  MPILogInitAll mpilog;
+
+  deallog << std::setprecision(12);
+
+  test();
+}
diff --git a/tests/lac/parallel_vector_16.with_64bit_indices=on.mpirun=4.output b/tests/lac/parallel_vector_16.with_64bit_indices=on.mpirun=4.output
new file mode 100644 (file)
index 0000000..0bf5c6d
--- /dev/null
@@ -0,0 +1,612 @@
+
+DEAL:0::numproc=4
+DEAL:0::Local range of proc 0: 4294967256 4294967298
+DEAL:0::vector norm: 55669139270.9
+Process #0
+Local range: [4294967256, 4294967298), global size: 4294967424
+Vector data:
+4294967256.000000000000
+4294967257.000000000000
+4294967258.000000000000
+4294967259.000000000000
+4294967260.000000000000
+4294967261.000000000000
+4294967262.000000000000
+4294967263.000000000000
+4294967264.000000000000
+4294967265.000000000000
+4294967266.000000000000
+4294967267.000000000000
+4294967268.000000000000
+4294967269.000000000000
+4294967270.000000000000
+4294967271.000000000000
+4294967272.000000000000
+4294967273.000000000000
+4294967274.000000000000
+4294967275.000000000000
+4294967276.000000000000
+4294967277.000000000000
+4294967278.000000000000
+4294967279.000000000000
+4294967280.000000000000
+4294967281.000000000000
+4294967282.000000000000
+4294967283.000000000000
+4294967284.000000000000
+4294967285.000000000000
+4294967286.000000000000
+4294967287.000000000000
+4294967288.000000000000
+4294967289.000000000000
+4294967290.000000000000
+4294967291.000000000000
+4294967292.000000000000
+4294967293.000000000000
+4294967294.000000000000
+4294967295.000000000000
+4294967296.000000000000
+4294967297.000000000000
+
+Process #0
+Local range: [4294967256, 4294967298), global size: 4294967424
+Vector data:
+4294967256.000000000000
+4294967257.000000000000
+4294967258.000000000000
+4294967259.000000000000
+4294967260.000000000000
+4294967261.000000000000
+4294967262.000000000000
+4294967263.000000000000
+4294967264.000000000000
+4294967265.000000000000
+4294967266.000000000000
+4294967267.000000000000
+4294967268.000000000000
+4294967269.000000000000
+4294967270.000000000000
+4294967271.000000000000
+4294967272.000000000000
+4294967273.000000000000
+4294967274.000000000000
+4294967275.000000000000
+4294967276.000000000000
+4294967277.000000000000
+4294967278.000000000000
+4294967279.000000000000
+4294967280.000000000000
+4294967281.000000000000
+4294967282.000000000000
+4294967283.000000000000
+4294967284.000000000000
+4294967285.000000000000
+4294967286.000000000000
+4294967287.000000000000
+4294967288.000000000000
+4294967289.000000000000
+4294967290.000000000000
+4294967291.000000000000
+4294967292.000000000000
+4294967293.000000000000
+4294967294.000000000000
+4294967295.000000000000
+4294967296.000000000000
+4294967297.000000000000
+
+Ghost entries (global index / value):
+(4294967298/4294967298.000000000000)
+
+Process #0
+Local range: [4294967256, 4294967298), global size: 4294967424
+Vector data:
+4294967256.000000000000
+4294967257.000000000000
+4294967258.000000000000
+4294967259.000000000000
+4294967260.000000000000
+4294967261.000000000000
+4294967262.000000000000
+4294967263.000000000000
+4294967264.000000000000
+4294967265.000000000000
+4294967266.000000000000
+4294967267.000000000000
+4294967268.000000000000
+4294967269.000000000000
+4294967270.000000000000
+4294967271.000000000000
+4294967272.000000000000
+4294967273.000000000000
+4294967274.000000000000
+4294967275.000000000000
+4294967276.000000000000
+4294967277.000000000000
+4294967278.000000000000
+4294967279.000000000000
+4294967280.000000000000
+4294967281.000000000000
+4294967282.000000000000
+4294967283.000000000000
+4294967284.000000000000
+4294967285.000000000000
+4294967286.000000000000
+4294967287.000000000000
+4294967288.000000000000
+4294967289.000000000000
+4294967290.000000000000
+4294967291.000000000000
+4294967292.000000000000
+4294967293.000000000000
+17179869024.000000000000
+34359738048.000000000000
+4294967296.000000000000
+17179869052.000000000000
+
+Ghost entries (global index / value):
+(4294967298/-17179869024.000000000000)
+
+DEAL:0::OK
+
+DEAL:1::Local range of proc 0: 4294967298 4294967340
+DEAL:1::vector norm: 55669139270.9
+Process #1
+Local range: [4294967298, 4294967340), global size: 4294967424
+Vector data:
+4294967298.000000000000
+4294967299.000000000000
+4294967300.000000000000
+4294967301.000000000000
+4294967302.000000000000
+4294967303.000000000000
+4294967304.000000000000
+4294967305.000000000000
+4294967306.000000000000
+4294967307.000000000000
+4294967308.000000000000
+4294967309.000000000000
+4294967310.000000000000
+4294967311.000000000000
+4294967312.000000000000
+4294967313.000000000000
+4294967314.000000000000
+4294967315.000000000000
+4294967316.000000000000
+4294967317.000000000000
+4294967318.000000000000
+4294967319.000000000000
+4294967320.000000000000
+4294967321.000000000000
+4294967322.000000000000
+4294967323.000000000000
+4294967324.000000000000
+4294967325.000000000000
+4294967326.000000000000
+4294967327.000000000000
+4294967328.000000000000
+4294967329.000000000000
+4294967330.000000000000
+4294967331.000000000000
+4294967332.000000000000
+4294967333.000000000000
+4294967334.000000000000
+4294967335.000000000000
+4294967336.000000000000
+4294967337.000000000000
+4294967338.000000000000
+4294967339.000000000000
+
+Process #1
+Local range: [4294967298, 4294967340), global size: 4294967424
+Vector data:
+4294967298.000000000000
+4294967299.000000000000
+4294967300.000000000000
+4294967301.000000000000
+4294967302.000000000000
+4294967303.000000000000
+4294967304.000000000000
+4294967305.000000000000
+4294967306.000000000000
+4294967307.000000000000
+4294967308.000000000000
+4294967309.000000000000
+4294967310.000000000000
+4294967311.000000000000
+4294967312.000000000000
+4294967313.000000000000
+4294967314.000000000000
+4294967315.000000000000
+4294967316.000000000000
+4294967317.000000000000
+4294967318.000000000000
+4294967319.000000000000
+4294967320.000000000000
+4294967321.000000000000
+4294967322.000000000000
+4294967323.000000000000
+4294967324.000000000000
+4294967325.000000000000
+4294967326.000000000000
+4294967327.000000000000
+4294967328.000000000000
+4294967329.000000000000
+4294967330.000000000000
+4294967331.000000000000
+4294967332.000000000000
+4294967333.000000000000
+4294967334.000000000000
+4294967335.000000000000
+4294967336.000000000000
+4294967337.000000000000
+4294967338.000000000000
+4294967339.000000000000
+
+Ghost entries (global index / value):
+(4294967294/4294967294.000000000000)
+(4294967295/4294967295.000000000000)
+(4294967297/4294967297.000000000000)
+
+Process #1
+Local range: [4294967298, 4294967340), global size: 4294967424
+Vector data:
+-17179869024.000000000000
+4294967299.000000000000
+4294967300.000000000000
+4294967301.000000000000
+4294967302.000000000000
+4294967303.000000000000
+4294967304.000000000000
+4294967305.000000000000
+4294967306.000000000000
+4294967307.000000000000
+4294967308.000000000000
+4294967309.000000000000
+4294967310.000000000000
+4294967311.000000000000
+4294967312.000000000000
+4294967313.000000000000
+4294967314.000000000000
+4294967315.000000000000
+4294967316.000000000000
+4294967317.000000000000
+4294967318.000000000000
+4294967319.000000000000
+4294967320.000000000000
+4294967321.000000000000
+4294967322.000000000000
+4294967323.000000000000
+4294967324.000000000000
+4294967325.000000000000
+4294967326.000000000000
+4294967327.000000000000
+4294967328.000000000000
+4294967329.000000000000
+4294967330.000000000000
+4294967331.000000000000
+4294967332.000000000000
+4294967333.000000000000
+4294967334.000000000000
+4294967335.000000000000
+4294967336.000000000000
+4294967337.000000000000
+4294967338.000000000000
+4294967339.000000000000
+
+Ghost entries (global index / value):
+(4294967294/17179869024.000000000000)
+(4294967295/34359738048.000000000000)
+(4294967297/17179869052.000000000000)
+
+DEAL:1::OK
+
+
+DEAL:2::Local range of proc 0: 4294967340 4294967382
+DEAL:2::vector norm: 55669139270.9
+Process #2
+Local range: [4294967340, 4294967382), global size: 4294967424
+Vector data:
+4294967340.000000000000
+4294967341.000000000000
+4294967342.000000000000
+4294967343.000000000000
+4294967344.000000000000
+4294967345.000000000000
+4294967346.000000000000
+4294967347.000000000000
+4294967348.000000000000
+4294967349.000000000000
+4294967350.000000000000
+4294967351.000000000000
+4294967352.000000000000
+4294967353.000000000000
+4294967354.000000000000
+4294967355.000000000000
+4294967356.000000000000
+4294967357.000000000000
+4294967358.000000000000
+4294967359.000000000000
+4294967360.000000000000
+4294967361.000000000000
+4294967362.000000000000
+4294967363.000000000000
+4294967364.000000000000
+4294967365.000000000000
+4294967366.000000000000
+4294967367.000000000000
+4294967368.000000000000
+4294967369.000000000000
+4294967370.000000000000
+4294967371.000000000000
+4294967372.000000000000
+4294967373.000000000000
+4294967374.000000000000
+4294967375.000000000000
+4294967376.000000000000
+4294967377.000000000000
+4294967378.000000000000
+4294967379.000000000000
+4294967380.000000000000
+4294967381.000000000000
+
+Process #2
+Local range: [4294967340, 4294967382), global size: 4294967424
+Vector data:
+4294967340.000000000000
+4294967341.000000000000
+4294967342.000000000000
+4294967343.000000000000
+4294967344.000000000000
+4294967345.000000000000
+4294967346.000000000000
+4294967347.000000000000
+4294967348.000000000000
+4294967349.000000000000
+4294967350.000000000000
+4294967351.000000000000
+4294967352.000000000000
+4294967353.000000000000
+4294967354.000000000000
+4294967355.000000000000
+4294967356.000000000000
+4294967357.000000000000
+4294967358.000000000000
+4294967359.000000000000
+4294967360.000000000000
+4294967361.000000000000
+4294967362.000000000000
+4294967363.000000000000
+4294967364.000000000000
+4294967365.000000000000
+4294967366.000000000000
+4294967367.000000000000
+4294967368.000000000000
+4294967369.000000000000
+4294967370.000000000000
+4294967371.000000000000
+4294967372.000000000000
+4294967373.000000000000
+4294967374.000000000000
+4294967375.000000000000
+4294967376.000000000000
+4294967377.000000000000
+4294967378.000000000000
+4294967379.000000000000
+4294967380.000000000000
+4294967381.000000000000
+
+Ghost entries (global index / value):
+(4294967294/4294967294.000000000000)
+(4294967295/4294967295.000000000000)
+(4294967297/4294967297.000000000000)
+(4294967298/4294967298.000000000000)
+
+Process #2
+Local range: [4294967340, 4294967382), global size: 4294967424
+Vector data:
+4294967340.000000000000
+4294967341.000000000000
+4294967342.000000000000
+4294967343.000000000000
+4294967344.000000000000
+4294967345.000000000000
+4294967346.000000000000
+4294967347.000000000000
+4294967348.000000000000
+4294967349.000000000000
+4294967350.000000000000
+4294967351.000000000000
+4294967352.000000000000
+4294967353.000000000000
+4294967354.000000000000
+4294967355.000000000000
+4294967356.000000000000
+4294967357.000000000000
+4294967358.000000000000
+4294967359.000000000000
+4294967360.000000000000
+4294967361.000000000000
+4294967362.000000000000
+4294967363.000000000000
+4294967364.000000000000
+4294967365.000000000000
+4294967366.000000000000
+4294967367.000000000000
+4294967368.000000000000
+4294967369.000000000000
+4294967370.000000000000
+4294967371.000000000000
+4294967372.000000000000
+4294967373.000000000000
+4294967374.000000000000
+4294967375.000000000000
+4294967376.000000000000
+4294967377.000000000000
+4294967378.000000000000
+4294967379.000000000000
+4294967380.000000000000
+4294967381.000000000000
+
+Ghost entries (global index / value):
+(4294967294/17179869024.000000000000)
+(4294967295/34359738048.000000000000)
+(4294967297/17179869052.000000000000)
+(4294967298/-17179869024.000000000000)
+
+DEAL:2::OK
+
+
+DEAL:3::Local range of proc 0: 4294967382 4294967424
+DEAL:3::vector norm: 55669139270.9
+Process #3
+Local range: [4294967382, 4294967424), global size: 4294967424
+Vector data:
+4294967382.000000000000
+4294967383.000000000000
+4294967384.000000000000
+4294967385.000000000000
+4294967386.000000000000
+4294967387.000000000000
+4294967388.000000000000
+4294967389.000000000000
+4294967390.000000000000
+4294967391.000000000000
+4294967392.000000000000
+4294967393.000000000000
+4294967394.000000000000
+4294967395.000000000000
+4294967396.000000000000
+4294967397.000000000000
+4294967398.000000000000
+4294967399.000000000000
+4294967400.000000000000
+4294967401.000000000000
+4294967402.000000000000
+4294967403.000000000000
+4294967404.000000000000
+4294967405.000000000000
+4294967406.000000000000
+4294967407.000000000000
+4294967408.000000000000
+4294967409.000000000000
+4294967410.000000000000
+4294967411.000000000000
+4294967412.000000000000
+4294967413.000000000000
+4294967414.000000000000
+4294967415.000000000000
+4294967416.000000000000
+4294967417.000000000000
+4294967418.000000000000
+4294967419.000000000000
+4294967420.000000000000
+4294967421.000000000000
+4294967422.000000000000
+4294967423.000000000000
+
+Process #3
+Local range: [4294967382, 4294967424), global size: 4294967424
+Vector data:
+4294967382.000000000000
+4294967383.000000000000
+4294967384.000000000000
+4294967385.000000000000
+4294967386.000000000000
+4294967387.000000000000
+4294967388.000000000000
+4294967389.000000000000
+4294967390.000000000000
+4294967391.000000000000
+4294967392.000000000000
+4294967393.000000000000
+4294967394.000000000000
+4294967395.000000000000
+4294967396.000000000000
+4294967397.000000000000
+4294967398.000000000000
+4294967399.000000000000
+4294967400.000000000000
+4294967401.000000000000
+4294967402.000000000000
+4294967403.000000000000
+4294967404.000000000000
+4294967405.000000000000
+4294967406.000000000000
+4294967407.000000000000
+4294967408.000000000000
+4294967409.000000000000
+4294967410.000000000000
+4294967411.000000000000
+4294967412.000000000000
+4294967413.000000000000
+4294967414.000000000000
+4294967415.000000000000
+4294967416.000000000000
+4294967417.000000000000
+4294967418.000000000000
+4294967419.000000000000
+4294967420.000000000000
+4294967421.000000000000
+4294967422.000000000000
+4294967423.000000000000
+
+Ghost entries (global index / value):
+(4294967294/4294967294.000000000000)
+(4294967295/4294967295.000000000000)
+(4294967297/4294967297.000000000000)
+(4294967298/4294967298.000000000000)
+
+Process #3
+Local range: [4294967382, 4294967424), global size: 4294967424
+Vector data:
+4294967382.000000000000
+4294967383.000000000000
+4294967384.000000000000
+4294967385.000000000000
+4294967386.000000000000
+4294967387.000000000000
+4294967388.000000000000
+4294967389.000000000000
+4294967390.000000000000
+4294967391.000000000000
+4294967392.000000000000
+4294967393.000000000000
+4294967394.000000000000
+4294967395.000000000000
+4294967396.000000000000
+4294967397.000000000000
+4294967398.000000000000
+4294967399.000000000000
+4294967400.000000000000
+4294967401.000000000000
+4294967402.000000000000
+4294967403.000000000000
+4294967404.000000000000
+4294967405.000000000000
+4294967406.000000000000
+4294967407.000000000000
+4294967408.000000000000
+4294967409.000000000000
+4294967410.000000000000
+4294967411.000000000000
+4294967412.000000000000
+4294967413.000000000000
+4294967414.000000000000
+4294967415.000000000000
+4294967416.000000000000
+4294967417.000000000000
+4294967418.000000000000
+4294967419.000000000000
+4294967420.000000000000
+4294967421.000000000000
+4294967422.000000000000
+4294967423.000000000000
+
+Ghost entries (global index / value):
+(4294967294/17179869024.000000000000)
+(4294967295/34359738048.000000000000)
+(4294967297/17179869052.000000000000)
+(4294967298/-17179869024.000000000000)
+
+DEAL:3::OK
+
diff --git a/tests/lac/parallel_vector_21.cc b/tests/lac/parallel_vector_21.cc
new file mode 100644 (file)
index 0000000..2f4e21c
--- /dev/null
@@ -0,0 +1,86 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+// Check that the data range representing ghosts is really initialized to zero
+// when doing reinit() from another vector and manually setting the local
+// range
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+void
+test()
+{
+  unsigned int my_id   = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int n_procs = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  IndexSet locally_owned(n_procs * 2);
+  locally_owned.add_range(my_id * 2, my_id * 2 + 2);
+  IndexSet ghost_set(n_procs * 2);
+  ghost_set.add_index(0);
+  ghost_set.add_index(2);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v(
+    locally_owned, ghost_set, MPI_COMM_WORLD);
+
+  // create vector without actually setting the entries since they will be
+  // overwritten soon anyway
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> v2;
+  v2.reinit(v, true);
+
+  // set locally owned range of v2 manually
+  Kokkos::parallel_for(v2.local_size(), KOKKOS_LAMBDA(int i) {v2.get_values()[i] = 1.;});
+  Kokkos::fence();
+
+  // add entries to ghost values
+  // Because of limitation in import, the IndexSet of the ReadWriteVector needs
+  // to have the local elements.
+  IndexSet workaround_set(locally_owned);
+  workaround_set.add_index(0);
+  workaround_set.add_index(2);
+  workaround_set.compress();
+  LinearAlgebra::ReadWriteVector<double> rw_vector(workaround_set);
+  rw_vector(0) += 1.;
+  rw_vector(2) += 1.;
+  v2.import(rw_vector, VectorOperation::add);
+
+  // now we should have the correct data, not some uninitialized trash that
+  // resided in the ghost range
+  v2.print(deallog.get_file_stream());
+
+  v2.update_ghost_values();
+  v2.print(deallog.get_file_stream());
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  MPILogInitAll log;
+  test();
+}
diff --git a/tests/lac/parallel_vector_21.mpirun=2.output b/tests/lac/parallel_vector_21.mpirun=2.output
new file mode 100644 (file)
index 0000000..71d2ddb
--- /dev/null
@@ -0,0 +1,23 @@
+
+Process #0
+Local range: [0, 2), global size: 4
+Vector data:
+3.000e+00 1.000e+00 
+Process #0
+Local range: [0, 2), global size: 4
+Vector data:
+3.000e+00 1.000e+00 
+Ghost entries (global index / value):
+(2/3.000e+00) 
+
+Process #1
+Local range: [2, 4), global size: 4
+Vector data:
+3.000e+00 1.000e+00 
+Process #1
+Local range: [2, 4), global size: 4
+Vector data:
+3.000e+00 1.000e+00 
+Ghost entries (global index / value):
+(0/3.000e+00) 
+
diff --git a/tests/lac/parallel_vector_22.cc b/tests/lac/parallel_vector_22.cc
new file mode 100644 (file)
index 0000000..43c2ddb
--- /dev/null
@@ -0,0 +1,184 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE at
+// the top level of the deal.II distribution.
+//
+// ---------------------------------------------------------------------
+
+
+// check LA::Vector::compress(VectorOperation::min/max) from ghosts
+
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+template <typename Number>
+double
+print_value(Number* values_dev, unsigned int index)
+{
+  static Kokkos::View<Number, Kokkos::HostSpace> cpu_value("cpu_value");
+  Kokkos::deep_copy(cpu_value, Kokkos::View<Number>(values_dev+index));
+  return cpu_value();
+}
+
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  // create vector
+  LinearAlgebra::distributed::Vector<double, MemorySpace::Device> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+  Kokkos::View<double*, MemorySpace::Device::kokkos_space> v_device(v.get_values(), local_relevant.n_elements());
+  const auto &partitioner = v.get_partitioner();
+
+  // set local values
+  Kokkos::deep_copy(Kokkos::subview(v_device, partitioner->global_to_local(myid * 2)), myid*2.0);
+  Kokkos::deep_copy(Kokkos::subview(v_device, partitioner->global_to_local(myid * 2+1)), myid*2.0+1.0);
+  v.compress(VectorOperation::add);
+  v *= 2.0;
+
+  // check setup of vectors
+  deallog << myid << ":"
+          << "first owned entry: "
+          << print_value(v.get_values(), partitioner->global_to_local(myid * 2))
+          << std::endl;
+  deallog << myid << ":"
+          << "second owned entry: "
+          << print_value(v.get_values(),
+                         partitioner->global_to_local(myid * 2 + 1))
+          << std::endl;
+
+  // set ghost dof on owning processor and maximize
+  if (myid != 0)
+    Kokkos::deep_copy(Kokkos::subview(v_device,
+                        partitioner->global_to_local(1)),
+                        7. * myid);
+  v.compress(VectorOperation::max);
+
+  // import ghosts onto all procs
+  v.update_ghost_values();
+
+  // check
+  deallog << myid << ":"
+          << "ghost entry after max from owner: "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+
+  // ghosts are set to zero
+  v.zero_out_ghost_values();
+
+  // minimize
+  v.compress(VectorOperation::min);
+  v.update_ghost_values();
+
+  // check
+  deallog << myid << ":"
+          << "ghost entry after min from zero: "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+
+  // set ghost dof on non-owning processors and minimize
+  v.zero_out_ghost_values();
+  if (myid == 0)
+    Kokkos::deep_copy(Kokkos::subview(v_device, partitioner->global_to_local(1)), -1);
+  v.compress(VectorOperation::min);
+  v.update_ghost_values();
+
+  // check
+  deallog << myid << ":"
+          << "ghost entry after min from : "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+
+  // set vector to 1, zeros in ghosts except on owner where -1. is set
+  v.zero_out_ghost_values();
+  v = 1.0;
+  if (myid == 0)
+    Kokkos::deep_copy(Kokkos::subview(v_device, partitioner->global_to_local(1)), -1);
+
+  // maximize
+  v.compress(VectorOperation::max);
+  v.update_ghost_values();
+
+  // even if only one value is set (-1. on owner), the other values
+  // contribute a "0" and maximization receives zero and returns it
+  deallog << myid << ":"
+          << "ghost entry after max and partly init: "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+
+  // however, if the ghost value is set on all processors, the
+  // maximum is -1:
+  v.zero_out_ghost_values();
+  v = 1.0;
+  Kokkos::deep_copy(Kokkos::subview(v_device, partitioner->global_to_local(1)), -1);
+  v.compress(VectorOperation::max);
+  v.update_ghost_values();
+  deallog << myid << ":"
+          << "ghost entry after max and full init: "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+
+  // what happens in case max is called two times and all values were smaller
+  // than zero
+  v.zero_out_ghost_values();
+  v = -1.0;
+  Kokkos::deep_copy(Kokkos::subview(v_device, partitioner->global_to_local(1)), -1);
+  v.compress(VectorOperation::max);
+  deallog << myid << ":"
+          << "ghost entry after first max: "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+  v.compress(VectorOperation::max);
+  deallog << myid << ":"
+          << "ghost entry after second max: "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  MPILogInitAll log;
+
+  test();
+}
diff --git a/tests/lac/parallel_vector_22.mpirun=4.output b/tests/lac/parallel_vector_22.mpirun=4.output
new file mode 100644 (file)
index 0000000..f1940df
--- /dev/null
@@ -0,0 +1,45 @@
+
+DEAL:0::numproc=4
+DEAL:0::0:first owned entry: 0.00000
+DEAL:0::0:second owned entry: 2.00000
+DEAL:0::0:ghost entry after max from owner: 21.0000
+DEAL:0::0:ghost entry after min from zero: 0.00000
+DEAL:0::0:ghost entry after min from : -1.00000
+DEAL:0::0:ghost entry after max and partly init: 0.00000
+DEAL:0::0:ghost entry after max and full init: -1.00000
+DEAL:0::0:ghost entry after first max: -1.00000
+DEAL:0::0:ghost entry after second max: 0.00000
+DEAL:0::OK
+
+DEAL:1::1:first owned entry: 4.00000
+DEAL:1::1:second owned entry: 6.00000
+DEAL:1::1:ghost entry after max from owner: 21.0000
+DEAL:1::1:ghost entry after min from zero: 0.00000
+DEAL:1::1:ghost entry after min from : -1.00000
+DEAL:1::1:ghost entry after max and partly init: 0.00000
+DEAL:1::1:ghost entry after max and full init: -1.00000
+DEAL:1::1:ghost entry after first max: 0.00000
+DEAL:1::1:ghost entry after second max: 0.00000
+
+
+DEAL:2::2:first owned entry: 8.00000
+DEAL:2::2:second owned entry: 10.0000
+DEAL:2::2:ghost entry after max from owner: 21.0000
+DEAL:2::2:ghost entry after min from zero: 0.00000
+DEAL:2::2:ghost entry after min from : -1.00000
+DEAL:2::2:ghost entry after max and partly init: 0.00000
+DEAL:2::2:ghost entry after max and full init: -1.00000
+DEAL:2::2:ghost entry after first max: 0.00000
+DEAL:2::2:ghost entry after second max: 0.00000
+
+
+DEAL:3::3:first owned entry: 12.0000
+DEAL:3::3:second owned entry: 14.0000
+DEAL:3::3:ghost entry after max from owner: 21.0000
+DEAL:3::3:ghost entry after min from zero: 0.00000
+DEAL:3::3:ghost entry after min from : -1.00000
+DEAL:3::3:ghost entry after max and partly init: 0.00000
+DEAL:3::3:ghost entry after max and full init: -1.00000
+DEAL:3::3:ghost entry after first max: 0.00000
+DEAL:3::3:ghost entry after second max: 0.00000
+
diff --git a/tests/lac/parallel_vector_23.cc b/tests/lac/parallel_vector_23.cc
new file mode 100644 (file)
index 0000000..264e286
--- /dev/null
@@ -0,0 +1,107 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE at
+// the top level of the deal.II distribution.
+//
+// ---------------------------------------------------------------------
+
+
+// check LA::Vector::compress(VectorOperation::min/max) from ghosts
+
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+#include <deal.II/lac/read_write_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include "../tests.h"
+
+
+template <typename Number>
+double
+print_value(Number* values_dev, unsigned int index)
+{
+  static Kokkos::View<Number, Kokkos::HostSpace> cpu_value("cpu_value");
+  Kokkos::deep_copy(cpu_value, Kokkos::View<Number>(values_dev+index));
+  return cpu_value();
+}
+
+
+
+void
+test()
+{
+  unsigned int myid    = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  unsigned int numproc = Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
+
+  if (myid == 0)
+    deallog << "numproc=" << numproc << std::endl;
+
+
+  // each processor owns 2 indices and all
+  // are ghosting element 1 (the second)
+  IndexSet local_owned(numproc * 2);
+  local_owned.add_range(myid * 2, myid * 2 + 2);
+  IndexSet local_relevant(numproc * 2);
+  local_relevant = local_owned;
+  local_relevant.add_range(1, 2);
+
+  // create vector
+  LinearAlgebra::distributed::Vector<double, MemorySpace::Device> v(
+    local_owned, local_relevant, MPI_COMM_WORLD);
+  const auto &partitioner = v.get_partitioner();
+
+  // the read write vector additionally has ghost elements
+  IndexSet                               read_write_owned(numproc * 2);
+  LinearAlgebra::ReadWriteVector<double> read_write_vector(local_relevant);
+
+  read_write_vector.local_element(0) = myid;
+  read_write_vector(1)               = 2. * myid;
+
+  v.import(read_write_vector, VectorOperation::max);
+  v.update_ghost_values();
+
+  deallog << myid << ":"
+          << "ghost entry after max: "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+
+  if (myid == 0)
+    read_write_vector(1) = -1.0;
+
+  v.import(read_write_vector, VectorOperation::min);
+  v.update_ghost_values();
+
+  deallog << myid << ":"
+          << "ghost entry after min: "
+          << print_value(v.get_values(), partitioner->global_to_local(1))
+          << std::endl;
+
+
+  if (myid == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  MPILogInitAll log;
+
+  test();
+}
diff --git a/tests/lac/parallel_vector_23.mpirun=4.output b/tests/lac/parallel_vector_23.mpirun=4.output
new file mode 100644 (file)
index 0000000..d1ab3a1
--- /dev/null
@@ -0,0 +1,17 @@
+
+DEAL:0::numproc=4
+DEAL:0::0:ghost entry after max: 6.00000
+DEAL:0::0:ghost entry after min: -1.00000
+DEAL:0::OK
+
+DEAL:1::1:ghost entry after max: 6.00000
+DEAL:1::1:ghost entry after min: -1.00000
+
+
+DEAL:2::2:ghost entry after max: 6.00000
+DEAL:2::2:ghost entry after min: -1.00000
+
+
+DEAL:3::3:ghost entry after max: 6.00000
+DEAL:3::3:ghost entry after min: -1.00000
+
diff --git a/tests/lac/parallel_vector_25.cc b/tests/lac/parallel_vector_25.cc
new file mode 100644 (file)
index 0000000..d081dc5
--- /dev/null
@@ -0,0 +1,95 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2018 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// check import from LA::d::Vector to another LA::d::Vector
+
+#include <deal.II/base/cuda.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.h>
+
+#include "../tests.h"
+
+void
+test()
+{
+  unsigned int       rank = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  const unsigned int size = 100;
+  const unsigned int local_size = 50;
+  const unsigned int ghost_size = 75;
+
+  IndexSet locally_owned(size);
+  if (rank == 0)
+    locally_owned.add_range(0, local_size);
+  else
+    locally_owned.add_range(size - local_size, size);
+  locally_owned.compress();
+
+  IndexSet ghost_indices(size);
+  if (rank == 0)
+    ghost_indices.add_range(0, ghost_size);
+  else
+    ghost_indices.add_range(size - ghost_size, size);
+  ghost_indices.size();
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::Host> vec_ref(
+    locally_owned, ghost_indices, MPI_COMM_WORLD);
+  for (unsigned int i = 0; i < local_size; ++i)
+    vec_ref.local_element(i) = i;
+  vec_ref.compress(VectorOperation::insert);
+
+  auto partitioner = vec_ref.get_partitioner();
+  LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA> vec_dev(
+    partitioner);
+  LinearAlgebra::distributed::Vector<double, MemorySpace::Host> vec_host(
+    partitioner);
+
+  // Assignment from Host to CUDA
+  vec_dev.import(vec_ref, VectorOperation::insert);
+
+  // Assignment from CUDA to Host
+  vec_host.import(vec_dev, VectorOperation::insert);
+
+  for (unsigned int i = 0; i < ghost_size; ++i)
+    {
+      AssertThrow(std::fabs(vec_ref.local_element(i) -
+                            vec_host.local_element(i)) < 1e-12,
+                  ExcInternalError());
+    }
+
+  if (rank == 0)
+    deallog << "OK" << std::endl;
+}
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int myid = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(myid));
+
+  if (myid == 0)
+    {
+      initlog();
+
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_25.mpirun=2.output b/tests/lac/parallel_vector_25.mpirun=2.output
new file mode 100644 (file)
index 0000000..be8d055
--- /dev/null
@@ -0,0 +1,2 @@
+
+DEAL:0::OK
diff --git a/tests/lac/parallel_vector_26.cc b/tests/lac/parallel_vector_26.cc
new file mode 100644 (file)
index 0000000..aea7f20
--- /dev/null
@@ -0,0 +1,79 @@
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2019 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE.md at
+// the top level directory of deal.II.
+//
+// ---------------------------------------------------------------------
+
+
+// Check for a bug where compress cannot be called for a LA::d::Vector<Host> in
+// a CUDA file
+
+#include <deal.II/base/index_set.h>
+#include <deal.II/base/utilities.h>
+
+#include <deal.II/lac/la_parallel_vector.templates.h>
+
+#include "../tests.h"
+
+
+void
+test()
+{
+  unsigned int       rank = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  const unsigned int total_size = 100;
+  const unsigned int ghost_size = 75;
+  const unsigned int local_size = 50;
+
+  IndexSet local_owned(total_size);
+  if (rank == 0)
+    local_owned.add_range(0, local_size);
+  else
+    local_owned.add_range(total_size - local_size, total_size);
+
+  IndexSet ghost_indices(total_size);
+  if (rank == 0)
+    ghost_indices.add_range(0, ghost_size);
+  else
+    ghost_indices.add_range(total_size - ghost_size, total_size);
+
+  LinearAlgebra::distributed::Vector<double, MemorySpace::Host> v(
+    local_owned, ghost_indices, MPI_COMM_WORLD);
+
+  for (unsigned int i = 0; i < ghost_size; ++i)
+    v.local_element(i) = i;
+  v.compress(VectorOperation::add);
+
+
+  if (rank == 0)
+    deallog << "OK" << std::endl;
+}
+
+
+
+int
+main(int argc, char **argv)
+{
+  Utilities::MPI::MPI_InitFinalize mpi_initialization(
+    argc, argv, testing_max_num_threads());
+
+  unsigned int rank = Utilities::MPI::this_mpi_process(MPI_COMM_WORLD);
+  deallog.push(Utilities::int_to_string(rank));
+
+  if (rank == 0)
+    {
+      initlog();
+      deallog << std::setprecision(4);
+      test();
+    }
+  else
+    test();
+}
diff --git a/tests/lac/parallel_vector_26.mpirun=2.output b/tests/lac/parallel_vector_26.mpirun=2.output
new file mode 100644 (file)
index 0000000..be8d055
--- /dev/null
@@ -0,0 +1,2 @@
+
+DEAL:0::OK

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.