Add MemorySpaceData and start to implement resize_val

diff --git a/include/deal.II/lac/la_sm_vector.h b/include/deal.II/lac/la_sm_vector.h
index 0888c831f00730e11ef171ba56bd9896560894ae..748b73db318cee8a60a5158b3658bdcdb96548a1 100644 (file)
--- a/include/deal.II/lac/la_sm_vector.h
+++ b/include/deal.II/lac/la_sm_vector.h
@@ -75,6 +75,32 @@ namespace LinearAlgebra
 {
   namespace SharedMPI
   {
+    template <typename Number>
+    struct MemorySpaceData
+    {
+      void
+      copy_to(Number *begin, std::size_t n_elements)
+      {
+        Assert(false, ExcNotImplemented());
+        (void)begin;
+        (void)n_elements;
+      }
+
+      void
+      copy_from(Number *begin, std::size_t n_elements)
+      {
+        Assert(false, ExcNotImplemented());
+        (void)begin;
+        (void)n_elements;
+      }
+
+      std::unique_ptr<Number[], std::function<void(Number *&)>> values;
+      MPI_Win *values_win = nullptr;
+
+      std::unique_ptr<Number[]> values_dev;
+    };
+
+
     template <typename Number, typename MemorySpace = MemorySpace::Host>
     class Vector : public ::dealii::LinearAlgebra::VectorSpaceVector<Number>,
                    public Subscriptor
@@ -451,13 +477,10 @@ namespace LinearAlgebra
 
       size_type allocated_size;
 
-      mutable ::dealii::MemorySpace::MemorySpaceData<Number, MemorySpace> data;
-
-      mutable std::shared_ptr<::dealii::parallel::internal::TBBPartitioner>
-        thread_loop_partitioner;
+      mutable MemorySpaceData<Number> data;
 
-      mutable ::dealii::MemorySpace::MemorySpaceData<Number, MemorySpace>
-        import_data;
+      // needed?
+      mutable MemorySpaceData<Number> import_data;
 
       mutable bool vector_is_ghosted;
 
@@ -487,92 +510,26 @@ namespace LinearAlgebra
       struct Policy
       {
         static inline typename Vector<Number, MemorySpace>::iterator
-        begin(::dealii::MemorySpace::MemorySpaceData<Number, MemorySpace> &)
+        begin(MemorySpaceData<Number> &)
         {
           Assert(false, ExcNotImplemented());
           return nullptr;
         }
 
         static inline typename Vector<Number, MemorySpace>::const_iterator
-        begin(
-          const ::dealii::MemorySpace::MemorySpaceData<Number, MemorySpace> &)
+        begin(const MemorySpaceData<Number> &)
         {
           Assert(false, ExcNotImplemented());
           return nullptr;
         }
 
         static inline Number *
-        get_values(
-          ::dealii::MemorySpace::MemorySpaceData<Number, MemorySpace> &)
+        get_values(MemorySpaceData<Number> &)
         {
           Assert(false, ExcNotImplemented());
           return nullptr;
         }
       };
-
-
-
-      template <typename Number>
-      struct Policy<Number, ::dealii::MemorySpace::Host>
-      {
-        static inline
-          typename Vector<Number, ::dealii::MemorySpace::Host>::iterator
-          begin(::dealii::MemorySpace::
-                  MemorySpaceData<Number, ::dealii::MemorySpace::Host> &data)
-        {
-          Assert(false, ExcNotImplemented());
-          return data.values.get();
-        }
-
-        static inline
-          typename Vector<Number, ::dealii::MemorySpace::Host>::const_iterator
-          begin(const ::dealii::MemorySpace::
-                  MemorySpaceData<Number, ::dealii::MemorySpace::Host> &data)
-        {
-          Assert(false, ExcNotImplemented());
-          return data.values.get();
-        }
-
-        static inline Number *
-        get_values(::dealii::MemorySpace::
-                     MemorySpaceData<Number, ::dealii::MemorySpace::Host> &data)
-        {
-          Assert(false, ExcNotImplemented());
-          return data.values.get();
-        }
-      };
-
-
-
-      template <typename Number>
-      struct Policy<Number, ::dealii::MemorySpace::CUDA>
-      {
-        static inline
-          typename Vector<Number, ::dealii::MemorySpace::CUDA>::iterator
-          begin(::dealii::MemorySpace::
-                  MemorySpaceData<Number, ::dealii::MemorySpace::CUDA> &data)
-        {
-          Assert(false, ExcNotImplemented());
-          return data.values_dev.get();
-        }
-
-        static inline
-          typename Vector<Number, ::dealii::MemorySpace::CUDA>::const_iterator
-          begin(const ::dealii::MemorySpace::
-                  MemorySpaceData<Number, ::dealii::MemorySpace::CUDA> &data)
-        {
-          Assert(false, ExcNotImplemented());
-          return data.values_dev.get();
-        }
-
-        static inline Number *
-        get_values(::dealii::MemorySpace::
-                     MemorySpaceData<Number, ::dealii::MemorySpace::CUDA> &data)
-        {
-          Assert(false, ExcNotImplemented());
-          return data.values_dev.get();
-        }
-      };
     } // namespace internal
 
 

diff --git a/include/deal.II/lac/la_sm_vector.templates.h b/include/deal.II/lac/la_sm_vector.templates.h
index 6049b90438d0b08cc72b3d3b3b2a37c2368f6457..fd4ca5de3b43844a77158395603eca5640ca5324 100644 (file)
--- a/include/deal.II/lac/la_sm_vector.templates.h
+++ b/include/deal.II/lac/la_sm_vector.templates.h
@@ -38,6 +38,61 @@ namespace LinearAlgebra
 {
   namespace SharedMPI
   {
+    namespace internal
+    {
+      template <typename Number, typename MemorySpaceType>
+      struct la_parallel_vector_templates_functions
+      {
+        using size_type = types::global_dof_index;
+
+        static void
+        resize_val(const types::global_dof_index new_alloc_size,
+                   types::global_dof_index &     allocated_size,
+                   MemorySpaceData<Number> &     data)
+        {
+          Assert(allocated_size == 0, ExcNotImplemented());
+          Assert(data.values == nullptr, ExcNotImplemented());
+
+          allocated_size = new_alloc_size;
+
+          // TODO
+          std::vector<Number *> data_others;
+          MPI_Comm              comm_shared;
+
+          data.values_win   = new MPI_Win;
+          Number *data_this = (Number *)malloc(0);
+          data_others.resize(Utilities::MPI::n_mpi_processes(comm_shared));
+
+
+          MPI_Info info;
+          MPI_Info_create(&info);
+          MPI_Info_set(info, "alloc_shared_noncontig", "true");
+
+          MPI_Win_allocate_shared(new_alloc_size * sizeof(Number),
+                                  sizeof(Number),
+                                  info,
+                                  comm_shared,
+                                  data_this,
+                                  data.values_win);
+
+          for (unsigned int i = 0;
+               i < Utilities::MPI::n_mpi_processes(comm_shared);
+               i++)
+            {
+              int      disp_unit;
+              MPI_Aint ssize;
+              MPI_Win_shared_query(
+                *data.values_win, i, &ssize, &disp_unit, &data_others[i]);
+            }
+
+          data.values = {
+            data_others[Utilities::MPI::this_mpi_process(comm_shared)],
+            [&data](Number *&) { MPI_Win_free(data.values_win); }};
+        }
+      };
+    } // namespace internal
+
+
     template <typename Number, typename MemorySpaceType>
     void
     Vector<Number, MemorySpaceType>::clear_mpi_requests()
@@ -51,8 +106,9 @@ namespace LinearAlgebra
     void
     Vector<Number, MemorySpaceType>::resize_val(const size_type new_alloc_size)
     {
-      Assert(false, ExcNotImplemented());
-      (void)new_alloc_size;
+      internal::la_parallel_vector_templates_functions<
+        Number,
+        MemorySpaceType>::resize_val(new_alloc_size, allocated_size, data);
     }
 
 
@@ -116,8 +172,26 @@ namespace LinearAlgebra
     Vector<Number, MemorySpaceType>::reinit(
       const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner_in)
     {
-      Assert(false, ExcNotImplemented());
-      (void)partitioner_in;
+      clear_mpi_requests();
+      partitioner = partitioner_in;
+
+      // set vector size and allocate memory
+      const size_type new_allocated_size =
+        partitioner->local_size() + partitioner->n_ghost_indices();
+      resize_val(new_allocated_size);
+
+      // initialize to zero
+      this->operator=(Number());
+
+
+      // do not reallocate import_data directly, but only upon request. It
+      // is only used as temporary storage for compress() and
+      // update_ghost_values, and we might have vectors where we never
+      // call these methods and hence do not need to have the storage.
+      import_data.values.reset();
+      import_data.values_dev.reset();
+
+      vector_is_ghosted = false;
     }