From: Bruno Turcksin <bruno.turcksin@gmail.com>
Date: Fri, 8 Feb 2019 14:44:36 +0000 (+0000)
Subject: Move initialization of import_indices_plain_dev into a separate function
X-Git-Tag: v9.1.0-rc1~349^2~2
X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=17a59dbcc69f88d87e048c515cd276619671f7b3;p=dealii.git

Move initialization of import_indices_plain_dev into a separate function
---

diff --git a/include/deal.II/base/partitioner.h b/include/deal.II/base/partitioner.h
index 2c0db0955f..c98454dcbe 100644
--- a/include/deal.II/base/partitioner.h
+++ b/include/deal.II/base/partitioner.h
@@ -571,6 +571,13 @@ namespace Utilities
                      << " elements for this partitioner.");
 
     private:
+      /**
+       * Initialize import_indices_plain_dev from import_indices_data. This
+       * function is only used when using CUDA-aware MPI.
+       */
+      void
+      initialize_import_indices_plain_dev() const;
+
       /**
        * The global size of the vector over all processors
        */
diff --git a/include/deal.II/base/partitioner.templates.h b/include/deal.II/base/partitioner.templates.h
index 582034a7bc..48432cfcbc 100644
--- a/include/deal.II/base/partitioner.templates.h
+++ b/include/deal.II/base/partitioner.templates.h
@@ -105,47 +105,9 @@ namespace Utilities
       // performance reasons as this can significantly decrease the number of
       // kernel launched. The indices are expanded the first time the function
       // is called.
-      if (std::is_same<MemorySpaceType, MemorySpace::CUDA>::value)
-        {
-          if (import_indices_plain_dev.size() == 0)
-            {
-              import_indices_plain_dev.reserve(n_import_targets);
-              for (unsigned int i = 0; i < n_import_targets; i++)
-                {
-                  // Expand the indices on the host
-                  std::vector<std::pair<unsigned int, unsigned int>>::
-                    const_iterator my_imports =
-                                     import_indices_data.begin() +
-                                     import_indices_chunks_by_rank_data[i],
-                                   end_my_imports =
-                                     import_indices_data.begin() +
-                                     import_indices_chunks_by_rank_data[i + 1];
-                  std::vector<unsigned int> import_indices_plain_host;
-                  for (; my_imports != end_my_imports; ++my_imports)
-                    {
-                      const unsigned int chunk_size =
-                        my_imports->second - my_imports->first;
-                      for (unsigned int j = 0; j < chunk_size; ++j)
-                        import_indices_plain_host.push_back(my_imports->first +
-                                                            j);
-                    }
-
-                  // Move the indices to the device
-                  import_indices_plain_dev.emplace_back(std::make_pair(
-                    std::unique_ptr<unsigned int[], void (*)(unsigned int *)>(
-                      nullptr,
-                      Utilities::CUDA::delete_device_data<unsigned int>),
-                    import_indices_plain_host.size()));
-
-                  import_indices_plain_dev[i].first.reset(
-                    Utilities::CUDA::allocate_device_data<unsigned int>(
-                      import_indices_plain_dev[i].second));
-                  Utilities::CUDA::copy_to_dev(
-                    import_indices_plain_host,
-                    import_indices_plain_dev[i].first.get());
-                }
-            }
-        }
+      if ((std::is_same<MemorySpaceType, MemorySpace::CUDA>::value) &&
+          (import_indices_plain_dev.size() == 0))
+        initialize_import_indices_plain_dev();
 #    endif
 
       for (unsigned int i = 0; i < n_import_targets; i++)
diff --git a/source/base/partitioner.cu b/source/base/partitioner.cu
index ed02193e89..4788e7571d 100644
--- a/source/base/partitioner.cu
+++ b/source/base/partitioner.cu
@@ -18,6 +18,50 @@
 
 DEAL_II_NAMESPACE_OPEN
 
+
+
+namespace Utilities
+{
+  namespace MPI
+  {
+    void
+    Partitioner::initialize_import_indices_plain_dev() const
+    {
+      const unsigned int n_import_targets = import_targets_data.size();
+      import_indices_plain_dev.reserve(n_import_targets);
+      for (unsigned int i = 0; i < n_import_targets; i++)
+        {
+          // Expand the indices on the host
+          std::vector<std::pair<unsigned int, unsigned int>>::const_iterator
+            my_imports = import_indices_data.begin() +
+                         import_indices_chunks_by_rank_data[i],
+            end_my_imports = import_indices_data.begin() +
+                             import_indices_chunks_by_rank_data[i + 1];
+          std::vector<unsigned int> import_indices_plain_host;
+          for (; my_imports != end_my_imports; ++my_imports)
+            {
+              const unsigned int chunk_size =
+                my_imports->second - my_imports->first;
+              for (unsigned int j = 0; j < chunk_size; ++j)
+                import_indices_plain_host.push_back(my_imports->first + j);
+            }
+
+          // Move the indices to the device
+          import_indices_plain_dev.emplace_back(std::make_pair(
+            std::unique_ptr<unsigned int[], void (*)(unsigned int *)>(
+              nullptr, Utilities::CUDA::delete_device_data<unsigned int>),
+            import_indices_plain_host.size()));
+
+          import_indices_plain_dev[i].first.reset(
+            Utilities::CUDA::allocate_device_data<unsigned int>(
+              import_indices_plain_dev[i].second));
+          Utilities::CUDA::copy_to_dev(import_indices_plain_host,
+                                       import_indices_plain_dev[i].first.get());
+        }
+    }
+  } // namespace MPI
+} // namespace Utilities
+
 // explicit instantiations from .templates.h file
 #include "partitioner.cuda.inst"