]> https://gitweb.dealii.org/ - dealii.git/commitdiff
enable MemorySpace in GMG transfer
authorTimo Heister <timo.heister@gmail.com>
Thu, 14 Nov 2024 11:32:51 +0000 (06:32 -0500)
committerTimo Heister <timo.heister@gmail.com>
Thu, 14 Nov 2024 11:38:47 +0000 (06:38 -0500)
cmake/config/template-arguments.in
include/deal.II/multigrid/mg_transfer.h
include/deal.II/multigrid/mg_transfer.templates.h
include/deal.II/multigrid/mg_transfer_global_coarsening.h
include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h
source/multigrid/mg_level_global_transfer.cc
source/multigrid/mg_level_global_transfer.inst.in
source/multigrid/mg_transfer_global_coarsening.inst.in
source/multigrid/mg_transfer_prebuilt.inst.in

index 96c0175391b04a8d9cbf567b0814e4b6b68f9265..19e7b3a571e0774a61831d788e6c65ce9786cb72 100644 (file)
@@ -169,18 +169,13 @@ EXTERNAL_PARALLEL_VECTORS := { @DEAL_II_EXPAND_TRILINOS_MPI_VECTOR@;
                                @DEAL_II_EXPAND_PETSC_MPI_BLOCKVECTOR@
                              }
 
-// TODO: I don't understand this one. LA::distributed::Vector does not have a
-// native matrix type, especially if we don't compile with Trilinos. Currently
-// only used: mg_transfer_prebuilt.inst.in and
-// mg_level_global_transfer.inst.in
-VECTORS_WITH_MATRIX := { Vector<double>;
+// Vectors we can do GMG with excluding the LinearAlgebra::distributed::Vector<> (which is handled separately):
+VECTORS_WITHOUT_LAVEC := { Vector<double>;
                          Vector<float> ;
 
                          BlockVector<double>;
                          BlockVector<float>;
 
-                         LinearAlgebra::distributed::Vector<double>;
-
                          @DEAL_II_EXPAND_TRILINOS_MPI_VECTOR@;
                          @DEAL_II_EXPAND_EPETRA_VECTOR@;
                          @DEAL_II_EXPAND_TPETRA_VECTOR_DOUBLE@;
index 909300fdbe5f1ea0bb92de0365308d8cbec2357f..9b038bc510338ce94655de77bc661cb3f5559fce 100644 (file)
@@ -407,9 +407,11 @@ private:
  * routines as compared to the %parallel vectors in the PETScWrappers and
  * TrilinosWrappers namespaces.
  */
-template <typename Number>
-class MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>
-  : public MGTransferBase<LinearAlgebra::distributed::Vector<Number>>
+template <typename Number, typename MemorySpace>
+class MGLevelGlobalTransfer<
+  LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+  : public MGTransferBase<
+      LinearAlgebra::distributed::Vector<Number, MemorySpace>>
 {
 public:
   /**
@@ -426,9 +428,10 @@ public:
    */
   template <int dim, typename Number2, int spacedim>
   void
-  copy_to_mg(const DoFHandler<dim, spacedim> &dof_handler,
-             MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &dst,
-             const LinearAlgebra::distributed::Vector<Number2> &src) const;
+  copy_to_mg(
+    const DoFHandler<dim, spacedim> &dof_handler,
+    MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>> &dst,
+    const LinearAlgebra::distributed::Vector<Number2, MemorySpace> &src) const;
 
   /**
    * Transfer from multi-level vector to normal vector.
@@ -440,9 +443,10 @@ public:
   template <int dim, typename Number2, int spacedim>
   void
   copy_from_mg(
-    const DoFHandler<dim, spacedim>             &dof_handler,
-    LinearAlgebra::distributed::Vector<Number2> &dst,
-    const MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &src) const;
+    const DoFHandler<dim, spacedim>                          &dof_handler,
+    LinearAlgebra::distributed::Vector<Number2, MemorySpace> &dst,
+    const MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+      &src) const;
 
   /**
    * Add a multi-level vector to a normal vector.
@@ -452,9 +456,10 @@ public:
   template <int dim, typename Number2, int spacedim>
   void
   copy_from_mg_add(
-    const DoFHandler<dim, spacedim>             &dof_handler,
-    LinearAlgebra::distributed::Vector<Number2> &dst,
-    const MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &src) const;
+    const DoFHandler<dim, spacedim>                          &dof_handler,
+    LinearAlgebra::distributed::Vector<Number2, MemorySpace> &dst,
+    const MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+      &src) const;
 
   /**
    * If this object operates on BlockVector objects, we need to describe how
@@ -493,10 +498,11 @@ protected:
    */
   template <int dim, typename Number2, int spacedim>
   void
-  copy_to_mg(const DoFHandler<dim, spacedim> &dof_handler,
-             MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &dst,
-             const LinearAlgebra::distributed::Vector<Number2>         &src,
-             const bool solution_transfer) const;
+  copy_to_mg(
+    const DoFHandler<dim, spacedim> &dof_handler,
+    MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>> &dst,
+    const LinearAlgebra::distributed::Vector<Number2, MemorySpace>         &src,
+    const bool solution_transfer) const;
 
   /**
    * Internal function to @p fill copy_indices*. Called by derived classes.
@@ -581,26 +587,27 @@ protected:
    * global vector for inserting into the level vectors. This vector is
    * populated with those entries.
    */
-  mutable LinearAlgebra::distributed::Vector<Number> ghosted_global_vector;
+  mutable LinearAlgebra::distributed::Vector<Number, MemorySpace>
+    ghosted_global_vector;
 
   /**
    * Same as above but used when working with solution vectors.
    */
-  mutable LinearAlgebra::distributed::Vector<Number>
+  mutable LinearAlgebra::distributed::Vector<Number, MemorySpace>
     solution_ghosted_global_vector;
 
   /**
    * In the function copy_from_mg, we access all level vectors with certain
    * ghost entries for inserting the result into a global vector.
    */
-  mutable MGLevelObject<LinearAlgebra::distributed::Vector<Number>>
+  mutable MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
     ghosted_level_vector;
 
   /**
    * Function to initialize internal level vectors.
    */
   std::function<void(const unsigned int,
-                     LinearAlgebra::distributed::Vector<Number> &)>
+                     LinearAlgebra::distributed::Vector<Number, MemorySpace> &)>
     initialize_dof_vector;
 
 private:
index 4b89ad6ee923583354c0192173eddd2dc3b518d4..effad1c943bd923f9fb33e17214b65635d4ef670 100644 (file)
@@ -394,27 +394,29 @@ MGLevelGlobalTransfer<VectorType>::assert_built(
 /* --------- MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector> -------
  */
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, typename Number2, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_to_mg(
-  const DoFHandler<dim, spacedim>                           &dof_handler,
-  MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &dst,
-  const LinearAlgebra::distributed::Vector<Number2>         &src) const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  copy_to_mg(
+    const DoFHandler<dim, spacedim> &dof_handler,
+    MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>> &dst,
+    const LinearAlgebra::distributed::Vector<Number2, MemorySpace> &src) const
 {
   assert_built(dof_handler);
   copy_to_mg(dof_handler, dst, src, false);
 }
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, typename Number2, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_to_mg(
-  const DoFHandler<dim, spacedim>                           &dof_handler,
-  MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &dst,
-  const LinearAlgebra::distributed::Vector<Number2>         &src,
-  const bool solution_transfer) const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  copy_to_mg(
+    const DoFHandler<dim, spacedim> &dof_handler,
+    MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>> &dst,
+    const LinearAlgebra::distributed::Vector<Number2, MemorySpace>         &src,
+    const bool solution_transfer) const
 {
   assert_built(dof_handler);
   LinearAlgebra::distributed::Vector<Number> &this_ghosted_global_vector =
@@ -513,13 +515,15 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_to_mg(
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, typename Number2, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_from_mg(
-  const DoFHandler<dim, spacedim>                                 &dof_handler,
-  LinearAlgebra::distributed::Vector<Number2>                     &dst,
-  const MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &src) const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  copy_from_mg(
+    const DoFHandler<dim, spacedim>                          &dof_handler,
+    LinearAlgebra::distributed::Vector<Number2, MemorySpace> &dst,
+    const MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+      &src) const
 {
   assert_built(dof_handler);
   AssertIndexRange(src.max_level(),
@@ -598,14 +602,15 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::copy_from_mg(
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, typename Number2, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
   copy_from_mg_add(
-    const DoFHandler<dim, spacedim>             &dof_handler,
-    LinearAlgebra::distributed::Vector<Number2> &dst,
-    const MGLevelObject<LinearAlgebra::distributed::Vector<Number>> &src) const
+    const DoFHandler<dim, spacedim>                          &dof_handler,
+    LinearAlgebra::distributed::Vector<Number2, MemorySpace> &dst,
+    const MGLevelObject<LinearAlgebra::distributed::Vector<Number, MemorySpace>>
+      &src) const
 {
   assert_built(dof_handler);
   // For non-DG: degrees of freedom in the refinement face may need special
@@ -645,19 +650,19 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
   set_component_to_block_map(const std::vector<unsigned int> &map)
 {
   component_to_block_map = map;
 }
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::assert_built(
-  const DoFHandler<dim, spacedim> &dof_handler) const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  assert_built(const DoFHandler<dim, spacedim> &dof_handler) const
 {
   (void)dof_handler;
   Assert(copy_indices.size() ==
index b982d60e66210526d03e3a4646d4f6db066cc9a6..76d469e23baacb33a141ef7909bf566a80e7983b 100644 (file)
@@ -15,6 +15,7 @@
 #ifndef dealii_mg_transfer_global_coarsening_h
 #define dealii_mg_transfer_global_coarsening_h
 
+#include <deal.II/base/memory_space.h>
 #include <deal.II/base/mg_level_object.h>
 #include <deal.II/base/mpi_remote_point_evaluation.h>
 #include <deal.II/base/vectorization.h>
@@ -50,7 +51,7 @@ namespace RepartitioningPolicyTools
   class Base;
 }
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 class MGTransferMF;
 #endif
 
@@ -227,7 +228,12 @@ public:
   static_assert(
     std::is_same_v<
       VectorType,
-      LinearAlgebra::distributed::Vector<typename VectorType::value_type>>,
+      LinearAlgebra::distributed::Vector<typename VectorType::value_type,
+                                         MemorySpace::Host>> ||
+      std::is_same_v<
+        VectorType,
+        LinearAlgebra::distributed::Vector<typename VectorType::value_type,
+                                           MemorySpace::Default>>,
     "This class is currently only implemented for vectors of "
     "type LinearAlgebra::distributed::Vector.");
 
@@ -286,17 +292,13 @@ protected:
    * Perform prolongation on vectors with correct ghosting.
    */
   virtual void
-  prolongate_and_add_internal(
-    LinearAlgebra::distributed::Vector<Number>       &dst,
-    const LinearAlgebra::distributed::Vector<Number> &src) const = 0;
+  prolongate_and_add_internal(VectorType &dst, const VectorType &src) const = 0;
 
   /**
    * Perform restriction on vectors with correct ghosting.
    */
   virtual void
-  restrict_and_add_internal(
-    LinearAlgebra::distributed::Vector<Number>       &dst,
-    const LinearAlgebra::distributed::Vector<Number> &src) const = 0;
+  restrict_and_add_internal(VectorType &dst, const VectorType &src) const = 0;
 
   /**
    * A wrapper around update_ghost_values() optimized in case the
@@ -360,7 +362,7 @@ protected:
   /**
    * Internal vector on which the actual prolongation/restriction is performed.
    */
-  mutable LinearAlgebra::distributed::Vector<Number> vec_coarse;
+  mutable VectorType vec_coarse;
 
   /**
    * Internal vector needed for collecting all degrees of freedom of the fine
@@ -369,7 +371,7 @@ protected:
    * accessed by the given vectors in the prolongate/restrict functions),
    * otherwise it is left at size zero.
    */
-  mutable LinearAlgebra::distributed::Vector<Number> vec_fine;
+  mutable VectorType vec_fine;
 
   /**
    * Bool indicating whether fine vector has relevant ghost values.
@@ -436,7 +438,12 @@ public:
   static_assert(
     std::is_same_v<
       VectorType,
-      LinearAlgebra::distributed::Vector<typename VectorType::value_type>>,
+      LinearAlgebra::distributed::Vector<typename VectorType::value_type,
+                                         MemorySpace::Host>> ||
+      std::is_same_v<
+        VectorType,
+        LinearAlgebra::distributed::Vector<typename VectorType::value_type,
+                                           MemorySpace::Default>>,
     "This class is currently only implemented for vectors of "
     "type LinearAlgebra::distributed::Vector.");
 
@@ -725,7 +732,7 @@ private:
 
   friend class internal::MGTwoLevelTransferImplementation;
 
-  friend class MGTransferMF<dim, Number>;
+  friend class MGTransferMF<dim, Number, typename VectorType::memory_space>;
 };
 
 
@@ -956,7 +963,7 @@ private:
    */
   std::vector<unsigned int> level_dof_indices_fine_ptrs;
 
-  friend class MGTransferMF<dim, Number>;
+  friend class MGTransferMF<dim, Number, typename VectorType::memory_space>;
 };
 
 
@@ -989,15 +996,15 @@ private:
  *
  * The implementation of this class is explained in detail in @cite munch2022gc.
  */
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 class MGTransferMF : public dealii::MGLevelGlobalTransfer<
-                       LinearAlgebra::distributed::Vector<Number>>
+                       LinearAlgebra::distributed::Vector<Number, MemorySpace>>
 {
 public:
   /**
    * Value type.
    */
-  using VectorType = LinearAlgebra::distributed::Vector<Number>;
+  using VectorType = LinearAlgebra::distributed::Vector<Number, MemorySpace>;
 
   /**
    * Default constructor.
@@ -1325,13 +1332,17 @@ private:
  */
 template <int dim, typename Number>
 class MGTransferBlockMF
-  : public MGTransferBlockMatrixFreeBase<dim, Number, MGTransferMF<dim, Number>>
+  : public MGTransferBlockMatrixFreeBase<
+      dim,
+      Number,
+      MGTransferMF<dim, Number, ::dealii::MemorySpace::Host>>
 {
 public:
   /**
    * Constructor.
    */
-  MGTransferBlockMF(const MGTransferMF<dim, Number> &transfer_operator);
+  MGTransferBlockMF(const MGTransferMF<dim, Number, ::dealii::MemorySpace::Host>
+                      &transfer_operator);
 
   /**
    * Constructor.
@@ -1388,27 +1399,30 @@ public:
   build(const std::vector<const DoFHandler<dim> *> &dof_handler);
 
 protected:
-  const MGTransferMF<dim, Number> &
+  const MGTransferMF<dim, Number, ::dealii::MemorySpace::Host> &
   get_matrix_free_transfer(const unsigned int b) const override;
 
 private:
   /**
    * Internal non-block version of transfer operation.
    */
-  std::vector<MGTransferMF<dim, Number>> transfer_operators_internal;
+  std::vector<MGTransferMF<dim, Number, ::dealii::MemorySpace::Host>>
+    transfer_operators_internal;
 
   /**
    * Non-block version of transfer operation.
    */
-  std::vector<ObserverPointer<const MGTransferMF<dim, Number>>>
+  std::vector<ObserverPointer<
+    const MGTransferMF<dim, Number, ::dealii::MemorySpace::Host>>>
     transfer_operators;
 };
 
 
 
 template <int dim, typename VectorType>
-using MGTransferGlobalCoarsening =
-  MGTransferMF<dim, typename VectorType::value_type>;
+using MGTransferGlobalCoarsening = MGTransferMF<dim,
+                                                typename VectorType::value_type,
+                                                ::dealii::MemorySpace::Host>;
 
 template <int dim, typename VectorType>
 using MGTransferBlockGlobalCoarsening =
@@ -1423,9 +1437,9 @@ using MGTransferBlockGlobalCoarsening =
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 template <typename MGTwoLevelTransferObject>
-MGTransferMF<dim, Number>::MGTransferMF(
+MGTransferMF<dim, Number, MemorySpace>::MGTransferMF(
   const MGLevelObject<MGTwoLevelTransferObject> &transfer,
   const std::function<void(const unsigned int, VectorType &)>
     &initialize_dof_vector)
@@ -1439,10 +1453,10 @@ MGTransferMF<dim, Number>::MGTransferMF(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 template <typename MGTwoLevelTransferObject>
 void
-MGTransferMF<dim, Number>::initialize_two_level_transfers(
+MGTransferMF<dim, Number, MemorySpace>::initialize_two_level_transfers(
   const MGLevelObject<MGTwoLevelTransferObject> &transfer)
 {
   this->initialize_transfer_references(transfer);
@@ -1450,10 +1464,10 @@ MGTransferMF<dim, Number>::initialize_two_level_transfers(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 template <typename MGTwoLevelTransferObject>
 void
-MGTransferMF<dim, Number>::initialize_transfer_references(
+MGTransferMF<dim, Number, MemorySpace>::initialize_transfer_references(
   const MGLevelObject<MGTwoLevelTransferObject> &transfer)
 {
   const unsigned int min_level = transfer.min_level();
@@ -1470,10 +1484,10 @@ MGTransferMF<dim, Number>::initialize_transfer_references(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 template <class InVector>
 void
-MGTransferMF<dim, Number>::initialize_dof_vector(
+MGTransferMF<dim, Number, MemorySpace>::initialize_dof_vector(
   const unsigned int level,
   VectorType        &vec,
   const InVector    &vec_reference,
@@ -1518,12 +1532,13 @@ MGTransferMF<dim, Number>::initialize_dof_vector(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 template <class InVector>
 void
-MGTransferMF<dim, Number>::copy_to_mg(const DoFHandler<dim>     &dof_handler,
-                                      MGLevelObject<VectorType> &dst,
-                                      const InVector            &src) const
+MGTransferMF<dim, Number, MemorySpace>::copy_to_mg(
+  const DoFHandler<dim>     &dof_handler,
+  MGLevelObject<VectorType> &dst,
+  const InVector            &src) const
 {
   assert_dof_handler(dof_handler);
 
@@ -1576,10 +1591,10 @@ MGTransferMF<dim, Number>::copy_to_mg(const DoFHandler<dim>     &dof_handler,
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 template <class OutVector>
 void
-MGTransferMF<dim, Number>::copy_from_mg(
+MGTransferMF<dim, Number, MemorySpace>::copy_from_mg(
   const DoFHandler<dim>           &dof_handler,
   OutVector                       &dst,
   const MGLevelObject<VectorType> &src) const
@@ -1630,11 +1645,12 @@ MGTransferMF<dim, Number>::copy_from_mg(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 template <class InVector>
 void
-MGTransferMF<dim, Number>::interpolate_to_mg(MGLevelObject<VectorType> &dst,
-                                             const InVector &src) const
+MGTransferMF<dim, Number, MemorySpace>::interpolate_to_mg(
+  MGLevelObject<VectorType> &dst,
+  const InVector            &src) const
 {
   DoFHandler<dim> dof_handler_dummy;
 
@@ -1643,12 +1659,13 @@ MGTransferMF<dim, Number>::interpolate_to_mg(MGLevelObject<VectorType> &dst,
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 template <class InVector>
 void
-MGTransferMF<dim, Number>::interpolate_to_mg(const DoFHandler<dim> &dof_handler,
-                                             MGLevelObject<VectorType> &dst,
-                                             const InVector &src) const
+MGTransferMF<dim, Number, MemorySpace>::interpolate_to_mg(
+  const DoFHandler<dim>     &dof_handler,
+  MGLevelObject<VectorType> &dst,
+  const InVector            &src) const
 {
   assert_dof_handler(dof_handler);
 
index f34bc2bb4e414b4fc364f31c7c37f4d437b6bdbd..09ed15f80d3ae91aef24a37e692731add157abb0 100644 (file)
@@ -18,6 +18,7 @@
 
 #include <deal.II/base/config.h>
 
+#include "deal.II/base/memory_space.h"
 #include <deal.II/base/mpi_consensus_algorithms.h>
 
 #include <deal.II/distributed/fully_distributed_tria.h>
@@ -1471,13 +1472,14 @@ namespace internal
     /**
      * Compute weights.
      */
-    template <int dim, typename Number>
+    template <int dim, typename Number, typename MemorySpace>
     static void
     setup_weights(
       const dealii::AffineConstraints<Number> &constraints_fine,
-      MGTwoLevelTransfer<dim, LinearAlgebra::distributed::Vector<Number>>
-                &transfer,
-      const bool is_feq)
+      MGTwoLevelTransfer<
+        dim,
+        LinearAlgebra::distributed::Vector<Number, MemorySpace>> &transfer,
+      const bool                                                  is_feq)
     {
       if (transfer.fine_element_is_continuous == false)
         return; // nothing to do
@@ -1613,7 +1615,7 @@ namespace internal
 
 
 
-    template <int dim, typename Number>
+    template <int dim, typename Number, typename MemorySpace>
     static void
     reinit_geometric_transfer(
       const DoFHandler<dim>                   &dof_handler_fine,
@@ -1622,8 +1624,9 @@ namespace internal
       const dealii::AffineConstraints<Number> &constraints_coarse,
       const unsigned int                       mg_level_fine,
       const unsigned int                       mg_level_coarse,
-      MGTwoLevelTransfer<dim, LinearAlgebra::distributed::Vector<Number>>
-        &transfer)
+      MGTwoLevelTransfer<
+        dim,
+        LinearAlgebra::distributed::Vector<Number, MemorySpace>> &transfer)
     {
       Assert((mg_level_fine == numbers::invalid_unsigned_int &&
               mg_level_coarse == numbers::invalid_unsigned_int) ||
@@ -2729,10 +2732,9 @@ namespace internal
     {
       const auto &vector_partitioner = vec.get_partitioner();
 
-      ArrayView<Number> ghost_array(
-        const_cast<LinearAlgebra::distributed::Vector<Number> &>(vec).begin() +
-          vector_partitioner->locally_owned_size(),
-        vector_partitioner->n_ghost_indices());
+      ArrayView<Number> ghost_array(const_cast<VectorType &>(vec).begin() +
+                                      vector_partitioner->locally_owned_size(),
+                                    vector_partitioner->n_ghost_indices());
 
       for (const auto &my_ghosts :
            embedded_partitioner->ghost_indices_within_larger_ghost_set())
@@ -4255,8 +4257,8 @@ MGTwoLevelTransferBase<VectorType>::zero_out_ghost_values(
 
 
 
-template <int dim, typename Number>
-MGTransferMF<dim, Number>::MGTransferMF()
+template <int dim, typename Number, typename MemorySpace>
+MGTransferMF<dim, Number, MemorySpace>::MGTransferMF()
 {
   this->transfer.clear();
   this->internal_transfer.clear();
@@ -4264,8 +4266,8 @@ MGTransferMF<dim, Number>::MGTransferMF()
 
 
 
-template <int dim, typename Number>
-MGTransferMF<dim, Number>::MGTransferMF(
+template <int dim, typename Number, typename MemorySpace>
+MGTransferMF<dim, Number, MemorySpace>::MGTransferMF(
   const MGConstrainedDoFs &mg_constrained_dofs)
 {
   this->transfer.clear();
@@ -4275,9 +4277,9 @@ MGTransferMF<dim, Number>::MGTransferMF(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::initialize_constraints(
+MGTransferMF<dim, Number, MemorySpace>::initialize_constraints(
   const MGConstrainedDoFs &mg_constrained_dofs)
 {
   this->mg_constrained_dofs = &mg_constrained_dofs;
@@ -4285,9 +4287,9 @@ MGTransferMF<dim, Number>::initialize_constraints(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::initialize_internal_transfer(
+MGTransferMF<dim, Number, MemorySpace>::initialize_internal_transfer(
   const DoFHandler<dim>                          &dof_handler,
   const ObserverPointer<const MGConstrainedDoFs> &mg_constrained_dofs)
 {
@@ -4317,41 +4319,48 @@ MGTransferMF<dim, Number>::initialize_internal_transfer(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 std::pair<const DoFHandler<dim> *, unsigned int>
-MGTransferMF<dim, Number>::get_dof_handler_fine() const
+MGTransferMF<dim, Number, MemorySpace>::get_dof_handler_fine() const
 {
   if (this->transfer.n_levels() <= 1)
     // single level: the information cannot be retrieved
     return {nullptr, numbers::invalid_unsigned_int};
 
-  if (const auto t = dynamic_cast<
-        const MGTwoLevelTransfer<dim,
-                                 LinearAlgebra::distributed::Vector<Number>> *>(
+  if (const auto t = dynamic_cast<const MGTwoLevelTransfer<
+        dim,
+        LinearAlgebra::distributed::Vector<Number, MemorySpace>> *>(
         this->transfer[this->transfer.max_level()].get()))
     {
       return {t->dof_handler_fine, t->mg_level_fine};
     }
-  else if (const auto t = dynamic_cast<const MGTwoLevelTransferNonNested<
-             dim,
-             LinearAlgebra::distributed::Vector<Number>> *>(
-             this->transfer[this->transfer.max_level()].get()))
-    {
-      return {t->dof_handler_fine, t->mg_level_fine};
-    }
-  else
+
+  if constexpr (std::is_same_v<MemorySpace, ::dealii::MemorySpace::Host>)
     {
-      DEAL_II_NOT_IMPLEMENTED();
-      return {nullptr, numbers::invalid_unsigned_int};
+      // MGTwoLevelTransferNonNested transfer is only instantiated for Host
+      // memory:
+      if (const auto t = dynamic_cast<const MGTwoLevelTransferNonNested<
+            dim,
+            LinearAlgebra::distributed::Vector<Number, MemorySpace>> *>(
+            this->transfer[this->transfer.max_level()].get()))
+        {
+          return {t->dof_handler_fine, t->mg_level_fine};
+        }
     }
+
+  {
+    DEAL_II_NOT_IMPLEMENTED();
+    return {nullptr, numbers::invalid_unsigned_int};
+  }
 }
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::fill_and_communicate_copy_indices_global_coarsening(
-  const DoFHandler<dim> &dof_handler_out)
+MGTransferMF<dim, Number, MemorySpace>::
+  fill_and_communicate_copy_indices_global_coarsening(
+    const DoFHandler<dim> &dof_handler_out)
 {
   const auto dof_handler_and_level_in = get_dof_handler_fine();
   const auto dof_handler_in           = dof_handler_and_level_in.first;
@@ -4448,9 +4457,9 @@ MGTransferMF<dim, Number>::fill_and_communicate_copy_indices_global_coarsening(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::build(
+MGTransferMF<dim, Number, MemorySpace>::build(
   const std::vector<std::shared_ptr<const Utilities::MPI::Partitioner>>
     &external_partitioners)
 {
@@ -4489,9 +4498,9 @@ MGTransferMF<dim, Number>::build(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::build(
+MGTransferMF<dim, Number, MemorySpace>::build(
   const std::function<void(const unsigned int, VectorType &)>
     &initialize_dof_vector)
 {
@@ -4506,7 +4515,8 @@ MGTransferMF<dim, Number>::build(
 
       for (unsigned int l = min_level; l <= max_level; ++l)
         {
-          LinearAlgebra::distributed::Vector<typename VectorType::value_type>
+          LinearAlgebra::distributed::Vector<typename VectorType::value_type,
+                                             MemorySpace>
             vector;
           initialize_dof_vector(l, vector);
           external_partitioners[l - min_level] = vector.get_partitioner();
@@ -4522,9 +4532,9 @@ MGTransferMF<dim, Number>::build(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::build(
+MGTransferMF<dim, Number, MemorySpace>::build(
   const DoFHandler<dim> &dof_handler,
   const std::vector<std::shared_ptr<const Utilities::MPI::Partitioner>>
     &external_partitioners)
@@ -4549,9 +4559,9 @@ MGTransferMF<dim, Number>::build(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::build(
+MGTransferMF<dim, Number, MemorySpace>::build(
   const DoFHandler<dim> &dof_handler,
   const std::function<void(const unsigned int, VectorType &)>
     &initialize_dof_vector)
@@ -4576,11 +4586,11 @@ MGTransferMF<dim, Number>::build(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::prolongate(const unsigned int to_level,
-                                      VectorType        &dst,
-                                      const VectorType  &src) const
+MGTransferMF<dim, Number, MemorySpace>::prolongate(const unsigned int to_level,
+                                                   VectorType        &dst,
+                                                   const VectorType  &src) const
 {
   dst = Number(0.0);
   prolongate_and_add(to_level, dst, src);
@@ -4588,31 +4598,33 @@ MGTransferMF<dim, Number>::prolongate(const unsigned int to_level,
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::prolongate_and_add(const unsigned int to_level,
-                                              VectorType        &dst,
-                                              const VectorType  &src) const
+MGTransferMF<dim, Number, MemorySpace>::prolongate_and_add(
+  const unsigned int to_level,
+  VectorType        &dst,
+  const VectorType  &src) const
 {
   this->transfer[to_level]->prolongate_and_add(dst, src);
 }
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::restrict_and_add(const unsigned int from_level,
-                                            VectorType        &dst,
-                                            const VectorType  &src) const
+MGTransferMF<dim, Number, MemorySpace>::restrict_and_add(
+  const unsigned int from_level,
+  VectorType        &dst,
+  const VectorType  &src) const
 {
   this->transfer[from_level]->restrict_and_add(dst, src);
 }
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 void
-MGTransferMF<dim, Number>::assert_dof_handler(
+MGTransferMF<dim, Number, MemorySpace>::assert_dof_handler(
   const DoFHandler<dim> &dof_handler_out) const
 {
 #ifndef DEBUG
@@ -4699,9 +4711,9 @@ MGTransferMF<dim, Number>::assert_dof_handler(
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 std::size_t
-MGTransferMF<dim, Number>::memory_consumption() const
+MGTransferMF<dim, Number, MemorySpace>::memory_consumption() const
 {
   std::size_t size = 0;
 
@@ -4716,26 +4728,26 @@ MGTransferMF<dim, Number>::memory_consumption() const
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 inline unsigned int
-MGTransferMF<dim, Number>::min_level() const
+MGTransferMF<dim, Number, MemorySpace>::min_level() const
 {
   return transfer.min_level();
 }
 
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 inline unsigned int
-MGTransferMF<dim, Number>::max_level() const
+MGTransferMF<dim, Number, MemorySpace>::max_level() const
 {
   return transfer.max_level();
 }
 
 
-template <int dim, typename Number>
+template <int dim, typename Number, typename MemorySpace>
 inline void
-MGTransferMF<dim, Number>::clear()
+MGTransferMF<dim, Number, MemorySpace>::clear()
 {
   MGLevelGlobalTransfer<VectorType>::clear();
 
@@ -4748,8 +4760,12 @@ MGTransferMF<dim, Number>::clear()
 
 template <int dim, typename Number>
 MGTransferBlockMF<dim, Number>::MGTransferBlockMF(
-  const MGTransferMF<dim, Number> &transfer_operator)
-  : MGTransferBlockMatrixFreeBase<dim, Number, MGTransferMF<dim, Number>>(true)
+  const MGTransferMF<dim, Number, ::dealii::MemorySpace::Host>
+    &transfer_operator)
+  : MGTransferBlockMatrixFreeBase<
+      dim,
+      Number,
+      MGTransferMF<dim, Number, ::dealii::MemorySpace::Host>>(true)
 {
   this->transfer_operators = {&transfer_operator};
 }
@@ -4759,7 +4775,10 @@ MGTransferBlockMF<dim, Number>::MGTransferBlockMF(
 template <int dim, typename Number>
 MGTransferBlockMF<dim, Number>::MGTransferBlockMF(
   const MGConstrainedDoFs &mg_constrained_dofs)
-  : MGTransferBlockMatrixFreeBase<dim, Number, MGTransferMF<dim, Number>>(true)
+  : MGTransferBlockMatrixFreeBase<
+      dim,
+      Number,
+      MGTransferMF<dim, Number, ::dealii::MemorySpace::Host>>(true)
 {
   initialize_constraints(mg_constrained_dofs);
 }
@@ -4769,7 +4788,10 @@ MGTransferBlockMF<dim, Number>::MGTransferBlockMF(
 template <int dim, typename Number>
 MGTransferBlockMF<dim, Number>::MGTransferBlockMF(
   const std::vector<MGConstrainedDoFs> &mg_constrained_dofs)
-  : MGTransferBlockMatrixFreeBase<dim, Number, MGTransferMF<dim, Number>>(false)
+  : MGTransferBlockMatrixFreeBase<
+      dim,
+      Number,
+      MGTransferMF<dim, Number, ::dealii::MemorySpace::Host>>(false)
 {
   initialize_constraints(mg_constrained_dofs);
 }
@@ -4843,7 +4865,7 @@ MGTransferBlockMF<dim, Number>::build(
 
 
 template <int dim, typename Number>
-const MGTransferMF<dim, Number> &
+const MGTransferMF<dim, Number, ::dealii::MemorySpace::Host> &
 MGTransferBlockMF<dim, Number>::get_matrix_free_transfer(
   const unsigned int b) const
 {
index 34a7b49a1e41a6455395108d135d3728ab0fd517..85943e942fdab1958470c01e7244e3824c21f62c 100644 (file)
@@ -275,10 +275,10 @@ namespace
   }
 } // namespace
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 template <int dim, int spacedim>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
   fill_and_communicate_copy_indices(const DoFHandler<dim, spacedim> &mg_dof)
 {
   const MPI_Comm mpi_communicator = mg_dof.get_mpi_communicator();
@@ -382,9 +382,10 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::clear()
+MGLevelGlobalTransfer<
+  LinearAlgebra::distributed::Vector<Number, MemorySpace>>::clear()
 {
   sizes.resize(0);
   copy_indices.clear();
@@ -404,9 +405,9 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::clear()
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 void
-MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
   print_indices(std::ostream &os) const
 {
   for (unsigned int level = 0; level < copy_indices.size(); ++level)
@@ -435,10 +436,10 @@ MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number>>::
 
 
 
-template <typename Number>
+template <typename Number, typename MemorySpace>
 std::size_t
-MGLevelGlobalTransfer<
-  LinearAlgebra::distributed::Vector<Number>>::memory_consumption() const
+MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number, MemorySpace>>::
+  memory_consumption() const
 {
   std::size_t result = sizeof(*this);
   result += MemoryConsumption::memory_consumption(sizes);
@@ -459,9 +460,4 @@ MGLevelGlobalTransfer<
 // explicit instantiation
 #include "mg_level_global_transfer.inst"
 
-// create an additional instantiation currently not supported by the automatic
-// template instantiation scheme
-template class MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<float>>;
-
-
 DEAL_II_NAMESPACE_CLOSE
index 8f6ee1a1e7bc8373f6c6f1bafec51faa4d8f11a8..3cbb47b81f870e5cbb8362a8e71e61eac6df2360 100644 (file)
 
 
 
-for (V1 : VECTORS_WITH_MATRIX)
+for (V1 : VECTORS_WITHOUT_LAVEC)
   {
     template class MGLevelGlobalTransfer<V1>;
   }
 
-for (deal_II_dimension : DIMENSIONS; V1 : VECTORS_WITH_MATRIX)
+for (S1 : REAL_SCALARS)
+  {
+    template class MGLevelGlobalTransfer<
+      LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>>;
+  }
+
+for (deal_II_dimension : DIMENSIONS; V1 : VECTORS_WITHOUT_LAVEC)
   {
     template void MGLevelGlobalTransfer<V1>::
       fill_and_communicate_copy_indices<deal_II_dimension, deal_II_dimension>(
@@ -43,42 +49,60 @@ for (deal_II_dimension : DIMENSIONS; V1, V2 : DEAL_II_VEC_TEMPLATES;
       const MGLevelObject<V1<S1>> &) const;
   }
 
-for (deal_II_dimension : DIMENSIONS)
+for (deal_II_dimension : DIMENSIONS; S1 : REAL_SCALARS)
   {
-    template void
-    MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<float>>::
+    template void MGLevelGlobalTransfer<
+      LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>>::
       fill_and_communicate_copy_indices<deal_II_dimension, deal_II_dimension>(
         const DoFHandler<deal_II_dimension, deal_II_dimension> &mg_dof);
   }
 
 for (deal_II_dimension : DIMENSIONS; S1, S2 : REAL_SCALARS)
   {
-    template void
-    MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<S1>>::copy_to_mg(
-      const DoFHandler<deal_II_dimension> &,
-      MGLevelObject<LinearAlgebra::distributed::Vector<S1>> &,
-      const LinearAlgebra::distributed::Vector<S2> &,
-      const bool) const;
+    template void MGLevelGlobalTransfer<
+      LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>>::
+      copy_to_mg(
+        const DoFHandler<deal_II_dimension> &,
+        MGLevelObject<LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>>
+          &,
+        const LinearAlgebra::distributed::Vector<S2, MemorySpace::Host> &,
+        const bool) const;
+
+    // template void
+    //     MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<S1,
+    //     MemorySpace::Default>>::copy_to_mg( const
+    //     DoFHandler<deal_II_dimension> &,
+    //   MGLevelObject<LinearAlgebra::distributed::Vector<S1,
+    //   MemorySpace::Host>>
+    //   &,
+    //  const LinearAlgebra::distributed::Vector<S2, MemorySpace::Host> &,
+    //  const bool) const;
   }
 
 for (deal_II_dimension : DIMENSIONS; S1, S2 : REAL_SCALARS)
   {
-    template void
-    MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<S1>>::copy_to_mg(
-      const DoFHandler<deal_II_dimension> &,
-      MGLevelObject<LinearAlgebra::distributed::Vector<S1>> &,
-      const LinearAlgebra::distributed::Vector<S2> &) const;
-    template void
-    MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<S1>>::copy_from_mg(
-      const DoFHandler<deal_II_dimension> &,
-      LinearAlgebra::distributed::Vector<S2> &,
-      const MGLevelObject<LinearAlgebra::distributed::Vector<S1>> &) const;
-    template void
-    MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<S1>>::
+    template void MGLevelGlobalTransfer<
+      LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>>::
+      copy_to_mg(
+        const DoFHandler<deal_II_dimension> &,
+        MGLevelObject<LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>>
+          &,
+        const LinearAlgebra::distributed::Vector<S2, MemorySpace::Host> &)
+        const;
+    template void MGLevelGlobalTransfer<
+      LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>>::
+      copy_from_mg(
+        const DoFHandler<deal_II_dimension> &,
+        LinearAlgebra::distributed::Vector<S2, MemorySpace::Host> &,
+        const MGLevelObject<
+          LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>> &) const;
+    template void MGLevelGlobalTransfer<
+      LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>>::
       copy_from_mg_add(
         const DoFHandler<deal_II_dimension> &,
-        LinearAlgebra::distributed::Vector<S2> &,
-        const MGLevelObject<LinearAlgebra::distributed::Vector<S1>> &) const;
+        LinearAlgebra::distributed::Vector<S2, MemorySpace::Host> &,
+        const MGLevelObject<
+          LinearAlgebra::distributed::Vector<S1, MemorySpace::Host>> &) const;
   }
 
 for (deal_II_dimension : DIMENSIONS)
index 058ac9fc0d48dbd59b2a635a6b7d6b7799095576..f4e40c0fdfbc9b58a26026c43e8afd09906d9289 100644 (file)
@@ -16,7 +16,7 @@
 for (S1 : REAL_SCALARS)
   {
     template class MGTwoLevelTransferBase<
-      LinearAlgebra::distributed::Vector<S1>>;
+      LinearAlgebra::distributed::Vector<S1, ::dealii::MemorySpace::Host>>;
   }
 
 for (deal_II_dimension : DIMENSIONS; S1 : REAL_SCALARS)
@@ -26,12 +26,16 @@ for (deal_II_dimension : DIMENSIONS; S1 : REAL_SCALARS)
     template class MGTwoLevelTransferNonNested<
       deal_II_dimension,
       LinearAlgebra::distributed::Vector<S1>>;
+
     template class MGTransferBlockMatrixFreeBase<
       deal_II_dimension,
       S1,
-      MGTransferMF<deal_II_dimension, S1>>;
-    template class MGTransferMF<deal_II_dimension, S1>;
+      MGTransferMF<deal_II_dimension, S1, ::dealii::MemorySpace::Host>>;
     template class MGTransferBlockMF<deal_II_dimension, S1>;
+
+    template class MGTransferMF<deal_II_dimension,
+                                S1,
+                                ::dealii::MemorySpace::Host>;
   }
 
 for (deal_II_dimension : DIMENSIONS; deal_II_space_dimension : SPACE_DIMENSIONS)
index 9881e1bc9ceb13d17460d015686bfb6cb9f9a8a7..330426dff3304905a8fb22a046797da892e400fc 100644 (file)
 
 
 
-for (V1 : VECTORS_WITH_MATRIX)
+for (V1 : VECTORS_WITHOUT_LAVEC)
   {
     template class MGTransferPrebuilt<V1>;
   }
 
-for (deal_II_dimension : DIMENSIONS; V1 : VECTORS_WITH_MATRIX)
+for (deal_II_dimension : DIMENSIONS; V1 : VECTORS_WITHOUT_LAVEC)
   {
     template void MGTransferPrebuilt<V1>::build<deal_II_dimension>(
       const DoFHandler<deal_II_dimension> &mg_dof);

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.