using kokkos_space = ::Kokkos::DefaultExecutionSpace::memory_space;
};
+#ifdef DEAL_II_WITH_CUDA
/**
* Structure describing CUDA memory space.
*/
- // FIXME Only enable if CUDA is enabled in deal.II.
using CUDA = Default;
-
+#endif
} // namespace MemorySpace
DEAL_II_NAMESPACE_CLOSE
* template is selected if number is not a complex data type, this
* function simply returns the given number.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
static constexpr DEAL_II_HOST_DEVICE const number &
conjugate(const number &x);
* general template is chosen for types not equal to std::complex, this
* function simply returns the square of the given number.
*
- * @note If the template type can be used in CUDA device code, the same holds true
+ * @note If the template type can be used in device code, the same holds true
* for this function.
*/
static constexpr DEAL_II_HOST_DEVICE real_type
#include <deal.II/base/config.h>
-#include <deal.II/base/cuda_size.h>
#include <deal.II/base/mpi_tags.h>
#include <deal.II/base/partitioner.h>
-#include <deal.II/lac/cuda_kernels.templates.h>
#include <deal.II/lac/la_parallel_vector.h>
#include <limits>
* Standard constructor. Creates an object that corresponds to the origin,
* i.e., all coordinates are set to zero.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE
Point();
* <tt>dim!=1</tt> as it would leave some components of the point
* coordinates uninitialized.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
explicit DEAL_II_HOST_DEVICE
Point(const Number x);
* coordinates uninitialized (if dim>2) or would not use some arguments (if
* dim<2).
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE
Point(const Number x, const Number y);
* point coordinates uninitialized (if dim>3) or would not use some
* arguments (if dim<3).
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE
Point(const Number x, const Number y, const Number z);
* that is zero in all coordinates except for a single 1 in the <tt>i</tt>th
* coordinate.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
static DEAL_II_HOST_DEVICE Point<dim, Number>
unit_vector(const unsigned int i);
/**
* Read access to the <tt>index</tt>th coordinate.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE Number
operator()(const unsigned int index) const;
/**
* Read and write access to the <tt>index</tt>th coordinate.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE Number &
operator()(const unsigned int index);
/**
* Add an offset given as Tensor<1,dim,Number> to a point.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE Point<dim, Number>
operator+(const Tensor<1, dim, Number> &) const;
* origin) and, consequently, the result is returned as a Tensor@<1,dim@>
* rather than as a Point@<dim@>.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE Tensor<1, dim, Number>
operator-(const Point<dim, Number> &) const;
* documentation of this class, the result is then naturally returned as a
* Point@<dim@> object rather than as a Tensor@<1,dim@>.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE Point<dim, Number>
operator-(const Tensor<1, dim, Number> &) const;
/**
* The opposite vector.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE Point<dim, Number>
operator-() const;
/**
* Multiply the current point by a factor.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso EnableIfScalar
*/
/**
* Divide the current point by a factor.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
DEAL_II_HOST_DEVICE Point<
/**
* Return the scalar product of the vectors representing two points.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE Number
operator*(const Tensor<1, dim, Number> &p) const;
* Tensor<rank,dim,Number>::norm_square() which returns the square of the
* Frobenius norm.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE typename numbers::NumberTraits<Number>::real_type
square() const;
* <tt>p</tt>, i.e. the $l_2$ norm of the difference between the
* vectors representing the two points.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE typename numbers::NumberTraits<Number>::real_type
distance(const Point<dim, Number> &p) const;
* Return the squared Euclidean distance of <tt>this</tt> point to the point
* <tt>p</tt>.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE typename numbers::NumberTraits<Number>::real_type
distance_square(const Point<dim, Number> &p) const;
/**
* Global operator scaling a point vector by a scalar.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relates Point
*/
/**
* Constructor. Set to zero.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE
Tensor();
* obviously requires that the @p OtherNumber type is convertible to @p
* Number.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE
/**
* Constructor, where the data is copied from a C-style array.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE
* This is the non-const conversion operator that returns a writable
* reference.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE
operator Number &();
*
* This is the const conversion operator that returns a read-only reference.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE operator const Number &() const;
* obviously requires that the @p OtherNumber type is convertible to @p
* Number.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
* copy constructor for Sacado::Rad::ADvar types automatically.
* See https://github.com/dealii/dealii/pull/5865.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE Tensor &
operator=(const Tensor<0, dim, Number> &rhs);
* This operator assigns a scalar to a tensor. This obviously requires
* that the @p OtherNumber type is convertible to @p Number.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Add another scalar.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Subtract another scalar.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Multiply the scalar with a <tt>factor</tt>.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Divide the scalar by <tt>factor</tt>.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Tensor with inverted entries.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE Tensor
operator-() const;
* Return the square of the Frobenius-norm of a tensor, i.e. the sum of the
* absolute squares of all entries.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE real_type
norm_square() const;
/**
* Constructor. Initialize all entries to zero.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE_ALWAYS_INLINE
Tensor();
/**
* A constructor where the data is copied from a C-style array.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE explicit Tensor(const array_type &initializer);
* either equal to @p Number, or is convertible to @p Number.
* Number.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename ElementType, typename MemorySpace>
constexpr DEAL_II_HOST_DEVICE explicit Tensor(
* obviously requires that the @p OtherNumber type is convertible to @p
* Number.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE
/**
* Read-Write access operator.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE value_type &
operator[](const unsigned int i);
/**
* Read-only access operator.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE const value_type &
operator[](const unsigned int i) const;
* This obviously requires that the @p OtherNumber type is convertible to @p
* Number.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Add another tensor.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Subtract another tensor.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
* Scale the tensor by <tt>factor</tt>, i.e. multiply all components by
* <tt>factor</tt>.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Scale the vector by <tt>1/factor</tt>.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename OtherNumber>
constexpr DEAL_II_HOST_DEVICE Tensor &
/**
* Unary minus operator. Negate all entries of a tensor.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE Tensor
operator-() const;
* the absolute squares of all entries. For the present case of rank-1
* tensors, this equals the usual <tt>l<sub>2</sub></tt> norm of the vector.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
DEAL_II_HOST_DEVICE
typename numbers::NumberTraits<Number>::real_type
* Return the square of the Frobenius-norm of a tensor, i.e. the sum of the
* absolute squares of all entries.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
constexpr DEAL_II_HOST_DEVICE
typename numbers::NumberTraits<Number>::real_type
* This constructor is for internal use. It provides a way
* to create constexpr constructors for Tensor<rank, dim, Number>
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*/
template <typename ArrayLike, std::size_t... Indices>
constexpr DEAL_II_HOST_DEVICE
* This function unwraps the underlying @p Number stored in the Tensor and
* multiplies @p object with it.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
* This function unwraps the underlying @p Number stored in the Tensor and
* multiplies @p object with it.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
* OtherNumber that are stored within the Tensor and multiplies them. It
* returns an unwrapped number of product type.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
/**
* Division of a tensor of rank 0 by a scalar number.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
/**
* Add two tensors of rank 0.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
/**
* Subtract two tensors of rank 0.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
* number, a complex floating point number, etc.) is allowed, see the
* documentation of EnableIfScalar for details.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
* number, a complex floating point number, etc.) is allowed, see the
* documentation of EnableIfScalar for details.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
* discussion on operator*() above for more information about template
* arguments and the return type.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
*
* @tparam rank The rank of both tensors.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
*
* @tparam rank The rank of both tensors.
*
- * @note This function can also be used in CUDA device code.
+ * @note This function can also be used in device code.
*
* @relatesalso Tensor
*/
#include <deal.II/base/config.h>
-#include <deal.II/base/cuda_size.h>
#include <deal.II/base/memory_consumption.h>
#include <deal.II/base/mpi_compute_index_owner_internal.h>
#include <deal.II/base/table.h>
* fail in some circumstances. Therefore, it is strongly recommended to
* not rely on this class to automatically detect the unsupported case.
*
- * <h4>CUDA support</h4>
+ * <h4>GPU support</h4>
*
- * This vector class supports two different memory spaces: Host and CUDA. By
- * default, the memory space is Host and all the data are allocated on the
- * CPU. When the memory space is CUDA, all the data is allocated on the GPU.
- * The operations on the vector are performed on the chosen memory space. *
- * From the host, there are two methods to access the elements of the Vector
- * when using the CUDA memory space:
+ * This vector class supports two different memory spaces: Host and Default.
+ * By default, the memory space is Host and all the data is allocated on the
+ * CPU. When the memory space is Default, all the data is allocated on
+ * Kokkos' default memory space. That means that if Kokkos was configured
+ * with a GPU backend, the data is allocated on a GPU. The operations on the
+ * vector are performed on the chosen memory space. From the host, there are
+ * two methods to access the elements of the Vector when using the Default
+ * memory space:
* <ul>
* <li> use get_values():
* @code
- * Vector<double, MemorySpace::CUDA> vector(local_range, comm);
+ * Vector<double, MemorySpace::Default> vector(local_range, comm);
* double* vector_dev = vector.get_values();
- * std::vector<double> vector_host(local_range.n_elements(), 1.);
- * Utilities::CUDA::copy_to_dev(vector_host, vector_dev);
+ * const int n_local_elements = local_range.n_elements();
+ * std::vector<double> vector_host(n_local_elements, 1.);
+ * Kokkos::deep_copy(Kokkos::View<double, Kokkos::HostSpace>(
+ * vector_host.data(), n_local_elements),
+ * Kokkos::View<double,
+ * MemorySpace::Default::kokkos_space>( vector_dev, n_local_elements));
* @endcode
* <li> use import():
* @code
- * Vector<double, MemorySpace::CUDA> vector(local_range, comm);
+ * Vector<double, MemorySpace::Default> vector(local_range, comm);
* ReadWriteVector<double> rw_vector(local_range);
* for (auto & val : rw_vector)
* val = 1.;
* necessary. Since an MPI communication may be performed, import needs to
* be called on all the processors.
*
- * @note By default, all the ranks will try to access the device 0. This is
- * fine is if you have one rank per node and one gpu per node. If you
- * have multiple GPUs on one node, we need each process to access a
- * different GPU. If each node has the same number of GPUs, this can be done
- * as follows:
- * <code> int n_devices = 0; cudaGetDeviceCount(&n_devices); int
- * device_id = my_rank % n_devices;
- * cudaSetDevice(device_id);
- * </code>
+ * @note By default, the GPU device id is chosen in a round-robin fashion
+ * according to the local MPI rank id. To choose a different device, Kokkos
+ * has to be initialized explicitly providing the respective devide id
+ * explicitly.
*
* <h4>MPI-3 shared-memory support</h4>
*
* MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL,
* &comm_sm);
* @endcode
- *
- * @see CUDAWrappers
*/
template <typename Number, typename MemorySpace = MemorySpace::Host>
class Vector : public ::dealii::LinearAlgebra::VectorSpaceVector<Number>,
static_assert(
std::is_same<MemorySpace, ::dealii::MemorySpace::Host>::value ||
- std::is_same<MemorySpace, ::dealii::MemorySpace::CUDA>::value,
- "MemorySpace should be Host or CUDA");
+ std::is_same<MemorySpace, ::dealii::MemorySpace::Default>::value,
+ "MemorySpace should be Host or Default");
/**
* @name 1: Basic Object-handling
*
* Must follow a call to the @p compress_start function.
*
- * When the MemorySpace is CUDA and MPI is not CUDA-aware, data changed on
- * the device after the call to compress_start will be lost.
+ * When the MemorySpace is Default and MPI is not GPU-aware, data changed
+ * on the device after the call to compress_start will be lost.
*/
void
compress_finish(VectorOperation::values operation);
* VectorOperation::values @p operation is used to decide if the elements
* in @p V should be added to the current vector or replace the current
* elements. The main purpose of this function is to get data from one
- * memory space, e.g. CUDA, to the other, e.g. the Host.
+ * memory space, e.g. Default, to the other, e.g. the Host.
*
* @note The partitioners of the two distributed vectors need to be the
* same as no MPI communication is performed.
* communication pattern is used multiple times. This can be used to
* improve performance.
*
- * @note If the MemorySpace is CUDA, the data in the ReadWriteVector will
+ * @note If the MemorySpace is Default, the data in the ReadWriteVector will
* be moved to the device.
*/
virtual void
*
* It holds that end() - begin() == locally_owned_size().
*
- * @note For the CUDA memory space, the iterator points to memory on the
- * device.
+ * @note For the Default memory space, the iterator might point to memory
+ * on the device.
*/
iterator
begin();
* Return constant iterator to the start of the locally owned elements
* of the vector.
*
- * @note For the CUDA memory space, the iterator points to memory on the
- * device.
+ * @note For the Default memory space, the iterator might point to memory
+ * on the device.
*/
const_iterator
begin() const;
* Return an iterator pointing to the element past the end of the array
* of locally owned entries.
*
- * @note For the CUDA memory space, the iterator points to memory on the
- * device.
+ * @note For the Default memory space, the iterator might point to memory
+ * on the device.
*/
iterator
end();
* Return a constant iterator pointing to the element past the end of
* the array of the locally owned entries.
*
- * @note For the CUDA memory space, the iterator points to memory on the
- * device.
+ * @note For the Default memory space, the iterator might point to memory
+ * on the device.
*/
const_iterator
end() const;
/**
* Return the pointer to the underlying raw array.
*
- * @note For the CUDA memory space, the pointer points to memory on the
- * device.
+ * @note For the Default memory space, the pointer might point to memory
+ * on the device.
*/
Number *
get_values() const;
*
* @pre The sizes of the @p indices and @p values arrays must be identical.
*
- * @note This function is not implemented for CUDA memory space.
+ * @note This function is not implemented for Default memory space.
*/
template <typename OtherNumber>
void
*/
DeclException0(ExcVectorTypeNotCompatible);
- /**
- * Attempt to perform an operation not implemented on the device.
- *
- * @ingroup Exceptions
- */
- DeclException0(ExcNotAllowedForCuda);
-
/**
* Exception
*/
#include <deal.II/base/config.h>
-#include <deal.II/base/cuda.h>
-#include <deal.II/base/cuda_size.h>
#include <deal.II/base/mpi.h>
#include <deal.II/lac/exceptions.h>
template <typename Number>
void
set_initial_guess(
- ::dealii::LinearAlgebra::distributed::Vector<Number, MemorySpace::CUDA>
+ ::dealii::LinearAlgebra::distributed::Vector<Number, MemorySpace::Default>
&vector)
{
// Choose a high-frequency mode consisting of numbers between 0 and 1
false) ||
(std::is_same<VectorType,
LinearAlgebra::distributed::
- Vector<NumberType, MemorySpace::CUDA>>::value ==
+ Vector<NumberType, MemorySpace::Default>>::value ==
false))))
temp_vector2.reinit(src, true);
else
* Tpetra uses Kokkos for thread-parallelism and chooses the execution and
* memory space automatically depending on Kokkos configuration. The
* priority is ranked from highest to lowest:
- * - Kokkos::Cuda
- * - Kokkos::OpenMP
- * - Kokkos::Threads
+ * - GPU backend
+ * - host parallel backend
* - Kokkos::Serial
*
- * In case Kokkos was configured with CUDA support, this class stores the
- * values in unified virtual memory space and performs its action on the
- * GPU. In particular, there is no need for manually synchronizing memory
- * between host and device.
+ * In case Kokkos was configured with GPU support, this class performs its
+ * actions on the GPU. In particular, there is no need for manually
+ * synchronizing memory between host and device.
*
* @ingroup TrilinosWrappers
* @ingroup Vectors
#include <deal.II/base/types.h>
#include <deal.II/base/vectorization.h>
-#include <deal.II/lac/cuda_kernels.h>
-#include <deal.II/lac/cuda_kernels.templates.h>
#include <deal.II/lac/vector_operation.h>
#include <cstdio>
projects / dealii.git / commitdiff